Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f4d569ec authored by Maryam Dehaini's avatar Maryam Dehaini
Browse files

Utilize the AssistContent#getWebUri when available 

Gets the assist content and, if the captured link is unavailable and
assist content contains a web uri, sets that as the browser link.

Bug: 349695493
Test: open in browser from docs
Flag: com.android.window.flags.enable_desktop_windowing_app_to_web

Change-Id: Id2cb79c95e1f6dff77ce7b66184b56e4264d85c5
parent 2faf7a96

libs/WindowManager/Shell/src/com/android/wm/shell/apptoweb/AssistContentRequester.kt

0 → 100644
+126 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2024 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package com.android.wm.shell.apptoweb



import android.app.ActivityTaskManager

import android.app.IActivityTaskManager

import android.app.IAssistDataReceiver

import android.app.assist.AssistContent

import android.content.Context

import android.graphics.Bitmap

import android.os.Bundle

import android.os.RemoteException

import android.util.Slog

import java.lang.ref.WeakReference

import java.util.Collections

import java.util.WeakHashMap

import java.util.concurrent.Executor



/**

 * Can be used to request the AssistContent from a provided task id, useful for getting the web uri

 * if provided from the task.

 */

class AssistContentRequester(

    context: Context,

    private val callBackExecutor: Executor,

    private val systemInteractionExecutor: Executor

) {

    interface Callback {

        // Called when the [AssistContent] of the requested task is available.

        fun onAssistContentAvailable(assistContent: AssistContent?)

    }



    private val activityTaskManager: IActivityTaskManager = ActivityTaskManager.getService()

    private val attributionTag: String? = context.attributionTag

    private val packageName: String = context.applicationContext.packageName



    // If system loses the callback, our internal cache of original callback will also get cleared.

    private val pendingCallbacks = Collections.synchronizedMap(WeakHashMap<Any, Callback>())



    /**

     * Request the [AssistContent] from the task with the provided id.

     *

     * @param taskId to query for the content.

     * @param callback to call when the content is available, called on the main thread.

     */

    fun requestAssistContent(taskId: Int, callback: Callback) {

        // ActivityTaskManager interaction here is synchronous, so call off the main thread.

        systemInteractionExecutor.execute {

            try {

                val success = activityTaskManager.requestAssistDataForTask(

                    AssistDataReceiver(callback, this),

                    taskId,

                    packageName,

                    attributionTag,

                    false /* fetchStructure */

                )

                if (!success) {

                    executeOnMainExecutor { callback.onAssistContentAvailable(null) }

                }

            } catch (e: RemoteException) {

                Slog.e(TAG, "Requesting assist content failed for task: $taskId", e)

            }

        }

    }



    private fun executeOnMainExecutor(callback: Runnable) {

        callBackExecutor.execute(callback)

    }



    private class AssistDataReceiver(

            callback: Callback,

            parent: AssistContentRequester

    ) : IAssistDataReceiver.Stub() {

        // The AssistDataReceiver binder callback object is passed to a system server, that may

        // keep hold of it for longer than the lifetime of the AssistContentRequester object,

        // potentially causing a memory leak. In the callback passed to the system server, only

        // keep a weak reference to the parent object and lookup its callback if it still exists.

        private val parentRef: WeakReference<AssistContentRequester>

        private val callbackKey = Any()



        init {

            parent.pendingCallbacks[callbackKey] = callback

            parentRef = WeakReference(parent)

        }



        override fun onHandleAssistData(data: Bundle?) {

            val content = data?.getParcelable(ASSIST_KEY_CONTENT, AssistContent::class.java)

            if (content == null) {

                Slog.d(TAG, "Received AssistData, but no AssistContent found")

                return

            }

            val requester = parentRef.get()

            if (requester != null) {

                val callback = requester.pendingCallbacks[callbackKey]

                if (callback != null) {

                    requester.executeOnMainExecutor { callback.onAssistContentAvailable(content) }

                } else {

                    Slog.d(TAG, "Callback received after calling UI was disposed of")

                }

            } else {

                Slog.d(TAG, "Callback received after Requester was collected")

            }

        }



        override fun onHandleAssistScreenshot(screenshot: Bitmap) {}

    }



    companion object {

        private const val TAG = "AssistContentRequester"

        private const val ASSIST_KEY_CONTENT = "content"

    }

}
 
 No newline at end of file

libs/WindowManager/Shell/src/com/android/wm/shell/dagger/WMShellModule.java

+12 −0
Original line number Diff line number Diff line
@@ -38,6 +38,7 @@ import com.android.wm.shell.ShellTaskOrganizer; 
import com.android.wm.shell.WindowManagerShellWrapper;

import com.android.wm.shell.activityembedding.ActivityEmbeddingController;

import com.android.wm.shell.apptoweb.AppToWebGenericLinksParser;

import com.android.wm.shell.apptoweb.AssistContentRequester;

import com.android.wm.shell.bubbles.BubbleController;

import com.android.wm.shell.bubbles.BubbleData;

import com.android.wm.shell.bubbles.BubbleDataRepository;
@@ -240,6 +241,7 @@ public abstract class WMShellModule { 
            RootTaskDisplayAreaOrganizer rootTaskDisplayAreaOrganizer,

            InteractionJankMonitor interactionJankMonitor,

            AppToWebGenericLinksParser genericLinksParser,

            AssistContentRequester assistContentRequester,

            MultiInstanceHelper multiInstanceHelper,

            Optional<DesktopTasksLimiter> desktopTasksLimiter,

            Optional<DesktopActivityOrientationChangeHandler> desktopActivityOrientationHandler) {
@@ -263,6 +265,7 @@ public abstract class WMShellModule { 
                    rootTaskDisplayAreaOrganizer,

                    interactionJankMonitor,

                    genericLinksParser,

                    assistContentRequester,

                    multiInstanceHelper,

                    desktopTasksLimiter,

                    desktopActivityOrientationHandler);
@@ -291,6 +294,15 @@ public abstract class WMShellModule { 
        return new AppToWebGenericLinksParser(context, mainExecutor);

    }



    @Provides

    static AssistContentRequester provideAssistContentRequester(

            Context context,

            @ShellMainThread ShellExecutor shellExecutor,

            @ShellBackgroundThread ShellExecutor bgExecutor

    ) {

        return new AssistContentRequester(context, shellExecutor, bgExecutor);

    }



    //

    // Freeform

    //

libs/WindowManager/Shell/src/com/android/wm/shell/windowdecor/DesktopModeWindowDecorViewModel.java

+8 −1
Original line number Diff line number Diff line
@@ -90,6 +90,7 @@ import com.android.wm.shell.R; 
import com.android.wm.shell.RootTaskDisplayAreaOrganizer;

import com.android.wm.shell.ShellTaskOrganizer;

import com.android.wm.shell.apptoweb.AppToWebGenericLinksParser;

import com.android.wm.shell.apptoweb.AssistContentRequester;

import com.android.wm.shell.common.DisplayChangeController;

import com.android.wm.shell.common.DisplayController;

import com.android.wm.shell.common.DisplayInsetsController;
@@ -182,6 +183,7 @@ public class DesktopModeWindowDecorViewModel implements WindowDecorViewModel { 
    private final Region mExclusionRegion = Region.obtain();

    private boolean mInImmersiveMode;

    private final String mSysUIPackageName;

    private final AssistContentRequester mAssistContentRequester;



    private final DisplayChangeController.OnDisplayChangingListener mOnDisplayChangingListener;

    private final ISystemGestureExclusionListener mGestureExclusionListener =
@@ -217,6 +219,7 @@ public class DesktopModeWindowDecorViewModel implements WindowDecorViewModel { 
            RootTaskDisplayAreaOrganizer rootTaskDisplayAreaOrganizer,

            InteractionJankMonitor interactionJankMonitor,

            AppToWebGenericLinksParser genericLinksParser,

            AssistContentRequester assistContentRequester,

            MultiInstanceHelper multiInstanceHelper,

            Optional<DesktopTasksLimiter> desktopTasksLimiter,

            Optional<DesktopActivityOrientationChangeHandler> activityOrientationChangeHandler
@@ -238,6 +241,7 @@ public class DesktopModeWindowDecorViewModel implements WindowDecorViewModel { 
                transitions,

                desktopTasksController,

                genericLinksParser,

                assistContentRequester,

                multiInstanceHelper,

                new DesktopModeWindowDecoration.Factory(),

                new InputMonitorFactory(),
@@ -267,6 +271,7 @@ public class DesktopModeWindowDecorViewModel implements WindowDecorViewModel { 
            Transitions transitions,

            Optional<DesktopTasksController> desktopTasksController,

            AppToWebGenericLinksParser genericLinksParser,

            AssistContentRequester assistContentRequester,

            MultiInstanceHelper multiInstanceHelper,

            DesktopModeWindowDecoration.Factory desktopModeWindowDecorFactory,

            InputMonitorFactory inputMonitorFactory,
@@ -304,6 +309,7 @@ public class DesktopModeWindowDecorViewModel implements WindowDecorViewModel { 
        mInteractionJankMonitor = interactionJankMonitor;

        mDesktopTasksLimiter = desktopTasksLimiter;

        mActivityOrientationChangeHandler = activityOrientationChangeHandler;

        mAssistContentRequester = assistContentRequester;

        mOnDisplayChangingListener = (displayId, fromRotation, toRotation, displayAreaInfo, t) -> {

            DesktopModeWindowDecoration decoration;

            RunningTaskInfo taskInfo;
@@ -626,7 +632,7 @@ public class DesktopModeWindowDecorViewModel implements WindowDecorViewModel { 
            } else if (id == R.id.caption_handle || id == R.id.open_menu_button) {

                if (!decoration.isHandleMenuActive()) {

                    moveTaskToFront(decoration.mTaskInfo);

                    decoration.createHandleMenu(mSplitScreenController);

                    decoration.createHandleMenu();

                }

            } else if (id == R.id.maximize_window) {

                // TODO(b/346441962): move click detection logic into the decor's
@@ -1270,6 +1276,7 @@ public class DesktopModeWindowDecorViewModel implements WindowDecorViewModel { 
                        mSyncQueue,

                        mRootTaskDisplayAreaOrganizer,

                        mGenericLinksParser,

                        mAssistContentRequester,

                        mMultiInstanceHelper);

        mWindowDecorByTaskId.put(taskInfo.taskId, windowDecoration);

libs/WindowManager/Shell/src/com/android/wm/shell/windowdecor/DesktopModeWindowDecoration.java

+34 −9
Original line number Diff line number Diff line
@@ -38,6 +38,7 @@ import android.annotation.Nullable; 
import android.annotation.SuppressLint;

import android.app.ActivityManager;

import android.app.WindowConfiguration.WindowingMode;

import android.app.assist.AssistContent;

import android.content.ComponentName;

import android.content.Context;

import android.content.pm.ActivityInfo;
@@ -76,6 +77,7 @@ import com.android.wm.shell.RootTaskDisplayAreaOrganizer; 
import com.android.wm.shell.ShellTaskOrganizer;

import com.android.wm.shell.apptoweb.AppToWebGenericLinksParser;

import com.android.wm.shell.apptoweb.AppToWebUtils;

import com.android.wm.shell.apptoweb.AssistContentRequester;

import com.android.wm.shell.common.DisplayController;

import com.android.wm.shell.common.DisplayLayout;

import com.android.wm.shell.common.MultiInstanceHelper;
@@ -151,6 +153,7 @@ public class DesktopModeWindowDecoration extends WindowDecoration<WindowDecorLin 
    private CharSequence mAppName;

    private CapturedLink mCapturedLink;

    private Uri mGenericLink;

    private Uri mWebUri;

    private Consumer<Uri> mOpenInBrowserClickListener;



    private ExclusionRegionListener mExclusionRegionListener;
@@ -159,6 +162,7 @@ public class DesktopModeWindowDecoration extends WindowDecoration<WindowDecorLin 
    private final MaximizeMenuFactory mMaximizeMenuFactory;

    private final HandleMenuFactory mHandleMenuFactory;

    private final AppToWebGenericLinksParser mGenericLinksParser;

    private final AssistContentRequester mAssistContentRequester;



    // Hover state for the maximize menu and button. The menu will remain open as long as either of

    // these is true. See {@link #onMaximizeHoverStateChanged()}.
@@ -185,16 +189,16 @@ public class DesktopModeWindowDecoration extends WindowDecoration<WindowDecorLin 
            SyncTransactionQueue syncQueue,

            RootTaskDisplayAreaOrganizer rootTaskDisplayAreaOrganizer,

            AppToWebGenericLinksParser genericLinksParser,

            AssistContentRequester assistContentRequester,

            MultiInstanceHelper multiInstanceHelper) {

        this (context, userContext, displayController, splitScreenController, taskOrganizer,

                taskInfo, taskSurface, handler, bgExecutor, choreographer, syncQueue,

                rootTaskDisplayAreaOrganizer, genericLinksParser, SurfaceControl.Builder::new,

                SurfaceControl.Transaction::new,  WindowContainerTransaction::new,

                SurfaceControl::new, new WindowManagerWrapper(

                rootTaskDisplayAreaOrganizer, genericLinksParser, assistContentRequester,

                SurfaceControl.Builder::new, SurfaceControl.Transaction::new,

                WindowContainerTransaction::new, SurfaceControl::new, new WindowManagerWrapper(

                        context.getSystemService(WindowManager.class)),

                new SurfaceControlViewHostFactory() {},

                DefaultMaximizeMenuFactory.INSTANCE, DefaultHandleMenuFactory.INSTANCE,

                multiInstanceHelper);

                new SurfaceControlViewHostFactory() {}, DefaultMaximizeMenuFactory.INSTANCE,

                DefaultHandleMenuFactory.INSTANCE, multiInstanceHelper);

    }



    DesktopModeWindowDecoration(
@@ -211,6 +215,7 @@ public class DesktopModeWindowDecoration extends WindowDecoration<WindowDecorLin 
            SyncTransactionQueue syncQueue,

            RootTaskDisplayAreaOrganizer rootTaskDisplayAreaOrganizer,

            AppToWebGenericLinksParser genericLinksParser,

            AssistContentRequester assistContentRequester,

            Supplier<SurfaceControl.Builder> surfaceControlBuilderSupplier,

            Supplier<SurfaceControl.Transaction> surfaceControlTransactionSupplier,

            Supplier<WindowContainerTransaction> windowContainerTransactionSupplier,
@@ -231,6 +236,7 @@ public class DesktopModeWindowDecoration extends WindowDecoration<WindowDecorLin 
        mSyncQueue = syncQueue;

        mRootTaskDisplayAreaOrganizer = rootTaskDisplayAreaOrganizer;

        mGenericLinksParser = genericLinksParser;

        mAssistContentRequester = assistContentRequester;

        mMaximizeMenuFactory = maximizeMenuFactory;

        mHandleMenuFactory = handleMenuFactory;

        mMultiInstanceHelper = multiInstanceHelper;
@@ -489,6 +495,8 @@ public class DesktopModeWindowDecoration extends WindowDecoration<WindowDecorLin 
        // Otherwise, return the generic link which is set to null if a generic link is unavailable.

        if (mCapturedLink != null && !mCapturedLink.mExpired) {

            return mCapturedLink.mUri;

        } else if (mWebUri != null) {

            return mWebUri;

        }

        return mGenericLink;

    }
@@ -994,18 +1002,32 @@ public class DesktopModeWindowDecoration extends WindowDecoration<WindowDecorLin 
    }



    /**

     * Create and display handle menu window.

     * Updates app info and creates and displays handle menu window.

     */

    void createHandleMenu() {

        // Requests assist content. When content is received, calls {@link #onAssistContentReceived}

        // which sets app info and creates the handle menu.

        mAssistContentRequester.requestAssistContent(

                mTaskInfo.taskId, this::onAssistContentReceived);

    }



    /**

     * Called when assist content is received. updates the saved links and creates the handle menu.

     */

    void createHandleMenu(SplitScreenController splitScreenController) {

    @VisibleForTesting

    void onAssistContentReceived(@Nullable AssistContent assistContent) {

        mWebUri = assistContent == null ? null : assistContent.getWebUri();

        loadAppInfoIfNeeded();

        updateGenericLink();



        // Create and display handle menu

        mHandleMenu = mHandleMenuFactory.create(

                this,

                mWindowManagerWrapper,

                mRelayoutParams.mLayoutResId,

                mAppIconBitmap,

                mAppName,

                splitScreenController,

                mSplitScreenController,

                DesktopModeStatus.canEnterDesktopMode(mContext),

                Flags.enableDesktopWindowingMultiInstanceFeatures()

                        && mMultiInstanceHelper
@@ -1018,6 +1040,7 @@ public class DesktopModeWindowDecoration extends WindowDecoration<WindowDecorLin 
        mWindowDecorViewHolder.onHandleMenuOpened();

        mHandleMenu.show(

                /* onToDesktopClickListener= */ () -> {

                    mOnToDesktopClickListener.accept(APP_HANDLE_MENU_BUTTON);

                    mOnToDesktopClickListener.accept(APP_HANDLE_MENU_BUTTON);

                    return Unit.INSTANCE;

                },
@@ -1340,6 +1363,7 @@ public class DesktopModeWindowDecoration extends WindowDecoration<WindowDecorLin 
                SyncTransactionQueue syncQueue,

                RootTaskDisplayAreaOrganizer rootTaskDisplayAreaOrganizer,

                AppToWebGenericLinksParser genericLinksParser,

                AssistContentRequester assistContentRequester,

                MultiInstanceHelper multiInstanceHelper) {

            return new DesktopModeWindowDecoration(

                    context,
@@ -1355,6 +1379,7 @@ public class DesktopModeWindowDecoration extends WindowDecoration<WindowDecorLin 
                    syncQueue,

                    rootTaskDisplayAreaOrganizer,

                    genericLinksParser,

                    assistContentRequester,

                    multiInstanceHelper);

        }

    }

libs/WindowManager/Shell/tests/unittest/src/com/android/wm/shell/windowdecor/DesktopModeWindowDecorViewModelTests.kt

+4 −1
Original line number Diff line number Diff line
@@ -73,6 +73,7 @@ import com.android.wm.shell.ShellTestCase 
import com.android.wm.shell.TestRunningTaskInfoBuilder

import com.android.wm.shell.TestShellExecutor

import com.android.wm.shell.apptoweb.AppToWebGenericLinksParser

import com.android.wm.shell.apptoweb.AssistContentRequester

import com.android.wm.shell.common.DisplayChangeController

import com.android.wm.shell.common.DisplayController

import com.android.wm.shell.common.DisplayInsetsController
@@ -165,6 +166,7 @@ class DesktopModeWindowDecorViewModelTests : ShellTestCase() { 
    @Mock private lateinit var mockInteractionJankMonitor: InteractionJankMonitor

    @Mock private lateinit var mockGenericLinksParser: AppToWebGenericLinksParser

    @Mock private lateinit var mockUserHandle: UserHandle

    @Mock private lateinit var mockAssistContentRequester: AssistContentRequester

    @Mock private lateinit var mockToast: Toast

    private val bgExecutor = TestShellExecutor()

    @Mock private lateinit var mockMultiInstanceHelper: MultiInstanceHelper
@@ -218,6 +220,7 @@ class DesktopModeWindowDecorViewModelTests : ShellTestCase() { 
                mockTransitions,

                Optional.of(mockDesktopTasksController),

                mockGenericLinksParser,

                mockAssistContentRequester,

                mockMultiInstanceHelper,

                mockDesktopModeWindowDecorFactory,

                mockInputMonitorFactory,
@@ -1131,7 +1134,7 @@ class DesktopModeWindowDecorViewModelTests : ShellTestCase() { 
        whenever(

            mockDesktopModeWindowDecorFactory.create(

                any(), any(), any(), any(), any(), eq(task), any(), any(), any(), any(), any(),

                any(), any(), any())

                any(), any(), any(), any())

        ).thenReturn(decoration)

        decoration.mTaskInfo = task

        whenever(decoration.isFocused).thenReturn(task.isFocused)