Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4964839e authored by Ming-Shin Lu's avatar Ming-Shin Lu
Browse files

Better IME transition while switching app with recents (4/N) 

This CL introduces TaskSnapshotController#snapshotImeFromAttachedTask to
attach IME screenshot when performing closing transition.

This can improve app transition without jank or flickering while the IME
insets control transits from closing task to the next task, we keep the
IME surface visibility by placing the IME screenshot with calling
DC#showImeScreenshot to be a part of task while animating
app transition, and then remove it with DC#removeImeScreenshotIfNeeded
when the transition animation finished or no longer used gracefully.
 
Bug: 166736352
Bug: 153145997
Bug: 172815805
Bug: 174222049
Bug: 167604724

Test: manual as below steps:
   1) Launch an app with focusing an editor (e.g. Dialer)
   2) Swipe down status bar and tap Settings icon.
   3) Verify that when doing task transition animation, app activity
      with IME keeps visible.
Test: manual as below steps:
   1) With 2-button or 3-button gesture, launch an app with focusing an
      editor to show soft-keyboard.
   2) Pressing home key
   3) Verify the IME screenshot keeps visible and animates with closing
      transition smoothly.

Change-Id: I6bef36c779a28777408576f57e5d1c67d5d48e3f
parent 4b305308

services/core/java/com/android/server/wm/DisplayContent.java

+127 −0
Original line number Diff line number Diff line
@@ -113,6 +113,8 @@ import static com.android.server.wm.DisplayContentProto.OPENING_APPS; 
import static com.android.server.wm.DisplayContentProto.RESUMED_ACTIVITY;

import static com.android.server.wm.DisplayContentProto.ROOT_DISPLAY_AREA;

import static com.android.server.wm.DisplayContentProto.SCREEN_ROTATION_ANIMATION;

import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_APP_TRANSITION;

import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_RECENTS;

import static com.android.server.wm.Task.ActivityState.RESUMED;

import static com.android.server.wm.WindowContainer.AnimationFlags.PARENTS;

import static com.android.server.wm.WindowContainer.AnimationFlags.TRANSITION;
@@ -163,6 +165,7 @@ import android.graphics.Rect; 
import android.graphics.RectF;

import android.graphics.Region;

import android.graphics.Region.Op;

import android.hardware.HardwareBuffer;

import android.hardware.display.DisplayManagerInternal;

import android.metrics.LogMaker;

import android.os.Binder;
@@ -606,6 +609,9 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp 
    /** If {@code true} hold off on modifying the animation layer of {@link #mImeLayeringTarget} */

    boolean mImeLayeringTargetWaitingAnim;



    /** The screenshot IME surface to place on the task while transitioning to the next task. */

    SurfaceControl mImeScreenshot;



    private final PointerEventDispatcher mPointerEventDispatcher;



    private final InsetsStateController mInsetsStateController;
@@ -3734,6 +3740,14 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp 
        if (target == mImeLayeringTarget && mImeLayeringTargetWaitingAnim == targetWaitingAnim) {

            return;

        }

        // Prepare the IME screenshot for the last IME target when its task is applying app

        // transition. This is for the better IME transition to keep IME visibility when

        // transitioning to the next task.

        if (mImeLayeringTarget != null && mImeLayeringTarget.isAnimating(PARENTS | TRANSITION,

                ANIMATION_TYPE_APP_TRANSITION | ANIMATION_TYPE_RECENTS)) {

            attachAndShowImeScreenshotOnTarget();

        }



        ProtoLog.i(WM_DEBUG_IME, "setInputMethodTarget %s", target);

        mImeLayeringTarget = target;

        mImeLayeringTargetWaitingAnim = targetWaitingAnim;
@@ -3768,6 +3782,109 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp 
        mImeControlTarget = target;

    }



    @VisibleForTesting

    void attachAndShowImeScreenshotOnTarget() {

        // No need to attach screenshot if the IME target not exists or screen is off.

        if (!isImeAttachedToApp() || !mWmService.mPolicy.isScreenOn()) {

            return;

        }



        final SurfaceControl.Transaction t = getPendingTransaction();

        // Prepare IME screenshot for the target if it allows to attach into.

        if (mInputMethodWindow != null && mInputMethodWindow.isVisible()) {

            final Task task = mImeLayeringTarget.getTask();

            // Re-new the IME screenshot when it does not exist or the size changed.

            final boolean renewImeSurface = mImeScreenshot == null

                    || mImeScreenshot.getWidth() != mInputMethodWindow.getFrame().width()

                    || mImeScreenshot.getHeight() != mInputMethodWindow.getFrame().height();

            if (task != null && !task.isHomeOrRecentsRootTask()) {

                SurfaceControl.ScreenshotHardwareBuffer imeBuffer = renewImeSurface

                        ? mWmService.mTaskSnapshotController.snapshotImeFromAttachedTask(task)

                        : null;

                if (imeBuffer != null) {

                    // Remove the last IME surface when the surface needs to renew.

                    removeImeSurfaceImmediately();

                    mImeScreenshot = createImeSurface(imeBuffer, t);

                }

            }

        }



        final boolean isValidSnapshot = mImeScreenshot != null && mImeScreenshot.isValid();

        // Showing the IME screenshot if the target has already in app transition stage.

        // Note that if the current IME insets is not showing, no need to show IME screenshot

        // to reflect the true IME insets visibility and the app task layout as possible.

        if (isValidSnapshot && getInsetsStateController().getImeSourceProvider().isImeShowing()) {

            if (DEBUG_INPUT_METHOD) {

                Slog.d(TAG, "show IME snapshot, ime target=" + mImeLayeringTarget);

            }

            t.show(mImeScreenshot);

        } else if (!isValidSnapshot) {

            removeImeSurfaceImmediately();

        }

    }



    @VisibleForTesting

    SurfaceControl createImeSurface(SurfaceControl.ScreenshotHardwareBuffer imeBuffer,

            Transaction t) {

        final HardwareBuffer buffer = imeBuffer.getHardwareBuffer();

        if (DEBUG_INPUT_METHOD) Slog.d(TAG, "create IME snapshot for "

                + mImeLayeringTarget + ", buff width=" + buffer.getWidth()

                + ", height=" + buffer.getHeight());

        final ActivityRecord activity = mImeLayeringTarget.mActivityRecord;

        final SurfaceControl imeSurface = mWmService.mSurfaceControlFactory.apply(null)

                .setName("IME-snapshot-surface")

                .setBufferSize(buffer.getWidth(), buffer.getHeight())

                .setFormat(buffer.getFormat())

                .setParent(activity.getSurfaceControl())

                .setCallsite("DisplayContent.attachAndShowImeScreenshotOnTarget")

                .build();

        // Make IME snapshot as trusted overlay

        InputMonitor.setTrustedOverlayInputInfo(imeSurface, t, getDisplayId(),

                "IME-snapshot-surface");

        Surface surface = mWmService.mSurfaceFactory.get();

        surface.copyFrom(imeSurface);

        surface.attachAndQueueBufferWithColorSpace(buffer, null);

        surface.release();

        t.setRelativeLayer(imeSurface, activity.getSurfaceControl(), 1);

        t.setPosition(imeSurface, mInputMethodWindow.getDisplayFrame().left,

                mInputMethodWindow.getDisplayFrame().top);

        return imeSurface;

    }



    /**

     * Shows the IME screenshot and attach to the IME target window.

     *

     * Used when the IME target window with IME visible is transitioning to the next target.

     * e.g. App transitioning or swiping this the task of the IME target window to recents app.

     */

    void showImeScreenshot() {

        attachAndShowImeScreenshotOnTarget();

    }



    /**

     * Removes the IME screenshot when necessary.

     *

     * Used when app transition animation finished or obsoleted screenshot surface like size

     * changed by rotation.

     */

    void removeImeScreenshotIfPossible() {

        if (mImeLayeringTarget == null

                || mImeLayeringTarget.mAttrs.type != TYPE_APPLICATION_STARTING

                && !mImeLayeringTarget.isAnimating(PARENTS | TRANSITION,

                ANIMATION_TYPE_APP_TRANSITION | ANIMATION_TYPE_RECENTS)) {

            removeImeSurfaceImmediately();

        }

    }



    /** Removes the IME screenshot immediately. */

    void removeImeSurfaceImmediately() {

        if (mImeScreenshot != null) {

            if (DEBUG_INPUT_METHOD) Slog.d(TAG, "remove IME snapshot");

            getSyncTransaction().remove(mImeScreenshot);

            mImeScreenshot = null;

        }

    }



    /**

     * The IME input target is the window which receives input from IME. It is also a candidate

     * which controls the visibility and animation of the input method window.
@@ -4037,6 +4154,16 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp 
        mWmService.mWindowPlacerLocked.performSurfacePlacement();

    }



    /**

     * Callbacks when the given type of {@link WindowContainer} animation finished running in the

     * hierarchy.

     */

    void onWindowAnimationFinished(int type) {

        if (type == ANIMATION_TYPE_APP_TRANSITION || type == ANIMATION_TYPE_RECENTS) {

            removeImeSurfaceImmediately();

        }

    }



    // TODO: Super unexpected long method that should be broken down...

    void applySurfaceChangesTransaction() {

        final WindowSurfacePlacer surfacePlacer = mWmService.mWindowPlacerLocked;

services/core/java/com/android/server/wm/TaskSnapshotController.java

+86 −31
Original line number Diff line number Diff line
@@ -35,6 +35,7 @@ import android.hardware.HardwareBuffer; 
import android.os.Environment;

import android.os.Handler;

import android.util.ArraySet;

import android.util.Pair;

import android.util.Slog;

import android.view.InsetsState;

import android.view.SurfaceControl;
@@ -275,39 +276,12 @@ class TaskSnapshotController { 
     */

    @VisibleForTesting

    boolean prepareTaskSnapshot(Task task, int pixelFormat, TaskSnapshot.Builder builder) {

        if (!mService.mPolicy.isScreenOn()) {

            if (DEBUG_SCREENSHOT) {

                Slog.i(TAG_WM, "Attempted to take screenshot while display was off.");

            }

            return false;

        }

        final ActivityRecord activity = findAppTokenForSnapshot(task);

        if (activity == null) {

            if (DEBUG_SCREENSHOT) {

                Slog.w(TAG_WM, "Failed to take screenshot. No visible windows for " + task);

            }

            return false;

        }

        if (activity.hasCommittedReparentToAnimationLeash()) {

            if (DEBUG_SCREENSHOT) {

                Slog.w(TAG_WM, "Failed to take screenshot. App is animating " + activity);

            }

            return false;

        }



        final WindowState mainWindow = activity.findMainWindow();

        if (mainWindow == null) {

            Slog.w(TAG_WM, "Failed to take screenshot. No main window for " + task);

        final Pair<ActivityRecord, WindowState> result = checkIfReadyToSnapshot(task);

        if (result == null) {

            return false;

        }

        if (activity.hasFixedRotationTransform()) {

            if (DEBUG_SCREENSHOT) {

                Slog.i(TAG_WM, "Skip taking screenshot. App has fixed rotation " + activity);

            }

            // The activity is in a temporal state that it has different rotation than the task.

            return false;

        }



        final ActivityRecord activity = result.first;

        final WindowState mainWindow = result.second;

        final Rect contentInsets = getSystemBarInsets(task.getBounds(),

                mainWindow.getInsetsStateWithVisibilityOverride());

        InsetUtils.addInsets(contentInsets, activity.getLetterboxInsets());
@@ -339,6 +313,50 @@ class TaskSnapshotController { 
        return true;

    }



    /**

     * Check if the state of the Task is appropriate to capture a snapshot, such like the task

     * snapshot or the associated IME surface snapshot.

     *

     * @param task the target task to capture the snapshot

     * @return Pair of (the top activity of the task, the main window of the task) if passed the

     * state checking. Returns {@code null} if the task state isn't ready to snapshot.

     */

    Pair<ActivityRecord, WindowState> checkIfReadyToSnapshot(Task task) {

        if (!mService.mPolicy.isScreenOn()) {

            if (DEBUG_SCREENSHOT) {

                Slog.i(TAG_WM, "Attempted to take screenshot while display was off.");

            }

            return null;

        }

        final ActivityRecord activity = findAppTokenForSnapshot(task);

        if (activity == null) {

            if (DEBUG_SCREENSHOT) {

                Slog.w(TAG_WM, "Failed to take screenshot. No visible windows for " + task);

            }

            return null;

        }

        if (activity.hasCommittedReparentToAnimationLeash()) {

            if (DEBUG_SCREENSHOT) {

                Slog.w(TAG_WM, "Failed to take screenshot. App is animating " + activity);

            }

            return null;

        }



        final WindowState mainWindow = activity.findMainWindow();

        if (mainWindow == null) {

            Slog.w(TAG_WM, "Failed to take screenshot. No main window for " + task);

            return null;

        }

        if (activity.hasFixedRotationTransform()) {

            if (DEBUG_SCREENSHOT) {

                Slog.i(TAG_WM, "Skip taking screenshot. App has fixed rotation " + activity);

            }

            // The activity is in a temporal state that it has different rotation than the task.

            return null;

        }

        return new Pair<>(activity, mainWindow);

    }



    @Nullable

    SurfaceControl.ScreenshotHardwareBuffer createTaskSnapshot(@NonNull Task task,

            TaskSnapshot.Builder builder) {
@@ -355,6 +373,43 @@ class TaskSnapshotController { 
        return createTaskSnapshot(task, scaleFraction, PixelFormat.RGBA_8888, null, builder);

    }



    @Nullable

    private SurfaceControl.ScreenshotHardwareBuffer createImeSnapshot(@NonNull Task task,

            int pixelFormat) {

        if (task.getSurfaceControl() == null) {

            if (DEBUG_SCREENSHOT) {

                Slog.w(TAG_WM, "Failed to take screenshot. No surface control for " + task);

            }

            return null;

        }

        final WindowState imeWindow = task.getDisplayContent().mInputMethodWindow;

        SurfaceControl.ScreenshotHardwareBuffer imeBuffer = null;

        if (imeWindow != null && imeWindow.isWinVisibleLw()) {

            final Rect bounds = imeWindow.getContainingFrame();

            bounds.offsetTo(0, 0);

            imeBuffer = SurfaceControl.captureLayersExcluding(imeWindow.getSurfaceControl(),

                    bounds, 1.0f, pixelFormat, null);

        }

        return imeBuffer;

    }



    /**

     * Create the snapshot of the IME surface on the task which used for placing on the closing

     * task to keep IME visibility while app transitioning.

     */

    @Nullable

    SurfaceControl.ScreenshotHardwareBuffer snapshotImeFromAttachedTask(@NonNull Task task) {

        // Check if the IME targets task ready to take the corresponding IME snapshot, if not,

        // means the task is not yet visible for some reasons and no need to snapshot IME surface.

        if (checkIfReadyToSnapshot(task) == null) {

            return null;

        }

        final int pixelFormat = mPersister.use16BitFormat()

                    ? PixelFormat.RGB_565

                    : PixelFormat.RGBA_8888;

        return createImeSnapshot(task, pixelFormat);

    }



    @Nullable

    SurfaceControl.ScreenshotHardwareBuffer createTaskSnapshot(@NonNull Task task,

            float scaleFraction, int pixelFormat, Point outTaskSize, TaskSnapshot.Builder builder) {

services/core/java/com/android/server/wm/WindowContainer.java

+7 −0
Original line number Diff line number Diff line
@@ -2682,6 +2682,10 @@ class WindowContainer<E extends WindowContainer> extends ConfigurationContainer< 
    protected void applyAnimationUnchecked(WindowManager.LayoutParams lp, boolean enter,

            @TransitionOldType int transit, boolean isVoiceInteraction,

            @Nullable ArrayList<WindowContainer> sources) {

        final Task task = asTask();

        if (task != null && !enter && !task.isHomeOrRecentsRootTask()) {

            mDisplayContent.showImeScreenshot();

        }

        final Pair<AnimationAdapter, AnimationAdapter> adapters = getAnimationAdapter(lp,

                transit, enter, isVoiceInteraction);

        AnimationAdapter adapter = adapters.first;
@@ -2826,6 +2830,9 @@ class WindowContainer<E extends WindowContainer> extends ConfigurationContainer< 
            mSurfaceAnimationSources.valueAt(i).onAnimationFinished(type, anim);

        }

        mSurfaceAnimationSources.clear();

        if (mDisplayContent != null) {

            mDisplayContent.onWindowAnimationFinished(type);

        }

    }



    /**

services/core/java/com/android/server/wm/WindowState.java

+2 −0
Original line number Diff line number Diff line
@@ -2166,6 +2166,8 @@ class WindowState extends WindowContainer<WindowState> implements WindowManagerP 


        final DisplayContent dc = getDisplayContent();

        if (isImeLayeringTarget()) {

            // Remove the IME screenshot surface if the layering target is not animating.

            dc.removeImeScreenshotIfPossible();

            // Make sure to set mImeLayeringTarget as null when the removed window is the

            // IME target, in case computeImeTarget may use the outdated target.

            dc.setImeLayeringTarget(null);

services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java

+62 −0
Original line number Diff line number Diff line
@@ -65,7 +65,11 @@ import static com.android.dx.mockito.inline.extended.ExtendedMockito.times; 
import static com.android.dx.mockito.inline.extended.ExtendedMockito.verify;

import static com.android.server.wm.DisplayContent.IME_TARGET_INPUT;

import static com.android.server.wm.DisplayContent.IME_TARGET_LAYERING;

import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_APP_TRANSITION;

import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_FIXED_TRANSFORM;

import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_RECENTS;

import static com.android.server.wm.WindowContainer.AnimationFlags.PARENTS;

import static com.android.server.wm.WindowContainer.AnimationFlags.TRANSITION;

import static com.android.server.wm.WindowContainer.POSITION_TOP;

import static com.android.server.wm.WindowManagerService.UPDATE_FOCUS_NORMAL;
@@ -81,6 +85,7 @@ import static org.junit.Assert.assertThat; 
import static org.junit.Assert.assertTrue;

import static org.mockito.ArgumentMatchers.anyInt;

import static org.mockito.ArgumentMatchers.eq;

import static org.mockito.Mockito.atLeast;

import static org.mockito.Mockito.atLeastOnce;

import static org.mockito.Mockito.clearInvocations;

import static org.mockito.Mockito.doAnswer;
@@ -1732,6 +1737,63 @@ public class DisplayContentTests extends WindowTestsBase { 
        verify(child1, never()).needsRelativeLayeringToIme();

    }



    @UseTestDisplay(addWindows = {W_INPUT_METHOD}, addAllCommonWindows = true)

    @Test

    public void testAttachAndShowImeScreenshotOnTarget() {

        // Preparation: Simulate screen state is on.

        spyOn(mWm.mPolicy);

        doReturn(true).when(mWm.mPolicy).isScreenOn();



        // Preparation: Simulate snapshot IME surface.

        spyOn(mWm.mTaskSnapshotController);

        doReturn(mock(SurfaceControl.ScreenshotHardwareBuffer.class)).when(

                mWm.mTaskSnapshotController).snapshotImeFromAttachedTask(any());

        final SurfaceControl imeSurface = mock(SurfaceControl.class);

        spyOn(imeSurface);

        doReturn(true).when(imeSurface).isValid();

        doReturn(imeSurface).when(mDisplayContent).createImeSurface(any(), any());



        // Preparation: Simulate snapshot Task.

        ActivityRecord act1 = createActivityRecord(mDisplayContent);

        final WindowState appWin1 = createWindow(null, TYPE_BASE_APPLICATION, act1, "appWin1");

        spyOn(appWin1);

        spyOn(appWin1.mWinAnimator);

        appWin1.setHasSurface(true);

        assertTrue(appWin1.canBeImeTarget());

        doReturn(true).when(appWin1.mWinAnimator).getShown();

        doReturn(true).when(appWin1.mActivityRecord).isSurfaceShowing();

        appWin1.mWinAnimator.mLastAlpha = 1f;



        // Test step 1: appWin1 is the current IME target and soft-keyboard is visible.

        mDisplayContent.computeImeTarget(true);

        assertEquals(appWin1, mDisplayContent.getImeTarget(IME_TARGET_LAYERING));

        spyOn(mDisplayContent.mInputMethodWindow);

        doReturn(true).when(mDisplayContent.mInputMethodWindow).isVisible();

        mDisplayContent.getInsetsStateController().getImeSourceProvider().setImeShowing(true);



        // Test step 2: Simulate launching appWin2 and appWin1 is in app transition.

        ActivityRecord act2 = createActivityRecord(mDisplayContent);

        final WindowState appWin2 = createWindow(null, TYPE_BASE_APPLICATION, act2, "appWin2");

        appWin2.setHasSurface(true);

        assertTrue(appWin2.canBeImeTarget());

        doReturn(true).when(appWin1).isAnimating(PARENTS | TRANSITION,

                ANIMATION_TYPE_APP_TRANSITION | ANIMATION_TYPE_RECENTS);



        // Test step 3: Verify appWin2 will be the next IME target and the IME snapshot surface will

        // be shown at this time.

        final Transaction t = mDisplayContent.getPendingTransaction();

        spyOn(t);

        mDisplayContent.setImeInputTarget(appWin2);

        mDisplayContent.computeImeTarget(true);

        assertEquals(appWin2, mDisplayContent.getImeTarget(IME_TARGET_LAYERING));

        assertTrue(mDisplayContent.isImeAttachedToApp());



        verify(mDisplayContent, atLeast(1)).attachAndShowImeScreenshotOnTarget();

        verify(mWm.mTaskSnapshotController).snapshotImeFromAttachedTask(appWin1.getTask());

        assertNotNull(mDisplayContent.mImeScreenshot);

        verify(t).show(mDisplayContent.mImeScreenshot);

    }



    private boolean isOptionsPanelAtRight(int displayId) {

        return (mWm.getPreferredOptionsPanelGravity(displayId) & Gravity.RIGHT) == Gravity.RIGHT;

    }