Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5351be9e authored by Ming-Shin Lu's avatar Ming-Shin Lu
Browse files

Refactoring IME snapshot with adopting shell-transition 

Extracts logic into the new ImeScreenshot class with using
TransitionController#{inTranstion, inRecentsTransition}
checks to attach/detach IME screenshot.

Bug: 201139553
Bug: 212570341
Test: atest DisplayContentTests#\
       testAttachAndShowImeScreenshotOnTarget
Test: manual as steps
   0) Settings -> Gestures -> enabling 3-buttons gesture mode
   1) Launch google search bar from the launcher to show IME
   2) Press home key and ensure IME screenshot is still visible
      when the google search activity is leaving w/w.o. enabling
      shell-transition.

Change-Id: I9228740cbf060a01fe985eb37c94c6abae6f79f8
parent 2773ac9c

services/core/java/com/android/server/wm/DisplayContent.java

+92 −59
Original line number Diff line number Diff line
@@ -633,9 +633,6 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp 
    @VisibleForTesting

    SurfaceControl mInputMethodSurfaceParent;



    /** The screenshot IME surface to place on the task while transitioning to the next task. */

    SurfaceControl mImeScreenshot;



    private final PointerEventDispatcher mPointerEventDispatcher;



    private final InsetsStateController mInsetsStateController;
@@ -4017,10 +4014,9 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp 
        // If the IME target is the input target, before it changes, prepare the IME screenshot

        // for the last IME target when its task is applying app transition. This is for the

        // better IME transition to keep IME visibility when transitioning to the next task.

        if (mImeLayeringTarget != null && mImeLayeringTarget.isAnimating(PARENTS | TRANSITION,

                ANIMATION_TYPE_APP_TRANSITION | ANIMATION_TYPE_RECENTS)

        if (mImeLayeringTarget != null && mImeLayeringTarget.inAppOrRecentsTransition()

                && mImeLayeringTarget == mImeInputTarget) {

            attachAndShowImeScreenshotOnTarget();

            showImeScreenshot();

        }



        ProtoLog.i(WM_DEBUG_IME, "setInputMethodTarget %s", target);
@@ -4068,71 +4064,108 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp 
        mImeControlTarget = target;

    }



    @VisibleForTesting

    void attachAndShowImeScreenshotOnTarget() {

        // No need to attach screenshot if the IME target not exists or screen is off.

        if (!shouldImeAttachedToApp() || !mWmService.mPolicy.isScreenOn()) {

            return;

    // ========== Begin of ImeScreenshot stuff ==========

    /** The screenshot IME surface to place on the task while transitioning to the next task. */

    ImeScreenshot mImeScreenshot;



    static final class ImeScreenshot {

        private WindowState mImeTarget;

        private SurfaceControl.Builder mSurfaceBuilder;

        private SurfaceControl mImeSurface;



        ImeScreenshot(SurfaceControl.Builder surfaceBuilder, @NonNull WindowState imeTarget) {

            mSurfaceBuilder = surfaceBuilder;

            mImeTarget = imeTarget;

        }



        final SurfaceControl.Transaction t = getPendingTransaction();

        private SurfaceControl createImeSurface(SurfaceControl.ScreenshotHardwareBuffer b,

                Transaction t) {

            final HardwareBuffer buffer = b.getHardwareBuffer();

            if (DEBUG_INPUT_METHOD) {

                Slog.d(TAG, "create IME snapshot for "

                        + mImeTarget + ", buff width=" + buffer.getWidth()

                        + ", height=" + buffer.getHeight());

            }

            final WindowState imeWindow = mImeTarget.getDisplayContent().mInputMethodWindow;

            final ActivityRecord activity = mImeTarget.mActivityRecord;

            final SurfaceControl imeSurface = mSurfaceBuilder

                    .setName("IME-snapshot-surface")

                    .setBLASTLayer()

                    .setFormat(buffer.getFormat())

                    .setParent(activity.getSurfaceControl())

                    .setCallsite("DisplayContent.attachAndShowImeScreenshotOnTarget")

                    .build();

            // Make IME snapshot as trusted overlay

            InputMonitor.setTrustedOverlayInputInfo(imeSurface, t, imeWindow.getDisplayId(),

                    "IME-snapshot-surface");

            t.setBuffer(imeSurface, buffer);

            t.setColorSpace(activity.mSurfaceControl, ColorSpace.get(ColorSpace.Named.SRGB));

            t.setLayer(imeSurface, 1);

            t.setPosition(imeSurface, imeWindow.getFrame().left - mImeTarget.getFrame().left,

                    imeWindow.getFrame().top - mImeTarget.getFrame().top);



            return imeSurface;

        }



        private void removeImeSurface(Transaction t) {

            if (mImeSurface != null) {

                if (DEBUG_INPUT_METHOD) Slog.d(TAG, "remove IME snapshot");

                t.remove(mImeSurface);

                mImeSurface = null;

            }

        }



        void attachAndShow(Transaction t) {

            final DisplayContent dc = mImeTarget.getDisplayContent();

            // Prepare IME screenshot for the target if it allows to attach into.

        if (mInputMethodWindow != null && mInputMethodWindow.isVisible()) {

            final Task task = mImeLayeringTarget.getTask();

            final Task task = mImeTarget.getTask();

            // Re-new the IME screenshot when it does not exist or the size changed.

            final boolean renewImeSurface = mImeScreenshot == null

                    || mImeScreenshot.getWidth() != mInputMethodWindow.getFrame().width()

                    || mImeScreenshot.getHeight() != mInputMethodWindow.getFrame().height();

            final boolean renewImeSurface = mImeSurface == null

                    || mImeSurface.getWidth() != dc.mInputMethodWindow.getFrame().width()

                    || mImeSurface.getHeight() != dc.mInputMethodWindow.getFrame().height();

            if (task != null && !task.isActivityTypeHomeOrRecents()) {

                SurfaceControl.ScreenshotHardwareBuffer imeBuffer = renewImeSurface

                        ? mWmService.mTaskSnapshotController.snapshotImeFromAttachedTask(task)

                        ? dc.mWmService.mTaskSnapshotController.snapshotImeFromAttachedTask(task)

                        : null;

                if (imeBuffer != null) {

                    // Remove the last IME surface when the surface needs to renew.

                    removeImeSurfaceImmediately();

                    mImeScreenshot = createImeSurface(imeBuffer, t);

                }

                    removeImeSurface(t);

                    mImeSurface = createImeSurface(imeBuffer, t);

                }

            }



        final boolean isValidSnapshot = mImeScreenshot != null && mImeScreenshot.isValid();

            final boolean isValidSnapshot = mImeSurface != null && mImeSurface.isValid();

            // Showing the IME screenshot if the target has already in app transition stage.

            // Note that if the current IME insets is not showing, no need to show IME screenshot

            // to reflect the true IME insets visibility and the app task layout as possible.

        if (isValidSnapshot && getInsetsStateController().getImeSourceProvider().isImeShowing()) {

            if (isValidSnapshot

                    && dc.getInsetsStateController().getImeSourceProvider().isImeShowing()) {

                if (DEBUG_INPUT_METHOD) {

                Slog.d(TAG, "show IME snapshot, ime target=" + mImeLayeringTarget);

                    Slog.d(TAG, "show IME snapshot, ime target=" + mImeTarget);

                }

            t.show(mImeScreenshot);

                t.show(mImeSurface);

            } else if (!isValidSnapshot) {

            removeImeSurfaceImmediately();

                removeImeSurface(t);

            }

        }



    @VisibleForTesting

    SurfaceControl createImeSurface(SurfaceControl.ScreenshotHardwareBuffer imeBuffer,

            Transaction t) {

        final HardwareBuffer buffer = imeBuffer.getHardwareBuffer();

        if (DEBUG_INPUT_METHOD) Slog.d(TAG, "create IME snapshot for "

                + mImeLayeringTarget + ", buff width=" + buffer.getWidth()

                + ", height=" + buffer.getHeight());

        final ActivityRecord activity = mImeLayeringTarget.mActivityRecord;

        final SurfaceControl imeSurface = mWmService.mSurfaceControlFactory.apply(null)

                .setName("IME-snapshot-surface")

                .setBLASTLayer()

                .setFormat(buffer.getFormat())

                .setParent(activity.getSurfaceControl())

                .setCallsite("DisplayContent.attachAndShowImeScreenshotOnTarget")

                .build();

        // Make IME snapshot as trusted overlay

        InputMonitor.setTrustedOverlayInputInfo(imeSurface, t, getDisplayId(),

                "IME-snapshot-surface");

        t.setBuffer(imeSurface, buffer);

        t.setColorSpace(mSurfaceControl, ColorSpace.get(ColorSpace.Named.SRGB));

        t.setLayer(imeSurface, 1);

        t.setPosition(imeSurface, mInputMethodWindow.getDisplayFrame().left,

                mInputMethodWindow.getDisplayFrame().top);

        return imeSurface;

        void detach(Transaction t) {

            removeImeSurface(t);

        }

    }



    private void attachAndShowImeScreenshotOnTarget() {

        // No need to attach screenshot if the IME target not exists or screen is off.

        if (!shouldImeAttachedToApp() || !mWmService.mPolicy.isScreenOn()) {

            return;

        }



        final SurfaceControl.Transaction t = getPendingTransaction();

        // Prepare IME screenshot for the target if it allows to attach into.

        if (mInputMethodWindow != null && mInputMethodWindow.isVisible()) {

            mImeScreenshot = new ImeScreenshot(

                    mWmService.mSurfaceControlFactory.apply(null), mImeLayeringTarget);

            mImeScreenshot.attachAndShow(t);

        }

    }



    /**
@@ -4154,8 +4187,7 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp 
    void removeImeScreenshotIfPossible() {

        if (mImeLayeringTarget == null

                || mImeLayeringTarget.mAttrs.type != TYPE_APPLICATION_STARTING

                && !mImeLayeringTarget.isAnimating(PARENTS | TRANSITION,

                ANIMATION_TYPE_APP_TRANSITION | ANIMATION_TYPE_RECENTS)) {

                && !mImeLayeringTarget.inAppOrRecentsTransition()) {

            removeImeSurfaceImmediately();

        }

    }
@@ -4164,10 +4196,11 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp 
    void removeImeSurfaceImmediately() {

        if (mImeScreenshot != null) {

            if (DEBUG_INPUT_METHOD) Slog.d(TAG, "remove IME snapshot");

            getSyncTransaction().remove(mImeScreenshot);

            mImeScreenshot.detach(getSyncTransaction());

            mImeScreenshot = null;

        }

    }

 // ========== End of ImeScreenshot stuff ==========



    /**

     * The IME input target is the window which receives input from IME. It is also a candidate

services/core/java/com/android/server/wm/Transition.java

+1 −0
Original line number Diff line number Diff line
@@ -574,6 +574,7 @@ class Transition extends Binder implements BLASTSyncEngine.TransactionReadyListe 
                            false /* disableImeIcon */);

                }

            }

            dc.removeImeSurfaceImmediately();

            dc.handleCompleteDeferredRemoval();

        }

    }

services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java

+8 −14
Original line number Diff line number Diff line
@@ -78,7 +78,6 @@ import static com.android.server.wm.ActivityTaskSupervisor.ON_TOP; 
import static com.android.server.wm.DisplayContent.IME_TARGET_LAYERING;

import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_APP_TRANSITION;

import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_FIXED_TRANSFORM;

import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_RECENTS;

import static com.android.server.wm.WindowContainer.AnimationFlags.PARENTS;

import static com.android.server.wm.WindowContainer.AnimationFlags.TRANSITION;

import static com.android.server.wm.WindowContainer.POSITION_TOP;
@@ -110,6 +109,7 @@ import android.graphics.Insets; 
import android.graphics.Point;

import android.graphics.Rect;

import android.graphics.Region;

import android.hardware.HardwareBuffer;

import android.hardware.display.VirtualDisplay;

import android.metrics.LogMaker;

import android.os.Binder;
@@ -1939,12 +1939,10 @@ public class DisplayContentTests extends WindowTestsBase { 


        // Preparation: Simulate snapshot IME surface.

        spyOn(mWm.mTaskSnapshotController);

        doReturn(mock(SurfaceControl.ScreenshotHardwareBuffer.class)).when(

                mWm.mTaskSnapshotController).snapshotImeFromAttachedTask(any());

        final SurfaceControl imeSurface = mock(SurfaceControl.class);

        spyOn(imeSurface);

        doReturn(true).when(imeSurface).isValid();

        doReturn(imeSurface).when(mDisplayContent).createImeSurface(any(), any());

        SurfaceControl.ScreenshotHardwareBuffer mockHwBuffer = mock(

                SurfaceControl.ScreenshotHardwareBuffer.class);

        doReturn(mock(HardwareBuffer.class)).when(mockHwBuffer).getHardwareBuffer();

        doReturn(mockHwBuffer).when(mWm.mTaskSnapshotController).snapshotImeFromAttachedTask(any());



        // Preparation: Simulate snapshot Task.

        ActivityRecord act1 = createActivityRecord(mDisplayContent);
@@ -1970,21 +1968,17 @@ public class DisplayContentTests extends WindowTestsBase { 
        final WindowState appWin2 = createWindow(null, TYPE_BASE_APPLICATION, act2, "appWin2");

        appWin2.setHasSurface(true);

        assertTrue(appWin2.canBeImeTarget());

        doReturn(true).when(appWin1).isAnimating(PARENTS | TRANSITION,

                ANIMATION_TYPE_APP_TRANSITION | ANIMATION_TYPE_RECENTS);

        doReturn(true).when(appWin1).inAppOrRecentsTransition();



        // Test step 3: Verify appWin2 will be the next IME target and the IME snapshot surface will

        // be shown at this time.

        final Transaction t = mDisplayContent.getPendingTransaction();

        spyOn(t);

        // be attached and shown on the display at this time.

        mDisplayContent.computeImeTarget(true);

        assertEquals(appWin2, mDisplayContent.getImeTarget(IME_TARGET_LAYERING));

        assertTrue(mDisplayContent.shouldImeAttachedToApp());



        verify(mDisplayContent, atLeast(1)).attachAndShowImeScreenshotOnTarget();

        verify(mDisplayContent, atLeast(1)).showImeScreenshot();

        verify(mWm.mTaskSnapshotController).snapshotImeFromAttachedTask(appWin1.getTask());

        assertNotNull(mDisplayContent.mImeScreenshot);

        verify(t).show(mDisplayContent.mImeScreenshot);

    }



    @UseTestDisplay(addWindows = {W_INPUT_METHOD}, addAllCommonWindows = true)

services/tests/wmtests/src/com/android/server/wm/MockSurfaceControlBuilder.java

+5 −1
Original line number Diff line number Diff line
@@ -20,6 +20,8 @@ import static org.mockito.Mockito.mock; 


import android.view.SurfaceControl;



import org.mockito.Mockito;



/**

 * Stubbed {@link SurfaceControl.Builder} class that returns a mocked SurfaceControl instance

 * that can be used for unit testing.
@@ -32,6 +34,8 @@ class MockSurfaceControlBuilder extends SurfaceControl.Builder { 


    @Override

    public SurfaceControl build() {

        return mock(SurfaceControl.class);

        SurfaceControl mockSurfaceControl = mock(SurfaceControl.class);

        Mockito.doReturn(true).when(mockSurfaceControl).isValid();

        return mockSurfaceControl;

    }

}

services/tests/wmtests/src/com/android/server/wm/StubTransaction.java

+9 −0
Original line number Diff line number Diff line
@@ -17,11 +17,13 @@ 
package com.android.server.wm;



import android.annotation.NonNull;

import android.annotation.Nullable;

import android.graphics.ColorSpace;

import android.graphics.GraphicBuffer;

import android.graphics.Matrix;

import android.graphics.Rect;

import android.graphics.Region;

import android.hardware.HardwareBuffer;

import android.os.IBinder;

import android.os.Parcel;

import android.view.InputWindowHandle;
@@ -266,6 +268,13 @@ public class StubTransaction extends SurfaceControl.Transaction { 
        return this;

    }



    @Override

    @NonNull

    public SurfaceControl.Transaction setBuffer(@NonNull SurfaceControl sc,

            @Nullable HardwareBuffer buffer) {

        return this;

    }



    @Override

    public SurfaceControl.Transaction setColorSpace(SurfaceControl sc, ColorSpace colorSpace) {

        return this;