Merge "Letterbox scrolling: fix double-tap letterbox reachability feature" into main (e4a34009) · Commits · e / os / android_frameworks_base

services/core/java/com/android/server/wm/Letterbox.java

+73 −24

Original line number	Diff line number	Diff line
		@@ -19,6 +19,7 @@ package com.android.server.wm;
		import static android.os.InputConstants.DEFAULT_DISPATCHING_TIMEOUT_MILLIS;
		import static android.view.SurfaceControl.HIDDEN;
		import static android.window.TaskConstants.TASK_CHILD_LAYER_LETTERBOX_BACKGROUND;
		import static android.window.TaskConstants.TASK_CHILD_LAYER_TASK_OVERLAY;

		import android.annotation.NonNull;
		import android.graphics.Color;
		@@ -37,6 +38,7 @@ import android.view.SurfaceControl;
		import android.view.WindowManager;

		import com.android.server.UiThread;
		import com.android.window.flags.Flags;

		import java.util.function.Supplier;

		@@ -66,6 +68,7 @@ public class Letterbox {
		// for overlaping an app window and letterbox surfaces.
		private final LetterboxSurface mFullWindowSurface = new LetterboxSurface("fullWindow");
		private final LetterboxSurface[] mSurfaces = { mLeft, mTop, mRight, mBottom };

		@NonNull
		private final AppCompatReachabilityPolicy mAppCompatReachabilityPolicy;
		@NonNull
		@@ -222,12 +225,14 @@ public class Letterbox {

		void onMovedToDisplay(int displayId) {
		for (LetterboxSurface surface : mSurfaces) {
		if (surface.mInputInterceptor != null) {
		surface.mInputInterceptor.mWindowHandle.displayId = displayId;
		setSurfaceDisplayID(surface, displayId);
		}
		setSurfaceDisplayID(mFullWindowSurface, displayId);
		}
		if (mFullWindowSurface.mInputInterceptor != null) {
		mFullWindowSurface.mInputInterceptor.mWindowHandle.displayId = displayId;

		private void setSurfaceDisplayID(LetterboxSurface surface, int displayId) {
		if (surface.mInputInterceptor != null) {
		surface.mInputInterceptor.mWindowHandle.displayId = displayId;
		}
		}

		@@ -242,14 +247,13 @@ public class Letterbox {
		private final class TapEventReceiver extends InputEventReceiver {

		private final GestureDetector mDoubleTapDetector;
		private final DoubleTapListener mDoubleTapListener;

		TapEventReceiver(InputChannel inputChannel, WindowManagerService wmService,
		Handler uiHandler) {
		super(inputChannel, uiHandler.getLooper());
		mDoubleTapListener = new DoubleTapListener(wmService);
		mDoubleTapDetector = new GestureDetector(wmService.mContext, mDoubleTapListener,
		uiHandler);
		final DoubleTapListener doubleTapListener = new DoubleTapListener(wmService);
		mDoubleTapDetector =
		new GestureDetector(wmService.mContext, doubleTapListener, uiHandler);
		}

		@Override
		@@ -293,7 +297,8 @@ public class Letterbox {
		InputInterceptor(String namePrefix, WindowState win) {
		mWmService = win.mWmService;
		mHandler = UiThread.getHandler();
		final String name = namePrefix + (win.mActivityRecord != null ? win.mActivityRecord : win);
		final String name = namePrefix
		+ (win.mActivityRecord != null ? win.mActivityRecord : win);
		mClientChannel = mWmService.mInputManager.createInputChannel(name);
		mInputEventReceiver = new TapEventReceiver(mClientChannel, mWmService, mHandler);

		@@ -303,12 +308,15 @@ public class Letterbox {
		win.getDisplayId());
		mWindowHandle.name = name;
		mWindowHandle.token = mToken;
		mWindowHandle.layoutParamsType = WindowManager.LayoutParams.TYPE_INPUT_CONSUMER;
		mWindowHandle.layoutParamsType = Flags.scrollingFromLetterbox()
		? WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY
		: WindowManager.LayoutParams.TYPE_INPUT_CONSUMER;
		mWindowHandle.dispatchingTimeoutMillis = DEFAULT_DISPATCHING_TIMEOUT_MILLIS;
		mWindowHandle.ownerPid = WindowManagerService.MY_PID;
		mWindowHandle.ownerUid = WindowManagerService.MY_UID;
		mWindowHandle.scaleFactor = 1.0f;
		mWindowHandle.inputConfig = InputConfig.NOT_FOCUSABLE \| InputConfig.SLIPPERY;
		mWindowHandle.inputConfig = InputConfig.NOT_FOCUSABLE
		\| (Flags.scrollingFromLetterbox() ? InputConfig.SPY : InputConfig.SLIPPERY);
		}

		void updateTouchableRegion(Rect frame) {
		@@ -341,6 +349,7 @@ public class Letterbox {

		private final String mType;
		private SurfaceControl mSurface;
		private SurfaceControl mInputSurface;
		private Color mColor;
		private boolean mHasWallpaperBackground;
		private SurfaceControl mParentSurface;
		@@ -373,22 +382,36 @@ public class Letterbox {
		.setColorSpaceAgnostic(mSurface, true);
		}

		private void createInputSurface(SurfaceControl.Transaction t) {
		mInputSurface = mSurfaceControlFactory.get()
		.setName("LetterboxInput - " + mType)
		.setFlags(HIDDEN)
		.setContainerLayer()
		.setOpaque(true)
		.setCallsite("LetterboxSurface.createInputSurface")
		.build();

		t.setLayer(mInputSurface, TASK_CHILD_LAYER_TASK_OVERLAY);
		}

		void attachInput(WindowState win) {
		if (mInputInterceptor != null) {
		mInputInterceptor.dispose();
		}
		// TODO(b/371179559): only detect double tap on LB surfaces not used for cutout area.
		// Potentially, the input interceptor may still be needed for slippery feature.
		mInputInterceptor = new InputInterceptor("Letterbox_" + mType + "_", win);
		}

		boolean isRemoved() {
		return mSurface != null \|\| mInputInterceptor != null;
		}

		public void remove() {
		if (mSurface != null) {
		mTransactionFactory.get().remove(mSurface).apply();
		mSurface = null;
		}
		if (mInputSurface != null) {
		mTransactionFactory.get().remove(mInputSurface).apply();
		mInputSurface = null;
		}
		if (mInputInterceptor != null) {
		mInputInterceptor.dispose();
		mInputInterceptor = null;
		@@ -415,28 +438,54 @@ public class Letterbox {
		createSurface(t);
		}

		if (Flags.scrollingFromLetterbox()
		&& mInputInterceptor != null
		&& mInputSurface == null) {
		createInputSurface(inputT);
		}

		mColor = mAppCompatLetterboxOverrides.getLetterboxBackgroundColor();
		mParentSurface = mParentSurfaceSupplier.get();
		t.setColor(mSurface, getRgbColorArray());
		t.setPosition(mSurface, mSurfaceFrameRelative.left, mSurfaceFrameRelative.top);
		t.setWindowCrop(mSurface, mSurfaceFrameRelative.width(),
		mSurfaceFrameRelative.height());
		t.reparent(mSurface, mParentSurface);
		setPositionAndReparent(t, mSurface);

		mHasWallpaperBackground = mAppCompatLetterboxOverrides
		.hasWallpaperBackgroundForLetterbox();
		updateAlphaAndBlur(t);

		t.show(mSurface);
		} else if (mSurface != null) {

		if (mInputSurface != null) {
		setPositionAndReparent(inputT, mInputSurface);
		inputT.setTrustedOverlay(mInputSurface, true);
		inputT.show(mInputSurface);
		}

		} else {
		if (mSurface != null) {
		t.hide(mSurface);
		}
		if (mSurface != null && mInputInterceptor != null) {
		if (mInputSurface != null) {
		inputT.hide(mInputSurface);
		}
		}

		SurfaceControl surfaceWithInput =
		Flags.scrollingFromLetterbox() ? mInputSurface : mSurface;
		if (surfaceWithInput != null && mInputInterceptor != null) {
		mInputInterceptor.updateTouchableRegion(mSurfaceFrameRelative);
		inputT.setInputWindowInfo(mSurface, mInputInterceptor.mWindowHandle);
		inputT.setInputWindowInfo(surfaceWithInput, mInputInterceptor.mWindowHandle);
		}
		}

		private void setPositionAndReparent(@NonNull SurfaceControl.Transaction t,
		@NonNull SurfaceControl surface) {
		t.setPosition(surface, mSurfaceFrameRelative.left, mSurfaceFrameRelative.top);
		t.setWindowCrop(surface, mSurfaceFrameRelative.width(),
		mSurfaceFrameRelative.height());
		t.reparent(surface, mParentSurface);
		}

		private void updateAlphaAndBlur(SurfaceControl.Transaction t) {
		if (!mHasWallpaperBackground) {
		// Opaque

services/core/java/com/android/server/wm/WindowState.java

+3 −0

Original line number	Diff line number	Diff line
		@@ -2777,6 +2777,9 @@ class WindowState extends WindowContainer<WindowState> implements WindowManagerP
		}

		if (mTmpRect.isEmpty()) {
		// TODO(b/371182877) If the app does not draw under cutout region, the touchable region
		// should not include cutout regions (if scrolling from letterbox feature is not desired
		// for this region).
		final Rect transformedBounds = mActivityRecord.getFixedRotationTransformDisplayBounds();
		if (transformedBounds != null) {
		// Task is in the same orientation as display, so the rotated bounds should be

services/tests/wmtests/src/com/android/server/wm/LetterboxAttachInputTest.java

0 → 100644

+178 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright 2024 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.server.wm;

		import static com.android.server.testutils.MockitoUtilsKt.eq;

		import static org.junit.Assert.assertFalse;
		import static org.junit.Assert.assertNotNull;
		import static org.junit.Assert.assertNull;
		import static org.junit.Assert.assertTrue;
		import static org.mockito.ArgumentMatchers.any;
		import static org.mockito.Mockito.clearInvocations;
		import static org.mockito.Mockito.doReturn;
		import static org.mockito.Mockito.mock;
		import static org.mockito.Mockito.never;
		import static org.mockito.Mockito.spy;
		import static org.mockito.Mockito.verify;

		import android.graphics.Color;
		import android.graphics.Point;
		import android.graphics.Rect;
		import android.os.InputConfig;
		import android.platform.test.annotations.DisableFlags;
		import android.platform.test.annotations.EnableFlags;
		import android.platform.test.annotations.Presubmit;
		import android.view.InputWindowHandle;
		import android.view.SurfaceControl;
		import android.view.WindowManager;

		import com.android.server.testutils.StubTransaction;
		import com.android.window.flags.Flags;

		import org.junit.Before;
		import org.junit.Test;
		import org.junit.runner.RunWith;
		import org.mockito.ArgumentCaptor;

		/**
		* Test class for {@link Letterbox}.
		* <p>
		* Build/Install/Run:
		* atest WmTests:LetterboxAttachInputTest
		*/
		@Presubmit
		@RunWith(WindowTestRunner.class)
		public class LetterboxAttachInputTest extends WindowTestsBase {

		private Letterbox mLetterbox;
		private LetterboxTest.SurfaceControlMocker mSurfaces;

		@Before
		public void setUp() throws Exception {
		mSurfaces = new LetterboxTest.SurfaceControlMocker();
		AppCompatLetterboxOverrides letterboxOverrides = mock(AppCompatLetterboxOverrides.class);
		doReturn(false).when(letterboxOverrides).shouldLetterboxHaveRoundedCorners();
		doReturn(Color.valueOf(Color.BLACK)).when(letterboxOverrides)
		.getLetterboxBackgroundColor();
		doReturn(false).when(letterboxOverrides).hasWallpaperBackgroundForLetterbox();
		doReturn(0).when(letterboxOverrides).getLetterboxWallpaperBlurRadiusPx();
		doReturn(0.5f).when(letterboxOverrides).getLetterboxWallpaperDarkScrimAlpha();
		mLetterbox = new Letterbox(mSurfaces, StubTransaction::new,
		mock(AppCompatReachabilityPolicy.class), letterboxOverrides,
		() -> mock(SurfaceControl.class));
		mTransaction = spy(StubTransaction.class);
		}

		@DisableFlags(Flags.FLAG_SCROLLING_FROM_LETTERBOX)
		@Test
		public void testSurface_createdHasSlipperyInput_scrollingFromLetterboxDisabled() {
		mLetterbox.layout(new Rect(0, 0, 10, 10), new Rect(0, 1, 10, 10), new Point(1000, 2000));

		attachInput();
		applySurfaceChanges();

		assertNotNull(mSurfaces.top);
		ArgumentCaptor<InputWindowHandle> handleCaptor =
		ArgumentCaptor.forClass(InputWindowHandle.class);
		verify(mTransaction).setInputWindowInfo(eq(mSurfaces.top), handleCaptor.capture());
		InputWindowHandle capturedHandle = handleCaptor.getValue();
		assertTrue((capturedHandle.inputConfig & InputConfig.SLIPPERY) != 0);
		assertFalse((capturedHandle.inputConfig & InputConfig.SPY) != 0);
		}

		@DisableFlags(Flags.FLAG_SCROLLING_FROM_LETTERBOX)
		@Test
		public void testInputSurface_notCreated_scrollingFromLetterboxDisabled() {
		mLetterbox.layout(new Rect(0, 0, 10, 10), new Rect(0, 1, 10, 10), new Point(1000, 2000));

		attachInput();
		applySurfaceChanges();

		assertNull(mSurfaces.topInput);
		}

		@EnableFlags(Flags.FLAG_SCROLLING_FROM_LETTERBOX)
		@Test
		public void testSurface_createdHasNoInput_scrollingFromLetterboxEnabled() {
		mLetterbox.layout(new Rect(0, 0, 10, 10), new Rect(0, 1, 10, 10), new Point(1000, 2000));

		attachInput();
		applySurfaceChanges();

		assertNotNull(mSurfaces.top);
		verify(mTransaction, never()).setInputWindowInfo(eq(mSurfaces.top), any());

		}

		@EnableFlags(Flags.FLAG_SCROLLING_FROM_LETTERBOX)
		@Test
		public void testInputSurface_createdHasSpyInput_scrollingFromLetterboxEnabled() {
		mLetterbox.layout(new Rect(0, 0, 10, 10), new Rect(0, 1, 10, 10), new Point(1000, 2000));

		attachInput();
		applySurfaceChanges();

		assertNotNull(mSurfaces.topInput);
		ArgumentCaptor<InputWindowHandle> handleCaptor =
		ArgumentCaptor.forClass(InputWindowHandle.class);
		verify(mTransaction).setInputWindowInfo(eq(mSurfaces.topInput), handleCaptor.capture());
		InputWindowHandle capturedHandle = handleCaptor.getValue();
		assertTrue((capturedHandle.inputConfig & InputConfig.SPY) != 0);
		assertFalse((capturedHandle.inputConfig & InputConfig.SLIPPERY) != 0);
		}

		@EnableFlags(Flags.FLAG_SCROLLING_FROM_LETTERBOX)
		@Test
		public void testInputSurfaceOrigin_applied_scrollingFromLetterboxEnabled() {
		mLetterbox.layout(new Rect(0, 0, 10, 10), new Rect(0, 1, 10, 10), new Point(1000, 2000));

		attachInput();
		applySurfaceChanges();

		verify(mTransaction).setPosition(mSurfaces.topInput, -1000, -2000);
		}

		@EnableFlags(Flags.FLAG_SCROLLING_FROM_LETTERBOX)
		@Test
		public void testInputSurfaceOrigin_changeCausesReapply_scrollingFromLetterboxEnabled() {
		mLetterbox.layout(new Rect(0, 0, 10, 10), new Rect(0, 1, 10, 10), new Point(1000, 2000));

		attachInput();
		applySurfaceChanges();
		clearInvocations(mTransaction);
		mLetterbox.layout(new Rect(0, 0, 10, 10), new Rect(0, 1, 10, 10), new Point(0, 0));

		assertTrue(mLetterbox.needsApplySurfaceChanges());

		applySurfaceChanges();

		verify(mTransaction).setPosition(mSurfaces.topInput, 0, 0);
		}

		private void applySurfaceChanges() {
		mLetterbox.applySurfaceChanges(/* syncTransaction */ mTransaction,
		/* pendingTransaction */ mTransaction);
		}

		private void attachInput() {
		final WindowManager.LayoutParams attrs = new WindowManager.LayoutParams();
		final WindowToken windowToken = createTestWindowToken(0, mDisplayContent);
		WindowState windowState = createWindowState(attrs, windowToken);
		mLetterbox.attachInput(windowState);
		}
		}

services/tests/wmtests/src/com/android/server/wm/LetterboxTest.java

+53 −25

Original line number	Diff line number	Diff line
		@@ -33,14 +33,19 @@ import static org.mockito.Mockito.when;
		import android.graphics.Color;
		import android.graphics.Point;
		import android.graphics.Rect;
		import android.platform.test.annotations.DisableFlags;
		import android.platform.test.annotations.EnableFlags;
		import android.platform.test.annotations.Presubmit;
		import android.platform.test.flag.junit.SetFlagsRule;
		import android.view.SurfaceControl;

		import androidx.test.filters.SmallTest;

		import com.android.server.testutils.StubTransaction;
		import com.android.window.flags.Flags;

		import org.junit.Before;
		import org.junit.Rule;
		import org.junit.Test;
		import org.mockito.invocation.InvocationOnMock;

		@@ -56,9 +61,12 @@ import java.util.function.Supplier;
		@Presubmit
		public class LetterboxTest {

		Letterbox mLetterbox;
		SurfaceControlMocker mSurfaces;
		SurfaceControl.Transaction mTransaction;
		@Rule
		public final SetFlagsRule mSetFlagsRule = new SetFlagsRule();

		private Letterbox mLetterbox;
		private SurfaceControlMocker mSurfaces;
		private SurfaceControl.Transaction mTransaction;

		private SurfaceControl mParentSurface = mock(SurfaceControl.class);
		private AppCompatLetterboxOverrides mLetterboxOverrides;
		@@ -183,6 +191,38 @@ public class LetterboxTest {
		verify(mTransaction).setPosition(mSurfaces.top, -1000, -2000);
		}

		@DisableFlags(Flags.FLAG_SCROLLING_FROM_LETTERBOX)
		@Test
		public void testSurface_created_scrollingFromLetterboxDisabled() {
		mLetterbox.layout(new Rect(0, 0, 10, 10), new Rect(0, 1, 10, 10), new Point(1000, 2000));
		applySurfaceChanges();
		assertNotNull(mSurfaces.top);
		}

		@DisableFlags(Flags.FLAG_SCROLLING_FROM_LETTERBOX)
		@Test
		public void testInputSurface_notCreated_scrollingFromLetterboxDisabled() {
		mLetterbox.layout(new Rect(0, 0, 10, 10), new Rect(0, 1, 10, 10), new Point(1000, 2000));
		applySurfaceChanges();
		assertNull(mSurfaces.topInput);
		}

		@EnableFlags(Flags.FLAG_SCROLLING_FROM_LETTERBOX)
		@Test
		public void testSurface_created_scrollingFromLetterboxEnabled() {
		mLetterbox.layout(new Rect(0, 0, 10, 10), new Rect(0, 1, 10, 10), new Point(1000, 2000));
		applySurfaceChanges();
		assertNotNull(mSurfaces.top);
		}

		@EnableFlags(Flags.FLAG_SCROLLING_FROM_LETTERBOX)
		@Test
		public void testInputSurface_notCreated_notAttachedInputAndScrollingFromLetterboxEnabled() {
		mLetterbox.layout(new Rect(0, 0, 10, 10), new Rect(0, 1, 10, 10), new Point(1000, 2000));
		applySurfaceChanges();
		assertNull(mSurfaces.topInput);
		}

		@Test
		public void testApplySurfaceChanges_setColor() {
		mLetterbox.layout(new Rect(0, 0, 10, 10), new Rect(0, 1, 10, 10), new Point(1000, 2000));
		@@ -283,15 +323,11 @@ public class LetterboxTest {
		/* pendingTransaction */ mTransaction);
		}

		class SurfaceControlMocker implements Supplier<SurfaceControl.Builder> {
		private SurfaceControl.Builder mLeftBuilder;
		public SurfaceControl left;
		static class SurfaceControlMocker implements Supplier<SurfaceControl.Builder> {
		private SurfaceControl.Builder mTopBuilder;
		public SurfaceControl top;
		private SurfaceControl.Builder mRightBuilder;
		public SurfaceControl right;
		private SurfaceControl.Builder mBottomBuilder;
		public SurfaceControl bottom;
		private SurfaceControl.Builder mTopInputBuilder;
		public SurfaceControl topInput;
		private SurfaceControl.Builder mFullWindowSurfaceBuilder;
		public SurfaceControl fullWindowSurface;

		@@ -300,32 +336,24 @@ public class LetterboxTest {
		final SurfaceControl.Builder builder = mock(SurfaceControl.Builder.class,
		InvocationOnMock::getMock);
		when(builder.setName(anyString())).then((i) -> {
		if (((String) i.getArgument(0)).contains("left")) {
		mLeftBuilder = (SurfaceControl.Builder) i.getMock();
		} else if (((String) i.getArgument(0)).contains("top")) {
		if (((String) i.getArgument(0)).contains("Letterbox - top")) {
		mTopBuilder = (SurfaceControl.Builder) i.getMock();
		} else if (((String) i.getArgument(0)).contains("right")) {
		mRightBuilder = (SurfaceControl.Builder) i.getMock();
		} else if (((String) i.getArgument(0)).contains("bottom")) {
		mBottomBuilder = (SurfaceControl.Builder) i.getMock();
		} else if (((String) i.getArgument(0)).contains("fullWindow")) {
		} else if (((String) i.getArgument(0)).contains("Letterbox - fullWindow")) {
		mFullWindowSurfaceBuilder = (SurfaceControl.Builder) i.getMock();
		} else if (((String) i.getArgument(0)).contains("LetterboxInput - top")) {
		mTopInputBuilder = (SurfaceControl.Builder) i.getMock();
		}
		return i.getMock();
		});

		doAnswer((i) -> {
		final SurfaceControl control = mock(SurfaceControl.class);
		if (i.getMock() == mLeftBuilder) {
		left = control;
		} else if (i.getMock() == mTopBuilder) {
		if (i.getMock() == mTopBuilder) {
		top = control;
		} else if (i.getMock() == mRightBuilder) {
		right = control;
		} else if (i.getMock() == mBottomBuilder) {
		bottom = control;
		} else if (i.getMock() == mFullWindowSurfaceBuilder) {
		fullWindowSurface = control;
		} else if (i.getMock() == mTopInputBuilder) {
		topInput = control;
		}
		return control;
		}).when(builder).build();