Merge "Add end-to-end test for full screen magnification continuous mouse following" into main (f25afeda) · Commits · e / os / android_frameworks_base

services/tests/servicestests/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -63,6 +63,7 @@ android_test {
		"androidx.test.runner",
		"androidx.test.rules",
		"androidx.test.ext.junit",
		"CtsAccessibilityCommon",
		"cts-wm-util",
		"platform-compat-test-rules",
		"platform-parametric-runner-lib",

services/tests/servicestests/AndroidManifest.xml

+15 −0

Original line number	Diff line number	Diff line
		@@ -129,6 +129,21 @@
		<application android:testOnly="true" android:debuggable="true">
		<uses-library android:name="android.test.runner"/>

		<service
		android:name="com.android.server.accessibility.integration.FullScreenMagnificationMouseFollowingTest$TestMagnificationAccessibilityService"
		android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE"
		android:exported="true">
		<intent-filter>
		<action android:name="android.accessibilityservice.AccessibilityService" />
		<category android:name="android.accessibilityservice.category.FEEDBACK_GENERIC" />
		</intent-filter>

		<meta-data
		android:name="android.accessibilityservice"
		android:resource="@xml/test_magnification_a11y_service" />
		</service>
		<activity android:name="com.android.server.accessibility.integration.FullScreenMagnificationMouseFollowingTest$TestActivity" />

		<service android:name="com.android.server.accounts.TestAccountType1AuthenticatorService"
		android:exported="false">
		<intent-filter>

services/tests/servicestests/res/xml/test_magnification_a11y_service.xml

0 → 100644

+24 −0

Original line number	Diff line number	Diff line
		<?xml version="1.0" encoding="utf-8"?>
		<!-- Copyright (C) 2025 The Android Open Source Project

		Licensed under the Apache License, Version 2.0 (the "License");
		you may not use this file except in compliance with the License.
		You may obtain a copy of the License at

		http://www.apache.org/licenses/LICENSE-2.0

		Unless required by applicable law or agreed to in writing, software
		distributed under the License is distributed on an "AS IS" BASIS,
		WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		See the License for the specific language governing permissions and
		limitations under the License.
		-->
		<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
		android:accessibilityEventTypes="typeAllMask"
		android:accessibilityFeedbackType="feedbackGeneric"
		android:canRetrieveWindowContent="true"
		android:canRequestTouchExplorationMode="true"
		android:canRequestEnhancedWebAccessibility="true"
		android:canRequestFilterKeyEvents="true"
		android:canControlMagnification="true"
		android:canPerformGestures="true"/>
		No newline at end of file

services/tests/servicestests/src/com/android/server/accessibility/integration/FullScreenMagnificationMouseFollowingTest.kt

0 → 100644

+378 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2025 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/
		package com.android.server.accessibility.integration

		import android.Manifest
		import android.accessibility.cts.common.AccessibilityDumpOnFailureRule
		import android.accessibility.cts.common.InstrumentedAccessibilityService
		import android.accessibility.cts.common.InstrumentedAccessibilityServiceTestRule
		import android.accessibilityservice.AccessibilityService
		import android.accessibilityservice.AccessibilityServiceInfo
		import android.accessibilityservice.MagnificationConfig
		import android.app.Activity
		import android.app.Instrumentation
		import android.app.UiAutomation
		import android.companion.virtual.VirtualDeviceManager
		import android.graphics.PointF
		import android.hardware.display.DisplayManager
		import android.hardware.display.VirtualDisplay
		import android.hardware.input.InputManager
		import android.hardware.input.VirtualMouse
		import android.hardware.input.VirtualMouseConfig
		import android.hardware.input.VirtualMouseRelativeEvent
		import android.os.Handler
		import android.os.Looper
		import android.os.OutcomeReceiver
		import android.platform.test.annotations.RequiresFlagsEnabled
		import android.testing.PollingCheck
		import android.view.Display
		import android.view.InputDevice
		import android.view.MotionEvent
		import android.virtualdevice.cts.common.VirtualDeviceRule
		import androidx.test.ext.junit.runners.AndroidJUnit4
		import androidx.test.platform.app.InstrumentationRegistry
		import com.android.server.accessibility.Flags
		import com.google.common.truth.Truth.assertThat
		import java.util.concurrent.CompletableFuture
		import java.util.concurrent.CountDownLatch
		import java.util.concurrent.TimeUnit
		import kotlin.math.abs
		import kotlin.time.Duration.Companion.milliseconds
		import kotlin.time.Duration.Companion.seconds
		import org.junit.After
		import org.junit.Before
		import org.junit.Rule
		import org.junit.Test
		import org.junit.rules.RuleChain
		import org.junit.runner.RunWith

		// Convenient extension functions for float.
		private const val EPS = 0.00001f
		private fun Float.nearEq(other: Float) = abs(this - other) < EPS
		private fun PointF.nearEq(other: PointF) = this.x.nearEq(other.x) && this.y.nearEq(other.y)

		/** End-to-end tests for full screen magnification following mouse cursor. */
		@RunWith(AndroidJUnit4::class)
		@RequiresFlagsEnabled(Flags.FLAG_ENABLE_MAGNIFICATION_FOLLOWS_MOUSE_WITH_POINTER_MOTION_FILTER)
		class FullScreenMagnificationMouseFollowingTest {

		private lateinit var instrumentation: Instrumentation
		private lateinit var uiAutomation: UiAutomation

		private val magnificationAccessibilityServiceRule =
		InstrumentedAccessibilityServiceTestRule<TestMagnificationAccessibilityService>(
		TestMagnificationAccessibilityService::class.java, false
		)
		private lateinit var service: TestMagnificationAccessibilityService

		// virtualDeviceRule tears down `virtualDevice` and `virtualDisplay`.
		// Note that CheckFlagsRule is a part of VirtualDeviceRule. See its javadoc.
		val virtualDeviceRule: VirtualDeviceRule =
		VirtualDeviceRule.withAdditionalPermissions(Manifest.permission.MANAGE_ACTIVITY_TASKS)
		private lateinit var virtualDevice: VirtualDeviceManager.VirtualDevice
		private lateinit var virtualDisplay: VirtualDisplay

		// Once created, it's our responsibility to close the mouse.
		private lateinit var virtualMouse: VirtualMouse

		@get:Rule
		val ruleChain: RuleChain =
		RuleChain.outerRule(virtualDeviceRule)
		.around(magnificationAccessibilityServiceRule)
		.around(AccessibilityDumpOnFailureRule())

		@Before
		fun setUp() {
		instrumentation = InstrumentationRegistry.getInstrumentation()
		uiAutomation =
		instrumentation.getUiAutomation(UiAutomation.FLAG_DONT_SUPPRESS_ACCESSIBILITY_SERVICES)
		uiAutomation.serviceInfo =
		uiAutomation.serviceInfo!!.apply {
		flags = flags or AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS
		}

		prepareVirtualDevices()

		launchTestActivityFullscreen(virtualDisplay.display.displayId)

		service = magnificationAccessibilityServiceRule.enableService()
		service.observingDisplayId = virtualDisplay.display.displayId
		}

		@After
		fun cleanUp() {
		if (this::virtualMouse.isInitialized) {
		virtualMouse.close()
		}
		}

		// Note on continuous movement:
		// Assume that the entire display is magnified, and the zoom level is z.
		// In continuous movement, mouse speed relative to the unscaled physical display is the same as
		// unmagnified speed. While, when a cursor moves from the left edge to the right edge of the
		// screen, the magnification center moves from the left bound to the right bound, which is
		// (display width) * (z - 1) / z.
		//
		// Similarly, when the mouse cursor moves by d in unscaled, display coordinates,
		// the magnification center moves by d * (z - 1) / z.

		@Test
		fun testContinuous_toBottomRight() {
		ensureMouseAtCenter()

		val controller = service.getMagnificationController(virtualDisplay.display.displayId)

		scaleTo(controller, 2f)
		assertMagnification(controller, scale = 2f, CENTER_X, CENTER_Y)

		// Move cursor by (10, 15)
		// This will move magnification center by (5, 7.5)
		sendMouseMove(10f, 15f)
		assertCursorLocation(CENTER_X + 10, CENTER_Y + 15)
		assertMagnification(controller, scale = 2f, CENTER_X + 5, CENTER_Y + 7.5f)

		// Move cursor to the rest of the way to the edge.
		sendMouseMove(DISPLAY_WIDTH - 10, DISPLAY_HEIGHT - 15)
		assertCursorLocation(DISPLAY_WIDTH - 1, DISPLAY_HEIGHT - 1)
		assertMagnification(controller, scale = 2f, DISPLAY_WIDTH * 3 / 4, DISPLAY_HEIGHT * 3 / 4)

		// Move cursor further won't move the magnification.
		sendMouseMove(100f, 100f)
		assertCursorLocation(DISPLAY_WIDTH - 1, DISPLAY_HEIGHT - 1)
		}

		@Test
		fun testContinuous_toTopLeft() {
		ensureMouseAtCenter()

		val controller = service.getMagnificationController(virtualDisplay.display.displayId)

		scaleTo(controller, 3f)
		assertMagnification(controller, scale = 3f, CENTER_X, CENTER_Y)

		// Move cursor by (-30, -15)
		// This will move magnification center by (-20, -10)
		sendMouseMove(-30f, -15f)
		assertCursorLocation(CENTER_X - 30, CENTER_Y - 15)
		assertMagnification(controller, scale = 3f, CENTER_X - 20, CENTER_Y - 10)

		// Move cursor to the rest of the way to the edge.
		sendMouseMove(-CENTER_X + 30, -CENTER_Y + 15)
		assertCursorLocation(0f, 0f)
		assertMagnification(controller, scale = 3f, DISPLAY_WIDTH / 6, DISPLAY_HEIGHT / 6)

		// Move cursor further won't move the magnification.
		sendMouseMove(-100f, -100f)
		assertCursorLocation(0f, 0f)
		assertMagnification(controller, scale = 3f, DISPLAY_WIDTH / 6, DISPLAY_HEIGHT / 6)
		}

		private fun ensureMouseAtCenter() {
		val displayCenter = PointF(320f, 240f)
		val cursorLocation = virtualMouse.cursorPosition
		if (!cursorLocation.nearEq(displayCenter)) {
		sendMouseMove(displayCenter.x - cursorLocation.x, displayCenter.y - cursorLocation.y)
		assertCursorLocation(320f, 240f)
		}
		}

		private fun sendMouseMove(dx: Float, dy: Float) {
		virtualMouse.sendRelativeEvent(
		VirtualMouseRelativeEvent.Builder().setRelativeX(dx).setRelativeY(dy).build()
		)
		}

		/**
		* Asserts that the cursor location is at the specified coordinates. The coordinates
		* are in the non-scaled, display coordinates.
		*/
		private fun assertCursorLocation(x: Float, y: Float) {
		PollingCheck.check("Wait for the cursor at ($x, $y)", CURSOR_TIMEOUT.inWholeMilliseconds) {
		service.lastObservedCursorLocation?.let { it.x.nearEq(x) && it.y.nearEq(y) } ?: false
		}
		}

		private fun scaleTo(controller: AccessibilityService.MagnificationController, scale: Float) {
		val config =
		MagnificationConfig.Builder()
		.setActivated(true)
		.setMode(MagnificationConfig.MAGNIFICATION_MODE_FULLSCREEN)
		.setScale(scale)
		.build()
		val setResult = BooleanArray(1)
		service.runOnServiceSync { setResult[0] = controller.setMagnificationConfig(config, false) }
		assertThat(setResult[0]).isTrue()
		}

		private fun assertMagnification(
		controller: AccessibilityService.MagnificationController,
		scale: Float = Float.NaN, centerX: Float = Float.NaN, centerY: Float = Float.NaN
		) {
		PollingCheck.check(
		"Wait for the magnification to scale=$scale, centerX=$centerX, centerY=$centerY",
		MAGNIFICATION_TIMEOUT.inWholeMilliseconds
		) check@{
		val actual = controller.getMagnificationConfig() ?: return@check false
		actual.isActivated &&
		(actual.mode == MagnificationConfig.MAGNIFICATION_MODE_FULLSCREEN) &&
		(scale.isNaN() \|\| scale.nearEq(actual.scale)) &&
		(centerX.isNaN() \|\| centerX.nearEq(actual.centerX)) &&
		(centerY.isNaN() \|\| centerY.nearEq(actual.centerY))
		}
		}

		/**
		* Sets up a virtual display and a virtual mouse for the test. The virtual mouse is associated
		* with the virtual display.
		*/
		private fun prepareVirtualDevices() {
		val deviceLatch = CountDownLatch(1)
		val im = instrumentation.context.getSystemService(InputManager::class.java)
		val inputDeviceListener =
		object : InputManager.InputDeviceListener {
		override fun onInputDeviceAdded(deviceId: Int) {
		onInputDeviceChanged(deviceId)
		}

		override fun onInputDeviceRemoved(deviceId: Int) {}

		override fun onInputDeviceChanged(deviceId: Int) {
		val device = im.getInputDevice(deviceId) ?: return
		if (device.vendorId == VIRTUAL_MOUSE_VENDOR_ID &&
		device.productId == VIRTUAL_MOUSE_PRODUCT_ID
		) {
		deviceLatch.countDown()
		}
		}
		}
		im.registerInputDeviceListener(inputDeviceListener, Handler(Looper.getMainLooper()))

		virtualDevice = virtualDeviceRule.createManagedVirtualDevice()
		virtualDisplay =
		virtualDeviceRule.createManagedVirtualDisplay(
		virtualDevice,
		VirtualDeviceRule
		.createDefaultVirtualDisplayConfigBuilder(
		DISPLAY_WIDTH.toInt(),
		DISPLAY_HEIGHT.toInt()
		)
		.setFlags(
		DisplayManager.VIRTUAL_DISPLAY_FLAG_PUBLIC
		or DisplayManager.VIRTUAL_DISPLAY_FLAG_TRUSTED
		or DisplayManager.VIRTUAL_DISPLAY_FLAG_OWN_CONTENT_ONLY
		)
		)!!
		virtualMouse =
		virtualDevice.createVirtualMouse(
		VirtualMouseConfig.Builder()
		.setVendorId(VIRTUAL_MOUSE_VENDOR_ID)
		.setProductId(VIRTUAL_MOUSE_PRODUCT_ID)
		.setAssociatedDisplayId(virtualDisplay.display.displayId)
		.setInputDeviceName("VirtualMouse")
		.build()
		)

		deviceLatch.await(UI_IDLE_GLOBAL_TIMEOUT.inWholeSeconds, TimeUnit.SECONDS)
		im.unregisterInputDeviceListener(inputDeviceListener)
		}

		/**
		* Launches a test (empty) activity and makes it fullscreen on the specified display. This
		* ensures that system bars are hidden and the full screen magnification enlarges the entire
		* display.
		*/
		private fun launchTestActivityFullscreen(displayId: Int) {
		val future = CompletableFuture<Void?>()
		val fullscreenCallback =
		object : OutcomeReceiver<Void, Throwable> {
		override fun onResult(result: Void?) {
		future.complete(null)
		}

		override fun onError(error: Throwable) {
		future.completeExceptionally(error)
		}
		}

		val activity =
		virtualDeviceRule.startActivityOnDisplaySync<TestActivity>(
		displayId,
		TestActivity::class.java
		)
		instrumentation.runOnMainSync {
		activity.requestFullscreenMode(
		Activity.FULLSCREEN_MODE_REQUEST_ENTER,
		fullscreenCallback
		)
		}
		future.get(UI_IDLE_GLOBAL_TIMEOUT.inWholeSeconds, TimeUnit.SECONDS)

		uiAutomation.waitForIdle(
		UI_IDLE_TIMEOUT.inWholeMilliseconds, UI_IDLE_GLOBAL_TIMEOUT.inWholeMilliseconds
		)
		}

		class TestMagnificationAccessibilityService : InstrumentedAccessibilityService() {
		private val lock = Any()

		var observingDisplayId = Display.INVALID_DISPLAY
		set(v) {
		synchronized(lock) { field = v }
		}

		var lastObservedCursorLocation: PointF? = null
		private set
		get() {
		synchronized(lock) {
		return field
		}
		}

		override fun onServiceConnected() {
		serviceInfo =
		getServiceInfo()!!.apply { setMotionEventSources(InputDevice.SOURCE_MOUSE) }

		super.onServiceConnected()
		}

		override fun onMotionEvent(event: MotionEvent) {
		super.onMotionEvent(event)

		synchronized(lock) {
		if (event.displayId == observingDisplayId) {
		lastObservedCursorLocation = PointF(event.x, event.y)
		}
		}
		}
		}

		class TestActivity : Activity()

		companion object {
		private const val VIRTUAL_MOUSE_VENDOR_ID = 123
		private const val VIRTUAL_MOUSE_PRODUCT_ID = 456

		private val CURSOR_TIMEOUT = 1.seconds
		private val MAGNIFICATION_TIMEOUT = 3.seconds
		private val UI_IDLE_TIMEOUT = 500.milliseconds
		private val UI_IDLE_GLOBAL_TIMEOUT = 5.seconds

		private const val DISPLAY_WIDTH = 640.0f
		private const val DISPLAY_HEIGHT = 480.0f
		private const val CENTER_X = DISPLAY_WIDTH / 2f
		private const val CENTER_Y = DISPLAY_HEIGHT / 2f
		}
		}