Merge "Introduce AccessibilityPointerMotionFilter" into main (25bf4c4f) · Commits · e / os / android_frameworks_native

services/inputflinger/PointerChoreographer.cpp

+28 −2

Original line number	Diff line number	Diff line
		@@ -19,6 +19,7 @@
		#include <android-base/logging.h>
		#include <android/configuration.h>
		#include <com_android_input_flags.h>
		#include <algorithm>
		#if defined(__ANDROID__)
		#include <gui/SurfaceComposerClient.h>
		#endif
		@@ -165,6 +166,7 @@ PointerChoreographer::PointerChoreographer(
		mNotifiedPointerDisplayId(ui::LogicalDisplayId::INVALID),
		mShowTouchesEnabled(false),
		mStylusPointerIconEnabled(false),
		mPointerMotionFilterEnabled(false),
		mCurrentFocusedDisplay(ui::LogicalDisplayId::DEFAULT),
		mIsWindowInfoListenerRegistered(false),
		mWindowInfoListener(sp<PointerChoreographerDisplayInfoListener>::make(this)),
		@@ -322,8 +324,10 @@ void PointerChoreographer::processPointerDeviceMotionEventLocked(NotifyMotionArg
		PointerControllerInterface& pc) {
		const float deltaX = newArgs.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_RELATIVE_X);
		const float deltaY = newArgs.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_RELATIVE_Y);

		vec2 unconsumedDelta = pc.move(deltaX, deltaY);
		vec2 filteredDelta =
		filterPointerMotionForAccessibilityLocked(pc.getPosition(), vec2{deltaX, deltaY},
		newArgs.displayId);
		vec2 unconsumedDelta = pc.move(filteredDelta.x, filteredDelta.y);
		if (com::android::input::flags::connected_displays_cursor() &&
		(std::abs(unconsumedDelta.x) > 0 \|\| std::abs(unconsumedDelta.y) > 0)) {
		handleUnconsumedDeltaLocked(pc, unconsumedDelta);
		@@ -638,6 +642,8 @@ void PointerChoreographer::dump(std::string& dump) {
		mShowTouchesEnabled ? "true" : "false");
		dump += StringPrintf(INDENT "Stylus PointerIcon Enabled: %s\n",
		mStylusPointerIconEnabled ? "true" : "false");
		dump += StringPrintf(INDENT "Accessibility Pointer Motion Filter Enabled: %s\n",
		mPointerMotionFilterEnabled ? "true" : "false");

		dump += INDENT "MousePointerControllers:\n";
		for (const auto& [displayId, mousePointerController] : mMousePointersByDisplay) {
		@@ -973,6 +979,11 @@ void PointerChoreographer::setFocusedDisplay(ui::LogicalDisplayId displayId) {
		mCurrentFocusedDisplay = displayId;
		}

		void PointerChoreographer::setAccessibilityPointerMotionFilterEnabled(bool enabled) {
		std::scoped_lock _l(getLock());
		mPointerMotionFilterEnabled = enabled;
		}

		PointerChoreographer::ControllerConstructor PointerChoreographer::getMouseControllerConstructor(
		ui::LogicalDisplayId displayId) {
		std::function<std::shared_ptr<PointerControllerInterface>()> ctor =
		@@ -1046,6 +1057,21 @@ PointerChoreographer::findDestinationDisplayLocked(const ui::LogicalDisplayId so
		return std::nullopt;
		}

		vec2 PointerChoreographer::filterPointerMotionForAccessibilityLocked(
		const vec2& current, const vec2& delta, const ui::LogicalDisplayId& displayId) {
		if (!mPointerMotionFilterEnabled) {
		return delta;
		}
		std::optional<vec2> filterResult =
		mPolicy.filterPointerMotionForAccessibility(current, delta, displayId);
		if (!filterResult.has_value()) {
		// Disable filter when there's any error.
		mPointerMotionFilterEnabled = false;
		return delta;
		}
		return *filterResult;
		}

		// --- PointerChoreographer::PointerChoreographerDisplayInfoListener ---

		void PointerChoreographer::PointerChoreographerDisplayInfoListener::onWindowInfosChanged(

services/inputflinger/PointerChoreographer.h

+11 −0

Original line number	Diff line number	Diff line
		@@ -90,6 +90,11 @@ public:
		* This method may be called on any thread (usually by the input manager on a binder thread).
		*/
		virtual void dump(std::string& dump) = 0;

		/**
		* Enables motion event filter before pointer coordinates are determined.
		*/
		virtual void setAccessibilityPointerMotionFilterEnabled(bool enabled) = 0;
		};

		class PointerChoreographer : public PointerChoreographerInterface {
		@@ -110,6 +115,7 @@ public:
		void setPointerIconVisibility(ui::LogicalDisplayId displayId, bool visible) override;
		void setFocusedDisplay(ui::LogicalDisplayId displayId) override;
		void setDisplayTopology(const DisplayTopologyGraph& displayTopologyGraph);
		void setAccessibilityPointerMotionFilterEnabled(bool enabled) override;

		void notifyInputDevicesChanged(const NotifyInputDevicesChangedArgs& args) override;
		void notifyKey(const NotifyKeyArgs& args) override;
		@@ -168,6 +174,10 @@ private:
		const DisplayTopologyPosition sourceBoundary,
		int32_t sourceCursorOffsetPx) const REQUIRES(getLock());

		vec2 filterPointerMotionForAccessibilityLocked(const vec2& current, const vec2& delta,
		const ui::LogicalDisplayId& displayId)
		REQUIRES(getLock());

		/* Topology is initialized with default-constructed value, which is an empty topology. Till we
		* receive setDisplayTopology call.
		* Meanwhile Choreographer will treat every display as independent disconnected display.
		@@ -228,6 +238,7 @@ private:
		std::vector<DisplayViewport> mViewports GUARDED_BY(getLock());
		bool mShowTouchesEnabled GUARDED_BY(getLock());
		bool mStylusPointerIconEnabled GUARDED_BY(getLock());
		bool mPointerMotionFilterEnabled GUARDED_BY(getLock());
		std::set<ui::LogicalDisplayId /displayId/> mDisplaysWithPointersHidden;
		ui::LogicalDisplayId mCurrentFocusedDisplay GUARDED_BY(getLock());

services/inputflinger/include/PointerChoreographerPolicyInterface.h

+10 −0

Original line number	Diff line number	Diff line
		@@ -61,6 +61,16 @@ public:

		/* Notifies that mouse cursor faded due to typing. */
		virtual void notifyMouseCursorFadedOnTyping() = 0;

		/**
		* Give accessibility a chance to filter motion event by pointer devices.
		* The return values denotes the delta x and y after filtering it.
		*
		* This call happens on the input hot path and it is extremely performance sensitive.
		* This also must not call back into native code.
		*/
		virtual std::optional<vec2> filterPointerMotionForAccessibility(
		const vec2& current, const vec2& delta, const ui::LogicalDisplayId& displayId) = 0;
		};

		} // namespace android

services/inputflinger/tests/InterfaceMocks.h

+3 −0

Original line number	Diff line number	Diff line
		@@ -191,6 +191,9 @@ public:
		(ui::LogicalDisplayId displayId, const vec2& position), (override));
		MOCK_METHOD(bool, isInputMethodConnectionActive, (), (override));
		MOCK_METHOD(void, notifyMouseCursorFadedOnTyping, (), (override));
		MOCK_METHOD(std::optional<vec2>, filterPointerMotionForAccessibility,
		(const vec2& current, const vec2& delta, const ui::LogicalDisplayId& displayId),
		(override));
		};

		class MockInputDevice : public InputDevice {

services/inputflinger/tests/PointerChoreographer_test.cpp

+83 −0

Original line number	Diff line number	Diff line
		@@ -1776,6 +1776,89 @@ TEST_F(PointerChoreographerTest, SetsPointerIconForMouseOnTwoDisplays) {
		firstMousePc->assertPointerIconNotSet();
		}

		TEST_F(PointerChoreographerTest, A11yPointerMotionFilterMouse) {
		mChoreographer.setDisplayViewports(createViewports({DISPLAY_ID}));
		mChoreographer.setDefaultMouseDisplayId(DISPLAY_ID);
		mChoreographer.notifyInputDevicesChanged(
		{/id=/0,
		{generateTestDeviceInfo(DEVICE_ID, AINPUT_SOURCE_MOUSE,
		ui::LogicalDisplayId::INVALID)}});
		auto pc = assertPointerControllerCreated(ControllerType::MOUSE);
		ASSERT_EQ(DISPLAY_ID, pc->getDisplayId());

		pc->setPosition(100, 200);
		mChoreographer.setAccessibilityPointerMotionFilterEnabled(true);

		EXPECT_CALL(mMockPolicy,
		filterPointerMotionForAccessibility(testing::Eq(vec2{100, 200}),
		testing::Eq(vec2{10.f, 20.f}),
		testing::Eq(DISPLAY_ID)))
		.Times(1)
		.WillOnce(testing::Return(vec2{4, 13}));

		mChoreographer.notifyMotion(
		MotionArgsBuilder(AMOTION_EVENT_ACTION_HOVER_MOVE, AINPUT_SOURCE_MOUSE)
		.pointer(MOUSE_POINTER)
		.deviceId(DEVICE_ID)
		.displayId(ui::LogicalDisplayId::INVALID)
		.build());

		// Cursor position is decided by filtered delta, but pointer coord's relative values are kept.
		pc->assertPosition(104, 213);
		mTestListener.assertNotifyMotionWasCalled(AllOf(WithCoords(104, 213), WithDisplayId(DISPLAY_ID),
		WithCursorPosition(104, 213),
		WithRelativeMotion(10, 20)));
		}

		TEST_F(PointerChoreographerTest, A11yPointerMotionFilterTouchpad) {
		mChoreographer.setDisplayViewports(createViewports({DISPLAY_ID}));
		mChoreographer.setDefaultMouseDisplayId(DISPLAY_ID);
		mChoreographer.notifyInputDevicesChanged(
		{/id=/0,
		{generateTestDeviceInfo(DEVICE_ID, AINPUT_SOURCE_MOUSE \| AINPUT_SOURCE_TOUCHPAD,
		ui::LogicalDisplayId::INVALID)}});
		auto pc = assertPointerControllerCreated(ControllerType::MOUSE);
		ASSERT_EQ(DISPLAY_ID, pc->getDisplayId());

		pc->setPosition(100, 200);
		mChoreographer.setAccessibilityPointerMotionFilterEnabled(true);

		EXPECT_CALL(mMockPolicy,
		filterPointerMotionForAccessibility(testing::Eq(vec2{100, 200}),
		testing::Eq(vec2{10.f, 20.f}),
		testing::Eq(DISPLAY_ID)))
		.Times(1)
		.WillOnce(testing::Return(vec2{4, 13}));

		mChoreographer.notifyMotion(
		MotionArgsBuilder(AMOTION_EVENT_ACTION_HOVER_MOVE, AINPUT_SOURCE_MOUSE)
		.pointer(TOUCHPAD_POINTER)
		.deviceId(DEVICE_ID)
		.displayId(ui::LogicalDisplayId::INVALID)
		.build());

		// Cursor position is decided by filtered delta, but pointer coord's relative values are kept.
		pc->assertPosition(104, 213);
		mTestListener.assertNotifyMotionWasCalled(AllOf(WithCoords(104, 213), WithDisplayId(DISPLAY_ID),
		WithCursorPosition(104, 213),
		WithRelativeMotion(10, 20)));
		}

		TEST_F(PointerChoreographerTest, A11yPointerMotionFilterNotFilterTouch) {
		mChoreographer.notifyInputDevicesChanged(
		{/id=/0, {generateTestDeviceInfo(DEVICE_ID, AINPUT_SOURCE_TOUCHSCREEN, DISPLAY_ID)}});
		mChoreographer.setAccessibilityPointerMotionFilterEnabled(true);

		EXPECT_CALL(mMockPolicy, filterPointerMotionForAccessibility).Times(0);

		mChoreographer.notifyMotion(
		MotionArgsBuilder(AMOTION_EVENT_ACTION_DOWN, AINPUT_SOURCE_TOUCHSCREEN)
		.pointer(FIRST_TOUCH_POINTER)
		.deviceId(DEVICE_ID)
		.displayId(DISPLAY_ID)
		.build());
		}

		using SkipPointerScreenshotForPrivacySensitiveDisplaysFixtureParam =
		std::tuple<std::string_view /name/, uint32_t /source/, ControllerType, PointerBuilder,
		std::function<void(PointerChoreographer&)>, int32_t /action/>;