Merge "Add multiple device resampling support to InputConsumerNoResampling with tests" into main (d0285f45) · Commits · e / os / android_frameworks_native

include/input/InputConsumerNoResampling.h

+18 −7

Original line number	Diff line number	Diff line
		@@ -16,6 +16,7 @@

		#pragma once

		#include <functional>
		#include <map>
		#include <memory>
		#include <optional>
		@@ -75,12 +76,13 @@ public:
		* the event is ready to consume.
		* @param looper needs to be sp and not shared_ptr because it inherits from
		* RefBase
		* @param resampler the resampling strategy to use. If null, no resampling will be
		* performed.
		* @param resamplerCreator callable that returns the resampling strategy to be used. If null, no
		* resampling will be performed. resamplerCreator must never return nullptr.
		*/
		explicit InputConsumerNoResampling(const std::shared_ptr<InputChannel>& channel,
		sp<Looper> looper, InputConsumerCallbacks& callbacks,
		std::unique_ptr<Resampler> resampler);
		explicit InputConsumerNoResampling(
		const std::shared_ptr<InputChannel>& channel, sp<Looper> looper,
		InputConsumerCallbacks& callbacks,
		std::function<std::unique_ptr<Resampler>()> resamplerCreator);

		~InputConsumerNoResampling();

		@@ -117,7 +119,13 @@ private:
		std::shared_ptr<InputChannel> mChannel;
		sp<Looper> mLooper;
		InputConsumerCallbacks& mCallbacks;
		std::unique_ptr<Resampler> mResampler;
		const std::function<std::unique_ptr<Resampler>()> mResamplerCreator;

		/**
		* A map to manage multidevice resampling. Each contained resampler is never null. This map is
		* only modified by handleMessages.
		*/
		std::map<DeviceId, std::unique_ptr<Resampler>> mResamplers;

		// Looper-related infrastructure
		/**
		@@ -190,7 +198,10 @@ private:
		/**
		* Batch messages that can be batched. When an unbatchable message is encountered, send it
		* to the InputConsumerCallbacks immediately. If there are batches remaining,
		* notify InputConsumerCallbacks.
		* notify InputConsumerCallbacks. If a resampleable ACTION_DOWN message is received, then a
		* resampler is inserted for that deviceId in mResamplers. If a resampleable ACTION_UP or
		* ACTION_CANCEL message is received then the resampler associated to that deviceId is erased
		* from mResamplers.
		*/
		void handleMessages(std::vector<InputMessage>&& messages);
		/**

include/input/Resampler.h

+0 −6

Original line number	Diff line number	Diff line
		@@ -91,12 +91,6 @@ private:
		}
		};

		/**
		* Keeps track of the previous MotionEvent deviceId to enable comparison between the previous
		* and the current deviceId.
		*/
		std::optional<DeviceId> mPreviousDeviceId;

		/**
		* Up to two latest samples from MotionEvent. Updated every time resampleMotionEvent is called.
		* Note: We store up to two samples in order to simplify the implementation. Although,

libs/input/InputConsumerNoResampling.cpp

+42 −23

Original line number	Diff line number	Diff line
		@@ -17,8 +17,6 @@
		#define LOG_TAG "InputConsumerNoResampling"
		#define ATRACE_TAG ATRACE_TAG_INPUT

		#include <chrono>

		#include <inttypes.h>

		#include <android-base/logging.h>
		@@ -39,6 +37,8 @@ namespace {

		using std::chrono::nanoseconds;

		using android::base::Result;

		/**
		* Log debug messages relating to the consumer end of the transport channel.
		* Enable this via "adb shell setprop log.tag.InputTransportConsumer DEBUG" (requires restart)
		@@ -169,24 +169,18 @@ InputMessage createTimelineMessage(int32_t inputEventId, nsecs_t gpuCompletedTim
		msg.body.timeline.graphicsTimeline[GraphicsTimeline::PRESENT_TIME] = presentTime;
		return msg;
		}

		bool isPointerEvent(const MotionEvent& motionEvent) {
		return (motionEvent.getSource() & AINPUT_SOURCE_CLASS_POINTER) == AINPUT_SOURCE_CLASS_POINTER;
		}
		} // namespace

		using android::base::Result;

		// --- InputConsumerNoResampling ---

		InputConsumerNoResampling::InputConsumerNoResampling(const std::shared_ptr<InputChannel>& channel,
		sp<Looper> looper,
		InputConsumerNoResampling::InputConsumerNoResampling(
		const std::shared_ptr<InputChannel>& channel, sp<Looper> looper,
		InputConsumerCallbacks& callbacks,
		std::unique_ptr<Resampler> resampler)
		std::function<std::unique_ptr<Resampler>()> resamplerCreator)
		: mChannel{channel},
		mLooper{looper},
		mCallbacks{callbacks},
		mResampler{std::move(resampler)},
		mResamplerCreator{std::move(resamplerCreator)},
		mFdEvents(0) {
		LOG_ALWAYS_FATAL_IF(mLooper == nullptr);
		mCallback = sp<LooperEventCallback>::make(
		@@ -319,7 +313,6 @@ void InputConsumerNoResampling::setFdEvents(int events) {
		}

		void InputConsumerNoResampling::handleMessages(std::vector<InputMessage>&& messages) {
		// TODO(b/297226446) : add resampling
		for (const InputMessage& msg : messages) {
		if (msg.header.type == InputMessage::Type::MOTION) {
		const int32_t action = msg.body.motion.action;
		@@ -329,12 +322,31 @@ void InputConsumerNoResampling::handleMessages(std::vector<InputMessage>&& messa
		action == AMOTION_EVENT_ACTION_HOVER_MOVE) &&
		(isFromSource(source, AINPUT_SOURCE_CLASS_POINTER) \|\|
		isFromSource(source, AINPUT_SOURCE_CLASS_JOYSTICK));

		const bool canResample = (mResamplerCreator != nullptr) &&
		(isFromSource(source, AINPUT_SOURCE_CLASS_POINTER));
		if (canResample) {
		if (action == AMOTION_EVENT_ACTION_DOWN) {
		if (std::unique_ptr<Resampler> resampler = mResamplerCreator();
		resampler != nullptr) {
		const auto [_, inserted] =
		mResamplers.insert(std::pair(deviceId, std::move(resampler)));
		LOG_IF(WARNING, !inserted) << deviceId << "already exists in mResamplers";
		}
		}
		}

		if (batchableEvent) {
		// add it to batch
		mBatches[deviceId].emplace(msg);
		} else {
		// consume all pending batches for this device immediately
		consumeBatchedInputEvents(deviceId, /requestedFrameTime=/std::nullopt);
		if (canResample &&
		(action == AMOTION_EVENT_ACTION_UP \|\| action == AMOTION_EVENT_ACTION_CANCEL)) {
		LOG_IF(INFO, mResamplers.erase(deviceId) == 0)
		<< deviceId << "does not exist in mResamplers";
		}
		handleMessage(msg);
		}
		} else {
		@@ -456,8 +468,13 @@ InputConsumerNoResampling::createBatchedMotionEvent(const nsecs_t requestedFrame
		std::queue<InputMessage>& messages) {
		std::unique_ptr<MotionEvent> motionEvent;
		std::optional<uint32_t> firstSeqForBatch;
		const nanoseconds resampleLatency =
		(mResampler != nullptr) ? mResampler->getResampleLatency() : nanoseconds{0};

		LOG_IF(FATAL, messages.empty()) << "messages queue is empty!";
		const DeviceId deviceId = messages.front().body.motion.deviceId;
		const auto resampler = mResamplers.find(deviceId);
		const nanoseconds resampleLatency = (resampler != mResamplers.cend())
		? resampler->second->getResampleLatency()
		: nanoseconds{0};
		const nanoseconds adjustedFrameTime = nanoseconds{requestedFrameTime} - resampleLatency;

		while (!messages.empty() &&
		@@ -474,15 +491,17 @@ InputConsumerNoResampling::createBatchedMotionEvent(const nsecs_t requestedFrame
		}
		messages.pop();
		}

		// Check if resampling should be performed.
		if (motionEvent != nullptr && isPointerEvent(*motionEvent) && mResampler != nullptr) {
		InputMessage* futureSample = nullptr;
		if (!messages.empty()) {
		futureSample = &messages.front();
		}
		mResampler->resampleMotionEvent(nanoseconds{requestedFrameTime}, *motionEvent,
		if ((motionEvent != nullptr) && (resampler != mResamplers.cend())) {
		resampler->second->resampleMotionEvent(nanoseconds{requestedFrameTime}, *motionEvent,
		futureSample);
		}

		return std::make_pair(std::move(motionEvent), firstSeqForBatch);
		}

libs/input/Resampler.cpp

+0 −5

Original line number	Diff line number	Diff line
		@@ -247,11 +247,6 @@ nanoseconds LegacyResampler::getResampleLatency() const {

		void LegacyResampler::resampleMotionEvent(nanoseconds frameTime, MotionEvent& motionEvent,
		const InputMessage* futureSample) {
		if (mPreviousDeviceId && *mPreviousDeviceId != motionEvent.getDeviceId()) {
		mLatestSamples.clear();
		}
		mPreviousDeviceId = motionEvent.getDeviceId();

		const nanoseconds resampleTime = frameTime - RESAMPLE_LATENCY;

		updateLatestSamples(motionEvent);

libs/input/tests/InputConsumer_test.cpp

+154 −6

Original line number	Diff line number	Diff line
		@@ -16,6 +16,9 @@

		#include <input/InputConsumerNoResampling.h>

		#include <gtest/gtest.h>

		#include <chrono>
		#include <memory>
		#include <optional>

		@@ -25,7 +28,9 @@
		#include <gmock/gmock.h>
		#include <gtest/gtest.h>
		#include <input/BlockingQueue.h>
		#include <input/Input.h>
		#include <input/InputEventBuilders.h>
		#include <input/Resampler.h>
		#include <utils/Looper.h>
		#include <utils/StrongPointer.h>

		@@ -37,8 +42,18 @@ using std::chrono::nanoseconds;

		using ::testing::AllOf;
		using ::testing::Matcher;
		using ::testing::Not;

		struct Pointer {
		int32_t id{0};
		ToolType toolType{ToolType::FINGER};
		float x{0.0f};
		float y{0.0f};
		bool isResampled{false};

		PointerBuilder asPointerBuilder() const {
		return PointerBuilder{id, toolType}.x(x).y(y).isResampled(isResampled);
		}
		};
		} // namespace

		class InputConsumerTest : public testing::Test, public InputConsumerCallbacks {
		@@ -47,9 +62,9 @@ protected:
		: mClientTestChannel{std::make_shared<TestInputChannel>("TestChannel")},
		mLooper{sp<Looper>::make(/allowNonCallbacks=/false)} {
		Looper::setForThread(mLooper);
		mConsumer =
		std::make_unique<InputConsumerNoResampling>(mClientTestChannel, mLooper, *this,
		std::make_unique<LegacyResampler>());
		mConsumer = std::make_unique<
		InputConsumerNoResampling>(mClientTestChannel, mLooper, *this,
		[]() { return std::make_unique<LegacyResampler>(); });
		}

		void invokeLooperCallback() const {
		@@ -71,6 +86,9 @@ protected:
		EXPECT_THAT(*motionEvent, matcher);
		}

		InputMessage nextPointerMessage(std::chrono::nanoseconds eventTime, DeviceId deviceId,
		int32_t action, const Pointer& pointer);

		std::shared_ptr<TestInputChannel> mClientTestChannel;
		sp<Looper> mLooper;
		std::unique_ptr<InputConsumerNoResampling> mConsumer;
		@@ -83,6 +101,7 @@ protected:
		BlockingQueue<std::unique_ptr<TouchModeEvent>> mTouchModeEvents;

		private:
		uint32_t mLastSeq{0};
		size_t mOnBatchedInputEventPendingInvocationCount{0};

		// InputConsumerCallbacks interface
		@@ -118,6 +137,19 @@ private:
		};
		};

		InputMessage InputConsumerTest::nextPointerMessage(std::chrono::nanoseconds eventTime,
		DeviceId deviceId, int32_t action,
		const Pointer& pointer) {
		++mLastSeq;
		return InputMessageBuilder{InputMessage::Type::MOTION, mLastSeq}
		.eventTime(eventTime.count())
		.deviceId(deviceId)
		.source(AINPUT_SOURCE_TOUCHSCREEN)
		.action(action)
		.pointer(pointer.asPointerBuilder())
		.build();
		}

		TEST_F(InputConsumerTest, MessageStreamBatchedInMotionEvent) {
		mClientTestChannel->enqueueMessage(InputMessageBuilder{InputMessage::Type::MOTION, /seq=/0}
		.eventTime(nanoseconds{0ms}.count())
		@@ -235,8 +267,7 @@ TEST_F(InputConsumerTest, BatchedEventsMultiDeviceConsumption) {
		.build());

		invokeLooperCallback();
		assertReceivedMotionEvent(AllOf(WithDeviceId(0), WithMotionAction(AMOTION_EVENT_ACTION_MOVE),
		Not(MotionEventIsResampled())));
		assertReceivedMotionEvent(AllOf(WithDeviceId(0), WithMotionAction(AMOTION_EVENT_ACTION_MOVE)));

		mClientTestChannel->assertFinishMessage(/seq=/0, /handled=/true);
		mClientTestChannel->assertFinishMessage(/seq=/4, /handled=/true);
		@@ -244,4 +275,121 @@ TEST_F(InputConsumerTest, BatchedEventsMultiDeviceConsumption) {
		mClientTestChannel->assertFinishMessage(/seq=/2, /handled=/true);
		mClientTestChannel->assertFinishMessage(/seq=/3, /handled=/true);
		}

		/**
		* The test supposes a 60Hz Vsync rate and a 200Hz input rate. The InputMessages are intertwined as
		* in a real use cases. The test's two devices should be resampled independently. Moreover, the
		* InputMessage stream layout for the test is:
		*
		* DOWN(0, 0ms)
		* MOVE(0, 5ms)
		* MOVE(0, 10ms)
		* DOWN(1, 15ms)
		*
		* CONSUME(16ms)
		*
		* MOVE(1, 20ms)
		* MOVE(1, 25ms)
		* MOVE(0, 30ms)
		*
		* CONSUME(32ms)
		*
		* MOVE(0, 35ms)
		* UP(1, 40ms)
		* UP(0, 45ms)
		*
		* CONSUME(48ms)
		*
		* The first field is device ID, and the second field is event time.
		*/
		TEST_F(InputConsumerTest, MultiDeviceResampling) {
		mClientTestChannel->enqueueMessage(nextPointerMessage(0ms, /deviceId=/0,
		AMOTION_EVENT_ACTION_DOWN,
		Pointer{.x = 0, .y = 0}));

		mClientTestChannel->assertNoSentMessages();

		invokeLooperCallback();
		assertReceivedMotionEvent(AllOf(WithDeviceId(0), WithMotionAction(AMOTION_EVENT_ACTION_DOWN),
		WithSampleCount(1)));

		mClientTestChannel->enqueueMessage(nextPointerMessage(5ms, /deviceId=/0,
		AMOTION_EVENT_ACTION_MOVE,
		Pointer{.x = 1.0f, .y = 2.0f}));
		mClientTestChannel->enqueueMessage(nextPointerMessage(10ms, /deviceId=/0,
		AMOTION_EVENT_ACTION_MOVE,
		Pointer{.x = 2.0f, .y = 4.0f}));
		mClientTestChannel->enqueueMessage(nextPointerMessage(15ms, /deviceId=/1,
		AMOTION_EVENT_ACTION_DOWN,
		Pointer{.x = 10.0f, .y = 10.0f}));

		invokeLooperCallback();
		mConsumer->consumeBatchedInputEvents(16'000'000 /ns/);

		assertReceivedMotionEvent(AllOf(WithDeviceId(1), WithMotionAction(AMOTION_EVENT_ACTION_DOWN),
		WithSampleCount(1)));
		assertReceivedMotionEvent(
		AllOf(WithDeviceId(0), WithMotionAction(AMOTION_EVENT_ACTION_MOVE), WithSampleCount(3),
		WithSample(/sampleIndex=/2,
		Sample{11ms,
		{PointerArgs{.x = 2.2f, .y = 4.4f, .isResampled = true}}})));

		mClientTestChannel->enqueueMessage(nextPointerMessage(20ms, /deviceId=/1,
		AMOTION_EVENT_ACTION_MOVE,
		Pointer{.x = 11.0f, .y = 12.0f}));
		mClientTestChannel->enqueueMessage(nextPointerMessage(25ms, /deviceId=/1,
		AMOTION_EVENT_ACTION_MOVE,
		Pointer{.x = 12.0f, .y = 14.0f}));
		mClientTestChannel->enqueueMessage(nextPointerMessage(30ms, /deviceId=/0,
		AMOTION_EVENT_ACTION_MOVE,
		Pointer{.x = 5.0f, .y = 6.0f}));

		invokeLooperCallback();
		assertOnBatchedInputEventPendingWasCalled();
		mConsumer->consumeBatchedInputEvents(32'000'000 /ns/);

		assertReceivedMotionEvent(
		AllOf(WithDeviceId(1), WithMotionAction(AMOTION_EVENT_ACTION_MOVE), WithSampleCount(3),
		WithSample(/sampleIndex=/2,
		Sample{27ms,
		{PointerArgs{.x = 12.4f, .y = 14.8f, .isResampled = true}}})));

		mClientTestChannel->enqueueMessage(nextPointerMessage(35ms, /deviceId=/0,
		AMOTION_EVENT_ACTION_MOVE,
		Pointer{.x = 8.0f, .y = 9.0f}));
		mClientTestChannel->enqueueMessage(nextPointerMessage(40ms, /deviceId=/1,
		AMOTION_EVENT_ACTION_UP,
		Pointer{.x = 12.0f, .y = 14.0f}));
		mClientTestChannel->enqueueMessage(nextPointerMessage(45ms, /deviceId=/0,
		AMOTION_EVENT_ACTION_UP,
		Pointer{.x = 8.0f, .y = 9.0f}));

		invokeLooperCallback();
		mConsumer->consumeBatchedInputEvents(48'000'000 /ns/);

		assertReceivedMotionEvent(
		AllOf(WithDeviceId(1), WithMotionAction(AMOTION_EVENT_ACTION_UP), WithSampleCount(1)));

		assertReceivedMotionEvent(
		AllOf(WithDeviceId(0), WithMotionAction(AMOTION_EVENT_ACTION_MOVE), WithSampleCount(3),
		WithSample(/sampleIndex=/2,
		Sample{37'500'000ns,
		{PointerArgs{.x = 9.5f, .y = 10.5f, .isResampled = true}}})));

		assertReceivedMotionEvent(
		AllOf(WithDeviceId(0), WithMotionAction(AMOTION_EVENT_ACTION_UP), WithSampleCount(1)));

		// The sequence order is based on the expected consumption. Each sequence number corresponds to
		// one of the previously enqueued messages.
		mClientTestChannel->assertFinishMessage(/seq=/1, /handled=/true);
		mClientTestChannel->assertFinishMessage(/seq=/4, /handled=/true);
		mClientTestChannel->assertFinishMessage(/seq=/2, /handled=/true);
		mClientTestChannel->assertFinishMessage(/seq=/3, /handled=/true);
		mClientTestChannel->assertFinishMessage(/seq=/5, /handled=/true);
		mClientTestChannel->assertFinishMessage(/seq=/6, /handled=/true);
		mClientTestChannel->assertFinishMessage(/seq=/9, /handled=/true);
		mClientTestChannel->assertFinishMessage(/seq=/7, /handled=/true);
		mClientTestChannel->assertFinishMessage(/seq=/8, /handled=/true);
		mClientTestChannel->assertFinishMessage(/seq=/10, /handled=/true);
		}
		} // namespace android