Add resampling to InputConsumerNoResampling (be9c544d) · Commits · e / os / android_frameworks_native

include/input/InputConsumerNoResampling.h

+18 −6

Original line number	Diff line number	Diff line
		@@ -17,6 +17,7 @@
		#pragma once

		#include <input/InputTransport.h>
		#include <input/Resampler.h>
		#include <utils/Looper.h>

		namespace android {
		@@ -47,13 +48,13 @@ public:
		/**
		* Consumes input events from an input channel.
		*
		* This is a re-implementation of InputConsumer that does not have resampling at the current moment.
		* A lot of the higher-level logic has been folded into this class, to make it easier to use.
		* In the legacy class, InputConsumer, the consumption logic was partially handled in the jni layer,
		* as well as various actions like adding the fd to the Choreographer.
		* This is a re-implementation of InputConsumer. At the moment it only supports resampling for
		* single pointer events. A lot of the higher-level logic has been folded into this class, to make
		* it easier to use. In the legacy class, InputConsumer, the consumption logic was partially handled
		* in the jni layer, as well as various actions like adding the fd to the Choreographer.
		*
		* TODO(b/297226446): use this instead of "InputConsumer":
		* - Add resampling to this class
		* - Add resampling for multiple pointer events.
		* - Allow various resampling strategies to be specified
		* - Delete the old "InputConsumer" and use this class instead, renaming it to "InputConsumer".
		* - Add tracing
		@@ -64,8 +65,18 @@ public:
		*/
		class InputConsumerNoResampling final {
		public:
		/**
		* @param callbacks are used to interact with InputConsumerNoResampling. They're called whenever
		* the event is ready to consume.
		* @param looper needs to be sp and not shared_ptr because it inherits from
		* RefBase
		* @param resampler the resampling strategy to use. If null, no resampling will be
		* performed.
		*/
		explicit InputConsumerNoResampling(const std::shared_ptr<InputChannel>& channel,
		sp<Looper> looper, InputConsumerCallbacks& callbacks);
		sp<Looper> looper, InputConsumerCallbacks& callbacks,
		std::unique_ptr<Resampler> resampler);

		~InputConsumerNoResampling();

		/**
		@@ -99,6 +110,7 @@ private:
		std::shared_ptr<InputChannel> mChannel;
		sp<Looper> mLooper;
		InputConsumerCallbacks& mCallbacks;
		std::unique_ptr<Resampler> mResampler;

		// Looper-related infrastructure
		/**

include/input/Resampler.h

0 → 100644

+115 −0

Original line number	Diff line number	Diff line
		/**
		* Copyright 2024 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#pragma once

		#include <chrono>
		#include <optional>

		#include <input/Input.h>
		#include <input/InputTransport.h>
		#include <input/RingBuffer.h>
		#include <utils/Timers.h>

		namespace android {

		/**
		* Resampler is an interface for resampling MotionEvents. Every resampling implementation
		* must use this interface to enable resampling inside InputConsumer's logic.
		*/
		struct Resampler {
		virtual ~Resampler() = default;

		/**
		* Tries to resample motionEvent at resampleTime. The provided resampleTime must be greater than
		* the latest sample time of motionEvent. It is not guaranteed that resampling occurs at
		* resampleTime. Interpolation may occur is futureSample is available. Otherwise, motionEvent
		* may be resampled by another method, or not resampled at all. Furthermore, it is the
		* implementer's responsibility to guarantee the following:
		* - If resampling occurs, a single additional sample should be added to motionEvent. That is,
		* if motionEvent had N samples before being passed to Resampler, then it will have N + 1
		* samples by the end of the resampling. No other field of motionEvent should be modified.
		* - If resampling does not occur, then motionEvent must not be modified in any way.
		*/
		virtual void resampleMotionEvent(const std::chrono::nanoseconds resampleTime,
		MotionEvent& motionEvent,
		const InputMessage* futureSample) = 0;
		};

		class LegacyResampler final : public Resampler {
		public:
		/**
		* Tries to resample `motionEvent` at `resampleTime` by adding a resampled sample at the end of
		* `motionEvent` with eventTime equal to `resampleTime` and pointer coordinates determined by
		* linear interpolation or linear extrapolation. An earlier `resampleTime` will be used if
		* extrapolation takes place and `resampleTime` is too far in the future. If `futureSample` is
		* not null, interpolation will occur. If `futureSample` is null and there is enough historical
		* data, LegacyResampler will extrapolate. Otherwise, no resampling takes place and
		* `motionEvent` is unmodified.
		*/
		void resampleMotionEvent(const std::chrono::nanoseconds resampleTime, MotionEvent& motionEvent,
		const InputMessage* futureSample) override;

		private:
		struct Pointer {
		PointerProperties properties;
		PointerCoords coords;
		};

		struct Sample {
		std::chrono::nanoseconds eventTime;
		Pointer pointer;

		Sample(const std::chrono::nanoseconds eventTime, const PointerProperties& properties,
		const PointerCoords& coords)
		: eventTime{eventTime}, pointer{properties, coords} {}
		};

		/**
		* Keeps track of the previous MotionEvent deviceId to enable comparison between the previous
		* and the current deviceId.
		*/
		std::optional<DeviceId> mPreviousDeviceId;

		/**
		* Up to two latest samples from MotionEvent. Updated every time resampleMotionEvent is called.
		* Note: We store up to two samples in order to simplify the implementation. Although,
		* calculations are possible with only one previous sample.
		*/
		RingBuffer<Sample> mLatestSamples{/capacity=/2};

		/**
		* Adds up to mLatestSamples.capacity() of motionEvent's latest samples to mLatestSamples. (If
		* motionEvent has fewer samples than mLatestSamples.capacity(), then the available samples are
		* added to mLatestSamples.)
		*/
		void updateLatestSamples(const MotionEvent& motionEvent);

		/**
		* May add a sample at the end of motionEvent with eventTime equal to resampleTime, and
		* interpolated coordinates between the latest motionEvent sample and futureSample.
		*/
		void interpolate(const std::chrono::nanoseconds resampleTime, MotionEvent& motionEvent,
		const InputMessage& futureSample) const;

		/**
		* May add a sample at the end of motionEvent by extrapolating from the latest two samples. The
		* added sample either has eventTime equal to resampleTime, or an earlier time if resampleTime
		* is too far in the future.
		*/
		void extrapolate(const std::chrono::nanoseconds resampleTime, MotionEvent& motionEvent) const;
		};
		} // namespace android
		No newline at end of file

libs/input/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -232,6 +232,7 @@ cc_library {
		"MotionPredictorMetricsManager.cpp",
		"PrintTools.cpp",
		"PropertyMap.cpp",
		"Resampler.cpp",
		"TfLiteMotionPredictor.cpp",
		"TouchVideoFrame.cpp",
		"VelocityControl.cpp",

libs/input/InputConsumerNoResampling.cpp

+22 −2

Original line number	Diff line number	Diff line
		@@ -17,6 +17,8 @@
		#define LOG_TAG "InputTransport"
		#define ATRACE_TAG ATRACE_TAG_INPUT

		#include <chrono>

		#include <inttypes.h>

		#include <android-base/logging.h>
		@@ -168,6 +170,10 @@ InputMessage createTimelineMessage(int32_t inputEventId, nsecs_t gpuCompletedTim
		return msg;
		}

		bool isPointerEvent(const MotionEvent& motionEvent) {
		return (motionEvent.getSource() & AINPUT_SOURCE_CLASS_POINTER) == AINPUT_SOURCE_CLASS_POINTER;
		}

		} // namespace

		using android::base::Result;
		@@ -177,8 +183,13 @@ using android::base::StringPrintf;

		InputConsumerNoResampling::InputConsumerNoResampling(const std::shared_ptr<InputChannel>& channel,
		sp<Looper> looper,
		InputConsumerCallbacks& callbacks)
		: mChannel(channel), mLooper(looper), mCallbacks(callbacks), mFdEvents(0) {
		InputConsumerCallbacks& callbacks,
		std::unique_ptr<Resampler> resampler)
		: mChannel(channel),
		mLooper(looper),
		mCallbacks(callbacks),
		mResampler(std::move(resampler)),
		mFdEvents(0) {
		LOG_ALWAYS_FATAL_IF(mLooper == nullptr);
		mCallback = sp<LooperEventCallback>::make(
		std::bind(&InputConsumerNoResampling::handleReceiveCallback, this,
		@@ -463,6 +474,15 @@ InputConsumerNoResampling::createBatchedMotionEvent(const nsecs_t frameTime,
		}
		messages.pop();
		}
		// Check if resampling should be performed.
		if (motionEvent != nullptr && isPointerEvent(*motionEvent) && mResampler != nullptr) {
		InputMessage* futureSample = nullptr;
		if (!messages.empty()) {
		futureSample = &messages.front();
		}
		mResampler->resampleMotionEvent(static_cast<std::chrono::nanoseconds>(frameTime),
		*motionEvent, futureSample);
		}
		return std::make_pair(std::move(motionEvent), firstSeqForBatch);
		}

libs/input/Resampler.cpp

0 → 100644

+151 −0

Original line number	Diff line number	Diff line
		/**
		* Copyright 2024 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#define LOG_TAG "LegacyResampler"

		#include <algorithm>
		#include <chrono>

		#include <android-base/logging.h>
		#include <android-base/properties.h>

		#include <input/Resampler.h>
		#include <utils/Timers.h>

		using std::chrono::nanoseconds;

		namespace android {

		namespace {

		const bool IS_DEBUGGABLE_BUILD =
		#if defined(__ANDROID__)
		android::base::GetBoolProperty("ro.debuggable", false);
		#else
		true;
		#endif

		bool debugResampling() {
		if (!IS_DEBUGGABLE_BUILD) {
		static const bool DEBUG_TRANSPORT_RESAMPLING =
		__android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Resampling",
		ANDROID_LOG_INFO);
		return DEBUG_TRANSPORT_RESAMPLING;
		}
		return __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Resampling", ANDROID_LOG_INFO);
		}

		constexpr std::chrono::milliseconds RESAMPLE_LATENCY{5};

		constexpr std::chrono::milliseconds RESAMPLE_MIN_DELTA{2};

		constexpr std::chrono::milliseconds RESAMPLE_MAX_DELTA{20};

		constexpr std::chrono::milliseconds RESAMPLE_MAX_PREDICTION{8};

		inline float lerp(float a, float b, float alpha) {
		return a + alpha * (b - a);
		}

		const PointerCoords calculateResampledCoords(const PointerCoords& a, const PointerCoords& b,
		const float alpha) {
		// Ensure the struct PointerCoords is initialized.
		PointerCoords resampledCoords{};
		resampledCoords.isResampled = true;
		resampledCoords.setAxisValue(AMOTION_EVENT_AXIS_X, lerp(a.getX(), b.getX(), alpha));
		resampledCoords.setAxisValue(AMOTION_EVENT_AXIS_Y, lerp(a.getY(), b.getY(), alpha));
		return resampledCoords;
		}
		} // namespace

		void LegacyResampler::updateLatestSamples(const MotionEvent& motionEvent) {
		const size_t motionEventSampleSize = motionEvent.getHistorySize() + 1;
		for (size_t i = 0; i < motionEventSampleSize; ++i) {
		Sample sample{static_cast<nanoseconds>(motionEvent.getHistoricalEventTime(i)),
		*motionEvent.getPointerProperties(0),
		motionEvent.getSamplePointerCoords()[i]};
		mLatestSamples.pushBack(sample);
		}
		}

		void LegacyResampler::interpolate(const nanoseconds resampleTime, MotionEvent& motionEvent,
		const InputMessage& futureSample) const {
		const Sample pastSample = mLatestSamples.back();
		const nanoseconds delta =
		static_cast<nanoseconds>(futureSample.body.motion.eventTime) - pastSample.eventTime;
		if (delta < RESAMPLE_MIN_DELTA) {
		LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too small: " << delta << "ns.";
		return;
		}
		const float alpha =
		std::chrono::duration<float, std::milli>(resampleTime - pastSample.eventTime) / delta;

		const PointerCoords resampledCoords =
		calculateResampledCoords(pastSample.pointer.coords,
		futureSample.body.motion.pointers[0].coords, alpha);
		motionEvent.addSample(resampleTime.count(), &resampledCoords, motionEvent.getId());
		}

		void LegacyResampler::extrapolate(const nanoseconds resampleTime, MotionEvent& motionEvent) const {
		if (mLatestSamples.size() < 2) {
		return;
		}
		const Sample pastSample = *(mLatestSamples.end() - 2);
		const Sample presentSample = *(mLatestSamples.end() - 1);
		const nanoseconds delta =
		static_cast<nanoseconds>(presentSample.eventTime - pastSample.eventTime);
		if (delta < RESAMPLE_MIN_DELTA) {
		LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too small: " << delta << "ns.";
		return;
		} else if (delta > RESAMPLE_MAX_DELTA) {
		LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too large: " << delta << "ns.";
		return;
		}
		// The farthest future time to which we can extrapolate. If the given resampleTime exceeds this,
		// we use this value as the resample time target.
		const nanoseconds farthestPrediction = static_cast<nanoseconds>(presentSample.eventTime) +
		std::min<nanoseconds>(delta / 2, RESAMPLE_MAX_PREDICTION);
		const nanoseconds newResampleTime =
		(resampleTime > farthestPrediction) ? (farthestPrediction) : (resampleTime);
		LOG_IF(INFO, debugResampling() && newResampleTime == farthestPrediction)
		<< "Resample time is too far in the future. Adjusting prediction from "
		<< (resampleTime - presentSample.eventTime) << " to "
		<< (farthestPrediction - presentSample.eventTime) << "ns.";
		const float alpha =
		std::chrono::duration<float, std::milli>(newResampleTime - pastSample.eventTime) /
		delta;

		const PointerCoords resampledCoords =
		calculateResampledCoords(pastSample.pointer.coords, presentSample.pointer.coords,
		alpha);
		motionEvent.addSample(newResampleTime.count(), &resampledCoords, motionEvent.getId());
		}

		void LegacyResampler::resampleMotionEvent(const nanoseconds resampleTime, MotionEvent& motionEvent,
		const InputMessage* futureSample) {
		if (mPreviousDeviceId && *mPreviousDeviceId != motionEvent.getDeviceId()) {
		mLatestSamples.clear();
		}
		mPreviousDeviceId = motionEvent.getDeviceId();
		updateLatestSamples(motionEvent);
		if (futureSample) {
		interpolate(resampleTime, motionEvent, *futureSample);
		} else {
		extrapolate(resampleTime, motionEvent);
		}
		LOG_IF(INFO, debugResampling()) << "Not resampled. Not enough data.";
		}
		} // namespace android