Merge "Refactor resampler logic to constrain MotionEvent mutation" into main (c5d58f50) · Commits · e / os / android_frameworks_native

include/input/Resampler.h

+27 −14

Original line number	Original line	Diff line number	Diff line
	@@ -44,7 +44,7 @@ struct Resampler {
	* samples by the end of the resampling. No other field of motionEvent should be modified.		* samples by the end of the resampling. No other field of motionEvent should be modified.
	* - If resampling does not occur, then motionEvent must not be modified in any way.		* - If resampling does not occur, then motionEvent must not be modified in any way.
	*/		*/
	virtual void resampleMotionEvent(const std::chrono::nanoseconds resampleTime,		virtual void resampleMotionEvent(std::chrono::nanoseconds resampleTime,
	MotionEvent& motionEvent,		MotionEvent& motionEvent,
	const InputMessage* futureSample) = 0;		const InputMessage* futureSample) = 0;
	};		};
	@@ -60,7 +60,7 @@ public:
	* data, LegacyResampler will extrapolate. Otherwise, no resampling takes place and		* data, LegacyResampler will extrapolate. Otherwise, no resampling takes place and
	* `motionEvent` is unmodified.		* `motionEvent` is unmodified.
	*/		*/
	void resampleMotionEvent(const std::chrono::nanoseconds resampleTime, MotionEvent& motionEvent,		void resampleMotionEvent(std::chrono::nanoseconds resampleTime, MotionEvent& motionEvent,
	const InputMessage* futureSample) override;		const InputMessage* futureSample) override;

	private:		private:
	@@ -72,10 +72,6 @@ private:
	struct Sample {		struct Sample {
	std::chrono::nanoseconds eventTime;		std::chrono::nanoseconds eventTime;
	Pointer pointer;		Pointer pointer;

	Sample(const std::chrono::nanoseconds eventTime, const PointerProperties& properties,
	const PointerCoords& coords)
	: eventTime{eventTime}, pointer{properties, coords} {}
	};		};

	/**		/**
	@@ -99,17 +95,34 @@ private:
	void updateLatestSamples(const MotionEvent& motionEvent);		void updateLatestSamples(const MotionEvent& motionEvent);

	/**		/**
	* May add a sample at the end of motionEvent with eventTime equal to resampleTime, and		* Checks if there are necessary conditions to interpolate. For example, interpolation cannot
	* interpolated coordinates between the latest motionEvent sample and futureSample.		* take place if samples are too far apart in time. mLatestSamples must have at least one sample
			* when canInterpolate is invoked.
			*/
			bool canInterpolate(const InputMessage& futureSample) const;

			/**
			* Returns a sample interpolated between the latest sample of mLatestSamples and futureSample,
			* if the conditions from canInterpolate are satisfied. Otherwise, returns nullopt.
			* mLatestSamples must have at least one sample when attemptInterpolation is called.
	*/		*/
	void interpolate(const std::chrono::nanoseconds resampleTime, MotionEvent& motionEvent,		std::optional<Sample> attemptInterpolation(std::chrono::nanoseconds resampleTime,
	const InputMessage& futureSample) const;		const InputMessage& futureSample) const;

	/**		/**
	* May add a sample at the end of motionEvent by extrapolating from the latest two samples. The		* Checks if there are necessary conditions to extrapolate. That is, there are at least two
	* added sample either has eventTime equal to resampleTime, or an earlier time if resampleTime		* samples in mLatestSamples, and delta is bounded within a time interval.
	* is too far in the future.
	*/		*/
	void extrapolate(const std::chrono::nanoseconds resampleTime, MotionEvent& motionEvent) const;		bool canExtrapolate() const;

			/**
			* Returns a sample extrapolated from the two samples of mLatestSamples, if the conditions from
			* canExtrapolate are satisfied. The returned sample either has eventTime equal to resampleTime,
			* or an earlier time if resampleTime is too far in the future. If canExtrapolate returns false,
			* this function returns nullopt.
			*/
			std::optional<Sample> attemptExtrapolation(std::chrono::nanoseconds resampleTime) const;

			inline static void addSampleToMotionEvent(const Sample& sample, MotionEvent& motionEvent);
	};		};
	} // namespace android		} // namespace android
			No newline at end of file

libs/input/Resampler.cpp

+72 −32

Original line number	Original line	Diff line number	Diff line
	@@ -60,8 +60,8 @@ inline float lerp(float a, float b, float alpha) {
	return a + alpha * (b - a);		return a + alpha * (b - a);
	}		}

	const PointerCoords calculateResampledCoords(const PointerCoords& a, const PointerCoords& b,		PointerCoords calculateResampledCoords(const PointerCoords& a, const PointerCoords& b,
	const float alpha) {		float alpha) {
	// We use the value of alpha to initialize resampledCoords with the latest sample information.		// We use the value of alpha to initialize resampledCoords with the latest sample information.
	PointerCoords resampledCoords = (alpha < 1.0f) ? a : b;		PointerCoords resampledCoords = (alpha < 1.0f) ? a : b;
	resampledCoords.isResampled = true;		resampledCoords.isResampled = true;
	@@ -72,52 +72,85 @@ const PointerCoords calculateResampledCoords(const PointerCoords& a, const Point
	} // namespace		} // namespace

	void LegacyResampler::updateLatestSamples(const MotionEvent& motionEvent) {		void LegacyResampler::updateLatestSamples(const MotionEvent& motionEvent) {
	const size_t motionEventSampleSize = motionEvent.getHistorySize() + 1;		const size_t numSamples = motionEvent.getHistorySize() + 1;
	for (size_t i = 0; i < motionEventSampleSize; ++i) {		for (size_t i = 0; i < numSamples; ++i) {
	Sample sample{static_cast<nanoseconds>(motionEvent.getHistoricalEventTime(i)),		mLatestSamples.pushBack(
	*motionEvent.getPointerProperties(0),		Sample{static_cast<nanoseconds>(motionEvent.getHistoricalEventTime(i)),
	motionEvent.getSamplePointerCoords()[i]};		Pointer{*motionEvent.getPointerProperties(0),
	mLatestSamples.pushBack(sample);		motionEvent.getSamplePointerCoords()[i]}});
	}		}
	}		}

	void LegacyResampler::interpolate(const nanoseconds resampleTime, MotionEvent& motionEvent,		bool LegacyResampler::canInterpolate(const InputMessage& futureSample) const {
	const InputMessage& futureSample) const {		LOG_IF(FATAL, mLatestSamples.empty())
	const Sample pastSample = mLatestSamples.back();		<< "Not resampled. mLatestSamples must not be empty to interpolate.";

			const Sample& pastSample = *(mLatestSamples.end() - 1);
	const nanoseconds delta =		const nanoseconds delta =
	static_cast<nanoseconds>(futureSample.body.motion.eventTime) - pastSample.eventTime;		static_cast<nanoseconds>(futureSample.body.motion.eventTime) - pastSample.eventTime;
	if (delta < RESAMPLE_MIN_DELTA) {		if (delta < RESAMPLE_MIN_DELTA) {
	LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too small: " << delta << "ns.";		LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too small: " << delta << "ns.";
	return;		return false;
			}
			return true;
			}

			std::optional<LegacyResampler::Sample> LegacyResampler::attemptInterpolation(
			nanoseconds resampleTime, const InputMessage& futureSample) const {
			if (!canInterpolate(futureSample)) {
			return std::nullopt;
	}		}
			LOG_IF(FATAL, mLatestSamples.empty())
			<< "Not resampled. mLatestSamples must not be empty to interpolate.";

			const Sample& pastSample = *(mLatestSamples.end() - 1);
			const nanoseconds delta =
			static_cast<nanoseconds>(futureSample.body.motion.eventTime) - pastSample.eventTime;
	const float alpha =		const float alpha =
	std::chrono::duration<float, std::milli>(resampleTime - pastSample.eventTime) / delta;		std::chrono::duration<float, std::milli>(resampleTime - pastSample.eventTime) / delta;

	const PointerCoords resampledCoords =		const PointerCoords resampledCoords =
	calculateResampledCoords(pastSample.pointer.coords,		calculateResampledCoords(pastSample.pointer.coords,
	futureSample.body.motion.pointers[0].coords, alpha);		futureSample.body.motion.pointers[0].coords, alpha);
	motionEvent.addSample(resampleTime.count(), &resampledCoords, motionEvent.getId());
			return Sample{resampleTime, Pointer{pastSample.pointer.properties, resampledCoords}};
	}		}

	void LegacyResampler::extrapolate(const nanoseconds resampleTime, MotionEvent& motionEvent) const {		bool LegacyResampler::canExtrapolate() const {
	if (mLatestSamples.size() < 2) {		if (mLatestSamples.size() < 2) {
	return;		LOG_IF(INFO, debugResampling()) << "Not resampled. Not enough data.";
			return false;
	}		}
	const Sample pastSample = *(mLatestSamples.end() - 2);
	const Sample presentSample = *(mLatestSamples.end() - 1);		const Sample& pastSample = *(mLatestSamples.end() - 2);
	const nanoseconds delta =		const Sample& presentSample = *(mLatestSamples.end() - 1);
	static_cast<nanoseconds>(presentSample.eventTime - pastSample.eventTime);
			const nanoseconds delta = presentSample.eventTime - pastSample.eventTime;
	if (delta < RESAMPLE_MIN_DELTA) {		if (delta < RESAMPLE_MIN_DELTA) {
	LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too small: " << delta << "ns.";		LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too small: " << delta << "ns.";
	return;		return false;
	} else if (delta > RESAMPLE_MAX_DELTA) {		} else if (delta > RESAMPLE_MAX_DELTA) {
	LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too large: " << delta << "ns.";		LOG_IF(INFO, debugResampling()) << "Not resampled. Delta is too large: " << delta << "ns.";
	return;		return false;
	}		}
			return true;
			}

			std::optional<LegacyResampler::Sample> LegacyResampler::attemptExtrapolation(
			nanoseconds resampleTime) const {
			if (!canExtrapolate()) {
			return std::nullopt;
			}
			LOG_IF(FATAL, mLatestSamples.size() < 2)
			<< "Not resampled. mLatestSamples must have at least two samples to extrapolate.";

			const Sample& pastSample = *(mLatestSamples.end() - 2);
			const Sample& presentSample = *(mLatestSamples.end() - 1);

			const nanoseconds delta = presentSample.eventTime - pastSample.eventTime;
	// The farthest future time to which we can extrapolate. If the given resampleTime exceeds this,		// The farthest future time to which we can extrapolate. If the given resampleTime exceeds this,
	// we use this value as the resample time target.		// we use this value as the resample time target.
	const nanoseconds farthestPrediction = static_cast<nanoseconds>(presentSample.eventTime) +		const nanoseconds farthestPrediction =
	std::min<nanoseconds>(delta / 2, RESAMPLE_MAX_PREDICTION);		presentSample.eventTime + std::min<nanoseconds>(delta / 2, RESAMPLE_MAX_PREDICTION);
	const nanoseconds newResampleTime =		const nanoseconds newResampleTime =
	(resampleTime > farthestPrediction) ? (farthestPrediction) : (resampleTime);		(resampleTime > farthestPrediction) ? (farthestPrediction) : (resampleTime);
	LOG_IF(INFO, debugResampling() && newResampleTime == farthestPrediction)		LOG_IF(INFO, debugResampling() && newResampleTime == farthestPrediction)
	@@ -127,25 +160,32 @@ void LegacyResampler::extrapolate(const nanoseconds resampleTime, MotionEvent& m
	const float alpha =		const float alpha =
	std::chrono::duration<float, std::milli>(newResampleTime - pastSample.eventTime) /		std::chrono::duration<float, std::milli>(newResampleTime - pastSample.eventTime) /
	delta;		delta;

	const PointerCoords resampledCoords =		const PointerCoords resampledCoords =
	calculateResampledCoords(pastSample.pointer.coords, presentSample.pointer.coords,		calculateResampledCoords(pastSample.pointer.coords, presentSample.pointer.coords,
	alpha);		alpha);
	motionEvent.addSample(newResampleTime.count(), &resampledCoords, motionEvent.getId());
			return Sample{newResampleTime, Pointer{presentSample.pointer.properties, resampledCoords}};
			}

			inline void LegacyResampler::addSampleToMotionEvent(const Sample& sample,
			MotionEvent& motionEvent) {
			motionEvent.addSample(sample.eventTime.count(), &sample.pointer.coords, motionEvent.getId());
	}		}

	void LegacyResampler::resampleMotionEvent(const nanoseconds resampleTime, MotionEvent& motionEvent,		void LegacyResampler::resampleMotionEvent(nanoseconds resampleTime, MotionEvent& motionEvent,
	const InputMessage* futureSample) {		const InputMessage* futureSample) {
	if (mPreviousDeviceId && *mPreviousDeviceId != motionEvent.getDeviceId()) {		if (mPreviousDeviceId && *mPreviousDeviceId != motionEvent.getDeviceId()) {
	mLatestSamples.clear();		mLatestSamples.clear();
	}		}
	mPreviousDeviceId = motionEvent.getDeviceId();		mPreviousDeviceId = motionEvent.getDeviceId();

	updateLatestSamples(motionEvent);		updateLatestSamples(motionEvent);
	if (futureSample) {
	interpolate(resampleTime, motionEvent, *futureSample);		const std::optional<Sample> sample = (futureSample != nullptr)
	} else {		? (attemptInterpolation(resampleTime, *futureSample))
	extrapolate(resampleTime, motionEvent);		: (attemptExtrapolation(resampleTime));
			if (sample.has_value()) {
			addSampleToMotionEvent(*sample, motionEvent);
	}		}
	LOG_IF(INFO, debugResampling()) << "Not resampled. Not enough data.";
	}		}
	} // namespace android		} // namespace android