Merge "Add support for TFLite motion prediction model." (e5b2c6f4) · Commits · e / os / android_frameworks_base

core/java/android/view/MotionPredictor.java

+44 −29

Original line number	Diff line number	Diff line
		@@ -22,26 +22,27 @@ import android.content.Context;
		import libcore.util.NativeAllocationRegistry;

		import java.util.Arrays;
		import java.util.Collections;
		import java.util.List;

		/**
		* Calculate motion predictions.
		*
		* Feed motion events to this class in order to generate the predicted events. The prediction
		* functionality may not be available on all devices. Check if a specific source is supported on a
		* given input device using #isPredictionAvailable.
		* Feed motion events to this class in order to generate predicted future events. The prediction
		* functionality may not be available on all devices: check if a specific source is supported on a
		* given input device using {@link #isPredictionAvailable}.
		*
		* Send all of the events that were received from the system here in order to generate complete,
		* accurate predictions. When processing the returned predictions, make sure to consider all of the
		* {@link MotionEvent#getHistoricalAxisValue historical samples}.
		* Send all of the events that were received from the system to {@link #record} to generate
		* complete, accurate predictions from {@link #predict}. When processing the returned predictions,
		* make sure to consider all of the {@link MotionEvent#getHistoricalAxisValue historical samples}.
		*/
		// Acts as a pass-through to the native MotionPredictor object.
		// Do not store any state in this Java layer, or add any business logic here. All of the
		// implementation details should go into the native MotionPredictor.
		// The context / resource access must be here rather than in native layer due to the lack of the
		// corresponding native API surface.
		public final class MotionPredictor {

		// This is a pass-through to the native MotionPredictor object (mPtr). Do not store any state or
		// add any business logic here -- all of the implementation details should go into the native
		// MotionPredictor (except for accessing the context/resources, which have no corresponding
		// native API).

		private static class RegistryHolder {
		public static final NativeAllocationRegistry REGISTRY =
		NativeAllocationRegistry.createMalloced(
		@@ -67,49 +68,63 @@ public final class MotionPredictor {

		/**
		* Record a movement so that in the future, a prediction for the current gesture can be
		* generated. Ensure to add all motions from the gesture of interest to generate the correct
		* prediction.
		* generated. Ensure to add all motions from the gesture of interest to generate correct
		* predictions.
		* @param event The received event
		*/
		public void record(@NonNull MotionEvent event) {
		if (!isPredictionEnabled()) {
		return;
		}
		nativeRecord(mPtr, event);
		}

		/**
		* Get predicted events for all gestures that have been provided to the 'record' function.
		* Get predicted events for all gestures that have been provided to {@link #record}.
		* If events from multiple devices were sent to 'record', this will produce a separate
		* {@link MotionEvent} for each device id. The returned list may be empty if no predictions for
		* any of the added events are available.
		* {@link MotionEvent} for each device. The returned list may be empty if no predictions for
		* any of the added events/devices are available.
		* Predictions may not reach the requested timestamp if the confidence in the prediction results
		* is low.
		*
		* @param predictionTimeNanos The time that the prediction should target, in the
		* {@link android.os.SystemClock#uptimeMillis} time base, but in nanoseconds.
		*
		* @return the list of predicted motion events, for each device id. Ensure to check the
		* historical data in addition to the latest ({@link MotionEvent#getX getX()},
		* {@link MotionEvent#getY getY()}) coordinates for smoothest prediction curves. Empty list is
		* returned if predictions are not supported, or not possible for the current set of gestures.
		* @return A list of predicted motion events, with at most one for each device observed by
		* {@link #record}. Be sure to check the historical data in addition to the latest
		* ({@link MotionEvent#getX getX()}, {@link MotionEvent#getY getY()}) coordinates for smooth
		* prediction curves. An empty list is returned if predictions are not supported, or not
		* possible for the current set of gestures.
		*/
		@NonNull
		public List<MotionEvent> predict(long predictionTimeNanos) {
		if (!isPredictionEnabled()) {
		return Collections.emptyList();
		}
		return Arrays.asList(nativePredict(mPtr, predictionTimeNanos));
		}

		/**
		* Check whether this device supports motion predictions for the given source type.
		*
		* @param deviceId The input device id
		* @param source The source of input events
		* @return True if the current device supports predictions, false otherwise.
		*/
		public boolean isPredictionAvailable(int deviceId, int source) {
		private boolean isPredictionEnabled() {
		// Device-specific override
		if (!mContext.getResources().getBoolean(
		com.android.internal.R.bool.config_enableMotionPrediction)) {
		return false;
		}
		return nativeIsPredictionAvailable(mPtr, deviceId, source);
		return true;
		}

		/**
		* Check whether a device supports motion predictions for a given source type.
		*
		* @param deviceId The input device id.
		* @param source The source of input events.
		* @return True if the current device supports predictions, false otherwise.
		*
		* @see MotionEvent#getDeviceId
		* @see MotionEvent#getSource
		*/
		public boolean isPredictionAvailable(int deviceId, int source) {
		return isPredictionEnabled() && nativeIsPredictionAvailable(mPtr, deviceId, source);
		}

		private static native long nativeInitialize(int offsetNanos);

core/jni/Android.bp

+3 −0

Original line number	Diff line number	Diff line
		@@ -326,6 +326,7 @@ cc_library_shared {
		"libdl",
		"libdl_android",
		"libtimeinstate",
		"libtflite",
		"server_configurable_flags",
		"libimage_io",
		"libjpegdecoder",
		@@ -343,7 +344,9 @@ cc_library_shared {
		header_libs: [
		"bionic_libc_platform_headers",
		"dnsproxyd_protocol_headers",
		"flatbuffer_headers",
		"libtextclassifier_hash_headers",
		"tensorflow_headers",
		],
		runtime_libs: [
		"libidmap2",

core/res/res/values/config.xml

+5 −14

Original line number	Diff line number	Diff line
		@@ -2383,22 +2383,13 @@
		display, this value should be true. -->
		<bool name="config_perDisplayFocusEnabled">false</bool>

		<!-- Whether the system enables motion prediction. Only enable this after confirming that the
		model works well on your device. To enable system-based prediction, set this value to true.
		-->
		<bool name="config_enableMotionPrediction">true</bool>
		<!-- Whether to use the system motion prediction model. Only set this value to true after
		confirming that the model works well on your device. -->
		<bool name="config_enableMotionPrediction">false</bool>

		<!-- Additional offset to use for motion prediction, in nanoseconds. A positive number indicates
		that the prediction will take place further in the future. For example, suppose a
		MotionEvent arrives with timestamp t=1, and the current expected presentation time is t=2.
		Typically, the prediction will target the presentation time, t=2. If you'd like to make
		prediction more aggressive, you could set the offset to a positive number.
		Setting the offset to 1 here would mean that the prediction will be done for time t=3.
		A negative number may also be provided, to make the prediction less aggressive. In general,
		the offset here should represent some built-in hardware delays that may not be accounted
		for by the "expected present time". See also:
		https://developer.android.com/reference/android/view/
		Choreographer.FrameTimeline#getExpectedPresentationTimeNanos() -->
		that the prediction will take place further in the future and, in general, should represent
		some built-in hardware delays that prediction should try to recover. -->
		<integer name="config_motionPredictionOffsetNanos">0</integer>

		<!-- Whether a software navigation bar should be shown. NOTE: in the future this may be

tests/Input/src/com/android/test/input/MotionPredictorTest.kt

+10 −9

Original line number	Diff line number	Diff line
		@@ -102,9 +102,9 @@ class MotionPredictorTest {
		* a prediction. Here, we send 2 events to the predictor and check the returned event.
		* Input:
		* t = 0 x = 0 y = 0
		* t = 1 x = 1 y = 2
		* t = 4 x = 10 y = 20
		* Output (expected):
		* t = 3 x = 3 y = 6
		* t = 12 x = 30 y = 60 ± error
		*
		* Historical data is ignored for simplicity.
		*/
		@@ -118,19 +118,20 @@ class MotionPredictorTest {
		// ACTION_DOWN t=0 x=0 y=0
		predictor.record(downEvent)

		eventTime += Duration.ofMillis(1)
		val moveEvent = getStylusMotionEvent(eventTime, ACTION_MOVE, /x=/1f, /y=/2f)
		eventTime += Duration.ofMillis(4)
		val moveEvent = getStylusMotionEvent(eventTime, ACTION_MOVE, /x=/10f, /y=/20f)
		// ACTION_MOVE t=1 x=1 y=2
		predictor.record(moveEvent)

		val predicted = predictor.predict(Duration.ofMillis(2).toNanos())
		val predicted = predictor.predict(Duration.ofMillis(8).toNanos())
		assertEquals(1, predicted.size)
		val event = predicted[0]
		assertNotNull(event)

		// Prediction will happen for t=3 (2 + 1, since offset is 1 and present time is 2)
		assertEquals(3, event.eventTime)
		assertEquals(3f, event.x, /delta=/0.001f)
		assertEquals(6f, event.y, /delta=/0.001f)
		// Prediction will happen for t=12 (since it is the next input interval after the requested
		// time, 8, plus the model offset, 1).
		assertEquals(12, event.eventTime)
		assertEquals(30f, event.x, /delta=/5f)
		assertEquals(60f, event.y, /delta=/15f)
		}
		}