Observe frame info from native input code - try 3 (7524d2f2) · Commits · e / os / android_frameworks_base

core/java/android/view/InputEventReceiver.java

+6 −11

Original line number	Diff line number	Diff line
		@@ -55,13 +55,13 @@ public abstract class InputEventReceiver {
		private static native void nativeDispose(long receiverPtr);
		private static native void nativeFinishInputEvent(long receiverPtr, int seq, boolean handled);
		private static native boolean nativeProbablyHasInput(long receiverPtr);
		private static native void nativeReportTimeline(long receiverPtr, int inputEventId,
		long gpuCompletedTime, long presentTime);
		private static native boolean nativeConsumeBatchedInputEvents(long receiverPtr,
		long frameTimeNanos);
		private static native IBinder nativeGetToken(long receiverPtr);
		private static native long nativeGetFrameMetricsObserver(long receiverPtr);
		private static native String nativeDump(long receiverPtr, String prefix);


		/**
		* Creates an input event receiver bound to the specified input channel.
		*
		@@ -234,15 +234,6 @@ public abstract class InputEventReceiver {
		event.recycleIfNeededAfterDispatch();
		}

		/**
		* Report the timing / latency information for a specific input event.
		*/
		public final void reportTimeline(int inputEventId, long gpuCompletedTime, long presentTime) {
		Trace.traceBegin(Trace.TRACE_TAG_INPUT, "reportTimeline");
		nativeReportTimeline(mReceiverPtr, inputEventId, gpuCompletedTime, presentTime);
		Trace.traceEnd(Trace.TRACE_TAG_INPUT);
		}

		/**
		* Consumes all pending batched input events.
		* Must be called on the same Looper thread to which the receiver is attached.
		@@ -265,6 +256,10 @@ public abstract class InputEventReceiver {
		return false;
		}

		protected final long getNativeFrameMetricsObserver() {
		return nativeGetFrameMetricsObserver(mReceiverPtr);
		}

		/**
		* @return Returns a token to identify the input channel.
		*/

core/java/android/view/ViewRootImpl.java

+11 −48

Original line number	Diff line number	Diff line
		@@ -22,7 +22,6 @@ import static android.app.WindowConfiguration.WINDOWING_MODE_MULTI_WINDOW;
		import static android.content.pm.ActivityInfo.OVERRIDE_SANDBOX_VIEW_BOUNDS_APIS;
		import static android.graphics.HardwareRenderer.SYNC_CONTEXT_IS_STOPPED;
		import static android.graphics.HardwareRenderer.SYNC_LOST_SURFACE_REWARD_IF_FOUND;
		import static android.os.IInputConstants.INVALID_INPUT_EVENT_ID;
		import static android.os.Trace.TRACE_TAG_VIEW;
		import static android.util.SequenceUtils.getInitSeq;
		import static android.util.SequenceUtils.isIncomingSeqStale;
		@@ -367,13 +366,6 @@ public final class ViewRootImpl implements ViewParent,
		*/
		private static final boolean MT_RENDERER_AVAILABLE = true;

		/**
		* Whether or not to report end-to-end input latency. Can be disabled temporarily as a
		* risk mitigation against potential jank caused by acquiring a weak reference
		* per frame.
		*/
		private static final boolean ENABLE_INPUT_LATENCY_TRACKING = true;

		/**
		* Whether the client (system UI) is handling the transient gesture and the corresponding
		* animation.
		@@ -1709,14 +1701,8 @@ public final class ViewRootImpl implements ViewParent,
		mInputQueueCallback.onInputQueueCreated(mInputQueue);
		}
		mInputEventReceiver = new WindowInputEventReceiver(inputChannel,
		Looper.myLooper());
		Looper.myLooper(), mAttachInfo.mThreadedRenderer);

		if (ENABLE_INPUT_LATENCY_TRACKING && mAttachInfo.mThreadedRenderer != null) {
		InputMetricsListener listener = new InputMetricsListener();
		mHardwareRendererObserver = new HardwareRendererObserver(
		listener, listener.data, mHandler, true /waitForPresentTime/);
		mAttachInfo.mThreadedRenderer.addObserver(mHardwareRendererObserver);
		}
		// Update unbuffered request when set the root view.
		mUnbufferedInputSource = mView.mUnbufferedInputSource;
		}
		@@ -10650,8 +10636,14 @@ public final class ViewRootImpl implements ViewParent,
		final TraversalRunnable mTraversalRunnable = new TraversalRunnable();

		final class WindowInputEventReceiver extends InputEventReceiver {
		public WindowInputEventReceiver(InputChannel inputChannel, Looper looper) {
		private final HardwareRenderer mRenderer;
		WindowInputEventReceiver(InputChannel inputChannel, Looper looper,
		HardwareRenderer renderer) {
		super(inputChannel, looper);
		mRenderer = renderer;
		if (mRenderer != null) {
		mRenderer.addObserver(getNativeFrameMetricsObserver());
		}
		}

		@Override
		@@ -10703,43 +10695,14 @@ public final class ViewRootImpl implements ViewParent,
		@Override
		public void dispose() {
		unscheduleConsumeBatchedInput();
		if (mRenderer != null) {
		mRenderer.removeObserver(getNativeFrameMetricsObserver());
		}
		super.dispose();
		}
		}
		private WindowInputEventReceiver mInputEventReceiver;

		final class InputMetricsListener
		implements HardwareRendererObserver.OnFrameMetricsAvailableListener {
		public long[] data = new long[FrameMetrics.Index.FRAME_STATS_COUNT];

		@Override
		public void onFrameMetricsAvailable(int dropCountSinceLastInvocation) {
		final int inputEventId = (int) data[FrameMetrics.Index.INPUT_EVENT_ID];
		if (inputEventId == INVALID_INPUT_EVENT_ID) {
		return;
		}
		final long presentTime = data[FrameMetrics.Index.DISPLAY_PRESENT_TIME];
		if (presentTime <= 0) {
		// Present time is not available for this frame. If the present time is not
		// available, we cannot compute end-to-end input latency metrics.
		return;
		}
		final long gpuCompletedTime = data[FrameMetrics.Index.GPU_COMPLETED];
		if (mInputEventReceiver == null) {
		return;
		}
		if (gpuCompletedTime >= presentTime) {
		final double discrepancyMs = (gpuCompletedTime - presentTime) * 1E-6;
		final long vsyncId = data[FrameMetrics.Index.FRAME_TIMELINE_VSYNC_ID];
		Log.w(TAG, "Not reporting timeline because gpuCompletedTime is " + discrepancyMs
		+ "ms ahead of presentTime. FRAME_TIMELINE_VSYNC_ID=" + vsyncId
		+ ", INPUT_EVENT_ID=" + inputEventId);
		// TODO(b/186664409): figure out why this sometimes happens
		return;
		}
		mInputEventReceiver.reportTimeline(inputEventId, gpuCompletedTime, presentTime);
		}
		}
		HardwareRendererObserver mHardwareRendererObserver;

		final class ConsumeBatchedInputRunnable implements Runnable {

core/jni/Android.bp

+3 −0

Original line number	Diff line number	Diff line
		@@ -55,6 +55,9 @@ cc_library_shared_for_libandroid_runtime {
		],

		cppflags: ["-Wno-conversion-null"],
		header_libs: [
		"libhwui_internal_headers",
		],

		product_variables: {
		eng: {

core/jni/android_view_InputEventReceiver.cpp

+163 −42

Original line number	Diff line number	Diff line
		@@ -23,6 +23,8 @@
		#include <android-base/logging.h>
		#include <android-base/stringprintf.h>
		#include <android_runtime/AndroidRuntime.h>
		#include <ftl/enum.h>
		#include <input/BlockingQueue.h>
		#include <input/InputConsumer.h>
		#include <input/InputTransport.h>
		#include <input/PrintTools.h>
		@@ -32,9 +34,12 @@
		#include <nativehelper/ScopedLocalRef.h>
		#include <utils/Looper.h>

		#include <cinttypes>
		#include <variant>
		#include <vector>

		#include "FrameInfo.h"
		#include "FrameMetricsObserver.h"
		#include "android_os_MessageQueue.h"
		#include "android_util_Binder.h"
		#include "android_view_InputChannel.h"
		@@ -85,20 +90,72 @@ std::string addPrefix(std::string str, std::string_view prefix) {
		}
		} // namespace

		class NativeInputEventReceiver final : public LooperCallback {
		class NativeInputEventReceiver;

		/**
		* This observer is allowed to outlive the NativeInputEventReceiver, so we must store the receiver
		* inside a wp.
		*/
		class InputFrameMetricsObserver : public uirenderer::FrameMetricsObserver {
		public:
		InputFrameMetricsObserver(wp<NativeInputEventReceiver> receiver)
		: FrameMetricsObserver(/waitForPresentTime=/true), mReceiver(receiver) {};
		void notify(const uirenderer::FrameInfoBuffer& buffer) override;

		private:
		wp<NativeInputEventReceiver> mReceiver;
		};

		enum class ReceiverMessageType : decltype(Message::what) {
		OUTBOUND_EVENTS_AVAILABLE,
		};

		/**
		* The interaction with NativeInputEventReceiver should be done on the main (looper from the
		* provided 'messageQueue') thread. However, there is one exception - the "enqueueTimeline"
		* function, which may be called on any thread.
		*
		* In practice, that means that main/ui thread will interact with NativeInputEventReceiver, and will
		* not obtain any locks, except for when handling outbound events. To receive the timeline
		* information, NativeInputEventReceiver uses FrameMetricsObserver, which notifies on the render
		* thread. To avoid blocking the render thread, the processing of timeline information inside
		* "enqueueTimeline" should be fast.
		*
		* To avoid using explicit locks in this class, thread-safe BlockingQueue is used for storing the
		* outbound events. All of the other processing should happen on the main thread and does not need
		* locking.
		*/
		class NativeInputEventReceiver final : public LooperCallback, public MessageHandler {
		public:
		NativeInputEventReceiver(JNIEnv* env, jobject receiverWeak,
		const std::shared_ptr<InputChannel>& inputChannel,
		const sp<MessageQueue>& messageQueue);

		status_t initialize();
		/**
		* Dispose the receiver. This is roughly equivalent to destroying the receiver. The reason we
		* can't just destroy the receiver is that there are other entities owning refs to this
		* receiver, including the looper (for fd callbacks), and other java callers.
		*/
		void dispose();
		status_t finishInputEvent(uint32_t seq, bool handled);
		bool probablyHasInput() const;
		status_t reportTimeline(int32_t inputEventId, nsecs_t gpuCompletedTime, nsecs_t presentTime);
		/**
		* Add a timeline message to the outbound queue to be sent out on the looper thread at some
		* point later. This function may be called on any thread.
		*
		* This function is guaranteed to return fast, thus making it safe for use in time-critical
		* paths.
		*
		* @param inputEventId the id of the input event
		* @param gpuCompletedTime the time at which renderthread finished rendering the frame
		* @param presentTime the time at which the frame was presented on the display
		*/
		void enqueueTimeline(int32_t inputEventId, nsecs_t gpuCompletedTime, nsecs_t presentTime);
		status_t consumeEvents(JNIEnv* env, bool consumeBatches, nsecs_t frameTime,
		bool* outConsumedBatch);
		sp<IBinder> getInputChannelToken() const;
		uirenderer::FrameMetricsObserver* getFrameMetricsObserver();
		std::string dump(const char* prefix) const;

		protected:
		@@ -117,19 +174,26 @@ private:
		typedef std::variant<Finish, Timeline> OutboundEvent;

		jobject mReceiverWeakGlobal;

		// The consumer is created in the constructor, and set to null when the receiver is disposed.
		// This provides the guarantee to the users of receiver that when the receiver is disposed,
		// there will no longer be any input events consumed by the receiver.
		std::unique_ptr<InputConsumer> mInputConsumer;
		sp<MessageQueue> mMessageQueue;
		PreallocatedInputEventFactory mInputEventFactory;
		bool mBatchedInputEventPending;
		const std::string mName;
		int mFdEvents;
		std::vector<OutboundEvent> mOutboundQueue;
		BlockingQueue<OutboundEvent> mOutboundQueue;

		sp<uirenderer::FrameMetricsObserver> mFrameMetricsObserver;
		void setFdEvents(int events);

		status_t processOutboundEvents();
		// From 'LooperCallback'
		int handleEvent(int receiveFd, int events, void* data) override;
		// From 'MessageHandler'
		void handleMessage(const Message& message) override;
		};

		NativeInputEventReceiver::NativeInputEventReceiver(
		@@ -169,6 +233,11 @@ void NativeInputEventReceiver::dispose() {
		setFdEvents(0);
		// Do not process any more events after the receiver has been disposed.
		mInputConsumer.reset();

		mMessageQueue->getLooper()->removeMessages(this);

		// At this point, the consumer has been destroyed, so no further input processing can be done
		// by this NativeInputEventReceiver object.
		}

		status_t NativeInputEventReceiver::finishInputEvent(uint32_t seq, bool handled) {
		@@ -180,7 +249,7 @@ status_t NativeInputEventReceiver::finishInputEvent(uint32_t seq, bool handled)
		.seq = seq,
		.handled = handled,
		};
		mOutboundQueue.push_back(finish);
		mOutboundQueue.emplace(finish);
		return processOutboundEvents();
		}

		@@ -191,7 +260,7 @@ bool NativeInputEventReceiver::probablyHasInput() const {
		return mInputConsumer->probablyHasInput();
		}

		status_t NativeInputEventReceiver::reportTimeline(int32_t inputEventId, nsecs_t gpuCompletedTime,
		void NativeInputEventReceiver::enqueueTimeline(int32_t inputEventId, nsecs_t gpuCompletedTime,
		nsecs_t presentTime) {
		if (kDebugDispatchCycle) {
		ALOGD("channel '%s' ~ %s", mName.c_str(), __func__);
		@@ -203,8 +272,15 @@ status_t NativeInputEventReceiver::reportTimeline(int32_t inputEventId, nsecs_t
		.inputEventId = inputEventId,
		.timeline = graphicsTimeline,
		};
		mOutboundQueue.push_back(timeline);
		return processOutboundEvents();
		mOutboundQueue.emplace(timeline);
		// We shouldn't be processing the incoming event directly here, since the call may come in on
		// any thread (normally, it would arrive on the render thread).
		// Instead, we notify the looper that there's pending data, and let the events be processed
		// as a Message on the looper thread.
		mMessageQueue->getLooper()
		->sendMessage(this,
		Message(ftl::to_underlying(
		ReceiverMessageType::OUTBOUND_EVENTS_AVAILABLE)));
		}

		void NativeInputEventReceiver::setFdEvents(int events) {
		@@ -255,32 +331,36 @@ status_t NativeInputEventReceiver::processOutboundEvents() {
		if (mInputConsumer == nullptr) {
		return DEAD_OBJECT;
		}
		while (!mOutboundQueue.empty()) {
		OutboundEvent& outbound = *mOutboundQueue.begin();
		status_t status;

		if (std::holds_alternative<Finish>(outbound)) {
		const Finish& finish = std::get<Finish>(outbound);
		while (true) {
		std::optional<OutboundEvent> outbound = mOutboundQueue.popWithTimeout(0ms);
		if (!outbound.has_value()) {
		break;
		}

		status_t status;
		if (std::holds_alternative<Finish>(*outbound)) {
		const Finish& finish = std::get<Finish>(*outbound);
		status = mInputConsumer->sendFinishedSignal(finish.seq, finish.handled);
		} else if (std::holds_alternative<Timeline>(outbound)) {
		const Timeline& timeline = std::get<Timeline>(outbound);
		} else if (std::holds_alternative<Timeline>(*outbound)) {
		const Timeline& timeline = std::get<Timeline>(*outbound);
		status = mInputConsumer->sendTimeline(timeline.inputEventId, timeline.timeline);
		} else {
		LOG_ALWAYS_FATAL("Unexpected event type in std::variant");
		status = BAD_VALUE;
		}
		if (status == OK) {
		// Successful send. Erase the entry and keep trying to send more
		mOutboundQueue.erase(mOutboundQueue.begin());
		// Successful send. Keep trying to send more
		continue;
		}

		// Publisher is busy, try again later. Keep this entry (do not erase)
		// Publisher is busy, try again later. Put the popped entry back into the queue.
		if (status == WOULD_BLOCK) {
		if (kDebugDispatchCycle) {
		ALOGD("channel '%s' ~ Remaining outbound events: %zu.", mName.c_str(),
		mOutboundQueue.size());
		}
		mOutboundQueue.emplace(*outbound);
		setFdEvents(ALOOPER_EVENT_INPUT \| ALOOPER_EVENT_OUTPUT);
		return WOULD_BLOCK; // try again later
		}
		@@ -518,6 +598,22 @@ sp<IBinder> NativeInputEventReceiver::getInputChannelToken() const {
		return mInputConsumer->getChannel()->getConnectionToken();
		}

		void NativeInputEventReceiver::handleMessage(const Message& message) {
		switch (ReceiverMessageType(message.what)) {
		case ReceiverMessageType::OUTBOUND_EVENTS_AVAILABLE: {
		processOutboundEvents();
		}
		}
		}

		uirenderer::FrameMetricsObserver* NativeInputEventReceiver::getFrameMetricsObserver() {
		// Lazy initialization, in case the user does not want to register the observer
		if (mFrameMetricsObserver == nullptr) {
		mFrameMetricsObserver = sp<InputFrameMetricsObserver>::make(this);
		}
		return mFrameMetricsObserver.get();
		}

		std::string NativeInputEventReceiver::dump(const char* prefix) const {
		std::string out;
		std::string consumerDump =
		@@ -526,27 +622,65 @@ std::string NativeInputEventReceiver::dump(const char* prefix) const {
		out += android::base::StringPrintf("mBatchedInputEventPending: %s\n",
		toString(mBatchedInputEventPending));
		out = out + "mOutboundQueue:\n";
		for (const OutboundEvent& outbound : mOutboundQueue) {
		out += mOutboundQueue.dump([](const OutboundEvent& outbound) {
		if (std::holds_alternative<Finish>(outbound)) {
		const Finish& finish = std::get<Finish>(outbound);
		out += android::base::StringPrintf(" Finish: seq=%" PRIu32 " handled=%s\n", finish.seq,
		return android::base::StringPrintf(" Finish: seq=%" PRIu32 " handled=%s\n", finish.seq,
		toString(finish.handled));
		} else if (std::holds_alternative<Timeline>(outbound)) {
		const Timeline& timeline = std::get<Timeline>(outbound);
		out += android::base::
		return android::base::
		StringPrintf(" Timeline: inputEventId=%" PRId32 " gpuCompletedTime=%" PRId64
		", presentTime=%" PRId64 "\n",
		timeline.inputEventId,
		timeline.timeline[GraphicsTimeline::GPU_COMPLETED_TIME],
		timeline.timeline[GraphicsTimeline::PRESENT_TIME]);
		}
		}
		return std::string("Invalid type found in OutboundEvent");
		});
		if (mOutboundQueue.empty()) {
		out = out + " <empty>\n";
		}
		return addPrefix(out, prefix);
		}

		void InputFrameMetricsObserver::notify(const uirenderer::FrameInfoBuffer& buffer) {
		const int64_t inputEventId =
		buffer[static_cast<size_t>(uirenderer::FrameInfoIndex::InputEventId)];
		if (inputEventId == android::os::IInputConstants::INVALID_INPUT_EVENT_ID) {
		return;
		}
		if (IdGenerator::getSource(inputEventId) != IdGenerator::Source::INPUT_READER) {
		// skip this event, it did not originate from hardware
		return;
		}

		const int64_t presentTime =
		buffer[static_cast<size_t>(uirenderer::FrameInfoIndex::DisplayPresentTime)];
		if (presentTime <= 0) {
		// Present time is not available for this frame. If the present time is not
		// available, we cannot compute end-to-end input latency metrics.
		return;
		}
		const int64_t gpuCompletedTime =
		buffer[static_cast<size_t>(uirenderer::FrameInfoIndex::GpuCompleted)];
		if (gpuCompletedTime >= presentTime) {
		const int64_t discrepancy = (gpuCompletedTime - presentTime);
		const int64_t vsyncId =
		buffer[static_cast<size_t>(uirenderer::FrameInfoIndex::FrameTimelineVsyncId)];
		LOG(ERROR) << "Not reporting timeline because gpuCompletedTime is " << discrepancy * 1E-6
		<< "ms ahead of presentTime. FRAME_TIMELINE_VSYNC_ID=" << vsyncId
		<< ", INPUT_EVENT_ID=" << inputEventId;
		return;
		}

		sp<NativeInputEventReceiver> receiver = mReceiver.promote();
		if (receiver == nullptr) {
		return;
		}
		receiver->enqueueTimeline(inputEventId, gpuCompletedTime, presentTime);
		}

		static jlong nativeInit(JNIEnv* env, jclass clazz, jobject receiverWeak,
		jobject inputChannelObj, jobject messageQueueObj) {
		std::shared_ptr<InputChannel> inputChannel =
		@@ -606,25 +740,6 @@ static bool nativeProbablyHasInput(JNIEnv* env, jclass clazz, jlong receiverPtr)
		return receiver->probablyHasInput();
		}

		static void nativeReportTimeline(JNIEnv* env, jclass clazz, jlong receiverPtr, jint inputEventId,
		jlong gpuCompletedTime, jlong presentTime) {
		if (IdGenerator::getSource(inputEventId) != IdGenerator::Source::INPUT_READER) {
		// skip this event, it did not originate from hardware
		return;
		}
		sp<NativeInputEventReceiver> receiver =
		reinterpret_cast<NativeInputEventReceiver*>(receiverPtr);
		status_t status = receiver->reportTimeline(inputEventId, gpuCompletedTime, presentTime);
		if (status == OK \|\| status == WOULD_BLOCK) {
		return; // normal operation
		}
		if (status != DEAD_OBJECT) {
		std::string message = android::base::StringPrintf("Failed to send timeline. status=%s(%d)",
		strerror(-status), status);
		jniThrowRuntimeException(env, message.c_str());
		}
		}

		static jboolean nativeConsumeBatchedInputEvents(JNIEnv* env, jclass clazz, jlong receiverPtr,
		jlong frameTimeNanos) {
		sp<NativeInputEventReceiver> receiver =
		@@ -648,6 +763,12 @@ static jobject nativeGetToken(JNIEnv* env, jclass clazz, jlong receiverPtr) {
		return javaObjectForIBinder(env, receiver->getInputChannelToken());
		}

		static jlong nativeGetFrameMetricsObserver(JNIEnv* env, jclass clazz, jlong receiverPtr) {
		sp<NativeInputEventReceiver> receiver =
		reinterpret_cast<NativeInputEventReceiver*>(receiverPtr);
		return reinterpret_cast<jlong>(receiver->getFrameMetricsObserver());
		}

		static jstring nativeDump(JNIEnv* env, jclass clazz, jlong receiverPtr, jstring prefix) {
		sp<NativeInputEventReceiver> receiver =
		reinterpret_cast<NativeInputEventReceiver*>(receiverPtr);
		@@ -663,9 +784,9 @@ static const JNINativeMethod gMethods[] = {
		{"nativeDispose", "(J)V", (void*)nativeDispose},
		{"nativeFinishInputEvent", "(JIZ)V", (void*)nativeFinishInputEvent},
		{"nativeProbablyHasInput", "(J)Z", (void*)nativeProbablyHasInput},
		{"nativeReportTimeline", "(JIJJ)V", (void*)nativeReportTimeline},
		{"nativeConsumeBatchedInputEvents", "(JJ)Z", (void*)nativeConsumeBatchedInputEvents},
		{"nativeGetToken", "(J)Landroid/os/IBinder;", (void*)nativeGetToken},
		{"nativeGetFrameMetricsObserver", "(J)J", (void*)nativeGetFrameMetricsObserver},
		{"nativeDump", "(JLjava/lang/String;)Ljava/lang/String;", (void*)nativeDump},
		};

tests/Input/Android.bp

+4 −0

Original line number	Diff line number	Diff line
		@@ -37,6 +37,7 @@ android_test {
		"flag-junit",
		"frameworks-base-testutils",
		"hamcrest-library",
		"inputtests_aidl-java",
		"junit-params",
		"kotlin-test",
		"mockito-kotlin-nodeps",
		@@ -50,6 +51,9 @@ android_test {
		"truth",
		"ui-trace-collector",
		],
		jni_libs: [
		"libinputtests_jni",
		],
		libs: [
		"android.test.mock.stubs.system",
		"android.test.base.stubs.system",