Support re-attaching the inline suggestion view to window (b46851c9) · Commits · e / os / android_frameworks_base

core/java/android/service/autofill/IInlineSuggestionUi.aidl

0 → 100644

+29 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package android.service.autofill;

		import android.service.autofill.ISurfacePackageResultCallback;

		/**
		* Interface to interact with a remote inline suggestion UI.
		*
		* @hide
		*/
		oneway interface IInlineSuggestionUi {
		void getSurfacePackage(ISurfacePackageResultCallback callback);
		void releaseSurfaceControlViewHost();
		}

core/java/android/service/autofill/IInlineSuggestionUiCallback.aidl

+4 −2

Original line number	Diff line number	Diff line
		@@ -18,17 +18,19 @@ package android.service.autofill;

		import android.content.IntentSender;
		import android.os.IBinder;
		import android.service.autofill.IInlineSuggestionUi;
		import android.view.SurfaceControlViewHost;

		/**
		* Interface to receive events from inline suggestions.
		* Interface to receive events from a remote inline suggestion UI.
		*
		* @hide
		*/
		oneway interface IInlineSuggestionUiCallback {
		void onClick();
		void onLongClick();
		void onContent(in SurfaceControlViewHost.SurfacePackage surface, int width, int height);
		void onContent(in IInlineSuggestionUi content, in SurfaceControlViewHost.SurfacePackage surface,
		int width, int height);
		void onError();
		void onTransferTouchFocusToImeWindow(in IBinder sourceInputToken, int displayId);
		void onStartIntentSender(in IntentSender intentSender);

core/java/android/service/autofill/ISurfacePackageResultCallback.aidl

0 → 100644

+28 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package android.service.autofill;

		import android.view.SurfaceControlViewHost;

		/**
		* Interface to receive a SurfaceControlViewHost.SurfacePackage.
		*
		* @hide
		*/
		oneway interface ISurfacePackageResultCallback {
		void onResult(in SurfaceControlViewHost.SurfacePackage result);
		}

core/java/android/service/autofill/InlineSuggestionRenderService.java

+113 −8

Original line number	Diff line number	Diff line
		@@ -33,6 +33,7 @@ import android.os.Looper;
		import android.os.RemoteCallback;
		import android.os.RemoteException;
		import android.util.Log;
		import android.util.LruCache;
		import android.util.Size;
		import android.view.Display;
		import android.view.SurfaceControlViewHost;
		@@ -40,6 +41,8 @@ import android.view.View;
		import android.view.ViewGroup;
		import android.view.WindowManager;

		import java.lang.ref.WeakReference;

		/**
		* A service that renders an inline presentation view given the {@link InlinePresentation}.
		*
		@@ -65,6 +68,27 @@ public abstract class InlineSuggestionRenderService extends Service {

		private IInlineSuggestionUiCallback mCallback;


		/**
		* A local LRU cache keeping references to the inflated {@link SurfaceControlViewHost}s, so
		* they can be released properly when no longer used. Each view needs to be tracked separately,
		* therefore for simplicity we use the hash code of the value object as key in the cache.
		*/
		private final LruCache<InlineSuggestionUiImpl, Boolean> mActiveInlineSuggestions =
		new LruCache<InlineSuggestionUiImpl, Boolean>(30) {
		@Override
		public void entryRemoved(boolean evicted, InlineSuggestionUiImpl key,
		Boolean oldValue,
		Boolean newValue) {
		if (evicted) {
		Log.w(TAG,
		"Hit max=100 entries in the cache. Releasing oldest one to make "
		+ "space.");
		key.releaseSurfaceControlViewHost();
		}
		}
		};

		/**
		* If the specified {@code width}/{@code height} is an exact value, then it will be returned as
		* is, otherwise the method tries to measure a size that is just large enough to fit the view
		@@ -169,8 +193,14 @@ public abstract class InlineSuggestionRenderService extends Service {
		return true;
		});

		sendResult(callback, host.getSurfacePackage(), measuredSize.getWidth(),
		measuredSize.getHeight());
		try {
		InlineSuggestionUiImpl uiImpl = new InlineSuggestionUiImpl(host, mHandler);
		mActiveInlineSuggestions.put(uiImpl, true);
		callback.onContent(new InlineSuggestionUiWrapper(uiImpl), host.getSurfacePackage(),
		measuredSize.getWidth(), measuredSize.getHeight());
		} catch (RemoteException e) {
		Log.w(TAG, "RemoteException calling onContent()");
		}
		} finally {
		updateDisplay(Display.DEFAULT_DISPLAY);
		}
		@@ -181,12 +211,87 @@ public abstract class InlineSuggestionRenderService extends Service {
		callback.sendResult(rendererInfo);
		}

		private void sendResult(@NonNull IInlineSuggestionUiCallback callback,
		@Nullable SurfaceControlViewHost.SurfacePackage surface, int width, int height) {
		/**
		* A wrapper class around the {@link InlineSuggestionUiImpl} to ensure it's not strongly
		* reference by the remote system server process.
		*/
		private static final class InlineSuggestionUiWrapper extends
		android.service.autofill.IInlineSuggestionUi.Stub {

		private final WeakReference<InlineSuggestionUiImpl> mUiImpl;

		InlineSuggestionUiWrapper(InlineSuggestionUiImpl uiImpl) {
		mUiImpl = new WeakReference<>(uiImpl);
		}

		@Override
		public void releaseSurfaceControlViewHost() {
		final InlineSuggestionUiImpl uiImpl = mUiImpl.get();
		if (uiImpl != null) {
		uiImpl.releaseSurfaceControlViewHost();
		}
		}

		@Override
		public void getSurfacePackage(ISurfacePackageResultCallback callback) {
		final InlineSuggestionUiImpl uiImpl = mUiImpl.get();
		if (uiImpl != null) {
		uiImpl.getSurfacePackage(callback);
		}
		}
		}

		/**
		* Keeps track of a SurfaceControlViewHost to ensure it's released when its lifecycle ends.
		*
		* <p>This class is thread safe, because all the outside calls are piped into a single
		* handler thread to be processed.
		*/
		private final class InlineSuggestionUiImpl {

		@Nullable
		private SurfaceControlViewHost mViewHost;
		@NonNull
		private final Handler mHandler;

		InlineSuggestionUiImpl(SurfaceControlViewHost viewHost, Handler handler) {
		this.mViewHost = viewHost;
		this.mHandler = handler;
		}

		/**
		* Call {@link SurfaceControlViewHost#release()} to release it. After this, this view is
		* not usable, and any further calls to the
		* {@link #getSurfacePackage(ISurfacePackageResultCallback)} will get {@code null} result.
		*/
		public void releaseSurfaceControlViewHost() {
		mHandler.post(() -> {
		if (mViewHost == null) {
		return;
		}
		Log.v(TAG, "Releasing inline suggestion view host");
		mViewHost.release();
		mViewHost = null;
		InlineSuggestionRenderService.this.mActiveInlineSuggestions.remove(
		InlineSuggestionUiImpl.this);
		Log.v(TAG, "Removed the inline suggestion from the cache, current size="
		+ InlineSuggestionRenderService.this.mActiveInlineSuggestions.size());
		});
		}

		/**
		* Sends back a new {@link android.view.SurfaceControlViewHost.SurfacePackage} if the view
		* is not released, {@code null} otherwise.
		*/
		public void getSurfacePackage(ISurfacePackageResultCallback callback) {
		Log.d(TAG, "getSurfacePackage");
		mHandler.post(() -> {
		try {
		callback.onContent(surface, width, height);
		callback.onResult(mViewHost == null ? null : mViewHost.getSurfacePackage());
		} catch (RemoteException e) {
		Log.w(TAG, "RemoteException calling onContent(" + surface + ")");
		Log.w(TAG, "RemoteException calling onSurfacePackage");
		}
		});
		}
		}

core/java/android/view/inputmethod/InlineSuggestion.java

+211 −59

Original line number	Diff line number	Diff line
		@@ -18,11 +18,13 @@ package android.view.inputmethod;

		import android.annotation.BinderThread;
		import android.annotation.CallbackExecutor;
		import android.annotation.MainThread;
		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.annotation.TestApi;
		import android.content.Context;
		import android.os.AsyncTask;
		import android.os.Handler;
		import android.os.Looper;
		import android.os.Parcel;
		import android.os.Parcelable;
		import android.os.RemoteException;
		@@ -42,26 +44,26 @@ import java.util.concurrent.Executor;
		import java.util.function.Consumer;

		/**
		* This class represents an inline suggestion which is made by one app
		* and can be embedded into the UI of another. Suggestions may contain
		* sensitive information not known to the host app which needs to be
		* protected from spoofing. To address that the suggestion view inflated
		* on demand for embedding is created in such a way that the hosting app
		* cannot introspect its content and cannot interact with it.
		* This class represents an inline suggestion which is made by one app and can be embedded into the
		* UI of another. Suggestions may contain sensitive information not known to the host app which
		* needs to be protected from spoofing. To address that the suggestion view inflated on demand for
		* embedding is created in such a way that the hosting app cannot introspect its content and cannot
		* interact with it.
		*/
		@DataClass(
		genEqualsHashCode = true,
		genToString = true,
		genHiddenConstDefs = true,
		@DataClass(genEqualsHashCode = true, genToString = true, genHiddenConstDefs = true,
		genHiddenConstructor = true)
		@DataClass.Suppress({"getContentProvider"})
		public final class InlineSuggestion implements Parcelable {

		private static final String TAG = "InlineSuggestion";

		private final @NonNull InlineSuggestionInfo mInfo;
		@NonNull
		private final InlineSuggestionInfo mInfo;

		private final @Nullable IInlineContentProvider mContentProvider;
		/**
		* @hide
		*/
		@Nullable
		private final IInlineContentProvider mContentProvider;

		/**
		* Used to keep a strong reference to the callback so it doesn't get garbage collected.
		@@ -69,7 +71,8 @@ public final class InlineSuggestion implements Parcelable {
		* @hide
		*/
		@DataClass.ParcelWith(InlineContentCallbackImplParceling.class)
		private @Nullable InlineContentCallbackImpl mInlineContentCallback;
		@Nullable
		private InlineContentCallbackImpl mInlineContentCallback;

		/**
		* Creates a new {@link InlineSuggestion}, for testing purpose.
		@@ -87,8 +90,7 @@ public final class InlineSuggestion implements Parcelable {
		*
		* @hide
		*/
		public InlineSuggestion(
		@NonNull InlineSuggestionInfo info,
		public InlineSuggestion(@NonNull InlineSuggestionInfo info,
		@Nullable IInlineContentProvider contentProvider) {
		this(info, contentProvider, /* inlineContentCallback */ null);
		}
		@@ -96,25 +98,30 @@ public final class InlineSuggestion implements Parcelable {
		/**
		* Inflates a view with the content of this suggestion at a specific size.
		*
		* <p> The size must be either 1) between the
		* {@link android.widget.inline.InlinePresentationSpec#getMinSize() min size} and the
		* {@link android.widget.inline.InlinePresentationSpec#getMaxSize() max size} of the
		* presentation spec returned by {@link InlineSuggestionInfo#getInlinePresentationSpec()},
		* or 2) {@link ViewGroup.LayoutParams#WRAP_CONTENT}. If the size is set to
		* {@link ViewGroup.LayoutParams#WRAP_CONTENT}, then the size of the inflated view will be just
		* large enough to fit the content, while still conforming to the min / max size specified by
		* the {@link android.widget.inline.InlinePresentationSpec}.
		* <p> Each dimension of the size must satisfy one of the following conditions:
		*
		* <ol>
		* <li>between {@link android.widget.inline.InlinePresentationSpec#getMinSize()} and
		* {@link android.widget.inline.InlinePresentationSpec#getMaxSize()} of the presentation spec
		* from {@code mInfo}
		* <li>{@link ViewGroup.LayoutParams#WRAP_CONTENT}
		* </ol>
		*
		* If the size is set to {@link
		* ViewGroup.LayoutParams#WRAP_CONTENT}, then the size of the inflated view will be just large
		* enough to fit the content, while still conforming to the min / max size specified by the
		* {@link android.widget.inline.InlinePresentationSpec}.
		*
		* <p> The caller can attach an {@link android.view.View.OnClickListener} and/or an
		* {@link android.view.View.OnLongClickListener} to the view in the
		* {@code callback} to receive click and long click events on the view.
		* {@link android.view.View.OnLongClickListener} to the view in the {@code callback} to receive
		* click and long click events on the view.
		*
		* @param context Context in which to inflate the view.
		* @param size The size at which to inflate the suggestion. For each dimension, it maybe
		* an exact value or {@link ViewGroup.LayoutParams#WRAP_CONTENT}.
		* @param callback Callback for receiving the inflated view, where the
		* {@link ViewGroup.LayoutParams} of the view is set as the actual size of
		* the underlying remote view.
		* @param size The size at which to inflate the suggestion. For each dimension, it maybe an
		* exact value or {@link ViewGroup.LayoutParams#WRAP_CONTENT}.
		* @param callback Callback for receiving the inflated view, where the {@link
		* ViewGroup.LayoutParams} of the view is set as the actual size of the
		* underlying remote view.
		* @throws IllegalArgumentException If an invalid argument is passed.
		* @throws IllegalStateException If this method is already called.
		*/
		@@ -130,9 +137,8 @@ public final class InlineSuggestion implements Parcelable {
		+ ", nor wrap_content");
		}
		mInlineContentCallback = getInlineContentCallback(context, callbackExecutor, callback);
		AsyncTask.THREAD_POOL_EXECUTOR.execute(() -> {
		if (mContentProvider == null) {
		callback.accept(/* view */ null);
		callbackExecutor.execute(() -> callback.accept(/* view */ null));
		return;
		}
		try {
		@@ -140,9 +146,8 @@ public final class InlineSuggestion implements Parcelable {
		new InlineContentCallbackWrapper(mInlineContentCallback));
		} catch (RemoteException e) {
		Slog.w(TAG, "Error creating suggestion content surface: " + e);
		callback.accept(/* view */ null);
		callbackExecutor.execute(() -> callback.accept(/* view */ null));
		}
		});
		}

		/**
		@@ -161,9 +166,14 @@ public final class InlineSuggestion implements Parcelable {
		if (mInlineContentCallback != null) {
		throw new IllegalStateException("Already called #inflate()");
		}
		return new InlineContentCallbackImpl(context, callbackExecutor, callback);
		return new InlineContentCallbackImpl(context, mContentProvider, callbackExecutor,
		callback);
		}

		/**
		* A wrapper class around the {@link InlineContentCallbackImpl} to ensure it's not strongly
		* reference by the remote system server process.
		*/
		private static final class InlineContentCallbackWrapper extends IInlineContentCallback.Stub {

		private final WeakReference<InlineContentCallbackImpl> mCallbackImpl;
		@@ -201,17 +211,68 @@ public final class InlineSuggestion implements Parcelable {
		}
		}

		/**
		* Handles the communication between the inline suggestion view in current (IME) process and
		* the remote view provided from the system server.
		*
		* <p>This class is thread safe, because all the outside calls are piped into a single
		* handler thread to be processed.
		*/
		private static final class InlineContentCallbackImpl {

		private final @NonNull Context mContext;
		private final @NonNull Executor mCallbackExecutor;
		private final @NonNull Consumer<InlineContentView> mCallback;
		private @Nullable InlineContentView mView;
		@NonNull
		private final Handler mMainHandler = new Handler(Looper.getMainLooper());

		@NonNull
		private final Context mContext;
		@Nullable
		private final IInlineContentProvider mInlineContentProvider;
		@NonNull
		private final Executor mCallbackExecutor;

		/**
		* Callback from the client (IME) that will receive the inflated suggestion view. It'll
		* only be called once when the view SurfacePackage is first sent back to the client. Any
		* updates to the view due to attach to window and detach from window events will be
		* handled under the hood, transparent from the client.
		*/
		@NonNull
		private final Consumer<InlineContentView> mCallback;

		/**
		* Indicates whether the first content has been received or not.
		*/
		private boolean mFirstContentReceived = false;

		/**
		* The client (IME) side view which internally wraps a remote view. It'll be set when
		* {@link #onContent(SurfaceControlViewHost.SurfacePackage, int, int)} is called, which
		* should only happen once in the lifecycle of this inline suggestion instance.
		*/
		@Nullable
		private InlineContentView mView;

		/**
		* The SurfacePackage pointing to the remote view. It's cached here to be sent to the next
		* available consumer.
		*/
		@Nullable
		private SurfaceControlViewHost.SurfacePackage mSurfacePackage;

		/**
		* The callback (from the {@link InlineContentView}) which consumes the surface package.
		* It's cached here to be called when the SurfacePackage is returned from the remote
		* view owning process.
		*/
		@Nullable
		private Consumer<SurfaceControlViewHost.SurfacePackage> mSurfacePackageConsumer;

		InlineContentCallbackImpl(@NonNull Context context,
		@Nullable IInlineContentProvider inlineContentProvider,
		@NonNull @CallbackExecutor Executor callbackExecutor,
		@NonNull Consumer<InlineContentView> callback) {
		mContext = context;
		mInlineContentProvider = inlineContentProvider;
		mCallbackExecutor = callbackExecutor;
		mCallback = callback;
		}
		@@ -219,28 +280,110 @@ public final class InlineSuggestion implements Parcelable {
		@BinderThread
		public void onContent(SurfaceControlViewHost.SurfacePackage content, int width,
		int height) {
		if (content == null) {
		mMainHandler.post(() -> handleOnContent(content, width, height));
		}

		@MainThread
		private void handleOnContent(SurfaceControlViewHost.SurfacePackage content, int width,
		int height) {
		if (!mFirstContentReceived) {
		handleOnFirstContentReceived(content, width, height);
		mFirstContentReceived = true;
		} else {
		handleOnSurfacePackage(content);
		}
		}

		/**
		* Called when the view content is returned for the first time.
		*/
		@MainThread
		private void handleOnFirstContentReceived(SurfaceControlViewHost.SurfacePackage content,
		int width, int height) {
		mSurfacePackage = content;
		if (mSurfacePackage == null) {
		mCallbackExecutor.execute(() -> mCallback.accept(/* view */null));
		} else {
		mView = new InlineContentView(mContext);
		mView.setLayoutParams(new ViewGroup.LayoutParams(width, height));
		mView.setChildSurfacePackage(content);
		mView.setChildSurfacePackageUpdater(getSurfacePackageUpdater());
		mCallbackExecutor.execute(() -> mCallback.accept(mView));
		}
		}

		/**
		* Called when any subsequent SurfacePackage is returned from the remote view owning
		* process.
		*/
		@MainThread
		private void handleOnSurfacePackage(SurfaceControlViewHost.SurfacePackage surfacePackage) {
		mSurfacePackage = surfacePackage;
		if (mSurfacePackage != null && mSurfacePackageConsumer != null) {
		mSurfacePackageConsumer.accept(mSurfacePackage);
		mSurfacePackageConsumer = null;
		}
		}

		@MainThread
		private void handleOnSurfacePackageReleased() {
		mSurfacePackage = null;
		try {
		mInlineContentProvider.onSurfacePackageReleased();
		} catch (RemoteException e) {
		Slog.w(TAG, "Error calling onSurfacePackageReleased(): " + e);
		}
		}

		@MainThread
		private void handleGetSurfacePackage(
		Consumer<SurfaceControlViewHost.SurfacePackage> consumer) {
		if (mSurfacePackage != null) {
		consumer.accept(mSurfacePackage);
		} else {
		mSurfacePackageConsumer = consumer;
		try {
		mInlineContentProvider.requestSurfacePackage();
		} catch (RemoteException e) {
		Slog.w(TAG, "Error calling getSurfacePackage(): " + e);
		consumer.accept(null);
		mSurfacePackageConsumer = null;
		}
		}
		}

		private InlineContentView.SurfacePackageUpdater getSurfacePackageUpdater() {
		return new InlineContentView.SurfacePackageUpdater() {
		@Override
		public void onSurfacePackageReleased() {
		mMainHandler.post(
		() -> InlineContentCallbackImpl.this.handleOnSurfacePackageReleased());
		}

		@Override
		public void getSurfacePackage(
		Consumer<SurfaceControlViewHost.SurfacePackage> consumer) {
		mMainHandler.post(
		() -> InlineContentCallbackImpl.this.handleGetSurfacePackage(consumer));
		}
		};
		}

		@BinderThread
		public void onClick() {
		mMainHandler.post(() -> {
		if (mView != null && mView.hasOnClickListeners()) {
		mView.callOnClick();
		}
		});
		}

		@BinderThread
		public void onLongClick() {
		mMainHandler.post(() -> {
		if (mView != null && mView.hasOnLongClickListeners()) {
		mView.performLongClick();
		}
		});
		}
		}

		@@ -262,6 +405,7 @@ public final class InlineSuggestion implements Parcelable {




		// Code below generated by codegen v1.0.15.
		//
		// DO NOT MODIFY!
		@@ -301,6 +445,14 @@ public final class InlineSuggestion implements Parcelable {
		return mInfo;
		}

		/**
		* @hide
		*/
		@DataClass.Generated.Member
		public @Nullable IInlineContentProvider getContentProvider() {
		return mContentProvider;
		}

		/**
		* Used to keep a strong reference to the callback so it doesn't get garbage collected.
		*
		@@ -421,7 +573,7 @@ public final class InlineSuggestion implements Parcelable {
		};

		@DataClass.Generated(
		time = 1587771173367L,
		time = 1588308946517L,
		codegenVersion = "1.0.15",
		sourceFile = "frameworks/base/core/java/android/view/inputmethod/InlineSuggestion.java",
		inputSignatures = "private static final java.lang.String TAG\nprivate final @android.annotation.NonNull android.view.inputmethod.InlineSuggestionInfo mInfo\nprivate final @android.annotation.Nullable com.android.internal.view.inline.IInlineContentProvider mContentProvider\nprivate @com.android.internal.util.DataClass.ParcelWith(android.view.inputmethod.InlineSuggestion.InlineContentCallbackImplParceling.class) @android.annotation.Nullable android.view.inputmethod.InlineSuggestion.InlineContentCallbackImpl mInlineContentCallback\npublic static @android.annotation.TestApi @android.annotation.NonNull android.view.inputmethod.InlineSuggestion newInlineSuggestion(android.view.inputmethod.InlineSuggestionInfo)\npublic void inflate(android.content.Context,android.util.Size,java.util.concurrent.Executor,java.util.function.Consumer<android.widget.inline.InlineContentView>)\nprivate static boolean isValid(int,int,int)\nprivate synchronized android.view.inputmethod.InlineSuggestion.InlineContentCallbackImpl getInlineContentCallback(android.content.Context,java.util.concurrent.Executor,java.util.function.Consumer<android.widget.inline.InlineContentView>)\nclass InlineSuggestion extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genEqualsHashCode=true, genToString=true, genHiddenConstDefs=true, genHiddenConstructor=true)")