Add translation APIs for virtual views. (e1b23cd4) · Commits · e / os / android_frameworks_base

core/api/current.txt

+2 −0

core/api/system-current.txt

+2 −0

core/java/android/view/View.java

+46 −4

Original line number	Original line	Diff line number	Diff line
	@@ -107,6 +107,7 @@ import android.util.AttributeSet;
	import android.util.FloatProperty;		import android.util.FloatProperty;
	import android.util.LayoutDirection;		import android.util.LayoutDirection;
	import android.util.Log;		import android.util.Log;
			import android.util.LongSparseArray;
	import android.util.LongSparseLongArray;		import android.util.LongSparseLongArray;
	import android.util.Pair;		import android.util.Pair;
	import android.util.Pools.SynchronizedPool;		import android.util.Pools.SynchronizedPool;
	@@ -30740,6 +30741,30 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
	return null;		return null;
	}		}

			/**
			* Returns a {@link ViewTranslationRequest} list which represents the content to be translated
			* in the virtual view. This is called if this view returned a virtual view structure
			* from {@link #onProvideContentCaptureStructure} and the system determined that those virtual
			* views were relevant for translation.
			*
			* <p>The default implementation does nothing.</p>
			*
			* @param virtualChildIds the virtual child ids which represents the child views in the virtual
			* view.
			* @param supportedFormats the supported translation formats. For now, the only possible value
			* is the {@link android.view.translation.TranslationSpec#DATA_FORMAT_TEXT}.
			* @param requestsCollector a {@link ViewTranslationRequest} collector that will be called
			* multiple times to collect the information to be translated in the virtual view. One
			* {@link ViewTranslationRequest} per virtual child. The {@link ViewTranslationRequest} must
			* contains the {@link AutofillId} corresponding to the virtualChildIds.
			*/
			@SuppressLint("NullableCollection")
			public void onCreateTranslationRequests(@NonNull long[] virtualChildIds,
			@NonNull @DataFormat int[] supportedFormats,
			@NonNull Consumer<ViewTranslationRequest> requestsCollector) {
			// no-op
			}

	/**		/**
	* Returns a {@link ViewTranslationCallback} that is used to display/hide the translated		* Returns a {@link ViewTranslationCallback} that is used to display/hide the translated
	* information. If the View supports displaying translated content, it should implement		* information. If the View supports displaying translated content, it should implement
	@@ -30781,6 +30806,21 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
	// no-op		// no-op
	}		}

			/**
			* Called when the content from {@link View#onCreateTranslationRequest} had been translated by
			* the TranslationService.
			*
			* <p> The default implementation does nothing.</p>
			*
			* @param response a {@link ViewTranslationResponse} SparseArray for the request that send by
			* {@link View#onCreateTranslationRequests} that contains the translated information which can
			* be shown in the view. The key of SparseArray is
			* the virtual child ids.
			*/
			public void onTranslationResponse(@NonNull LongSparseArray<ViewTranslationResponse> response) {
			// no-op
			}

	/**		/**
	* Dispatch to collect the {@link ViewTranslationRequest}s for translation purpose by traversing		* Dispatch to collect the {@link ViewTranslationRequest}s for translation purpose by traversing
	* the hierarchy when the app requests ui translation. Typically, this method should only be		* the hierarchy when the app requests ui translation. Typically, this method should only be
	@@ -30788,7 +30828,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
	* classes should override {@link View#onCreateTranslationRequest}. When requested to start the		* classes should override {@link View#onCreateTranslationRequest}. When requested to start the
	* ui translation, the system will call this method to traverse the view hierarchy to call		* ui translation, the system will call this method to traverse the view hierarchy to call
	* {@link View#onCreateTranslationRequest} to build {@link ViewTranslationRequest}s and create a		* {@link View#onCreateTranslationRequest} to build {@link ViewTranslationRequest}s and create a
	* {@link android.view.translation.Translator} to translate the requests.		* {@link android.view.translation.Translator} to translate the requests. All the
			* {@link ViewTranslationRequest}s will be added when the traversal is done.
	*		*
	* <p> The default implementation will call {@link View#onCreateTranslationRequest} to build		* <p> The default implementation will call {@link View#onCreateTranslationRequest} to build
	* {@link ViewTranslationRequest} if the view should be translated. </p>		* {@link ViewTranslationRequest} if the view should be translated. </p>
	@@ -30808,14 +30849,15 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
	@NonNull List<ViewTranslationRequest> requests) {		@NonNull List<ViewTranslationRequest> requests) {
	AutofillId autofillId = getAutofillId();		AutofillId autofillId = getAutofillId();
	if (viewIds.containsKey(autofillId)) {		if (viewIds.containsKey(autofillId)) {
	ViewTranslationRequest request = null;
	if (viewIds.get(autofillId) == null) {		if (viewIds.get(autofillId) == null) {
	request = onCreateTranslationRequest(supportedFormats);		ViewTranslationRequest request = onCreateTranslationRequest(supportedFormats);
	if (request != null && request.getKeys().size() > 0) {		if (request != null && request.getKeys().size() > 0) {
	requests.add(request);		requests.add(request);
	}		}
	} else {		} else {
	// TODO: handle virtual view		onCreateTranslationRequests(viewIds.get(autofillId), supportedFormats, request -> {
			requests.add(request);
			});
	}		}
	}		}
	}		}

core/java/android/view/translation/UiTranslationController.java

+78 −16

Original line number	Original line	Diff line number	Diff line
	@@ -31,6 +31,7 @@ import android.os.HandlerThread;
	import android.os.Process;		import android.os.Process;
	import android.util.ArrayMap;		import android.util.ArrayMap;
	import android.util.Log;		import android.util.Log;
			import android.util.LongSparseArray;
	import android.util.Pair;		import android.util.Pair;
	import android.util.SparseArray;		import android.util.SparseArray;
	import android.util.SparseIntArray;		import android.util.SparseIntArray;
	@@ -238,29 +239,90 @@ public class UiTranslationController {
	final SparseArray<ViewTranslationResponse> translatedResult =		final SparseArray<ViewTranslationResponse> translatedResult =
	response.getViewTranslationResponses();		response.getViewTranslationResponses();
	final SparseArray<ViewTranslationResponse> viewsResult = new SparseArray<>();		final SparseArray<ViewTranslationResponse> viewsResult = new SparseArray<>();
	final SparseArray<ViewTranslationResponse> virtualViewsResult = new SparseArray<>();		final SparseArray<LongSparseArray<ViewTranslationResponse>> virtualViewsResult =
	final List<AutofillId> viewIds = new ArrayList<>();		new SparseArray<>();
			// TODO: use another structure to prevent autoboxing?
			final List<Integer> viewIds = new ArrayList<>();

	for (int i = 0; i < translatedResult.size(); i++) {		for (int i = 0; i < translatedResult.size(); i++) {
	final ViewTranslationResponse result = translatedResult.valueAt(i);		final ViewTranslationResponse result = translatedResult.valueAt(i);
	final AutofillId autofillId = result.getAutofillId();		final AutofillId autofillId = result.getAutofillId();
			if (!viewIds.contains(autofillId.getViewId())) {
			viewIds.add(autofillId.getViewId());
			}
	if (autofillId.isNonVirtual()) {		if (autofillId.isNonVirtual()) {
	viewsResult.put(translatedResult.keyAt(i), result);		viewsResult.put(translatedResult.keyAt(i), result);
	viewIds.add(autofillId);
	} else {		} else {
	virtualViewsResult.put(translatedResult.keyAt(i), result);		final boolean isVirtualViewAdded =
			virtualViewsResult.indexOfKey(autofillId.getViewId()) > 0;
			final LongSparseArray<ViewTranslationResponse> childIds =
			isVirtualViewAdded ? virtualViewsResult.get(autofillId.getViewId())
			: new LongSparseArray<>();
			childIds.put(autofillId.getVirtualChildLongId(), result);
			if (!isVirtualViewAdded) {
			virtualViewsResult.put(autofillId.getViewId(), childIds);
			}
	}		}
	}		}
			// Traverse tree and get views by the responsed AutofillId
			findViewsTraversalByAutofillIds(viewIds);

	if (viewsResult.size() > 0) {		if (viewsResult.size() > 0) {
	onTranslationCompleted(viewsResult, viewIds);		onTranslationCompleted(viewsResult);
			}
			if (virtualViewsResult.size() > 0) {
			onVirtualViewTranslationCompleted(virtualViewsResult);
			}
			}

			/**
			* The method is used to handle the translation result for the vertual views.
			*/
			private void onVirtualViewTranslationCompleted(
			SparseArray<LongSparseArray<ViewTranslationResponse>> translatedResult) {
			if (!mActivity.isResumed()) {
			if (DEBUG) {
			Log.v(TAG, "onTranslationCompleted: Activity is not resumed.");
			}
			return;
			}
			synchronized (mLock) {
			if (mCurrentState == STATE_UI_TRANSLATION_FINISHED) {
			Log.w(TAG, "onTranslationCompleted: the translation state is finished now. "
			+ "Skip to show the translated text.");
			return;
			}
			for (int i = 0; i < translatedResult.size(); i++) {
			final AutofillId autofillId = new AutofillId(translatedResult.keyAt(i));
			final View view = mViews.get(autofillId).get();
			if (view == null) {
			Log.w(TAG, "onTranslationCompleted: the view for autofill id " + autofillId
			+ " may be gone.");
			continue;
			}
			final LongSparseArray<ViewTranslationResponse> virtualChildResponse =
			translatedResult.valueAt(i);
			mActivity.runOnUiThread(() -> {
			if (view.getViewTranslationCallback() == null) {
			if (DEBUG) {
			Log.d(TAG, view + " doesn't support showing translation because of "
			+ "null ViewTranslationCallback.");
			}
			return;
			}
			view.onTranslationResponse(virtualChildResponse);
			if (view.getViewTranslationCallback() != null) {
			view.getViewTranslationCallback().onShowTranslation(view);
			}
			});
			}
	}		}
	//TODO(b/177960696): call virtual views onTranslationCompleted()
	}		}

	/**		/**
	* The method is used to handle the translation result for non-vertual views.		* The method is used to handle the translation result for non-vertual views.
	*/		*/
	private void onTranslationCompleted(SparseArray<ViewTranslationResponse> translatedResult,		private void onTranslationCompleted(SparseArray<ViewTranslationResponse> translatedResult) {
	List<AutofillId> viewIds) {
	if (!mActivity.isResumed()) {		if (!mActivity.isResumed()) {
	if (DEBUG) {		if (DEBUG) {
	Log.v(TAG, "onTranslationCompleted: Activity is not resumed.");		Log.v(TAG, "onTranslationCompleted: Activity is not resumed.");
	@@ -277,8 +339,6 @@ public class UiTranslationController {
	+ "Skip to show the translated text.");		+ "Skip to show the translated text.");
	return;		return;
	}		}
	// Traverse tree and get views by the responsed AutofillId
	findViewsTraversalByAutofillIds(viewIds);
	for (int i = 0; i < resultCount; i++) {		for (int i = 0; i < resultCount; i++) {
	final ViewTranslationResponse response = translatedResult.valueAt(i);		final ViewTranslationResponse response = translatedResult.valueAt(i);
	final AutofillId autofillId = response.getAutofillId();		final AutofillId autofillId = response.getAutofillId();
	@@ -300,7 +360,9 @@ public class UiTranslationController {
	return;		return;
	}		}
	view.onTranslationResponse(response);		view.onTranslationResponse(response);
			if (view.getViewTranslationCallback() != null) {
	view.getViewTranslationCallback().onShowTranslation(view);		view.getViewTranslationCallback().onShowTranslation(view);
			}
	});		});
	}		}
	}		}
	@@ -359,7 +421,7 @@ public class UiTranslationController {
	childs = new long[childCount];		childs = new long[childCount];
	viewIds.put(virtualViewAutofillId, childs);		viewIds.put(virtualViewAutofillId, childs);
	}		}
	int end = childs.length;		int end = childs.length - 1;
	childs[end] = autofillId.getVirtualChildLongId();		childs[end] = autofillId.getVirtualChildLongId();
	}		}
	}		}
	@@ -403,7 +465,7 @@ public class UiTranslationController {
	return new int[] {TranslationSpec.DATA_FORMAT_TEXT};		return new int[] {TranslationSpec.DATA_FORMAT_TEXT};
	}		}

	private void findViewsTraversalByAutofillIds(List<AutofillId> sourceViewIds) {		private void findViewsTraversalByAutofillIds(List<Integer> sourceViewIds) {
	final ArrayList<ViewRootImpl> roots =		final ArrayList<ViewRootImpl> roots =
	WindowManagerGlobal.getInstance().getRootViews(mActivity.getActivityToken());		WindowManagerGlobal.getInstance().getRootViews(mActivity.getActivityToken());
	for (int rootNum = 0; rootNum < roots.size(); rootNum++) {		for (int rootNum = 0; rootNum < roots.size(); rootNum++) {
	@@ -417,7 +479,7 @@ public class UiTranslationController {
	}		}

	private void findViewsTraversalByAutofillIds(ViewGroup viewGroup,		private void findViewsTraversalByAutofillIds(ViewGroup viewGroup,
	List<AutofillId> sourceViewIds) {		List<Integer> sourceViewIds) {
	final int childCount = viewGroup.getChildCount();		final int childCount = viewGroup.getChildCount();
	for (int i = 0; i < childCount; ++i) {		for (int i = 0; i < childCount; ++i) {
	final View child = viewGroup.getChildAt(i);		final View child = viewGroup.getChildAt(i);
	@@ -429,9 +491,9 @@ public class UiTranslationController {
	}		}
	}		}

	private void addViewIfNeeded(List<AutofillId> sourceViewIds, View view) {		private void addViewIfNeeded(List<Integer> sourceViewIds, View view) {
	final AutofillId autofillId = view.getAutofillId();		final AutofillId autofillId = view.getAutofillId();
	if (sourceViewIds.contains(autofillId)) {		if (sourceViewIds.contains(autofillId.getViewId()) && !mViews.containsKey(autofillId)) {
	mViews.put(autofillId, new WeakReference<>(view));		mViews.put(autofillId, new WeakReference<>(view));
	}		}
	}		}

core/java/android/webkit/WebView.java

+19 −0

Original line number	Original line	Diff line number	Diff line
	@@ -45,6 +45,7 @@ import android.os.StrictMode;
	import android.print.PrintDocumentAdapter;		import android.print.PrintDocumentAdapter;
	import android.util.AttributeSet;		import android.util.AttributeSet;
	import android.util.Log;		import android.util.Log;
			import android.util.LongSparseArray;
	import android.util.SparseArray;		import android.util.SparseArray;
	import android.view.DragEvent;		import android.view.DragEvent;
	import android.view.KeyEvent;		import android.view.KeyEvent;
	@@ -64,6 +65,9 @@ import android.view.inputmethod.EditorInfo;
	import android.view.inputmethod.InputConnection;		import android.view.inputmethod.InputConnection;
	import android.view.inspector.InspectableProperty;		import android.view.inspector.InspectableProperty;
	import android.view.textclassifier.TextClassifier;		import android.view.textclassifier.TextClassifier;
			import android.view.translation.TranslationSpec.DataFormat;
			import android.view.translation.ViewTranslationRequest;
			import android.view.translation.ViewTranslationResponse;
	import android.widget.AbsoluteLayout;		import android.widget.AbsoluteLayout;

	import java.io.BufferedWriter;		import java.io.BufferedWriter;
	@@ -73,6 +77,7 @@ import java.lang.annotation.RetentionPolicy;
	import java.util.List;		import java.util.List;
	import java.util.Map;		import java.util.Map;
	import java.util.concurrent.Executor;		import java.util.concurrent.Executor;
			import java.util.function.Consumer;

	/**		/**
	* A View that displays web pages.		* A View that displays web pages.
	@@ -2854,6 +2859,20 @@ public class WebView extends AbsoluteLayout
	return mProvider.getViewDelegate().isVisibleToUserForAutofill(virtualId);		return mProvider.getViewDelegate().isVisibleToUserForAutofill(virtualId);
	}		}

			@Override
			@Nullable
			public void onCreateTranslationRequests(@NonNull long[] virtualChildIds,
			@NonNull @DataFormat int[] supportedFormats,
			@NonNull Consumer<ViewTranslationRequest> requestsCollector) {
			mProvider.getViewDelegate().onCreateTranslationRequests(virtualChildIds, supportedFormats,
			requestsCollector);
			}

			@Override
			public void onTranslationResponse(@NonNull LongSparseArray<ViewTranslationResponse> response) {
			mProvider.getViewDelegate().onTranslationResponse(response);
			}

	/** @hide */		/** @hide */
	@Override		@Override
	public void onInitializeAccessibilityNodeInfoInternal(AccessibilityNodeInfo info) {		public void onInitializeAccessibilityNodeInfoInternal(AccessibilityNodeInfo info) {