Merge "Add translation APIs for virtual views." into sc-dev (2e3ea6a2) · Commits · e / os / android_frameworks_base

core/api/current.txt

+2 −0

Original line number	Diff line number	Diff line
		@@ -48771,6 +48771,7 @@ package android.view {
		method protected int[] onCreateDrawableState(int);
		method public android.view.inputmethod.InputConnection onCreateInputConnection(android.view.inputmethod.EditorInfo);
		method @Nullable public android.view.translation.ViewTranslationRequest onCreateTranslationRequest(@NonNull int[]);
		method public void onCreateTranslationRequests(@NonNull long[], @NonNull int[], @NonNull java.util.function.Consumer<android.view.translation.ViewTranslationRequest>);
		method @CallSuper protected void onDetachedFromWindow();
		method protected void onDisplayHint(int);
		method public boolean onDragEvent(android.view.DragEvent);
		@@ -48816,6 +48817,7 @@ package android.view {
		method public boolean onTouchEvent(android.view.MotionEvent);
		method public boolean onTrackballEvent(android.view.MotionEvent);
		method public void onTranslationResponse(@NonNull android.view.translation.ViewTranslationResponse);
		method public void onTranslationResponse(@NonNull android.util.LongSparseArray<android.view.translation.ViewTranslationResponse>);
		method @CallSuper public void onVisibilityAggregated(boolean);
		method protected void onVisibilityChanged(@NonNull android.view.View, int);
		method public void onWindowFocusChanged(boolean);

core/api/system-current.txt

+2 −0

Original line number	Diff line number	Diff line
		@@ -14725,6 +14725,7 @@ package android.webkit {
		method public default boolean onCheckIsTextEditor();
		method public void onConfigurationChanged(android.content.res.Configuration);
		method public android.view.inputmethod.InputConnection onCreateInputConnection(android.view.inputmethod.EditorInfo);
		method @Nullable public default void onCreateTranslationRequests(@NonNull long[], @NonNull int[], @NonNull java.util.function.Consumer<android.view.translation.ViewTranslationRequest>);
		method public void onDetachedFromWindow();
		method public boolean onDragEvent(android.view.DragEvent);
		method public void onDraw(android.graphics.Canvas);
		@@ -14749,6 +14750,7 @@ package android.webkit {
		method public void onStartTemporaryDetach();
		method public boolean onTouchEvent(android.view.MotionEvent);
		method public boolean onTrackballEvent(android.view.MotionEvent);
		method public default void onTranslationResponse(@NonNull android.util.LongSparseArray<android.view.translation.ViewTranslationResponse>);
		method public void onVisibilityChanged(android.view.View, int);
		method public void onWindowFocusChanged(boolean);
		method public void onWindowVisibilityChanged(int);

core/java/android/view/View.java

+46 −4

Original line number	Diff line number	Diff line
		@@ -107,6 +107,7 @@ import android.util.AttributeSet;
		import android.util.FloatProperty;
		import android.util.LayoutDirection;
		import android.util.Log;
		import android.util.LongSparseArray;
		import android.util.LongSparseLongArray;
		import android.util.Pair;
		import android.util.Pools.SynchronizedPool;
		@@ -30740,6 +30741,30 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
		return null;
		}

		/**
		* Returns a {@link ViewTranslationRequest} list which represents the content to be translated
		* in the virtual view. This is called if this view returned a virtual view structure
		* from {@link #onProvideContentCaptureStructure} and the system determined that those virtual
		* views were relevant for translation.
		*
		* <p>The default implementation does nothing.</p>
		*
		* @param virtualChildIds the virtual child ids which represents the child views in the virtual
		* view.
		* @param supportedFormats the supported translation formats. For now, the only possible value
		* is the {@link android.view.translation.TranslationSpec#DATA_FORMAT_TEXT}.
		* @param requestsCollector a {@link ViewTranslationRequest} collector that will be called
		* multiple times to collect the information to be translated in the virtual view. One
		* {@link ViewTranslationRequest} per virtual child. The {@link ViewTranslationRequest} must
		* contains the {@link AutofillId} corresponding to the virtualChildIds.
		*/
		@SuppressLint("NullableCollection")
		public void onCreateTranslationRequests(@NonNull long[] virtualChildIds,
		@NonNull @DataFormat int[] supportedFormats,
		@NonNull Consumer<ViewTranslationRequest> requestsCollector) {
		// no-op
		}

		/**
		* Returns a {@link ViewTranslationCallback} that is used to display/hide the translated
		* information. If the View supports displaying translated content, it should implement
		@@ -30781,6 +30806,21 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
		// no-op
		}

		/**
		* Called when the content from {@link View#onCreateTranslationRequest} had been translated by
		* the TranslationService.
		*
		* <p> The default implementation does nothing.</p>
		*
		* @param response a {@link ViewTranslationResponse} SparseArray for the request that send by
		* {@link View#onCreateTranslationRequests} that contains the translated information which can
		* be shown in the view. The key of SparseArray is
		* the virtual child ids.
		*/
		public void onTranslationResponse(@NonNull LongSparseArray<ViewTranslationResponse> response) {
		// no-op
		}

		/**
		* Dispatch to collect the {@link ViewTranslationRequest}s for translation purpose by traversing
		* the hierarchy when the app requests ui translation. Typically, this method should only be
		@@ -30788,7 +30828,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
		* classes should override {@link View#onCreateTranslationRequest}. When requested to start the
		* ui translation, the system will call this method to traverse the view hierarchy to call
		* {@link View#onCreateTranslationRequest} to build {@link ViewTranslationRequest}s and create a
		* {@link android.view.translation.Translator} to translate the requests.
		* {@link android.view.translation.Translator} to translate the requests. All the
		* {@link ViewTranslationRequest}s will be added when the traversal is done.
		*
		* <p> The default implementation will call {@link View#onCreateTranslationRequest} to build
		* {@link ViewTranslationRequest} if the view should be translated. </p>
		@@ -30808,14 +30849,15 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
		@NonNull List<ViewTranslationRequest> requests) {
		AutofillId autofillId = getAutofillId();
		if (viewIds.containsKey(autofillId)) {
		ViewTranslationRequest request = null;
		if (viewIds.get(autofillId) == null) {
		request = onCreateTranslationRequest(supportedFormats);
		ViewTranslationRequest request = onCreateTranslationRequest(supportedFormats);
		if (request != null && request.getKeys().size() > 0) {
		requests.add(request);
		}
		} else {
		// TODO: handle virtual view
		onCreateTranslationRequests(viewIds.get(autofillId), supportedFormats, request -> {
		requests.add(request);
		});
		}
		}
		}

core/java/android/view/translation/UiTranslationController.java

+78 −16

Original line number	Diff line number	Diff line
		@@ -31,6 +31,7 @@ import android.os.HandlerThread;
		import android.os.Process;
		import android.util.ArrayMap;
		import android.util.Log;
		import android.util.LongSparseArray;
		import android.util.Pair;
		import android.util.SparseArray;
		import android.util.SparseIntArray;
		@@ -238,29 +239,90 @@ public class UiTranslationController {
		final SparseArray<ViewTranslationResponse> translatedResult =
		response.getViewTranslationResponses();
		final SparseArray<ViewTranslationResponse> viewsResult = new SparseArray<>();
		final SparseArray<ViewTranslationResponse> virtualViewsResult = new SparseArray<>();
		final List<AutofillId> viewIds = new ArrayList<>();
		final SparseArray<LongSparseArray<ViewTranslationResponse>> virtualViewsResult =
		new SparseArray<>();
		// TODO: use another structure to prevent autoboxing?
		final List<Integer> viewIds = new ArrayList<>();

		for (int i = 0; i < translatedResult.size(); i++) {
		final ViewTranslationResponse result = translatedResult.valueAt(i);
		final AutofillId autofillId = result.getAutofillId();
		if (!viewIds.contains(autofillId.getViewId())) {
		viewIds.add(autofillId.getViewId());
		}
		if (autofillId.isNonVirtual()) {
		viewsResult.put(translatedResult.keyAt(i), result);
		viewIds.add(autofillId);
		} else {
		virtualViewsResult.put(translatedResult.keyAt(i), result);
		final boolean isVirtualViewAdded =
		virtualViewsResult.indexOfKey(autofillId.getViewId()) > 0;
		final LongSparseArray<ViewTranslationResponse> childIds =
		isVirtualViewAdded ? virtualViewsResult.get(autofillId.getViewId())
		: new LongSparseArray<>();
		childIds.put(autofillId.getVirtualChildLongId(), result);
		if (!isVirtualViewAdded) {
		virtualViewsResult.put(autofillId.getViewId(), childIds);
		}
		}
		}
		// Traverse tree and get views by the responsed AutofillId
		findViewsTraversalByAutofillIds(viewIds);

		if (viewsResult.size() > 0) {
		onTranslationCompleted(viewsResult, viewIds);
		onTranslationCompleted(viewsResult);
		}
		if (virtualViewsResult.size() > 0) {
		onVirtualViewTranslationCompleted(virtualViewsResult);
		}
		}

		/**
		* The method is used to handle the translation result for the vertual views.
		*/
		private void onVirtualViewTranslationCompleted(
		SparseArray<LongSparseArray<ViewTranslationResponse>> translatedResult) {
		if (!mActivity.isResumed()) {
		if (DEBUG) {
		Log.v(TAG, "onTranslationCompleted: Activity is not resumed.");
		}
		return;
		}
		synchronized (mLock) {
		if (mCurrentState == STATE_UI_TRANSLATION_FINISHED) {
		Log.w(TAG, "onTranslationCompleted: the translation state is finished now. "
		+ "Skip to show the translated text.");
		return;
		}
		for (int i = 0; i < translatedResult.size(); i++) {
		final AutofillId autofillId = new AutofillId(translatedResult.keyAt(i));
		final View view = mViews.get(autofillId).get();
		if (view == null) {
		Log.w(TAG, "onTranslationCompleted: the view for autofill id " + autofillId
		+ " may be gone.");
		continue;
		}
		final LongSparseArray<ViewTranslationResponse> virtualChildResponse =
		translatedResult.valueAt(i);
		mActivity.runOnUiThread(() -> {
		if (view.getViewTranslationCallback() == null) {
		if (DEBUG) {
		Log.d(TAG, view + " doesn't support showing translation because of "
		+ "null ViewTranslationCallback.");
		}
		return;
		}
		view.onTranslationResponse(virtualChildResponse);
		if (view.getViewTranslationCallback() != null) {
		view.getViewTranslationCallback().onShowTranslation(view);
		}
		});
		}
		}
		//TODO(b/177960696): call virtual views onTranslationCompleted()
		}

		/**
		* The method is used to handle the translation result for non-vertual views.
		*/
		private void onTranslationCompleted(SparseArray<ViewTranslationResponse> translatedResult,
		List<AutofillId> viewIds) {
		private void onTranslationCompleted(SparseArray<ViewTranslationResponse> translatedResult) {
		if (!mActivity.isResumed()) {
		if (DEBUG) {
		Log.v(TAG, "onTranslationCompleted: Activity is not resumed.");
		@@ -277,8 +339,6 @@ public class UiTranslationController {
		+ "Skip to show the translated text.");
		return;
		}
		// Traverse tree and get views by the responsed AutofillId
		findViewsTraversalByAutofillIds(viewIds);
		for (int i = 0; i < resultCount; i++) {
		final ViewTranslationResponse response = translatedResult.valueAt(i);
		final AutofillId autofillId = response.getAutofillId();
		@@ -300,7 +360,9 @@ public class UiTranslationController {
		return;
		}
		view.onTranslationResponse(response);
		if (view.getViewTranslationCallback() != null) {
		view.getViewTranslationCallback().onShowTranslation(view);
		}
		});
		}
		}
		@@ -359,7 +421,7 @@ public class UiTranslationController {
		childs = new long[childCount];
		viewIds.put(virtualViewAutofillId, childs);
		}
		int end = childs.length;
		int end = childs.length - 1;
		childs[end] = autofillId.getVirtualChildLongId();
		}
		}
		@@ -403,7 +465,7 @@ public class UiTranslationController {
		return new int[] {TranslationSpec.DATA_FORMAT_TEXT};
		}

		private void findViewsTraversalByAutofillIds(List<AutofillId> sourceViewIds) {
		private void findViewsTraversalByAutofillIds(List<Integer> sourceViewIds) {
		final ArrayList<ViewRootImpl> roots =
		WindowManagerGlobal.getInstance().getRootViews(mActivity.getActivityToken());
		for (int rootNum = 0; rootNum < roots.size(); rootNum++) {
		@@ -417,7 +479,7 @@ public class UiTranslationController {
		}

		private void findViewsTraversalByAutofillIds(ViewGroup viewGroup,
		List<AutofillId> sourceViewIds) {
		List<Integer> sourceViewIds) {
		final int childCount = viewGroup.getChildCount();
		for (int i = 0; i < childCount; ++i) {
		final View child = viewGroup.getChildAt(i);
		@@ -429,9 +491,9 @@ public class UiTranslationController {
		}
		}

		private void addViewIfNeeded(List<AutofillId> sourceViewIds, View view) {
		private void addViewIfNeeded(List<Integer> sourceViewIds, View view) {
		final AutofillId autofillId = view.getAutofillId();
		if (sourceViewIds.contains(autofillId)) {
		if (sourceViewIds.contains(autofillId.getViewId()) && !mViews.containsKey(autofillId)) {
		mViews.put(autofillId, new WeakReference<>(view));
		}
		}

core/java/android/webkit/WebView.java

+19 −0

Original line number	Diff line number	Diff line
		@@ -45,6 +45,7 @@ import android.os.StrictMode;
		import android.print.PrintDocumentAdapter;
		import android.util.AttributeSet;
		import android.util.Log;
		import android.util.LongSparseArray;
		import android.util.SparseArray;
		import android.view.DragEvent;
		import android.view.KeyEvent;
		@@ -64,6 +65,9 @@ import android.view.inputmethod.EditorInfo;
		import android.view.inputmethod.InputConnection;
		import android.view.inspector.InspectableProperty;
		import android.view.textclassifier.TextClassifier;
		import android.view.translation.TranslationSpec.DataFormat;
		import android.view.translation.ViewTranslationRequest;
		import android.view.translation.ViewTranslationResponse;
		import android.widget.AbsoluteLayout;

		import java.io.BufferedWriter;
		@@ -73,6 +77,7 @@ import java.lang.annotation.RetentionPolicy;
		import java.util.List;
		import java.util.Map;
		import java.util.concurrent.Executor;
		import java.util.function.Consumer;

		/**
		* A View that displays web pages.
		@@ -2854,6 +2859,20 @@ public class WebView extends AbsoluteLayout
		return mProvider.getViewDelegate().isVisibleToUserForAutofill(virtualId);
		}

		@Override
		@Nullable
		public void onCreateTranslationRequests(@NonNull long[] virtualChildIds,
		@NonNull @DataFormat int[] supportedFormats,
		@NonNull Consumer<ViewTranslationRequest> requestsCollector) {
		mProvider.getViewDelegate().onCreateTranslationRequests(virtualChildIds, supportedFormats,
		requestsCollector);
		}

		@Override
		public void onTranslationResponse(@NonNull LongSparseArray<ViewTranslationResponse> response) {
		mProvider.getViewDelegate().onTranslationResponse(response);
		}

		/** @hide */
		@Override
		public void onInitializeAccessibilityNodeInfoInternal(AccessibilityNodeInfo info) {