Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e1b23cd4 authored by Joanne Chung's avatar Joanne Chung
Browse files

Add translation APIs for virtual views.

Bug: 178046780
Test: manual
Test: atest CtsTranslationTestCases
CTS-Coverage-Bug: 177960696

Change-Id: I99caed0d42b3a164a25c2707eba7057c42c19140
parent 948bd864
Loading
Loading
Loading
Loading
+2 −0
Original line number Original line Diff line number Diff line
@@ -48771,6 +48771,7 @@ package android.view {
    method protected int[] onCreateDrawableState(int);
    method protected int[] onCreateDrawableState(int);
    method public android.view.inputmethod.InputConnection onCreateInputConnection(android.view.inputmethod.EditorInfo);
    method public android.view.inputmethod.InputConnection onCreateInputConnection(android.view.inputmethod.EditorInfo);
    method @Nullable public android.view.translation.ViewTranslationRequest onCreateTranslationRequest(@NonNull int[]);
    method @Nullable public android.view.translation.ViewTranslationRequest onCreateTranslationRequest(@NonNull int[]);
    method public void onCreateTranslationRequests(@NonNull long[], @NonNull int[], @NonNull java.util.function.Consumer<android.view.translation.ViewTranslationRequest>);
    method @CallSuper protected void onDetachedFromWindow();
    method @CallSuper protected void onDetachedFromWindow();
    method protected void onDisplayHint(int);
    method protected void onDisplayHint(int);
    method public boolean onDragEvent(android.view.DragEvent);
    method public boolean onDragEvent(android.view.DragEvent);
@@ -48816,6 +48817,7 @@ package android.view {
    method public boolean onTouchEvent(android.view.MotionEvent);
    method public boolean onTouchEvent(android.view.MotionEvent);
    method public boolean onTrackballEvent(android.view.MotionEvent);
    method public boolean onTrackballEvent(android.view.MotionEvent);
    method public void onTranslationResponse(@NonNull android.view.translation.ViewTranslationResponse);
    method public void onTranslationResponse(@NonNull android.view.translation.ViewTranslationResponse);
    method public void onTranslationResponse(@NonNull android.util.LongSparseArray<android.view.translation.ViewTranslationResponse>);
    method @CallSuper public void onVisibilityAggregated(boolean);
    method @CallSuper public void onVisibilityAggregated(boolean);
    method protected void onVisibilityChanged(@NonNull android.view.View, int);
    method protected void onVisibilityChanged(@NonNull android.view.View, int);
    method public void onWindowFocusChanged(boolean);
    method public void onWindowFocusChanged(boolean);
+2 −0
Original line number Original line Diff line number Diff line
@@ -14719,6 +14719,7 @@ package android.webkit {
    method public default boolean onCheckIsTextEditor();
    method public default boolean onCheckIsTextEditor();
    method public void onConfigurationChanged(android.content.res.Configuration);
    method public void onConfigurationChanged(android.content.res.Configuration);
    method public android.view.inputmethod.InputConnection onCreateInputConnection(android.view.inputmethod.EditorInfo);
    method public android.view.inputmethod.InputConnection onCreateInputConnection(android.view.inputmethod.EditorInfo);
    method @Nullable public default void onCreateTranslationRequests(@NonNull long[], @NonNull int[], @NonNull java.util.function.Consumer<android.view.translation.ViewTranslationRequest>);
    method public void onDetachedFromWindow();
    method public void onDetachedFromWindow();
    method public boolean onDragEvent(android.view.DragEvent);
    method public boolean onDragEvent(android.view.DragEvent);
    method public void onDraw(android.graphics.Canvas);
    method public void onDraw(android.graphics.Canvas);
@@ -14743,6 +14744,7 @@ package android.webkit {
    method public void onStartTemporaryDetach();
    method public void onStartTemporaryDetach();
    method public boolean onTouchEvent(android.view.MotionEvent);
    method public boolean onTouchEvent(android.view.MotionEvent);
    method public boolean onTrackballEvent(android.view.MotionEvent);
    method public boolean onTrackballEvent(android.view.MotionEvent);
    method public default void onTranslationResponse(@NonNull android.util.LongSparseArray<android.view.translation.ViewTranslationResponse>);
    method public void onVisibilityChanged(android.view.View, int);
    method public void onVisibilityChanged(android.view.View, int);
    method public void onWindowFocusChanged(boolean);
    method public void onWindowFocusChanged(boolean);
    method public void onWindowVisibilityChanged(int);
    method public void onWindowVisibilityChanged(int);
+46 −4
Original line number Original line Diff line number Diff line
@@ -107,6 +107,7 @@ import android.util.AttributeSet;
import android.util.FloatProperty;
import android.util.FloatProperty;
import android.util.LayoutDirection;
import android.util.LayoutDirection;
import android.util.Log;
import android.util.Log;
import android.util.LongSparseArray;
import android.util.LongSparseLongArray;
import android.util.LongSparseLongArray;
import android.util.Pair;
import android.util.Pair;
import android.util.Pools.SynchronizedPool;
import android.util.Pools.SynchronizedPool;
@@ -30740,6 +30741,30 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
        return null;
        return null;
    }
    }
    /**
     * Returns a {@link ViewTranslationRequest} list which represents the content to be translated
     * in the virtual view. This is called if this view returned a virtual view structure
     * from {@link #onProvideContentCaptureStructure} and the system determined that those virtual
     * views were relevant for translation.
     *
     * <p>The default implementation does nothing.</p>
     *
     * @param virtualChildIds the virtual child ids which represents the child views in the virtual
     * view.
     * @param supportedFormats the supported translation formats. For now, the only possible value
     * is the {@link android.view.translation.TranslationSpec#DATA_FORMAT_TEXT}.
     * @param requestsCollector a {@link ViewTranslationRequest} collector that will be called
     * multiple times to collect the information to be translated in the virtual view. One
     * {@link ViewTranslationRequest} per virtual child. The {@link ViewTranslationRequest} must
     * contains the {@link AutofillId} corresponding to the virtualChildIds.
     */
    @SuppressLint("NullableCollection")
    public void onCreateTranslationRequests(@NonNull long[] virtualChildIds,
            @NonNull @DataFormat int[] supportedFormats,
            @NonNull Consumer<ViewTranslationRequest> requestsCollector) {
        // no-op
    }
    /**
    /**
     * Returns a {@link ViewTranslationCallback} that is used to display/hide the translated
     * Returns a {@link ViewTranslationCallback} that is used to display/hide the translated
     * information. If the View supports displaying translated content, it should implement
     * information. If the View supports displaying translated content, it should implement
@@ -30781,6 +30806,21 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
        // no-op
        // no-op
    }
    }
    /**
     * Called when the content from {@link View#onCreateTranslationRequest} had been translated by
     * the TranslationService.
     *
     * <p> The default implementation does nothing.</p>
     *
     * @param response a {@link ViewTranslationResponse} SparseArray for the request that send by
     * {@link View#onCreateTranslationRequests} that contains the translated information which can
     * be shown in the view. The key of SparseArray is
     * the virtual child ids.
     */
    public void onTranslationResponse(@NonNull LongSparseArray<ViewTranslationResponse> response) {
        // no-op
    }
    /**
    /**
     * Dispatch to collect the {@link ViewTranslationRequest}s for translation purpose by traversing
     * Dispatch to collect the {@link ViewTranslationRequest}s for translation purpose by traversing
     * the hierarchy when the app requests ui translation. Typically, this method should only be
     * the hierarchy when the app requests ui translation. Typically, this method should only be
@@ -30788,7 +30828,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
     * classes should override {@link View#onCreateTranslationRequest}. When requested to start the
     * classes should override {@link View#onCreateTranslationRequest}. When requested to start the
     * ui translation, the system will call this method to traverse the view hierarchy to call
     * ui translation, the system will call this method to traverse the view hierarchy to call
     * {@link View#onCreateTranslationRequest} to build {@link ViewTranslationRequest}s and create a
     * {@link View#onCreateTranslationRequest} to build {@link ViewTranslationRequest}s and create a
     * {@link android.view.translation.Translator} to translate the requests.
     * {@link android.view.translation.Translator} to translate the requests. All the
     * {@link ViewTranslationRequest}s will be added when the traversal is done.
     *
     *
     * <p> The default implementation will call {@link View#onCreateTranslationRequest} to build
     * <p> The default implementation will call {@link View#onCreateTranslationRequest} to build
     * {@link ViewTranslationRequest} if the view should be translated. </p>
     * {@link ViewTranslationRequest} if the view should be translated. </p>
@@ -30808,14 +30849,15 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
            @NonNull List<ViewTranslationRequest> requests) {
            @NonNull List<ViewTranslationRequest> requests) {
        AutofillId autofillId = getAutofillId();
        AutofillId autofillId = getAutofillId();
        if (viewIds.containsKey(autofillId)) {
        if (viewIds.containsKey(autofillId)) {
            ViewTranslationRequest request = null;
            if (viewIds.get(autofillId) == null) {
            if (viewIds.get(autofillId) == null) {
                request = onCreateTranslationRequest(supportedFormats);
                ViewTranslationRequest request = onCreateTranslationRequest(supportedFormats);
                if (request != null && request.getKeys().size() > 0) {
                if (request != null && request.getKeys().size() > 0) {
                    requests.add(request);
                    requests.add(request);
                }
                }
            } else {
            } else {
                // TODO: handle virtual view
                onCreateTranslationRequests(viewIds.get(autofillId), supportedFormats, request -> {
                    requests.add(request);
                });
            }
            }
        }
        }
    }
    }
+78 −16
Original line number Original line Diff line number Diff line
@@ -31,6 +31,7 @@ import android.os.HandlerThread;
import android.os.Process;
import android.os.Process;
import android.util.ArrayMap;
import android.util.ArrayMap;
import android.util.Log;
import android.util.Log;
import android.util.LongSparseArray;
import android.util.Pair;
import android.util.Pair;
import android.util.SparseArray;
import android.util.SparseArray;
import android.util.SparseIntArray;
import android.util.SparseIntArray;
@@ -238,29 +239,90 @@ public class UiTranslationController {
        final SparseArray<ViewTranslationResponse> translatedResult =
        final SparseArray<ViewTranslationResponse> translatedResult =
                response.getViewTranslationResponses();
                response.getViewTranslationResponses();
        final SparseArray<ViewTranslationResponse> viewsResult = new SparseArray<>();
        final SparseArray<ViewTranslationResponse> viewsResult = new SparseArray<>();
        final SparseArray<ViewTranslationResponse> virtualViewsResult = new SparseArray<>();
        final SparseArray<LongSparseArray<ViewTranslationResponse>> virtualViewsResult =
        final List<AutofillId> viewIds = new ArrayList<>();
                new SparseArray<>();
        // TODO: use another structure to prevent autoboxing?
        final List<Integer> viewIds = new ArrayList<>();

        for (int i = 0; i < translatedResult.size(); i++) {
        for (int i = 0; i < translatedResult.size(); i++) {
            final ViewTranslationResponse result = translatedResult.valueAt(i);
            final ViewTranslationResponse result = translatedResult.valueAt(i);
            final AutofillId autofillId = result.getAutofillId();
            final AutofillId autofillId = result.getAutofillId();
            if (!viewIds.contains(autofillId.getViewId())) {
                viewIds.add(autofillId.getViewId());
            }
            if (autofillId.isNonVirtual()) {
            if (autofillId.isNonVirtual()) {
                viewsResult.put(translatedResult.keyAt(i), result);
                viewsResult.put(translatedResult.keyAt(i), result);
                viewIds.add(autofillId);
            } else {
            } else {
                virtualViewsResult.put(translatedResult.keyAt(i), result);
                final boolean isVirtualViewAdded =
                        virtualViewsResult.indexOfKey(autofillId.getViewId()) > 0;
                final LongSparseArray<ViewTranslationResponse> childIds =
                        isVirtualViewAdded ? virtualViewsResult.get(autofillId.getViewId())
                                : new LongSparseArray<>();
                childIds.put(autofillId.getVirtualChildLongId(), result);
                if (!isVirtualViewAdded) {
                    virtualViewsResult.put(autofillId.getViewId(), childIds);
                }
            }
            }
        }
        }
        // Traverse tree and get views by the responsed AutofillId
        findViewsTraversalByAutofillIds(viewIds);

        if (viewsResult.size() > 0) {
        if (viewsResult.size() > 0) {
            onTranslationCompleted(viewsResult, viewIds);
            onTranslationCompleted(viewsResult);
        }
        if (virtualViewsResult.size() > 0) {
            onVirtualViewTranslationCompleted(virtualViewsResult);
        }
    }

    /**
     * The method is used to handle the translation result for the vertual views.
     */
    private void onVirtualViewTranslationCompleted(
            SparseArray<LongSparseArray<ViewTranslationResponse>> translatedResult) {
        if (!mActivity.isResumed()) {
            if (DEBUG) {
                Log.v(TAG, "onTranslationCompleted: Activity is not resumed.");
            }
            return;
        }
        synchronized (mLock) {
            if (mCurrentState == STATE_UI_TRANSLATION_FINISHED) {
                Log.w(TAG, "onTranslationCompleted: the translation state is finished now. "
                        + "Skip to show the translated text.");
                return;
            }
            for (int i = 0; i < translatedResult.size(); i++) {
                final AutofillId autofillId = new AutofillId(translatedResult.keyAt(i));
                final View view = mViews.get(autofillId).get();
                if (view == null) {
                    Log.w(TAG, "onTranslationCompleted: the view for autofill id " + autofillId
                            + " may be gone.");
                    continue;
                }
                final LongSparseArray<ViewTranslationResponse> virtualChildResponse =
                        translatedResult.valueAt(i);
                mActivity.runOnUiThread(() -> {
                    if (view.getViewTranslationCallback() == null) {
                        if (DEBUG) {
                            Log.d(TAG, view + " doesn't support showing translation because of "
                                    + "null ViewTranslationCallback.");
                        }
                        return;
                    }
                    view.onTranslationResponse(virtualChildResponse);
                    if (view.getViewTranslationCallback() != null) {
                        view.getViewTranslationCallback().onShowTranslation(view);
                    }
                });
            }
        }
        }
        //TODO(b/177960696): call virtual views onTranslationCompleted()
    }
    }


    /**
    /**
     * The method is used to handle the translation result for non-vertual views.
     * The method is used to handle the translation result for non-vertual views.
     */
     */
    private void onTranslationCompleted(SparseArray<ViewTranslationResponse> translatedResult,
    private void onTranslationCompleted(SparseArray<ViewTranslationResponse> translatedResult) {
            List<AutofillId> viewIds) {
        if (!mActivity.isResumed()) {
        if (!mActivity.isResumed()) {
            if (DEBUG) {
            if (DEBUG) {
                Log.v(TAG, "onTranslationCompleted: Activity is not resumed.");
                Log.v(TAG, "onTranslationCompleted: Activity is not resumed.");
@@ -277,8 +339,6 @@ public class UiTranslationController {
                        + "Skip to show the translated text.");
                        + "Skip to show the translated text.");
                return;
                return;
            }
            }
            // Traverse tree and get views by the responsed AutofillId
            findViewsTraversalByAutofillIds(viewIds);
            for (int i = 0; i < resultCount; i++) {
            for (int i = 0; i < resultCount; i++) {
                final ViewTranslationResponse response = translatedResult.valueAt(i);
                final ViewTranslationResponse response = translatedResult.valueAt(i);
                final AutofillId autofillId = response.getAutofillId();
                final AutofillId autofillId = response.getAutofillId();
@@ -300,7 +360,9 @@ public class UiTranslationController {
                        return;
                        return;
                    }
                    }
                    view.onTranslationResponse(response);
                    view.onTranslationResponse(response);
                    if (view.getViewTranslationCallback() != null) {
                        view.getViewTranslationCallback().onShowTranslation(view);
                        view.getViewTranslationCallback().onShowTranslation(view);
                    }
                });
                });
            }
            }
        }
        }
@@ -359,7 +421,7 @@ public class UiTranslationController {
                        childs = new long[childCount];
                        childs = new long[childCount];
                        viewIds.put(virtualViewAutofillId, childs);
                        viewIds.put(virtualViewAutofillId, childs);
                    }
                    }
                    int end = childs.length;
                    int end = childs.length - 1;
                    childs[end] = autofillId.getVirtualChildLongId();
                    childs[end] = autofillId.getVirtualChildLongId();
                }
                }
            }
            }
@@ -403,7 +465,7 @@ public class UiTranslationController {
        return new int[] {TranslationSpec.DATA_FORMAT_TEXT};
        return new int[] {TranslationSpec.DATA_FORMAT_TEXT};
    }
    }


    private void findViewsTraversalByAutofillIds(List<AutofillId> sourceViewIds) {
    private void findViewsTraversalByAutofillIds(List<Integer> sourceViewIds) {
        final ArrayList<ViewRootImpl> roots =
        final ArrayList<ViewRootImpl> roots =
                WindowManagerGlobal.getInstance().getRootViews(mActivity.getActivityToken());
                WindowManagerGlobal.getInstance().getRootViews(mActivity.getActivityToken());
        for (int rootNum = 0; rootNum < roots.size(); rootNum++) {
        for (int rootNum = 0; rootNum < roots.size(); rootNum++) {
@@ -417,7 +479,7 @@ public class UiTranslationController {
    }
    }


    private void findViewsTraversalByAutofillIds(ViewGroup viewGroup,
    private void findViewsTraversalByAutofillIds(ViewGroup viewGroup,
            List<AutofillId> sourceViewIds) {
            List<Integer> sourceViewIds) {
        final int childCount = viewGroup.getChildCount();
        final int childCount = viewGroup.getChildCount();
        for (int i = 0; i < childCount; ++i) {
        for (int i = 0; i < childCount; ++i) {
            final View child = viewGroup.getChildAt(i);
            final View child = viewGroup.getChildAt(i);
@@ -429,9 +491,9 @@ public class UiTranslationController {
        }
        }
    }
    }


    private void addViewIfNeeded(List<AutofillId> sourceViewIds, View view) {
    private void addViewIfNeeded(List<Integer> sourceViewIds, View view) {
        final AutofillId autofillId = view.getAutofillId();
        final AutofillId autofillId = view.getAutofillId();
        if (sourceViewIds.contains(autofillId)) {
        if (sourceViewIds.contains(autofillId.getViewId()) && !mViews.containsKey(autofillId)) {
            mViews.put(autofillId, new WeakReference<>(view));
            mViews.put(autofillId, new WeakReference<>(view));
        }
        }
    }
    }
+19 −0
Original line number Original line Diff line number Diff line
@@ -45,6 +45,7 @@ import android.os.StrictMode;
import android.print.PrintDocumentAdapter;
import android.print.PrintDocumentAdapter;
import android.util.AttributeSet;
import android.util.AttributeSet;
import android.util.Log;
import android.util.Log;
import android.util.LongSparseArray;
import android.util.SparseArray;
import android.util.SparseArray;
import android.view.DragEvent;
import android.view.DragEvent;
import android.view.KeyEvent;
import android.view.KeyEvent;
@@ -64,6 +65,9 @@ import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.InputConnection;
import android.view.inputmethod.InputConnection;
import android.view.inspector.InspectableProperty;
import android.view.inspector.InspectableProperty;
import android.view.textclassifier.TextClassifier;
import android.view.textclassifier.TextClassifier;
import android.view.translation.TranslationSpec.DataFormat;
import android.view.translation.ViewTranslationRequest;
import android.view.translation.ViewTranslationResponse;
import android.widget.AbsoluteLayout;
import android.widget.AbsoluteLayout;


import java.io.BufferedWriter;
import java.io.BufferedWriter;
@@ -73,6 +77,7 @@ import java.lang.annotation.RetentionPolicy;
import java.util.List;
import java.util.List;
import java.util.Map;
import java.util.Map;
import java.util.concurrent.Executor;
import java.util.concurrent.Executor;
import java.util.function.Consumer;


/**
/**
 * A View that displays web pages.
 * A View that displays web pages.
@@ -2854,6 +2859,20 @@ public class WebView extends AbsoluteLayout
        return mProvider.getViewDelegate().isVisibleToUserForAutofill(virtualId);
        return mProvider.getViewDelegate().isVisibleToUserForAutofill(virtualId);
    }
    }


    @Override
    @Nullable
    public void onCreateTranslationRequests(@NonNull long[] virtualChildIds,
            @NonNull @DataFormat int[] supportedFormats,
            @NonNull Consumer<ViewTranslationRequest> requestsCollector) {
        mProvider.getViewDelegate().onCreateTranslationRequests(virtualChildIds, supportedFormats,
                requestsCollector);
    }

    @Override
    public void onTranslationResponse(@NonNull LongSparseArray<ViewTranslationResponse> response) {
        mProvider.getViewDelegate().onTranslationResponse(response);
    }

    /** @hide */
    /** @hide */
    @Override
    @Override
    public void onInitializeAccessibilityNodeInfoInternal(AccessibilityNodeInfo info) {
    public void onInitializeAccessibilityNodeInfoInternal(AccessibilityNodeInfo info) {
Loading