Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b4f328a2 authored by Yohei Yukawa's avatar Yohei Yukawa
Browse files

Reenable CursorAnchorInfo API with ActivityView 

With my previous CL [1], CursorAnchorInfo API was globally disabled
for cross-display scenario including ActivityView scenario.

This CL slightly relaxes the above condition so that IMEs can rely on
CursorAnchorInfo APIs again to interact with apps running inside
ActivityView.

The basic idea here is keeping reporting relevant information from
ActivityView to InputMethodManagerService (IMMS) so that IMMS can take
the display hierarchy because of ActivityView into account.  As long
as IMMS has the up-to-date hierarchical information, IMMS can tell
InputMethodManager (IMM) running in the IME client process about the
missing coordinate transformation information from the virtual display
space to the outer display space where the IME is actually shown.

Note that there was a similar fix for AccessibilityService that keeps
reporting ActivityView location to WindowManagerService (WMS) [2].
Ideally we should be able to share the logic, but to do so we need to
introduce a generalized callback mechanism into WMS so that IMMS can
be notified when a cetain coordinate transform matrix has changed.
For Q, this CL implements IMMS's own mechanism to keep track of
ActivityView hierarchy instead of introducing a direct dependency from
WMS to IMMS.

For R+, most likely we may want to reconsider how ActivityView should
be implemented.

There should be no behavior change in this CL if ActivityView is not
involved.

 [1]: Ie2f7a5117cff3a13ad5c5806fd4b3abef7569549
      3d2cc0ff
 [2]: I38da5b84a11890bf0f4a57eb9d5b7e71bdcc16a9
      d8ec9386

Fix: 115693908
Test: atest CtsWindowManagerDeviceTestCases:ActivityViewTest#testInputMethod
Change-Id: Id0411a80456182111bb5b681c6d1230b58e7ec2e
parent e4410a13

core/java/android/app/ActivityView.java

+21 −1
Original line number Diff line number Diff line
@@ -28,6 +28,7 @@ import android.content.ComponentName; 
import android.content.Context;

import android.content.Intent;

import android.graphics.Insets;

import android.graphics.Matrix;

import android.graphics.Region;

import android.hardware.display.DisplayManager;

import android.hardware.display.VirtualDisplay;
@@ -51,6 +52,7 @@ import android.view.ViewGroup; 
import android.view.ViewParent;

import android.view.WindowManager;

import android.view.WindowManagerGlobal;

import android.view.inputmethod.InputMethodManager;



import dalvik.system.CloseGuard;
@@ -320,6 +322,14 @@ public class ActivityView extends ViewGroup { 
        updateLocationAndTapExcludeRegion();

    }



    private void clearActivityViewGeometryForIme() {

        if (mVirtualDisplay == null) {

            return;

        }

        final int displayId = mVirtualDisplay.getDisplay().getDisplayId();

        mContext.getSystemService(InputMethodManager.class).reportActivityView(displayId, null);

    }



    @Override

    public void onLayout(boolean changed, int l, int t, int r, int b) {

        mSurfaceView.layout(0 /* left */, 0 /* top */, r - l /* right */, b - t /* bottom */);
@@ -347,8 +357,17 @@ public class ActivityView extends ViewGroup { 
            if (x != mLocationInWindow[0] || y != mLocationInWindow[1]) {

                x = mLocationInWindow[0];

                y = mLocationInWindow[1];

                final int displayId = mVirtualDisplay.getDisplay().getDisplayId();

                WindowManagerGlobal.getWindowSession().updateDisplayContentLocation(

                        getWindow(), x, y, mVirtualDisplay.getDisplay().getDisplayId());

                        getWindow(), x, y, displayId);



                // Also report this geometry information to InputMethodManagerService.

                // TODO(b/115693908): Unify this logic into the above WMS-based one.

                final Matrix matrix = new Matrix();

                matrix.set(getMatrix());

                matrix.postTranslate(x, y);

                mContext.getSystemService(InputMethodManager.class)

                        .reportActivityView(displayId, matrix);

            }

            updateTapExcludeRegion(x, y);

        } catch (RemoteException e) {
@@ -408,6 +427,7 @@ public class ActivityView extends ViewGroup { 
            if (mVirtualDisplay != null) {

                mVirtualDisplay.setDisplayState(false);

            }

            clearActivityViewGeometryForIme();

            cleanTapExcludeRegion();

        }

    }

core/java/android/view/inputmethod/InputMethodManager.java

+94 −1
Original line number Diff line number Diff line
@@ -33,6 +33,7 @@ import android.content.ComponentName; 
import android.content.ContentResolver;

import android.content.Context;

import android.content.pm.PackageManager;

import android.graphics.Matrix;

import android.graphics.Rect;

import android.inputmethodservice.InputMethodService;

import android.os.Binder;
@@ -425,6 +426,17 @@ public final class InputMethodManager { 
     */

    private CursorAnchorInfo mCursorAnchorInfo = null;



    /**

     * A special {@link Matrix} that can be provided by the system when this instance is running

     * inside a virtual display that is managed by {@link android.app.ActivityView}.

     *

     * <p>If this is non-{@code null}, {@link #updateCursorAnchorInfo(View, CursorAnchorInfo)}

     * should be adjusted with this {@link Matrix}.</p>

     *

     * <p>{@code null} when not used.</p>

     */

    private Matrix mActivityViewToScreenMatrix = null;



    // -----------------------------------------------------------



    /**
@@ -473,6 +485,7 @@ public final class InputMethodManager { 
    static final int MSG_REPORT_FULLSCREEN_MODE = 10;

    static final int MSG_REPORT_PRE_RENDERED = 15;

    static final int MSG_APPLY_IME_VISIBILITY = 20;

    static final int MSG_UPDATE_ACTIVITY_VIEW_TO_SCREEN_MATRIX = 30;



    private static boolean isAutofillUIShowing(View servedView) {

        AutofillManager afm = servedView.getContext().getSystemService(AutofillManager.class);
@@ -579,6 +592,7 @@ public final class InputMethodManager { 
                        mCurMethod = res.method;

                        mCurId = res.id;

                        mBindSequence = res.sequence;

                        mActivityViewToScreenMatrix = res.getActivityViewToScreenMatrix();

                    }

                    startInputInner(StartInputReason.BOUND_TO_IMMS, null, 0, 0, 0);

                    return;
@@ -686,6 +700,48 @@ public final class InputMethodManager { 
                    }

                    return;

                }

                case MSG_UPDATE_ACTIVITY_VIEW_TO_SCREEN_MATRIX: {

                    final float[] matrixValues = (float[]) msg.obj;

                    final int bindSequence = msg.arg1;

                    synchronized (mH) {

                        if (mBindSequence != bindSequence) {

                            return;

                        }

                        if (matrixValues == null) {

                            // That this app is unbound from the parent ActivityView. In this case,

                            // calling updateCursorAnchorInfo() isn't safe.  Only clear the matrix.

                            mActivityViewToScreenMatrix = null;

                            return;

                        }



                        final float[] currentValues = new float[9];

                        mActivityViewToScreenMatrix.getValues(currentValues);

                        if (Arrays.equals(currentValues, matrixValues)) {

                            return;

                        }

                        mActivityViewToScreenMatrix.setValues(matrixValues);



                        if (mCursorAnchorInfo == null || mCurMethod == null

                                || mServedInputConnectionWrapper == null) {

                            return;

                        }

                        final boolean isMonitoring = (mRequestUpdateCursorAnchorInfoMonitorMode

                                & InputConnection.CURSOR_UPDATE_MONITOR) != 0;

                        if (!isMonitoring) {

                            return;

                        }

                        // Since the host ActivityView is moved, we need to issue

                        // IMS#updateCursorAnchorInfo() again.

                        try {

                            mCurMethod.updateCursorAnchorInfo(

                                    CursorAnchorInfo.createForAdditionalParentMatrix(

                                            mCursorAnchorInfo, mActivityViewToScreenMatrix));

                        } catch (RemoteException e) {

                            Log.w(TAG, "IME died: " + mCurId, e);

                        }

                    }

                    return;

                }

            }

        }

    }
@@ -777,6 +833,11 @@ public final class InputMethodManager { 
                    .sendToTarget();

        }



        @Override

        public void updateActivityViewToScreenMatrix(int bindSequence, float[] matrixValues) {

            mH.obtainMessage(MSG_UPDATE_ACTIVITY_VIEW_TO_SCREEN_MATRIX, bindSequence, 0,

                    matrixValues).sendToTarget();

        }

    };



    final InputConnection mDummyInputConnection = new BaseInputConnection(this, false);
@@ -1192,6 +1253,7 @@ public final class InputMethodManager { 
    @UnsupportedAppUsage

    void finishInputLocked() {

        mNextServedView = null;

        mActivityViewToScreenMatrix = null;

        if (mServedView != null) {

            if (DEBUG) Log.v(TAG, "FINISH INPUT: mServedView=" + dumpViewInfo(mServedView));

            mServedView = null;
@@ -1668,6 +1730,7 @@ public final class InputMethodManager { 
                            + InputMethodDebug.startInputFlagsToString(startInputFlags));

                    return false;

                }

                mActivityViewToScreenMatrix = res.getActivityViewToScreenMatrix();

                if (res.id != null) {

                    setInputChannelLocked(res.channel);

                    mBindSequence = res.sequence;
@@ -2200,7 +2263,13 @@ public final class InputMethodManager { 
            }

            if (DEBUG) Log.v(TAG, "updateCursorAnchorInfo: " + cursorAnchorInfo);

            try {

                if (mActivityViewToScreenMatrix != null) {

                    mCurMethod.updateCursorAnchorInfo(

                            CursorAnchorInfo.createForAdditionalParentMatrix(

                                    cursorAnchorInfo, mActivityViewToScreenMatrix));

                } else {

                    mCurMethod.updateCursorAnchorInfo(cursorAnchorInfo);

                }

                mCursorAnchorInfo = cursorAnchorInfo;

                // Clear immediate bit (if any).

                mRequestUpdateCursorAnchorInfoMonitorMode &=
@@ -2778,6 +2847,30 @@ public final class InputMethodManager { 
        }

    }



    /**

     * An internal API for {@link android.app.ActivityView} to report where its embedded virtual

     * display is placed.

     *

     * @param childDisplayId Display ID of the embedded virtual display.

     * @param matrix {@link Matrix} to convert virtual display screen coordinates to

     *               the host screen coordinates. {@code null} to clear the relationship.

     * @hide

     */

    public void reportActivityView(int childDisplayId, @Nullable Matrix matrix) {

        try {

            final float[] matrixValues;

            if (matrix == null) {

                matrixValues = null;

            } else {

                matrixValues = new float[9];

                matrix.getValues(matrixValues);

            }

            mService.reportActivityView(mClient, childDisplayId, matrixValues);

        } catch (RemoteException e) {

            throw e.rethrowFromSystemServer();

        }

    }



    /**

     * Force switch to the last used input method and subtype. If the last input method didn't have

     * any subtypes, the framework will simply switch to the last input method with no subtype

core/java/com/android/internal/view/IInputMethodClient.aidl

+1 −0
Original line number Diff line number Diff line
@@ -31,4 +31,5 @@ oneway interface IInputMethodClient { 
    void reportFullscreenMode(boolean fullscreen);

    void reportPreRendered(in EditorInfo info);

    void applyImeVisibility(boolean setVisible);

    void updateActivityViewToScreenMatrix(int bindSequence, in float[] matrixValues);

}

core/java/com/android/internal/view/IInputMethodManager.aidl

+3 −0
Original line number Diff line number Diff line
@@ -68,4 +68,7 @@ interface IInputMethodManager { 
    // This is kept due to @UnsupportedAppUsage.

    // TODO(Bug 113914148): Consider removing this.

    int getInputMethodWindowVisibleHeight();



    void reportActivityView(in IInputMethodClient parentClient, int childDisplayId,

            in float[] matrixValues);

}

core/java/com/android/internal/view/InputBindResult.java

+31 −2
Original line number Diff line number Diff line
@@ -19,10 +19,12 @@ package com.android.internal.view; 
import static java.lang.annotation.RetentionPolicy.SOURCE;



import android.annotation.IntDef;

import android.annotation.Nullable;

import android.annotation.UnsupportedAppUsage;

import android.content.ComponentName;

import android.content.Intent;

import android.content.ServiceConnection;

import android.graphics.Matrix;

import android.os.IBinder;

import android.os.Parcel;

import android.os.Parcelable;
@@ -192,13 +194,37 @@ public final class InputBindResult implements Parcelable { 
     */

    public final int sequence;



    @Nullable

    private final float[] mActivityViewToScreenMatrixValues;



    /**

     * @return {@link Matrix} that corresponds to {@link #mActivityViewToScreenMatrixValues}.

     *         {@code null} if {@link #mActivityViewToScreenMatrixValues} is {@code null}.

     */

    @Nullable

    public Matrix getActivityViewToScreenMatrix() {

        if (mActivityViewToScreenMatrixValues == null) {

            return null;

        }

        final Matrix matrix = new Matrix();

        matrix.setValues(mActivityViewToScreenMatrixValues);

        return matrix;

    }



    public InputBindResult(@ResultCode int _result,

            IInputMethodSession _method, InputChannel _channel, String _id, int _sequence) {

            IInputMethodSession _method, InputChannel _channel, String _id, int _sequence,

            @Nullable Matrix activityViewToScreenMatrix) {

        result = _result;

        method = _method;

        channel = _channel;

        id = _id;

        sequence = _sequence;

        if (activityViewToScreenMatrix == null) {

            mActivityViewToScreenMatrixValues = null;

        } else {

            mActivityViewToScreenMatrixValues = new float[9];

            activityViewToScreenMatrix.getValues(mActivityViewToScreenMatrixValues);

        }

    }



    InputBindResult(Parcel source) {
@@ -211,12 +237,14 @@ public final class InputBindResult implements Parcelable { 
        }

        id = source.readString();

        sequence = source.readInt();

        mActivityViewToScreenMatrixValues = source.createFloatArray();

    }



    @Override

    public String toString() {

        return "InputBindResult{result=" + getResultString() + " method="+ method + " id=" + id

                + " sequence=" + sequence

                + " activityViewToScreenMatrix=" + getActivityViewToScreenMatrix()

                + "}";

    }
@@ -238,6 +266,7 @@ public final class InputBindResult implements Parcelable { 
        }

        dest.writeString(id);

        dest.writeInt(sequence);

        dest.writeFloatArray(mActivityViewToScreenMatrixValues);

    }



    /**
@@ -302,7 +331,7 @@ public final class InputBindResult implements Parcelable { 
    }



    private static InputBindResult error(@ResultCode int result) {

        return new InputBindResult(result, null, null, null, -1);

        return new InputBindResult(result, null, null, null, -1, null);

    }



    /**