Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 48b25653 authored by Winson Chung's avatar Winson Chung
Browse files

Expose task id and activity component in task snapshot and assist structure 

- This would allow the AiAi service to cross-reference assist data with
  streaming text data and snapshots
- Pre-fill the task id/activity when receiving the assist data from the
  activity and remove unnecessary autofill santization down the line

Bug: 117268952
Test: adb shell dumpsys window all
Test: atest CtsAutoFillServiceTestCases
Test: atest CtsAssistTestCases

Change-Id: I0d0d2c85426777cc77397716db34b520593db100
parent b8d66722

core/java/android/app/ActivityManager.java

+19 −4
Original line number Diff line number Diff line
@@ -1751,6 +1751,8 @@ public class ActivityManager { 
     */

    public static class TaskSnapshot implements Parcelable {



        // Top activity in task when snapshot was taken

        private final ComponentName mTopActivityComponent;

        private final GraphicBuffer mSnapshot;

        private final int mOrientation;

        private final Rect mContentInsets;
@@ -1765,9 +1767,11 @@ public class ActivityManager { 
        private final int mSystemUiVisibility;

        private final boolean mIsTranslucent;



        public TaskSnapshot(GraphicBuffer snapshot, int orientation, Rect contentInsets,

                boolean reducedResolution, float scale, boolean isRealSnapshot, int windowingMode,

                int systemUiVisibility, boolean isTranslucent) {

        public TaskSnapshot(@NonNull ComponentName topActivityComponent, GraphicBuffer snapshot,

                int orientation, Rect contentInsets, boolean reducedResolution, float scale,

                boolean isRealSnapshot, int windowingMode, int systemUiVisibility,

                boolean isTranslucent) {

            mTopActivityComponent = topActivityComponent;

            mSnapshot = snapshot;

            mOrientation = orientation;

            mContentInsets = new Rect(contentInsets);
@@ -1780,6 +1784,7 @@ public class ActivityManager { 
        }



        private TaskSnapshot(Parcel source) {

            mTopActivityComponent = ComponentName.readFromParcel(source);

            mSnapshot = source.readParcelable(null /* classLoader */);

            mOrientation = source.readInt();

            mContentInsets = source.readParcelable(null /* classLoader */);
@@ -1791,6 +1796,13 @@ public class ActivityManager { 
            mIsTranslucent = source.readBoolean();

        }



        /**

         * @return The top activity component for the task at the point this snapshot was taken.

         */

        public ComponentName getTopActivityComponent() {

            return mTopActivityComponent;

        }



        /**

         * @return The graphic buffer representing the screenshot.

         */
@@ -1871,6 +1883,7 @@ public class ActivityManager { 


        @Override

        public void writeToParcel(Parcel dest, int flags) {

            ComponentName.writeToParcel(mTopActivityComponent, dest);

            dest.writeParcelable(mSnapshot, 0);

            dest.writeInt(mOrientation);

            dest.writeParcelable(mContentInsets, 0);
@@ -1886,7 +1899,9 @@ public class ActivityManager { 
        public String toString() {

            final int width = mSnapshot != null ? mSnapshot.getWidth() : 0;

            final int height = mSnapshot != null ? mSnapshot.getHeight() : 0;

            return "TaskSnapshot{mSnapshot=" + mSnapshot + " (" + width + "x" + height + ")"

            return "TaskSnapshot{"

                    + " mTopActivityComponent=" + mTopActivityComponent.flattenToShortString()

                    + " mSnapshot=" + mSnapshot + " (" + width + "x" + height + ")"

                    + " mOrientation=" + mOrientation

                    + " mContentInsets=" + mContentInsets.toShortString()

                    + " mReducedResolution=" + mReducedResolution + " mScale=" + mScale

core/java/android/app/assist/AssistStructure.java

+47 −28
Original line number Diff line number Diff line
@@ -61,37 +61,39 @@ import java.util.List; 
 * <a href="/guide/topics/text/autofill">Autofill Framework</a> guides.

 */

public class AssistStructure implements Parcelable {

    static final String TAG = "AssistStructure";

    private static final String TAG = "AssistStructure";



    static final boolean DEBUG_PARCEL = false;

    static final boolean DEBUG_PARCEL_CHILDREN = false;

    static final boolean DEBUG_PARCEL_TREE = false;

    private static final boolean DEBUG_PARCEL = false;

    private static final boolean DEBUG_PARCEL_CHILDREN = false;

    private static final boolean DEBUG_PARCEL_TREE = false;



    static final int VALIDATE_WINDOW_TOKEN = 0x11111111;

    static final int VALIDATE_VIEW_TOKEN = 0x22222222;

    private static final int VALIDATE_WINDOW_TOKEN = 0x11111111;

    private static final int VALIDATE_VIEW_TOKEN = 0x22222222;



    boolean mHaveData;

    private boolean mHaveData;



    ComponentName mActivityComponent;

    // The task id and component of the activity which this assist structure is for

    private int mTaskId;

    private ComponentName mActivityComponent;

    private boolean mIsHomeActivity;

    private int mFlags;

    private int mAutofillFlags;



    final ArrayList<WindowNode> mWindowNodes = new ArrayList<>();

    private final ArrayList<WindowNode> mWindowNodes = new ArrayList<>();



    final ArrayList<ViewNodeBuilder> mPendingAsyncChildren = new ArrayList<>();

    private final ArrayList<ViewNodeBuilder> mPendingAsyncChildren = new ArrayList<>();



    SendChannel mSendChannel;

    IBinder mReceiveChannel;

    private SendChannel mSendChannel;

    private IBinder mReceiveChannel;



    Rect mTmpRect = new Rect();

    private Rect mTmpRect = new Rect();



    boolean mSanitizeOnWrite = false;

    private boolean mSanitizeOnWrite = false;

    private long mAcquisitionStartTime;

    private long mAcquisitionEndTime;



    static final int TRANSACTION_XFER = Binder.FIRST_CALL_TRANSACTION+1;

    static final String DESCRIPTOR = "android.app.AssistStructure";

    private static final int TRANSACTION_XFER = Binder.FIRST_CALL_TRANSACTION+1;

    private static final String DESCRIPTOR = "android.app.AssistStructure";



    /** @hide */

    public void setAcquisitionStartTime(long acquisitionStartTime) {
@@ -197,7 +199,6 @@ public class AssistStructure implements Parcelable { 
        ParcelTransferWriter(AssistStructure as, Parcel out) {

            mSanitizeOnWrite = as.mSanitizeOnWrite;

            mWriteStructure = as.waitForReady();

            ComponentName.writeToParcel(as.mActivityComponent, out);

            out.writeInt(as.mFlags);

            out.writeInt(as.mAutofillFlags);

            out.writeLong(as.mAcquisitionStartTime);
@@ -353,7 +354,6 @@ public class AssistStructure implements Parcelable { 


        void go() {

            fetchData();

            mActivityComponent = ComponentName.readFromParcel(mCurParcel);

            mFlags = mCurParcel.readInt();

            mAutofillFlags = mCurParcel.readInt();

            mAcquisitionStartTime = mCurParcel.readLong();
@@ -2129,7 +2129,6 @@ public class AssistStructure implements Parcelable { 
    /** @hide */

    public AssistStructure(Activity activity, boolean forAutoFill, int flags) {

        mHaveData = true;

        mActivityComponent = activity.getComponentName();

        mFlags = flags;

        ArrayList<ViewRootImpl> views = WindowManagerGlobal.getInstance().getRootViews(

                activity.getActivityToken());
@@ -2145,12 +2144,13 @@ public class AssistStructure implements Parcelable { 


    public AssistStructure() {

        mHaveData = true;

        mActivityComponent = null;

        mFlags = 0;

    }



    /** @hide */

    public AssistStructure(Parcel in) {

        mTaskId = in.readInt();

        mActivityComponent = ComponentName.readFromParcel(in);

        mIsHomeActivity = in.readInt() == 1;

        mReceiveChannel = in.readStrongBinder();

    }
@@ -2171,7 +2171,10 @@ public class AssistStructure implements Parcelable { 
            Log.i(TAG, "dump(): calling ensureData() first");

            ensureData();

        }

        Log.i(TAG, "Activity: " + mActivityComponent.flattenToShortString());

        Log.i(TAG, "Task id: " + mTaskId);

        Log.i(TAG, "Activity: " + (mActivityComponent != null 

                ? mActivityComponent.flattenToShortString()

                : null));

        Log.i(TAG, "Sanitize on write: " + mSanitizeOnWrite);

        Log.i(TAG, "Flags: " + mFlags);

        final int N = getWindowNodeCount();
@@ -2283,23 +2286,37 @@ public class AssistStructure implements Parcelable { 
    }



    /**

     * Return the activity this AssistStructure came from.

     * Sets the task id is associated with the activity from which this AssistStructure was

     * generated.

     * @hide

     */

    public ComponentName getActivityComponent() {

        ensureData();

        return mActivityComponent;

    public void setTaskId(int taskId) {

        mTaskId = taskId;

    }



    /**

     * Called by Autofill server when app forged a different value.

     *

     * @return The task id for the associated activity.

     * @hide

     */

    public int getTaskId() {

        return mTaskId;

    }



    /**

     * Sets the activity that is associated with this AssistStructure.

     * @hide

     */

    public void setActivityComponent(ComponentName componentName) {

        ensureData();

        mActivityComponent = componentName;

    }



    /**

     * Return the activity this AssistStructure came from.

     */

    public ComponentName getActivityComponent() {

        return mActivityComponent;

    }



    /** @hide */

    public int getFlags() {

        return mFlags;
@@ -2393,6 +2410,8 @@ public class AssistStructure implements Parcelable { 


    @Override

    public void writeToParcel(Parcel out, int flags) {

        out.writeInt(mTaskId);

        ComponentName.writeToParcel(mActivityComponent, out);

        out.writeInt(mIsHomeActivity ? 1 : 0);

        if (mHaveData) {

            // This object holds its data.  We want to write a send channel that the

core/tests/coretests/src/android/app/assist/AssistStructureTest.java

+0 −10
Original line number Diff line number Diff line
@@ -133,8 +133,6 @@ public class AssistStructureTest { 
    private void assertStructureWithManySmallViews(AssistStructure structure, int expectedSize) {

        int i = 0;

        try {

            assertPackageName(structure);



            assertThat(structure.getWindowNodeCount()).isEqualTo(1);



            ViewNode rootView = structure.getWindowNodeAt(0).getRootViewNode();
@@ -188,8 +186,6 @@ public class AssistStructureTest { 


    private void assertStructureWithOneBigView(AssistStructure structure) {

        try {

            assertPackageName(structure);



            assertThat(structure.getWindowNodeCount()).isEqualTo(1);



            ViewNode rootView = structure.getWindowNodeAt(0).getRootViewNode();
@@ -275,12 +271,6 @@ public class AssistStructureTest { 
        assertThat(hint.charAt(BIG_VIEW_SIZE - 1)).isEqualTo(BIG_VIEW_CHAR);

    }



    private void assertPackageName(AssistStructure structure) {

        assertThat(structure.getActivityComponent()).isEqualTo(

                new ComponentName("com.android.frameworks.coretests",

                        "android.app.assist.EmptyLayoutActivity"));

    }



    private AssistStructure cloneThroughParcel(AssistStructure structure) {

        Parcel parcel = Parcel.obtain();

proto/src/task_snapshot.proto

+1 −0
Original line number Diff line number Diff line
@@ -31,4 +31,5 @@ 
     int32 windowing_mode = 7;

     int32 system_ui_visibility = 8;

     bool is_translucent = 9;

     string top_activity_component = 10;

 }
 
 No newline at end of file

services/autofill/java/com/android/server/autofill/Session.java

+0 −12
Original line number Diff line number Diff line
@@ -281,18 +281,6 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState 
                    return;

                }



                // Sanitize structure before it's sent to service.

                final ComponentName componentNameFromApp = structure.getActivityComponent();

                if (componentNameFromApp == null || !mComponentName.getPackageName()

                        .equals(componentNameFromApp.getPackageName())) {

                    Slog.w(TAG, "Activity " + mComponentName + " forged different component on "

                            + "AssistStructure: " + componentNameFromApp);

                    structure.setActivityComponent(mComponentName);

                    mMetricsLogger.write(newLogMaker(MetricsEvent.AUTOFILL_FORGED_COMPONENT_ATTEMPT)

                            .addTaggedData(MetricsEvent.FIELD_AUTOFILL_FORGED_COMPONENT_NAME,

                                    componentNameFromApp == null ? "null"

                                            : componentNameFromApp.flattenToShortString()));

                }

                // Flags used to start the session.

                int flags = structure.getFlags();