Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 24d7173c authored by Felipe Leme's avatar Felipe Leme
Browse files

Very initial field detection prototype.

A.K.A "OMG, It's full of TODOs!"

Test: cts-tradefed run commandAndExit cts-dev -m CtsAutoFillServiceTestCases -t android.autofillservice.cts.FieldsDetectionTest

Bug: 67867469

Change-Id: I7c8f7c3e35ccbae0134e2a446b7b44e1e57261fc
parent 223d49f5
Loading
Loading
Loading
Loading
+10 −0
Original line number Diff line number Diff line
@@ -35544,6 +35544,7 @@ package android.provider {
    field public static final java.lang.String ALLOWED_GEOLOCATION_ORIGINS = "allowed_geolocation_origins";
    field public static final deprecated java.lang.String ALLOW_MOCK_LOCATION = "mock_location";
    field public static final java.lang.String ANDROID_ID = "android_id";
    field public static final java.lang.String AUTOFILL_FEATURE_FIELD_DETECTION = "autofill_field_detection";
    field public static final java.lang.String AUTOFILL_SERVICE = "autofill_service";
    field public static final deprecated java.lang.String BACKGROUND_DATA = "background_data";
    field public static final deprecated java.lang.String BLUETOOTH_ON = "bluetooth_on";
@@ -37599,6 +37600,13 @@ package android.service.autofill {
    method public android.service.autofill.Dataset.Builder setValue(android.view.autofill.AutofillId, android.view.autofill.AutofillValue, java.util.regex.Pattern, android.widget.RemoteViews);
  }
  public final class FieldsDetection implements android.os.Parcelable {
    ctor public FieldsDetection(android.view.autofill.AutofillId, java.lang.String, java.lang.String);
    method public int describeContents();
    method public void writeToParcel(android.os.Parcel, int);
    field public static final android.os.Parcelable.Creator<android.service.autofill.FieldsDetection> CREATOR;
  }
  public final class FillCallback {
    method public void onFailure(java.lang.CharSequence);
    method public void onSuccess(android.service.autofill.FillResponse);
@@ -37624,6 +37632,7 @@ package android.service.autofill {
    method public java.util.Map<android.view.autofill.AutofillId, java.lang.String> getChangedFields();
    method public android.os.Bundle getClientState();
    method public java.lang.String getDatasetId();
    method public java.util.Map<java.lang.String, java.lang.Integer> getDetectedFields();
    method public java.util.Set<java.lang.String> getIgnoredDatasetIds();
    method public java.util.Map<android.view.autofill.AutofillId, java.util.Set<java.lang.String>> getManuallyEnteredField();
    method public java.util.Set<java.lang.String> getSelectedDatasetIds();
@@ -37661,6 +37670,7 @@ package android.service.autofill {
    method public android.service.autofill.FillResponse.Builder disableAutofill(long);
    method public android.service.autofill.FillResponse.Builder setAuthentication(android.view.autofill.AutofillId[], android.content.IntentSender, android.widget.RemoteViews);
    method public android.service.autofill.FillResponse.Builder setClientState(android.os.Bundle);
    method public android.service.autofill.FillResponse.Builder setFieldsDetection(android.service.autofill.FieldsDetection);
    method public android.service.autofill.FillResponse.Builder setFlags(int);
    method public android.service.autofill.FillResponse.Builder setIgnoredIds(android.view.autofill.AutofillId...);
    method public android.service.autofill.FillResponse.Builder setSaveInfo(android.service.autofill.SaveInfo);
+9 −0
Original line number Diff line number Diff line
@@ -5308,6 +5308,15 @@ public final class Settings {
        @TestApi
        public static final String AUTOFILL_SERVICE = "autofill_service";

        /**
         * Experimental autofill feature.
         *
         * <p>TODO(b/67867469): remove once feature is finished
         * @hide
         */
        @TestApi
        public static final String AUTOFILL_FEATURE_FIELD_DETECTION = "autofill_field_detection";

        /**
         * @deprecated Use {@link android.provider.Settings.Global#DEVICE_PROVISIONED} instead
         */
+127 −0
Original line number Diff line number Diff line
/*
 * Copyright 2017 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package android.service.autofill;

import android.annotation.TestApi;
import android.os.Parcel;
import android.os.Parcelable;
import android.view.autofill.AutofillId;

/**
 * Class by service to improve autofillable fields detection by tracking the meaning of fields
 * manually edited by the user (when they match values provided by the service).
 *
 * TODO(b/67867469):
 *  - proper javadoc
 *  - unhide / remove testApi
 *  - add FieldsDetection management so service can set it just once and reference it in further
 *    calls to improve performance (and also API to refresh it)
 *  - rename to FieldsDetectionInfo or FieldClassification? (same for CTS tests)
 *  - add FieldsDetectionUnitTest once API is well-defined
 * @hide
 */
@TestApi
public final class FieldsDetection implements Parcelable {

    private final AutofillId mFieldId;
    private final String mRemoteId;
    private final String mValue;

    /**
     * Creates a field detection for just one field / value pair.
     *
     * @param fieldId autofill id of the field in the screen.
     * @param remoteId id used by the service to identify the field later.
     * @param value field value known to the service.
     *
     * TODO(b/67867469):
     *  - proper javadoc
     *  - change signature to allow more fields / values / match methods
     *    - might also need to use a builder, where the constructor is the id for the fieldsdetector
     *    - might need id for values as well
     *  - add @NonNull / check it / add unit tests
     *  - make 'value' input more generic so it can accept distance-based match and other matches
     *  - throw exception if field value is less than X characters (somewhere between 7-10)
     *  - make sure to limit total number of fields to around 10 or so
     *  - use AutofillValue instead of String (so it can compare dates, for example)
     */
    public FieldsDetection(AutofillId fieldId, String remoteId, String value) {
        mFieldId = fieldId;
        mRemoteId = remoteId;
        mValue = value;
    }

    /** @hide */
    public AutofillId getFieldId() {
        return mFieldId;
    }

    /** @hide */
    public String getRemoteId() {
        return mRemoteId;
    }

    /** @hide */
    public String getValue() {
        return mValue;
    }

    /////////////////////////////////////
    // Object "contract" methods. //
    /////////////////////////////////////
    @Override
    public String toString() {
        // Cannot disclose remoteId or value because they could contain PII
        return new StringBuilder("FieldsDetection: [field=").append(mFieldId)
                .append(", remoteId_length=").append(mRemoteId.length())
                .append(", value_length=").append(mValue.length())
                .append("]").toString();
    }

    /////////////////////////////////////
    // Parcelable "contract" methods. //
    /////////////////////////////////////

    @Override
    public int describeContents() {
        return 0;
    }

    @Override
    public void writeToParcel(Parcel parcel, int flags) {
        parcel.writeParcelable(mFieldId, flags);
        parcel.writeString(mRemoteId);
        parcel.writeString(mValue);
    }

    public static final Parcelable.Creator<FieldsDetection> CREATOR =
            new Parcelable.Creator<FieldsDetection>() {
        @Override
        public FieldsDetection createFromParcel(Parcel parcel) {
            // TODO(b/67867469): remove comment below if it does not use a builder at the end
            // Always go through the builder to ensure the data ingested by
            // the system obeys the contract of the builder to avoid attacks
            // using specially crafted parcels.
            return new FieldsDetection(parcel.readParcelable(null), parcel.readString(),
                    parcel.readString());
        }

        @Override
        public FieldsDetection[] newArray(int size) {
            return new FieldsDetection[size];
        }
    };
}
+54 −3
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ package android.service.autofill;
import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.TestApi;
import android.content.IntentSender;
import android.os.Bundle;
import android.os.Parcel;
@@ -164,6 +165,10 @@ public final class FillEventHistory implements Parcelable {
                        dest.writeStringList(event.mManuallyFilledDatasetIds.get(j));
                    }
                }
                dest.writeString(event.mDetectedRemoteId);
                if (event.mDetectedRemoteId != null) {
                    dest.writeInt(event.mDetectedFieldScore);
                }
            }
        }
    }
@@ -226,6 +231,7 @@ public final class FillEventHistory implements Parcelable {
         * <p>See {@link android.view.autofill.AutofillManager} for more information about autofill
         * contexts.
         */
        // TODO(b/67867469): update with field detection behavior
        public static final int TYPE_CONTEXT_COMMITTED = 4;

        /** @hide */
@@ -253,6 +259,9 @@ public final class FillEventHistory implements Parcelable {
        @Nullable private final ArrayList<AutofillId> mManuallyFilledFieldIds;
        @Nullable private final ArrayList<ArrayList<String>> mManuallyFilledDatasetIds;

        @Nullable private final String mDetectedRemoteId;
        private final int mDetectedFieldScore;

        /**
         * Returns the type of the event.
         *
@@ -354,6 +363,39 @@ public final class FillEventHistory implements Parcelable {
            return changedFields;
        }

        /**
         * Gets the results of the last {@link FieldsDetection} request.
         *
         * @return map of edit-distance match ({@code 0} means full match,
         * {@code 1} means 1 character different, etc...) by remote id (as set in the
         * {@link FieldsDetection} constructor), or {@code null} if none of the user-input values
         * matched the requested detection.
         *
         * <p><b>Note: </b>Only set on events of type {@link #TYPE_CONTEXT_COMMITTED}, when the
         * service requested {@link FillResponse.Builder#setFieldsDetection(FieldsDetection) fields
         * detection}.
         *
         * TODO(b/67867469):
         *  - improve javadoc
         *  - refine score meaning (for example, should 1 be different of -1?)
         *  - mention when it's set
         *  - unhide
         *  - unhide / remove testApi
         *  - add @NonNull / check it / add unit tests
         *
         * @hide
         */
        @TestApi
        @NonNull public Map<String, Integer> getDetectedFields() {
            if (mDetectedRemoteId == null || mDetectedFieldScore == -1) {
                return Collections.emptyMap();
            }

            final ArrayMap<String, Integer> map = new ArrayMap<>(1);
            map.put(mDetectedRemoteId, mDetectedFieldScore);
            return map;
        }

        /**
         * Returns which fields were available on datasets provided by the service but manually
         * entered by the user.
@@ -430,7 +472,6 @@ public final class FillEventHistory implements Parcelable {
         * and belonged to datasets.
         * @param manuallyFilledDatasetIds The ids of datasets that had values matching the
         * respective entry on {@code manuallyFilledFieldIds}.
         *
         * @throws IllegalArgumentException If the length of {@code changedFieldIds} and
         * {@code changedDatasetIds} doesn't match.
         * @throws IllegalArgumentException If the length of {@code manuallyFilledFieldIds} and
@@ -438,13 +479,15 @@ public final class FillEventHistory implements Parcelable {
         *
         * @hide
         */
        // TODO(b/67867469): document detection field parameters once stable
        public Event(int eventType, @Nullable String datasetId, @Nullable Bundle clientState,
                @Nullable List<String> selectedDatasetIds,
                @Nullable ArraySet<String> ignoredDatasetIds,
                @Nullable ArrayList<AutofillId> changedFieldIds,
                @Nullable ArrayList<String> changedDatasetIds,
                @Nullable ArrayList<AutofillId> manuallyFilledFieldIds,
                @Nullable ArrayList<ArrayList<String>> manuallyFilledDatasetIds) {
                @Nullable ArrayList<ArrayList<String>> manuallyFilledDatasetIds,
                @Nullable String detectedRemoteId, int detectedFieldScore) {
            mEventType = Preconditions.checkArgumentInRange(eventType, 0, TYPE_CONTEXT_COMMITTED,
                    "eventType");
            mDatasetId = datasetId;
@@ -467,6 +510,8 @@ public final class FillEventHistory implements Parcelable {
            }
            mManuallyFilledFieldIds = manuallyFilledFieldIds;
            mManuallyFilledDatasetIds = manuallyFilledDatasetIds;
            mDetectedRemoteId = detectedRemoteId;
            mDetectedFieldScore = detectedFieldScore;
        }

        @Override
@@ -479,6 +524,8 @@ public final class FillEventHistory implements Parcelable {
                    + ", changedDatasetsIds=" + mChangedDatasetIds
                    + ", manuallyFilledFieldIds=" + mManuallyFilledFieldIds
                    + ", manuallyFilledDatasetIds=" + mManuallyFilledDatasetIds
                    + ", detectedRemoteId=" + mDetectedRemoteId
                    + ", detectedFieldScore=" + mDetectedFieldScore
                    + "]";
        }
    }
@@ -514,11 +561,15 @@ public final class FillEventHistory implements Parcelable {
                        } else {
                            manuallyFilledDatasetIds = null;
                        }
                        final String detectedRemoteId = parcel.readString();
                        final int detectedFieldScore = detectedRemoteId == null ? -1
                                : parcel.readInt();

                        selection.addEvent(new Event(eventType, datasetId, clientState,
                                selectedDatasetIds, ignoredDatasets,
                                changedFieldIds, changedDatasetIds,
                                manuallyFilledFieldIds, manuallyFilledDatasetIds));
                                manuallyFilledFieldIds, manuallyFilledDatasetIds,
                                detectedRemoteId, detectedFieldScore));
                    }
                    return selection;
                }
+40 −5
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ import static android.view.autofill.Helper.sDebug;
import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.TestApi;
import android.app.Activity;
import android.content.IntentSender;
import android.content.pm.ParceledListSlice;
@@ -75,6 +76,7 @@ public final class FillResponse implements Parcelable {
    private final @Nullable AutofillId[] mAuthenticationIds;
    private final @Nullable AutofillId[] mIgnoredIds;
    private final long mDisableDuration;
    private final @Nullable FieldsDetection mFieldsDetection;
    private final int mFlags;
    private int mRequestId;

@@ -87,6 +89,7 @@ public final class FillResponse implements Parcelable {
        mAuthenticationIds = builder.mAuthenticationIds;
        mIgnoredIds = builder.mIgnoredIds;
        mDisableDuration = builder.mDisableDuration;
        mFieldsDetection = builder.mFieldsDetection;
        mFlags = builder.mFlags;
        mRequestId = INVALID_REQUEST_ID;
    }
@@ -131,6 +134,11 @@ public final class FillResponse implements Parcelable {
        return mDisableDuration;
    }

    /** @hide */
    public @Nullable FieldsDetection getFieldsDetection() {
        return mFieldsDetection;
    }

    /** @hide */
    public int getFlags() {
        return mFlags;
@@ -167,6 +175,7 @@ public final class FillResponse implements Parcelable {
        private AutofillId[] mAuthenticationIds;
        private AutofillId[] mIgnoredIds;
        private long mDisableDuration;
        private FieldsDetection mFieldsDetection;
        private int mFlags;
        private boolean mDestroyed;

@@ -314,6 +323,25 @@ public final class FillResponse implements Parcelable {
            return this;
        }

        /**
         * TODO(b/67867469):
         *  - javadoc it
         *  - javadoc how to check results
         *  - unhide
         *  - unhide / remove testApi
         *  - throw exception (and document) if response has datasets or saveinfo
         *  - throw exception (and document) if id on fieldsDetection is ignored
         *
         * @hide
         */
        @TestApi
        public Builder setFieldsDetection(@NonNull FieldsDetection fieldsDetection) {
            throwIfDestroyed();
            throwIfDisableAutofillCalled();
            mFieldsDetection = Preconditions.checkNotNull(fieldsDetection);
            return this;
        }

        /**
         * Sets flags changing the response behavior.
         *
@@ -365,7 +393,8 @@ public final class FillResponse implements Parcelable {
            if (duration <= 0) {
                throw new IllegalArgumentException("duration must be greater than 0");
            }
            if (mAuthentication != null || mDatasets != null || mSaveInfo != null) {
            if (mAuthentication != null || mDatasets != null || mSaveInfo != null
                    || mFieldsDetection != null) {
                throw new IllegalStateException("disableAutofill() must be the only method called");
            }

@@ -388,11 +417,11 @@ public final class FillResponse implements Parcelable {
         */
        public FillResponse build() {
            throwIfDestroyed();

            if (mAuthentication == null && mDatasets == null && mSaveInfo == null
                    && mDisableDuration == 0) {
                throw new IllegalStateException("need to provide at least one DataSet or a "
                        + "SaveInfo or an authentication with a presentation or disable autofill");
                    && mDisableDuration == 0 && mFieldsDetection == null) {
                throw new IllegalStateException("need to provide: at least one DataSet, or a "
                        + "SaveInfo, or an authentication with a presentation, "
                        + "or a FieldsDetection, or disable autofill");
            }
            mDestroyed = true;
            return new FillResponse(this);
@@ -430,6 +459,7 @@ public final class FillResponse implements Parcelable {
                .append(", ignoredIds=").append(Arrays.toString(mIgnoredIds))
                .append(", disableDuration=").append(mDisableDuration)
                .append(", flags=").append(mFlags)
                .append(", fieldDetection=").append(mFieldsDetection)
                .append("]")
                .toString();
    }
@@ -453,6 +483,7 @@ public final class FillResponse implements Parcelable {
        parcel.writeParcelable(mPresentation, flags);
        parcel.writeParcelableArray(mIgnoredIds, flags);
        parcel.writeLong(mDisableDuration);
        parcel.writeParcelable(mFieldsDetection, flags);
        parcel.writeInt(mFlags);
        parcel.writeInt(mRequestId);
    }
@@ -488,6 +519,10 @@ public final class FillResponse implements Parcelable {
            if (disableDuration > 0) {
                builder.disableAutofill(disableDuration);
            }
            final FieldsDetection fieldsDetection = parcel.readParcelable(null);
            if (fieldsDetection != null) {
                builder.setFieldsDetection(fieldsDetection);
            }
            builder.setFlags(parcel.readInt());

            final FillResponse response = builder.build();
Loading