Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit cb172a1b authored by Oli Lan's avatar Oli Lan Committed by Automerger Merge Worker
Browse files

Merge "Add text classification results to clip description and clip data."... 

Merge "Add text classification results to clip description and clip data." into sc-dev am: fe4a8f58

Original change: https://googleplex-android-review.googlesource.com/c/platform/frameworks/base/+/13727468

Change-Id: Ibb4b42afcbced682c083d85549ce0d34eade2df3
parents fadd59bc fe4a8f58

core/api/current.txt

+7 −0
Original line number Diff line number Diff line
@@ -9910,6 +9910,7 @@ package android.content { 
    method public String getHtmlText();

    method public android.content.Intent getIntent();

    method public CharSequence getText();

    method @Nullable public android.view.textclassifier.TextLinks getTextLinks();

    method public android.net.Uri getUri();

  }









@@ -9919,6 +9920,8 @@ package android.content {













    method public static boolean compareMimeTypes(String, String);















    method public int describeContents();















    method public String[] filterMimeTypes(String);













    method public int getClassificationStatus();













    method @FloatRange(from=0.0, to=1.0) public float getConfidenceScore(@NonNull String);















    method public android.os.PersistableBundle getExtras();















    method public CharSequence getLabel();















    method public String getMimeType(int);









@@ -9928,6 +9931,9 @@ package android.content {













    method public boolean isStyledText();















    method public void setExtras(android.os.PersistableBundle);















    method public void writeToParcel(android.os.Parcel, int);













    field public static final int CLASSIFICATION_COMPLETE = 3; // 0x3













    field public static final int CLASSIFICATION_NOT_COMPLETE = 1; // 0x1













    field public static final int CLASSIFICATION_NOT_PERFORMED = 2; // 0x2















    field @NonNull public static final android.os.Parcelable.Creator<android.content.ClipDescription> CREATOR;















    field public static final String MIMETYPE_TEXT_HTML = "text/html";















    field public static final String MIMETYPE_TEXT_INTENT = "text/vnd.android.intent";










@@ -51916,6 +51922,7 @@ package android.view.textclassifier {













    field public static final String TYPE_PHONE = "phone";















    field public static final String TYPE_UNKNOWN = "";















    field public static final String TYPE_URL = "url";













    field public static final String WIDGET_TYPE_CLIPBOARD = "clipboard";















    field public static final String WIDGET_TYPE_CUSTOM_EDITTEXT = "customedit";















    field public static final String WIDGET_TYPE_CUSTOM_TEXTVIEW = "customview";















    field public static final String WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW = "nosel-customview";

























core/java/android/content/ClipData.java



+29
−0






























Original line number
Diff line number
Diff line








@@ -21,6 +21,7 @@ import static android.content.ContentResolver.SCHEME_ANDROID_RESOURCE;













import static android.content.ContentResolver.SCHEME_CONTENT;















import static android.content.ContentResolver.SCHEME_FILE;





























import android.annotation.Nullable;















import android.compat.annotation.UnsupportedAppUsage;















import android.content.pm.ActivityInfo;















import android.content.res.AssetFileDescriptor;









@@ -38,6 +39,7 @@ import android.text.TextUtils;













import android.text.style.URLSpan;















import android.util.Log;















import android.util.proto.ProtoOutputStream;













import android.view.textclassifier.TextLinks;































import com.android.internal.util.ArrayUtils;


























@@ -204,6 +206,7 @@ public class ClipData implements Parcelable {













        Uri mUri;















        // Additional activity info resolved by the system















        ActivityInfo mActivityInfo;













        private TextLinks mTextLinks;































        /** @hide */















        public Item(Item other) {










@@ -331,6 +334,29 @@ public class ClipData implements Parcelable {













            mActivityInfo = info;















        }





























        /**













         * Returns the results of text classification run on the raw text contained in this item,













         * if it was performed, and if any entities were found in the text. Classification is













         * generally only performed on the first item in clip data, and only if the text is below a













         * certain length.













         *













         * <p>Returns {@code null} if classification was not performed, or if no entities were













         * found in the text.













         *













         * @see ClipDescription#getConfidenceScore(String)













         */













        @Nullable













        public TextLinks getTextLinks() {













            return mTextLinks;













        }



























        /**













         * @hide













         */













        public void setTextLinks(TextLinks textLinks) {













            mTextLinks = textLinks;













        }





























        /**















         * Turn this item into text, regardless of the type of data it















         * actually contains.










@@ -1183,6 +1209,7 @@ public class ClipData implements Parcelable {













            dest.writeTypedObject(item.mIntent, flags);















            dest.writeTypedObject(item.mUri, flags);















            dest.writeTypedObject(item.mActivityInfo, flags);













            dest.writeTypedObject(item.mTextLinks, flags);















        }















    }

























@@ -1201,8 +1228,10 @@ public class ClipData implements Parcelable {













            Intent intent = in.readTypedObject(Intent.CREATOR);















            Uri uri = in.readTypedObject(Uri.CREATOR);















            ActivityInfo info = in.readTypedObject(ActivityInfo.CREATOR);













            TextLinks textLinks = in.readTypedObject(TextLinks.CREATOR);















            Item item = new Item(text, htmlText, intent, uri);















            item.setActivityInfo(info);













            item.setTextLinks(textLinks);















            mItems.add(item);















        }















    }

































core/java/android/content/ClipDescription.java



+110
−0






























Original line number
Diff line number
Diff line








@@ -16,16 +16,25 @@





























package android.content;





























import android.annotation.FloatRange;













import android.annotation.IntDef;















import android.annotation.NonNull;













import android.os.Bundle;















import android.os.Parcel;















import android.os.Parcelable;















import android.os.PersistableBundle;















import android.text.TextUtils;













import android.util.ArrayMap;















import android.util.TimeUtils;















import android.util.proto.ProtoOutputStream;













import android.view.textclassifier.TextClassifier;













import android.view.textclassifier.TextLinks;





























import java.lang.annotation.Retention;













import java.lang.annotation.RetentionPolicy;















import java.util.ArrayList;















import java.util.Arrays;













import java.util.Map;































/**















 * Meta-data describing the contents of a {@link ClipData}.  Provides enough










@@ -115,12 +124,39 @@ public class ClipDescription implements Parcelable {













     */















    public static final String EXTRA_ACTIVITY_OPTIONS = "android.intent.extra.ACTIVITY_OPTIONS";





























    /** @hide */













    @Retention(RetentionPolicy.SOURCE)













    @IntDef(value =













            { CLASSIFICATION_NOT_COMPLETE, CLASSIFICATION_NOT_PERFORMED, CLASSIFICATION_COMPLETE})













    @interface ClassificationStatus {}



























    /**













     * Value returned by {@link #getConfidenceScore(String)} if text classification has not been













     * completed on the associated clip. This will be always be the case if the clip has not been













     * copied to clipboard, or if there is no associated clip.













     */













    public static final int CLASSIFICATION_NOT_COMPLETE = 1;



























    /**













     * Value returned by {@link #getConfidenceScore(String)} if text classification was not and will













     * not be performed on the associated clip. This may be the case if the clip does not contain













     * text in its first item, or if the text is too long.













     */













    public static final int CLASSIFICATION_NOT_PERFORMED = 2;



























    /**













     * Value returned by {@link #getConfidenceScore(String)} if text classification has been













     * completed.













     */













    public static final int CLASSIFICATION_COMPLETE = 3;































    final CharSequence mLabel;















    private final ArrayList<String> mMimeTypes;















    private PersistableBundle mExtras;















    private long mTimeStamp;















    private boolean mIsStyledText;













    private final ArrayMap<String, Float> mEntityConfidence = new ArrayMap<>();













    private int mClassificationStatus = CLASSIFICATION_NOT_COMPLETE;































    /**















     * Create a new clip.










@@ -346,6 +382,61 @@ public class ClipDescription implements Parcelable {













        mIsStyledText = isStyledText;















    }





























    /**













     * Sets the current status of text classification for the associated clip.













     *













     * @hide













     */













    public void setClassificationStatus(@ClassificationStatus int status) {













        mClassificationStatus = status;













    }



























    /**













     * Returns a score indicating confidence that an instance of the given entity is present in the













     * first item of the clip data, if that item is plain text and text classification has been













     * performed. The value ranges from 0 (low confidence) to 1 (high confidence). 0 indicates that













     * the entity was not found in the classified text.













     *













     * <p>Entities should be as defined in the {@link TextClassifier} class, such as













     * {@link TextClassifier#TYPE_ADDRESS}, {@link TextClassifier#TYPE_URL}, or













     * {@link TextClassifier#TYPE_EMAIL}.













     *













     * <p>If the result is positive for any entity, the full classification result as a













     * {@link TextLinks} object may be obtained using the {@link ClipData.Item#getTextLinks()}













     * method.













     *













     * @throws IllegalStateException if {@link #getClassificationStatus()} is not













     * {@link #CLASSIFICATION_COMPLETE}













     */













    @FloatRange(from = 0.0, to = 1.0)













    public float getConfidenceScore(@NonNull @TextClassifier.EntityType String entity) {













        if (mClassificationStatus != CLASSIFICATION_COMPLETE) {













            throw new IllegalStateException("Classification not complete");













        }













        return mEntityConfidence.getOrDefault(entity, 0f);













    }



























    /**













     * Returns {@link #CLASSIFICATION_COMPLETE} if text classification has been performed on the













     * associated {@link ClipData}. If this is the case then {@link #getConfidenceScore} may be used













     * to retrieve information about entities within the text. Otherwise, returns













     * {@link #CLASSIFICATION_NOT_COMPLETE} if classification has not yet returned results, or













     * {@link #CLASSIFICATION_NOT_PERFORMED} if classification was not attempted (e.g. because the













     * text was too long).













     */













    public @ClassificationStatus int getClassificationStatus() {













        return mClassificationStatus;













    }



























    /**













     * @hide













     */













    public void setConfidenceScores(Map<String, Float> confidences) {













        mEntityConfidence.clear();













        mEntityConfidence.putAll(confidences);













        mClassificationStatus = CLASSIFICATION_COMPLETE;













    }





























    @Override















    public String toString() {















        StringBuilder b = new StringBuilder(128);










@@ -451,6 +542,23 @@ public class ClipDescription implements Parcelable {













        dest.writePersistableBundle(mExtras);















        dest.writeLong(mTimeStamp);















        dest.writeBoolean(mIsStyledText);













        dest.writeInt(mClassificationStatus);













        dest.writeBundle(confidencesToBundle());













    }



























    private Bundle confidencesToBundle() {













        Bundle bundle = new Bundle();













        int size = mEntityConfidence.size();













        for (int i = 0; i < size; i++) {













            bundle.putFloat(mEntityConfidence.keyAt(i), mEntityConfidence.valueAt(i));













        }













        return bundle;













    }



























    private void readBundleToConfidences(Bundle bundle) {













        for (String key : bundle.keySet()) {













            mEntityConfidence.put(key, bundle.getFloat(key));













        }















    }































    ClipDescription(Parcel in) {









@@ -459,6 +567,8 @@ public class ClipDescription implements Parcelable {













        mExtras = in.readPersistableBundle();















        mTimeStamp = in.readLong();















        mIsStyledText = in.readBoolean();













        mClassificationStatus = in.readInt();













        readBundleToConfidences(in.readBundle());















    }































    public static final @android.annotation.NonNull Parcelable.Creator<ClipDescription> CREATOR =

































core/java/android/view/textclassifier/TextClassifier.java



+3
−1






























Original line number
Diff line number
Diff line








@@ -145,7 +145,7 @@ public interface TextClassifier {













    @StringDef({WIDGET_TYPE_TEXTVIEW, WIDGET_TYPE_EDITTEXT, WIDGET_TYPE_UNSELECTABLE_TEXTVIEW,















            WIDGET_TYPE_WEBVIEW, WIDGET_TYPE_EDIT_WEBVIEW, WIDGET_TYPE_CUSTOM_TEXTVIEW,















            WIDGET_TYPE_CUSTOM_EDITTEXT, WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW,













            WIDGET_TYPE_NOTIFICATION, WIDGET_TYPE_UNKNOWN})













            WIDGET_TYPE_NOTIFICATION, WIDGET_TYPE_CLIPBOARD, WIDGET_TYPE_UNKNOWN })















    @interface WidgetType {}































    /** The widget involved in the text classification context is a standard









@@ -172,6 +172,8 @@ public interface TextClassifier {













    String WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW = "nosel-customview";















    /** The widget involved in the text classification context is a notification */















    String WIDGET_TYPE_NOTIFICATION = "notification";













    /** The text classification context is for use with the system clipboard. */













    String WIDGET_TYPE_CLIPBOARD = "clipboard";















    /** The widget involved in the text classification context is of an unknown/unspecified type. */















    String WIDGET_TYPE_UNKNOWN = "unknown";

















































services/core/java/com/android/server/clipboard/ClipboardService.java



+89
−0






























Original line number
Diff line number
Diff line








@@ -22,6 +22,7 @@ import android.Manifest;













import android.annotation.NonNull;















import android.annotation.Nullable;















import android.annotation.UserIdInt;













import android.annotation.WorkerThread;















import android.app.ActivityManagerInternal;















import android.app.AppGlobals;















import android.app.AppOpsManager;









@@ -43,6 +44,8 @@ import android.content.pm.PackageManager;













import android.content.pm.UserInfo;















import android.net.Uri;















import android.os.Binder;













import android.os.Handler;













import android.os.HandlerThread;















import android.os.IBinder;















import android.os.IUserManager;















import android.os.Parcel;









@@ -55,10 +58,15 @@ import android.os.UserManager;













import android.provider.DeviceConfig;















import android.provider.Settings;















import android.text.TextUtils;













import android.util.ArrayMap;















import android.util.Slog;















import android.util.SparseArray;















import android.util.SparseBooleanArray;















import android.view.autofill.AutofillManagerInternal;













import android.view.textclassifier.TextClassificationContext;













import android.view.textclassifier.TextClassificationManager;













import android.view.textclassifier.TextClassifier;













import android.view.textclassifier.TextLinks;















import android.widget.Toast;































import com.android.internal.R;










@@ -170,6 +178,8 @@ public class ClipboardService extends SystemService {













    // DeviceConfig properties















    private static final String PROPERTY_SHOW_ACCESS_NOTIFICATIONS = "show_access_notifications";















    private static final boolean DEFAULT_SHOW_ACCESS_NOTIFICATIONS = true;













    private static final String PROPERTY_MAX_CLASSIFICATION_LENGTH = "max_classification_length";













    private static final int DEFAULT_MAX_CLASSIFICATION_LENGTH = 400;































    private final ActivityManagerInternal mAmInternal;















    private final IUriGrantsManager mUgm;









@@ -180,8 +190,10 @@ public class ClipboardService extends SystemService {













    private final AppOpsManager mAppOps;















    private final ContentCaptureManagerInternal mContentCaptureInternal;















    private final AutofillManagerInternal mAutofillInternal;













    private final TextClassificationManager mTextClassificationManager;















    private final IBinder mPermissionOwner;















    private final HostClipboardMonitor mHostClipboardMonitor;













    private final Handler mWorkerHandler;































    @GuardedBy("mLock")















    private final SparseArray<PerUserClipboard> mClipboards = new SparseArray<>();









@@ -189,6 +201,9 @@ public class ClipboardService extends SystemService {













    @GuardedBy("mLock")















    private boolean mShowAccessNotifications = DEFAULT_SHOW_ACCESS_NOTIFICATIONS;





























    @GuardedBy("mLock")













    private int mMaxClassificationLength = DEFAULT_MAX_CLASSIFICATION_LENGTH;





























    private final Object mLock = new Object();































    /**









@@ -206,6 +221,8 @@ public class ClipboardService extends SystemService {













        mAppOps = (AppOpsManager) getContext().getSystemService(Context.APP_OPS_SERVICE);















        mContentCaptureInternal = LocalServices.getService(ContentCaptureManagerInternal.class);















        mAutofillInternal = LocalServices.getService(AutofillManagerInternal.class);













        mTextClassificationManager = (TextClassificationManager)













                getContext().getSystemService(Context.TEXT_CLASSIFICATION_SERVICE);















        final IBinder permOwner = mUgmInternal.newUriPermissionOwner("clipboard");















        mPermissionOwner = permOwner;















        if (IS_EMULATOR) {









@@ -232,6 +249,10 @@ public class ClipboardService extends SystemService {













        updateConfig();















        DeviceConfig.addOnPropertiesChangedListener(DeviceConfig.NAMESPACE_CLIPBOARD,















                getContext().getMainExecutor(), properties -> updateConfig());



























        HandlerThread workerThread = new HandlerThread(TAG);













        workerThread.start();













        mWorkerHandler = workerThread.getThreadHandler();















    }































    @Override









@@ -250,6 +271,8 @@ public class ClipboardService extends SystemService {













        synchronized (mLock) {















            mShowAccessNotifications = DeviceConfig.getBoolean(DeviceConfig.NAMESPACE_CLIPBOARD,















                    PROPERTY_SHOW_ACCESS_NOTIFICATIONS, DEFAULT_SHOW_ACCESS_NOTIFICATIONS);













            mMaxClassificationLength = DeviceConfig.getInt(DeviceConfig.NAMESPACE_CLIPBOARD,













                    PROPERTY_MAX_CLASSIFICATION_LENGTH, DEFAULT_MAX_CLASSIFICATION_LENGTH);















        }















    }


























@@ -592,6 +615,10 @@ public class ClipboardService extends SystemService {













            }















        }





























        if (clip != null) {













            startClassificationLocked(clip);













        }





























        // Update this user















        final int userId = UserHandle.getUserId(uid);















        setPrimaryClipInternalLocked(getClipboardLocked(userId), clip, uid, sourcePackage);










@@ -691,6 +718,68 @@ public class ClipboardService extends SystemService {













        }















    }





























    @GuardedBy("mLock")













    private void startClassificationLocked(@NonNull ClipData clip) {













        TextClassifier classifier;













        final long ident = Binder.clearCallingIdentity();













        try {













            classifier = mTextClassificationManager.createTextClassificationSession(













                    new TextClassificationContext.Builder(













                            getContext().getPackageName(),













                            TextClassifier.WIDGET_TYPE_CLIPBOARD













                    ).build()













            );













        } finally {













            Binder.restoreCallingIdentity(ident);













        }



























        if (clip.getItemCount() == 0) {













            clip.getDescription().setClassificationStatus(













                    ClipDescription.CLASSIFICATION_NOT_PERFORMED);













            return;













        }













        CharSequence text = clip.getItemAt(0).getText();













        if (TextUtils.isEmpty(text) || text.length() > mMaxClassificationLength













                || text.length() > classifier.getMaxGenerateLinksTextLength()) {













            clip.getDescription().setClassificationStatus(













                    ClipDescription.CLASSIFICATION_NOT_PERFORMED);













            return;













        }



























        mWorkerHandler.post(() -> doClassification(text, clip, classifier));













    }



























    @WorkerThread













    private void doClassification(













            CharSequence text, ClipData clip, TextClassifier classifier) {













        TextLinks.Request request = new TextLinks.Request.Builder(text).build();













        TextLinks links;













        try {













            links = classifier.generateLinks(request);













        } finally {













            classifier.destroy();













        }



























        // Find the highest confidence for each entity in the text.













        ArrayMap<String, Float> confidences = new ArrayMap<>();













        for (TextLinks.TextLink link : links.getLinks()) {













            for (int i = 0; i < link.getEntityCount(); i++) {













                String entity = link.getEntity(i);













                float conf = link.getConfidenceScore(entity);













                if (conf > confidences.getOrDefault(entity, 0f)) {













                    confidences.put(entity, conf);













                }













            }













        }



























        synchronized (mLock) {













            clip.getDescription().setConfidenceScores(confidences);













            if (!links.getLinks().isEmpty()) {













                clip.getItemAt(0).setTextLinks(links);













            }













        }













    }





























    private boolean isDeviceLocked(@UserIdInt int userId) {















        final long token = Binder.clearCallingIdentity();















        try {