Merge "Add text classification results to clip description and clip data." into sc-dev (fe4a8f58) · Commits · e / os / android_frameworks_base

core/api/current.txt

+7 −0

Original line number	Diff line number	Diff line
		@@ -9910,6 +9910,7 @@ package android.content {
		method public String getHtmlText();
		method public android.content.Intent getIntent();
		method public CharSequence getText();
		method @Nullable public android.view.textclassifier.TextLinks getTextLinks();
		method public android.net.Uri getUri();
		}

		@@ -9919,6 +9920,8 @@ package android.content {
		method public static boolean compareMimeTypes(String, String);
		method public int describeContents();
		method public String[] filterMimeTypes(String);
		method public int getClassificationStatus();
		method @FloatRange(from=0.0, to=1.0) public float getConfidenceScore(@NonNull String);
		method public android.os.PersistableBundle getExtras();
		method public CharSequence getLabel();
		method public String getMimeType(int);
		@@ -9928,6 +9931,9 @@ package android.content {
		method public boolean isStyledText();
		method public void setExtras(android.os.PersistableBundle);
		method public void writeToParcel(android.os.Parcel, int);
		field public static final int CLASSIFICATION_COMPLETE = 3; // 0x3
		field public static final int CLASSIFICATION_NOT_COMPLETE = 1; // 0x1
		field public static final int CLASSIFICATION_NOT_PERFORMED = 2; // 0x2
		field @NonNull public static final android.os.Parcelable.Creator<android.content.ClipDescription> CREATOR;
		field public static final String MIMETYPE_TEXT_HTML = "text/html";
		field public static final String MIMETYPE_TEXT_INTENT = "text/vnd.android.intent";
		@@ -51917,6 +51923,7 @@ package android.view.textclassifier {
		field public static final String TYPE_PHONE = "phone";
		field public static final String TYPE_UNKNOWN = "";
		field public static final String TYPE_URL = "url";
		field public static final String WIDGET_TYPE_CLIPBOARD = "clipboard";
		field public static final String WIDGET_TYPE_CUSTOM_EDITTEXT = "customedit";
		field public static final String WIDGET_TYPE_CUSTOM_TEXTVIEW = "customview";
		field public static final String WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW = "nosel-customview";

core/java/android/content/ClipData.java

+29 −0

Original line number	Diff line number	Diff line
		@@ -21,6 +21,7 @@ import static android.content.ContentResolver.SCHEME_ANDROID_RESOURCE;
		import static android.content.ContentResolver.SCHEME_CONTENT;
		import static android.content.ContentResolver.SCHEME_FILE;

		import android.annotation.Nullable;
		import android.compat.annotation.UnsupportedAppUsage;
		import android.content.pm.ActivityInfo;
		import android.content.res.AssetFileDescriptor;
		@@ -38,6 +39,7 @@ import android.text.TextUtils;
		import android.text.style.URLSpan;
		import android.util.Log;
		import android.util.proto.ProtoOutputStream;
		import android.view.textclassifier.TextLinks;

		import com.android.internal.util.ArrayUtils;

		@@ -204,6 +206,7 @@ public class ClipData implements Parcelable {
		Uri mUri;
		// Additional activity info resolved by the system
		ActivityInfo mActivityInfo;
		private TextLinks mTextLinks;

		/** @hide */
		public Item(Item other) {
		@@ -331,6 +334,29 @@ public class ClipData implements Parcelable {
		mActivityInfo = info;
		}

		/**
		* Returns the results of text classification run on the raw text contained in this item,
		* if it was performed, and if any entities were found in the text. Classification is
		* generally only performed on the first item in clip data, and only if the text is below a
		* certain length.
		*
		* <p>Returns {@code null} if classification was not performed, or if no entities were
		* found in the text.
		*
		* @see ClipDescription#getConfidenceScore(String)
		*/
		@Nullable
		public TextLinks getTextLinks() {
		return mTextLinks;
		}

		/**
		* @hide
		*/
		public void setTextLinks(TextLinks textLinks) {
		mTextLinks = textLinks;
		}

		/**
		* Turn this item into text, regardless of the type of data it
		* actually contains.
		@@ -1183,6 +1209,7 @@ public class ClipData implements Parcelable {
		dest.writeTypedObject(item.mIntent, flags);
		dest.writeTypedObject(item.mUri, flags);
		dest.writeTypedObject(item.mActivityInfo, flags);
		dest.writeTypedObject(item.mTextLinks, flags);
		}
		}

		@@ -1201,8 +1228,10 @@ public class ClipData implements Parcelable {
		Intent intent = in.readTypedObject(Intent.CREATOR);
		Uri uri = in.readTypedObject(Uri.CREATOR);
		ActivityInfo info = in.readTypedObject(ActivityInfo.CREATOR);
		TextLinks textLinks = in.readTypedObject(TextLinks.CREATOR);
		Item item = new Item(text, htmlText, intent, uri);
		item.setActivityInfo(info);
		item.setTextLinks(textLinks);
		mItems.add(item);
		}
		}

core/java/android/content/ClipDescription.java

+110 −0

Original line number	Diff line number	Diff line
		@@ -16,16 +16,25 @@

		package android.content;

		import android.annotation.FloatRange;
		import android.annotation.IntDef;
		import android.annotation.NonNull;
		import android.os.Bundle;
		import android.os.Parcel;
		import android.os.Parcelable;
		import android.os.PersistableBundle;
		import android.text.TextUtils;
		import android.util.ArrayMap;
		import android.util.TimeUtils;
		import android.util.proto.ProtoOutputStream;
		import android.view.textclassifier.TextClassifier;
		import android.view.textclassifier.TextLinks;

		import java.lang.annotation.Retention;
		import java.lang.annotation.RetentionPolicy;
		import java.util.ArrayList;
		import java.util.Arrays;
		import java.util.Map;

		/**
		* Meta-data describing the contents of a {@link ClipData}. Provides enough
		@@ -115,12 +124,39 @@ public class ClipDescription implements Parcelable {
		*/
		public static final String EXTRA_ACTIVITY_OPTIONS = "android.intent.extra.ACTIVITY_OPTIONS";

		/** @hide */
		@Retention(RetentionPolicy.SOURCE)
		@IntDef(value =
		{ CLASSIFICATION_NOT_COMPLETE, CLASSIFICATION_NOT_PERFORMED, CLASSIFICATION_COMPLETE})
		@interface ClassificationStatus {}

		/**
		* Value returned by {@link #getConfidenceScore(String)} if text classification has not been
		* completed on the associated clip. This will be always be the case if the clip has not been
		* copied to clipboard, or if there is no associated clip.
		*/
		public static final int CLASSIFICATION_NOT_COMPLETE = 1;

		/**
		* Value returned by {@link #getConfidenceScore(String)} if text classification was not and will
		* not be performed on the associated clip. This may be the case if the clip does not contain
		* text in its first item, or if the text is too long.
		*/
		public static final int CLASSIFICATION_NOT_PERFORMED = 2;

		/**
		* Value returned by {@link #getConfidenceScore(String)} if text classification has been
		* completed.
		*/
		public static final int CLASSIFICATION_COMPLETE = 3;

		final CharSequence mLabel;
		private final ArrayList<String> mMimeTypes;
		private PersistableBundle mExtras;
		private long mTimeStamp;
		private boolean mIsStyledText;
		private final ArrayMap<String, Float> mEntityConfidence = new ArrayMap<>();
		private int mClassificationStatus = CLASSIFICATION_NOT_COMPLETE;

		/**
		* Create a new clip.
		@@ -346,6 +382,61 @@ public class ClipDescription implements Parcelable {
		mIsStyledText = isStyledText;
		}

		/**
		* Sets the current status of text classification for the associated clip.
		*
		* @hide
		*/
		public void setClassificationStatus(@ClassificationStatus int status) {
		mClassificationStatus = status;
		}

		/**
		* Returns a score indicating confidence that an instance of the given entity is present in the
		* first item of the clip data, if that item is plain text and text classification has been
		* performed. The value ranges from 0 (low confidence) to 1 (high confidence). 0 indicates that
		* the entity was not found in the classified text.
		*
		* <p>Entities should be as defined in the {@link TextClassifier} class, such as
		* {@link TextClassifier#TYPE_ADDRESS}, {@link TextClassifier#TYPE_URL}, or
		* {@link TextClassifier#TYPE_EMAIL}.
		*
		* <p>If the result is positive for any entity, the full classification result as a
		* {@link TextLinks} object may be obtained using the {@link ClipData.Item#getTextLinks()}
		* method.
		*
		* @throws IllegalStateException if {@link #getClassificationStatus()} is not
		* {@link #CLASSIFICATION_COMPLETE}
		*/
		@FloatRange(from = 0.0, to = 1.0)
		public float getConfidenceScore(@NonNull @TextClassifier.EntityType String entity) {
		if (mClassificationStatus != CLASSIFICATION_COMPLETE) {
		throw new IllegalStateException("Classification not complete");
		}
		return mEntityConfidence.getOrDefault(entity, 0f);
		}

		/**
		* Returns {@link #CLASSIFICATION_COMPLETE} if text classification has been performed on the
		* associated {@link ClipData}. If this is the case then {@link #getConfidenceScore} may be used
		* to retrieve information about entities within the text. Otherwise, returns
		* {@link #CLASSIFICATION_NOT_COMPLETE} if classification has not yet returned results, or
		* {@link #CLASSIFICATION_NOT_PERFORMED} if classification was not attempted (e.g. because the
		* text was too long).
		*/
		public @ClassificationStatus int getClassificationStatus() {
		return mClassificationStatus;
		}

		/**
		* @hide
		*/
		public void setConfidenceScores(Map<String, Float> confidences) {
		mEntityConfidence.clear();
		mEntityConfidence.putAll(confidences);
		mClassificationStatus = CLASSIFICATION_COMPLETE;
		}

		@Override
		public String toString() {
		StringBuilder b = new StringBuilder(128);
		@@ -451,6 +542,23 @@ public class ClipDescription implements Parcelable {
		dest.writePersistableBundle(mExtras);
		dest.writeLong(mTimeStamp);
		dest.writeBoolean(mIsStyledText);
		dest.writeInt(mClassificationStatus);
		dest.writeBundle(confidencesToBundle());
		}

		private Bundle confidencesToBundle() {
		Bundle bundle = new Bundle();
		int size = mEntityConfidence.size();
		for (int i = 0; i < size; i++) {
		bundle.putFloat(mEntityConfidence.keyAt(i), mEntityConfidence.valueAt(i));
		}
		return bundle;
		}

		private void readBundleToConfidences(Bundle bundle) {
		for (String key : bundle.keySet()) {
		mEntityConfidence.put(key, bundle.getFloat(key));
		}
		}

		ClipDescription(Parcel in) {
		@@ -459,6 +567,8 @@ public class ClipDescription implements Parcelable {
		mExtras = in.readPersistableBundle();
		mTimeStamp = in.readLong();
		mIsStyledText = in.readBoolean();
		mClassificationStatus = in.readInt();
		readBundleToConfidences(in.readBundle());
		}

		public static final @android.annotation.NonNull Parcelable.Creator<ClipDescription> CREATOR =

core/java/android/view/textclassifier/TextClassifier.java

+3 −1

Original line number	Diff line number	Diff line
		@@ -145,7 +145,7 @@ public interface TextClassifier {
		@StringDef({WIDGET_TYPE_TEXTVIEW, WIDGET_TYPE_EDITTEXT, WIDGET_TYPE_UNSELECTABLE_TEXTVIEW,
		WIDGET_TYPE_WEBVIEW, WIDGET_TYPE_EDIT_WEBVIEW, WIDGET_TYPE_CUSTOM_TEXTVIEW,
		WIDGET_TYPE_CUSTOM_EDITTEXT, WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW,
		WIDGET_TYPE_NOTIFICATION, WIDGET_TYPE_UNKNOWN})
		WIDGET_TYPE_NOTIFICATION, WIDGET_TYPE_CLIPBOARD, WIDGET_TYPE_UNKNOWN })
		@interface WidgetType {}

		/** The widget involved in the text classification context is a standard
		@@ -172,6 +172,8 @@ public interface TextClassifier {
		String WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW = "nosel-customview";
		/** The widget involved in the text classification context is a notification */
		String WIDGET_TYPE_NOTIFICATION = "notification";
		/** The text classification context is for use with the system clipboard. */
		String WIDGET_TYPE_CLIPBOARD = "clipboard";
		/** The widget involved in the text classification context is of an unknown/unspecified type. */
		String WIDGET_TYPE_UNKNOWN = "unknown";

services/core/java/com/android/server/clipboard/ClipboardService.java

+89 −0

Original line number	Diff line number	Diff line
		@@ -22,6 +22,7 @@ import android.Manifest;
		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.annotation.UserIdInt;
		import android.annotation.WorkerThread;
		import android.app.ActivityManagerInternal;
		import android.app.AppGlobals;
		import android.app.AppOpsManager;
		@@ -43,6 +44,8 @@ import android.content.pm.PackageManager;
		import android.content.pm.UserInfo;
		import android.net.Uri;
		import android.os.Binder;
		import android.os.Handler;
		import android.os.HandlerThread;
		import android.os.IBinder;
		import android.os.IUserManager;
		import android.os.Parcel;
		@@ -55,10 +58,15 @@ import android.os.UserManager;
		import android.provider.DeviceConfig;
		import android.provider.Settings;
		import android.text.TextUtils;
		import android.util.ArrayMap;
		import android.util.Slog;
		import android.util.SparseArray;
		import android.util.SparseBooleanArray;
		import android.view.autofill.AutofillManagerInternal;
		import android.view.textclassifier.TextClassificationContext;
		import android.view.textclassifier.TextClassificationManager;
		import android.view.textclassifier.TextClassifier;
		import android.view.textclassifier.TextLinks;
		import android.widget.Toast;

		import com.android.internal.R;
		@@ -170,6 +178,8 @@ public class ClipboardService extends SystemService {
		// DeviceConfig properties
		private static final String PROPERTY_SHOW_ACCESS_NOTIFICATIONS = "show_access_notifications";
		private static final boolean DEFAULT_SHOW_ACCESS_NOTIFICATIONS = true;
		private static final String PROPERTY_MAX_CLASSIFICATION_LENGTH = "max_classification_length";
		private static final int DEFAULT_MAX_CLASSIFICATION_LENGTH = 400;

		private final ActivityManagerInternal mAmInternal;
		private final IUriGrantsManager mUgm;
		@@ -180,8 +190,10 @@ public class ClipboardService extends SystemService {
		private final AppOpsManager mAppOps;
		private final ContentCaptureManagerInternal mContentCaptureInternal;
		private final AutofillManagerInternal mAutofillInternal;
		private final TextClassificationManager mTextClassificationManager;
		private final IBinder mPermissionOwner;
		private final HostClipboardMonitor mHostClipboardMonitor;
		private final Handler mWorkerHandler;

		@GuardedBy("mLock")
		private final SparseArray<PerUserClipboard> mClipboards = new SparseArray<>();
		@@ -189,6 +201,9 @@ public class ClipboardService extends SystemService {
		@GuardedBy("mLock")
		private boolean mShowAccessNotifications = DEFAULT_SHOW_ACCESS_NOTIFICATIONS;

		@GuardedBy("mLock")
		private int mMaxClassificationLength = DEFAULT_MAX_CLASSIFICATION_LENGTH;

		private final Object mLock = new Object();

		/**
		@@ -206,6 +221,8 @@ public class ClipboardService extends SystemService {
		mAppOps = (AppOpsManager) getContext().getSystemService(Context.APP_OPS_SERVICE);
		mContentCaptureInternal = LocalServices.getService(ContentCaptureManagerInternal.class);
		mAutofillInternal = LocalServices.getService(AutofillManagerInternal.class);
		mTextClassificationManager = (TextClassificationManager)
		getContext().getSystemService(Context.TEXT_CLASSIFICATION_SERVICE);
		final IBinder permOwner = mUgmInternal.newUriPermissionOwner("clipboard");
		mPermissionOwner = permOwner;
		if (IS_EMULATOR) {
		@@ -232,6 +249,10 @@ public class ClipboardService extends SystemService {
		updateConfig();
		DeviceConfig.addOnPropertiesChangedListener(DeviceConfig.NAMESPACE_CLIPBOARD,
		getContext().getMainExecutor(), properties -> updateConfig());

		HandlerThread workerThread = new HandlerThread(TAG);
		workerThread.start();
		mWorkerHandler = workerThread.getThreadHandler();
		}

		@Override
		@@ -250,6 +271,8 @@ public class ClipboardService extends SystemService {
		synchronized (mLock) {
		mShowAccessNotifications = DeviceConfig.getBoolean(DeviceConfig.NAMESPACE_CLIPBOARD,
		PROPERTY_SHOW_ACCESS_NOTIFICATIONS, DEFAULT_SHOW_ACCESS_NOTIFICATIONS);
		mMaxClassificationLength = DeviceConfig.getInt(DeviceConfig.NAMESPACE_CLIPBOARD,
		PROPERTY_MAX_CLASSIFICATION_LENGTH, DEFAULT_MAX_CLASSIFICATION_LENGTH);
		}
		}

		@@ -592,6 +615,10 @@ public class ClipboardService extends SystemService {
		}
		}

		if (clip != null) {
		startClassificationLocked(clip);
		}

		// Update this user
		final int userId = UserHandle.getUserId(uid);
		setPrimaryClipInternalLocked(getClipboardLocked(userId), clip, uid, sourcePackage);
		@@ -691,6 +718,68 @@ public class ClipboardService extends SystemService {
		}
		}

		@GuardedBy("mLock")
		private void startClassificationLocked(@NonNull ClipData clip) {
		TextClassifier classifier;
		final long ident = Binder.clearCallingIdentity();
		try {
		classifier = mTextClassificationManager.createTextClassificationSession(
		new TextClassificationContext.Builder(
		getContext().getPackageName(),
		TextClassifier.WIDGET_TYPE_CLIPBOARD
		).build()
		);
		} finally {
		Binder.restoreCallingIdentity(ident);
		}

		if (clip.getItemCount() == 0) {
		clip.getDescription().setClassificationStatus(
		ClipDescription.CLASSIFICATION_NOT_PERFORMED);
		return;
		}
		CharSequence text = clip.getItemAt(0).getText();
		if (TextUtils.isEmpty(text) \|\| text.length() > mMaxClassificationLength
		\|\| text.length() > classifier.getMaxGenerateLinksTextLength()) {
		clip.getDescription().setClassificationStatus(
		ClipDescription.CLASSIFICATION_NOT_PERFORMED);
		return;
		}

		mWorkerHandler.post(() -> doClassification(text, clip, classifier));
		}

		@WorkerThread
		private void doClassification(
		CharSequence text, ClipData clip, TextClassifier classifier) {
		TextLinks.Request request = new TextLinks.Request.Builder(text).build();
		TextLinks links;
		try {
		links = classifier.generateLinks(request);
		} finally {
		classifier.destroy();
		}

		// Find the highest confidence for each entity in the text.
		ArrayMap<String, Float> confidences = new ArrayMap<>();
		for (TextLinks.TextLink link : links.getLinks()) {
		for (int i = 0; i < link.getEntityCount(); i++) {
		String entity = link.getEntity(i);
		float conf = link.getConfidenceScore(entity);
		if (conf > confidences.getOrDefault(entity, 0f)) {
		confidences.put(entity, conf);
		}
		}
		}

		synchronized (mLock) {
		clip.getDescription().setConfidenceScores(confidences);
		if (!links.getLinks().isEmpty()) {
		clip.getItemAt(0).setTextLinks(links);
		}
		}
		}

		private boolean isDeviceLocked(@UserIdInt int userId) {
		final long token = Binder.clearCallingIdentity();
		try {