Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 7e8237e6 authored by Tony Mak's avatar Tony Mak
Browse files

Notify TextClassifier an app is reading the clipboard 

Send TextClassifier an event when an app is reading the clipboard.
The event contains both the source package and the destination package.

Bug: 177898188

Will add an automated test on the text classifier side.

Test: Manual.
1. Copy and paste. Verify that an event is sent to textclassifier by
enable verbose logging.
2. Paste on the same app again, no event is sent.
3. Verify that no events are genereated for apps that read
   the clipboard in the background, e.g. keyboard and autofill.

Change-Id: Ide849bf5a0a56271ec8e70d47faabbc47a74332d
parent a7e6b948

core/java/android/view/textclassifier/TextClassifierEvent.java

+9 −1
Original line number Diff line number Diff line
@@ -86,7 +86,8 @@ public abstract class TextClassifierEvent implements Parcelable { 
            TYPE_ACTIONS_SHOWN, TYPE_LINK_CLICKED, TYPE_OVERTYPE, TYPE_COPY_ACTION,

            TYPE_PASTE_ACTION, TYPE_CUT_ACTION, TYPE_SHARE_ACTION, TYPE_SMART_ACTION,

            TYPE_SELECTION_DRAG, TYPE_SELECTION_DESTROYED, TYPE_OTHER_ACTION, TYPE_SELECT_ALL,

            TYPE_SELECTION_RESET, TYPE_MANUAL_REPLY, TYPE_ACTIONS_GENERATED, TYPE_LINKS_GENERATED})

            TYPE_SELECTION_RESET, TYPE_MANUAL_REPLY, TYPE_ACTIONS_GENERATED, TYPE_LINKS_GENERATED,

            TYPE_READ_CLIPBOARD})

    public @interface Type {

        // For custom event types, use range 1,000,000+.

    }
@@ -135,6 +136,13 @@ public abstract class TextClassifierEvent implements Parcelable { 
    public static final int TYPE_ACTIONS_GENERATED = 20;

    /** Some text links were generated.*/

    public static final int TYPE_LINKS_GENERATED = 21;

    /**

     * Read a clipboard.

     * TODO: Make this public.

     *

     * @hide

     */

    public static final int TYPE_READ_CLIPBOARD = 22;



    @Category

    private final int mEventCategory;

services/core/java/com/android/server/clipboard/ClipboardService.java

+64 −15
Original line number Diff line number Diff line
@@ -45,6 +45,7 @@ import android.content.pm.PackageManager; 
import android.content.pm.UserInfo;

import android.net.Uri;

import android.os.Binder;

import android.os.Bundle;

import android.os.Handler;

import android.os.HandlerThread;

import android.os.IBinder;
@@ -71,6 +72,7 @@ import android.view.autofill.AutofillManagerInternal; 
import android.view.textclassifier.TextClassificationContext;

import android.view.textclassifier.TextClassificationManager;

import android.view.textclassifier.TextClassifier;

import android.view.textclassifier.TextClassifierEvent;

import android.view.textclassifier.TextLinks;

import android.widget.Toast;
@@ -344,9 +346,18 @@ public class ClipboardService extends SystemService { 
        /** Uids that have already triggered a toast notification for {@link #primaryClip} */

        final SparseBooleanArray mNotifiedUids = new SparseBooleanArray();



        /**

         * Uids that have already triggered a notification to text classifier for

         * {@link #primaryClip}.

         */

        final SparseBooleanArray mNotifiedTextClassifierUids = new SparseBooleanArray();



        final HashSet<String> activePermissionOwners

                = new HashSet<String>();



        /** The text classifier session that is used to annotate the text in the primary clip. */

        TextClassifier mTextClassifier;



        PerUserClipboard(int userId) {

            this.userId = userId;

        }
@@ -504,6 +515,7 @@ public class ClipboardService extends SystemService { 
                addActiveOwnerLocked(intendingUid, pkg);

                PerUserClipboard clipboard = getClipboardLocked(intendingUserId);

                showAccessNotificationLocked(pkg, intendingUid, intendingUserId, clipboard);

                notifyTextClassifierLocked(clipboard, pkg, intendingUid);

                return clipboard.primaryClip;

            }

        }
@@ -724,6 +736,7 @@ public class ClipboardService extends SystemService { 
        }

        clipboard.primaryClip = clip;

        clipboard.mNotifiedUids.clear();

        clipboard.mNotifiedTextClassifierUids.clear();

        if (clip != null) {

            clipboard.primaryClipUid = uid;

            clipboard.mPrimaryClipPackage = sourcePackage;
@@ -763,6 +776,11 @@ public class ClipboardService extends SystemService { 


    @GuardedBy("mLock")

    private void startClassificationLocked(@NonNull ClipData clip, @UserIdInt int userId) {

        if (clip.getItemCount() == 0) {

            clip.getDescription().setClassificationStatus(

                    ClipDescription.CLASSIFICATION_NOT_PERFORMED);

            return;

        }

        TextClassifier classifier;

        final long ident = Binder.clearCallingIdentity();

        try {
@@ -776,12 +794,6 @@ public class ClipboardService extends SystemService { 
        } finally {

            Binder.restoreCallingIdentity(ident);

        }



        if (clip.getItemCount() == 0) {

            clip.getDescription().setClassificationStatus(

                    ClipDescription.CLASSIFICATION_NOT_PERFORMED);

            return;

        }

        CharSequence text = clip.getItemAt(0).getText();

        if (TextUtils.isEmpty(text) || text.length() > mMaxClassificationLength

                || text.length() > classifier.getMaxGenerateLinksTextLength()) {
@@ -789,7 +801,7 @@ public class ClipboardService extends SystemService { 
                    ClipDescription.CLASSIFICATION_NOT_PERFORMED);

            return;

        }



        getClipboardLocked(userId).mTextClassifier = classifier;

        mWorkerHandler.post(() -> doClassification(text, clip, classifier));

    }
@@ -797,12 +809,7 @@ public class ClipboardService extends SystemService { 
    private void doClassification(

            CharSequence text, ClipData clip, TextClassifier classifier) {

        TextLinks.Request request = new TextLinks.Request.Builder(text).build();

        TextLinks links;

        try {

            links = classifier.generateLinks(request);

        } finally {

            classifier.destroy();

        }

        TextLinks links = classifier.generateLinks(request);



        // Find the highest confidence for each entity in the text.

        ArrayMap<String, Float> confidences = new ArrayMap<>();
@@ -1123,6 +1130,48 @@ public class ClipboardService extends SystemService { 
                && item.getIntent() == null;

    }



    /** Potentially notifies the text classifier that an app is accessing a text clip. */

    @GuardedBy("mLock")

    private void notifyTextClassifierLocked(

            PerUserClipboard clipboard, String callingPackage, int callingUid) {

        if (clipboard.primaryClip == null) {

            return;

        }

        ClipData.Item item = clipboard.primaryClip.getItemAt(0);

        if (item == null) {

            return;

        }

        if (!isText(clipboard.primaryClip)) {

            return;

        }

        TextClassifier textClassifier = clipboard.mTextClassifier;

        // Don't notify text classifier if we haven't used it to annotate the text in the clip.

        if (textClassifier == null) {

            return;

        }

        // Don't notify text classifier if the app reading the clipboard does not have the focus.

        if (!mWm.isUidFocused(callingUid)) {

            return;

        }

        // Don't notify text classifier again if already notified for this uid and clip.

        if (clipboard.mNotifiedTextClassifierUids.get(callingUid)) {

            return;

        }

        clipboard.mNotifiedTextClassifierUids.put(callingUid, true);

        Binder.withCleanCallingIdentity(() -> {

            TextClassifierEvent.TextLinkifyEvent pasteEvent =

                    new TextClassifierEvent.TextLinkifyEvent.Builder(

                            TextClassifierEvent.TYPE_READ_CLIPBOARD)

                            .setEventContext(new TextClassificationContext.Builder(

                                    callingPackage, TextClassifier.WIDGET_TYPE_CLIPBOARD)

                                    .build())

                            .setExtras(

                                    Bundle.forPair("source_package", clipboard.mPrimaryClipPackage))

                            .build();

            textClassifier.onTextClassifierEvent(pasteEvent);

        });

    }



    private TextClassificationManager createTextClassificationManagerAsUser(@UserIdInt int userId) {

        Context context = getContext().createContextAsUser(UserHandle.of(userId), /* flags= */ 0);

        return context.getSystemService(TextClassificationManager.class);