Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d2010f26 authored by Satakshi's avatar Satakshi
Browse files

Screenshot Notification Smart Action: AiAi and - Sys UI integration 

This change creates ScreenshotNotificationSmartActionsProvider, which is overridden
in GoogleSystemUI.
Calling AiAi is guarded by a device config flag created in cl/277143225.

Test: Manually tested the code in this CL and corresponding change in SystemUIGoogle.
Took a screenshot and verified that AiAi gets invoked and the screenshot notification
shows smart actions.
Ran new tests added in this CL 'atest ScreenshotNotificationSmartActionsTest'
Bug: 143556894

Change-Id: I439a4be9aac53fb02b566ae4d438afe3edf2b37a
parent 3ad8b97d

core/java/com/android/internal/config/sysui/SystemUiDeviceConfigFlags.java

+14 −0
Original line number Diff line number Diff line
@@ -47,6 +47,20 @@ public final class SystemUiDeviceConfigFlags { 
     */

    public static final String NAS_MAX_SUGGESTIONS = "nas_max_suggestions";



    // Flags related to screenshot intelligence



    /**

     * (bool) Whether to enable smart actions in screenshot notifications.

     */

    public static final String ENABLE_SCREENSHOT_NOTIFICATION_SMART_ACTIONS =

            "enable_screenshot_notification_smart_actions";



    /**

     * (int) Timeout value in ms to get smart actions for screenshot notification.

     */

    public static final String SCREENSHOT_NOTIFICATION_SMART_ACTIONS_TIMEOUT_MS =

            "screenshot_notification_smart_actions_timeout_ms";



    // Flags related to controls



    /**

packages/SystemUI/AndroidManifest.xml

+3 −0
Original line number Diff line number Diff line
@@ -60,6 +60,9 @@ 
    <uses-permission android:name="android.permission.GET_APP_OPS_STATS" />

    <uses-permission android:name="android.permission.USE_RESERVED_DISK" />



    <!-- to invoke ContentSuggestionsService -->

    <uses-permission android:name="android.permission.MANAGE_CONTENT_SUGGESTIONS"/>



    <!-- Networking and telephony -->

    <uses-permission android:name="android.permission.BLUETOOTH" />

    <uses-permission android:name="android.permission.BLUETOOTH_ADMIN" />

packages/SystemUI/src/com/android/systemui/SystemUIFactory.java

+10 −0
Original line number Diff line number Diff line
@@ -33,6 +33,7 @@ import com.android.systemui.dagger.SystemUIRootComponent; 
import com.android.systemui.keyguard.DismissCallbackRegistry;

import com.android.systemui.plugins.FalsingManager;

import com.android.systemui.plugins.statusbar.StatusBarStateController;

import com.android.systemui.screenshot.ScreenshotNotificationSmartActionsProvider;

import com.android.systemui.statusbar.KeyguardIndicationController;

import com.android.systemui.statusbar.NotificationMediaManager;

import com.android.systemui.statusbar.notification.NotificationWakeUpCoordinator;
@@ -124,6 +125,15 @@ public class SystemUIFactory { 
        return new StatusBarKeyguardViewManager(context, viewMediatorCallback, lockPatternUtils);

    }



    /**

     * Creates an instance of ScreenshotNotificationSmartActionsProvider.

     * This method is overridden in vendor specific implementation of Sys UI.

     */

    public ScreenshotNotificationSmartActionsProvider

            createScreenshotNotificationSmartActionsProvider() {

        return new ScreenshotNotificationSmartActionsProvider();

    }



    public KeyguardBouncer createKeyguardBouncer(Context context, ViewMediatorCallback callback,

            LockPatternUtils lockPatternUtils, ViewGroup container,

            DismissCallbackRegistry dismissCallbackRegistry,

packages/SystemUI/src/com/android/systemui/screenshot/GlobalScreenshot.java

+107 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ 
package com.android.systemui.screenshot;



import static android.content.Context.NOTIFICATION_SERVICE;

import static android.os.AsyncTask.THREAD_POOL_EXECUTOR;

import static android.provider.DeviceConfig.NAMESPACE_SYSTEMUI;

import static android.view.WindowManager.LayoutParams.LAYOUT_IN_DISPLAY_CUTOUT_MODE_ALWAYS;
@@ -29,7 +30,9 @@ import android.animation.AnimatorSet; 
import android.animation.ValueAnimator;

import android.animation.ValueAnimator.AnimatorUpdateListener;

import android.annotation.Nullable;

import android.app.ActivityManager;

import android.app.ActivityOptions;

import android.app.ActivityTaskManager;

import android.app.Notification;

import android.app.Notification.BigPictureStyle;

import android.app.NotificationManager;
@@ -42,6 +45,7 @@ import android.content.ComponentName; 
import android.content.ContentResolver;

import android.content.Context;

import android.content.Intent;

import android.content.pm.UserInfo;

import android.content.res.Configuration;

import android.content.res.Resources;

import android.graphics.Bitmap;
@@ -64,7 +68,10 @@ import android.os.Looper; 
import android.os.Message;

import android.os.PowerManager;

import android.os.Process;

import android.os.RemoteException;

import android.os.SystemClock;

import android.os.UserHandle;

import android.os.UserManager;

import android.provider.DeviceConfig;

import android.provider.MediaStore;

import android.text.TextUtils;
@@ -82,9 +89,12 @@ import android.view.animation.Interpolator; 
import android.widget.ImageView;

import android.widget.Toast;



import com.android.internal.annotations.VisibleForTesting;

import com.android.internal.config.sysui.SystemUiDeviceConfigFlags;

import com.android.internal.messages.nano.SystemMessageProto.SystemMessage;

import com.android.systemui.R;

import com.android.systemui.SystemUI;

import com.android.systemui.SystemUIFactory;

import com.android.systemui.dagger.qualifiers.MainResources;

import com.android.systemui.shared.system.ActivityManagerWrapper;

import com.android.systemui.statusbar.phone.StatusBar;
@@ -97,8 +107,11 @@ import java.io.IOException; 
import java.io.OutputStream;

import java.text.DateFormat;

import java.text.SimpleDateFormat;

import java.util.Collections;

import java.util.Date;

import java.util.List;

import java.util.Optional;

import java.util.concurrent.CompletableFuture;

import java.util.concurrent.ExecutionException;

import java.util.concurrent.TimeUnit;

import java.util.concurrent.TimeoutException;
@@ -156,6 +169,8 @@ public class GlobalScreenshot { 
        private final BigPictureStyle mNotificationStyle;

        private final int mImageWidth;

        private final int mImageHeight;

        private final Handler mHandler;

        private final ScreenshotNotificationSmartActionsProvider mSmartActionsProvider;



        SaveImageInBackgroundTask(Context context, SaveImageInBackgroundData data,

                NotificationManager nManager) {
@@ -167,6 +182,11 @@ public class GlobalScreenshot { 
            String imageDate = new SimpleDateFormat("yyyyMMdd-HHmmss").format(new Date(mImageTime));

            mImageFileName = String.format(SCREENSHOT_FILE_NAME_TEMPLATE, imageDate);



            // Initialize screenshot notification smart actions provider.

            mHandler = new Handler();

            mSmartActionsProvider =

                SystemUIFactory.getInstance().createScreenshotNotificationSmartActionsProvider();



            // Create the large notification icon

            mImageWidth = data.image.getWidth();

            mImageHeight = data.image.getHeight();
@@ -242,6 +262,23 @@ public class GlobalScreenshot { 
            mNotificationStyle.bigLargeIcon((Bitmap) null);

        }



        private int getUserHandleOfForegroundApplication(Context context) {

            // This logic matches

            // com.android.systemui.statusbar.phone.PhoneStatusBarPolicy#updateManagedProfile

            try {

                return ActivityTaskManager.getService().getLastResumedActivityUserId();

            } catch (RemoteException e) {

                Slog.w(TAG, "getUserHandleOfForegroundApplication: ", e);

                return context.getUserId();

            }

        }



        private boolean isManagedProfile(Context context) {

            UserManager manager = UserManager.get(context);

            UserInfo info = manager.getUserInfo(getUserHandleOfForegroundApplication(context));

            return info.isManagedProfile();

        }



        /**

         * Generates a new hardware bitmap with specified values, copying the content from the

         * passed in bitmap.
@@ -268,6 +305,12 @@ public class GlobalScreenshot { 


            Context context = mParams.context;

            Bitmap image = mParams.image;

            boolean smartActionsEnabled = DeviceConfig.getBoolean(DeviceConfig.NAMESPACE_SYSTEMUI,

                    SystemUiDeviceConfigFlags.ENABLE_SCREENSHOT_NOTIFICATION_SMART_ACTIONS, true);

            CompletableFuture<List<Notification.Action>> smartActionsFuture = getSmartActionsFuture(

                    context, image, mSmartActionsProvider, mHandler, smartActionsEnabled,

                    isManagedProfile(context));



            Resources r = context.getResources();



            try {
@@ -378,6 +421,18 @@ public class GlobalScreenshot { 
                mParams.imageUri = uri;

                mParams.image = null;

                mParams.errorMsgResId = 0;



                if (smartActionsEnabled) {

                    int timeoutMs = DeviceConfig.getInt(DeviceConfig.NAMESPACE_SYSTEMUI,

                            SystemUiDeviceConfigFlags

                                    .SCREENSHOT_NOTIFICATION_SMART_ACTIONS_TIMEOUT_MS,

                            1000);

                    List<Notification.Action> smartActions = getSmartActions(smartActionsFuture,

                            timeoutMs);

                    for (Notification.Action action : smartActions) {

                        mNotificationBuilder.addAction(action);

                    }

                }

            } catch (Exception e) {

                // IOException/UnsupportedOperationException may be thrown if external storage is

                // not mounted
@@ -1039,6 +1094,58 @@ public class GlobalScreenshot { 
        nManager.notify(SystemMessage.NOTE_GLOBAL_SCREENSHOT, n);

    }



    @VisibleForTesting

    static CompletableFuture<List<Notification.Action>> getSmartActionsFuture(Context context,

            Bitmap image, ScreenshotNotificationSmartActionsProvider smartActionsProvider,

            Handler handler, boolean smartActionsEnabled, boolean isManagedProfile) {

        if (!smartActionsEnabled) {

            Slog.i(TAG, "Screenshot Intelligence not enabled, returning empty list.");

            return CompletableFuture.completedFuture(Collections.emptyList());

        }

        if (image.getConfig() != Bitmap.Config.HARDWARE) {

            Slog.w(TAG, String.format(

                    "Bitmap expected: Hardware, Bitmap found: %s. Returning empty list.",

                    image.getConfig()));

            return CompletableFuture.completedFuture(Collections.emptyList());

        }



        Slog.d(TAG, "Screenshot from a managed profile: " + isManagedProfile);

        CompletableFuture<List<Notification.Action>> smartActionsFuture;

        try {

            ActivityManager.RunningTaskInfo runningTask =

                    ActivityManagerWrapper.getInstance().getRunningTask();

            ComponentName componentName =

                    (runningTask != null && runningTask.topActivity != null)

                            ? runningTask.topActivity

                            : new ComponentName("", "");

            smartActionsFuture = smartActionsProvider.getActions(image, context,

                    THREAD_POOL_EXECUTOR,

                    handler,

                    componentName,

                    isManagedProfile);

        } catch (Throwable e) {

            smartActionsFuture = CompletableFuture.completedFuture(Collections.emptyList());

            Slog.e(TAG, "Failed to get future for screenshot notification smart actions.", e);

        }

        return smartActionsFuture;

    }



    @VisibleForTesting

    static List<Notification.Action> getSmartActions(

            CompletableFuture<List<Notification.Action>> smartActionsFuture, int timeoutMs) {

        try {

            long startTimeMs = SystemClock.uptimeMillis();

            List<Notification.Action> actions = smartActionsFuture.get(timeoutMs,

                    TimeUnit.MILLISECONDS);

            Slog.d(TAG, String.format("Wait time for smart actions: %d ms",

                    SystemClock.uptimeMillis() - startTimeMs));

            return actions;

        } catch (Throwable e) {

            Slog.e(TAG, "Failed to obtain screenshot notification smart actions.", e);

            return Collections.emptyList();

        }

    }



    /**

     * Receiver to proxy the share or edit intent, used to clean up the notification and send

     * appropriate signals to the system (ie. to dismiss the keyguard if necessary).

packages/SystemUI/src/com/android/systemui/screenshot/ScreenshotNotificationSmartActionsProvider.java

0 → 100644
+58 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package com.android.systemui.screenshot;



import android.app.Notification;

import android.content.ComponentName;

import android.content.Context;

import android.graphics.Bitmap;

import android.os.Handler;

import android.util.Log;



import java.util.Collections;

import java.util.List;

import java.util.concurrent.CompletableFuture;

import java.util.concurrent.Executor;



/**

 * This class can be overridden by a vendor-specific sys UI implementation,

 * in order to provide smart actions in the screenshot notification.

 */

public class ScreenshotNotificationSmartActionsProvider {

    private static final String TAG = "ScreenshotActions";



    /**

     * Default implementation that returns an empty list.

     * This method is overridden in vendor-specific Sys UI implementation.

     *

     * @param bitmap           The bitmap of the screenshot. The bitmap config must be {@link

     *                         HARDWARE}.

     * @param context          The current app {@link Context}.

     * @param executor         A {@link Executor} that can be used to execute tasks in parallel.

     * @param handler          A {@link Handler} to possibly run UI-thread code.

     * @param componentName    Contains package and activity class names where the screenshot was

     *                         taken. This is used as an additional signal to generate and rank more

     *                         relevant actions.

     * @param isManagedProfile The screenshot was taken for a work profile app.

     */

    public CompletableFuture<List<Notification.Action>> getActions(Bitmap bitmap, Context context,

            Executor executor, Handler handler, ComponentName componentName,

            boolean isManagedProfile) {

        Log.d(TAG, "Returning empty smart action list.");

        return CompletableFuture.completedFuture(Collections.emptyList());

    }

}