Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 898602ef authored by Winson Chung's avatar Winson Chung Committed by Android (Google) Code Review
Browse files

Merge changes from topic "back_gesture_model_qpr" into rvc-qpr-dev 

* changes:
  Add flag to get the name of the Model file from the flag in case we want to try out different models. In addition, we should not log touches in the middle of the screen for logging so moving the block at the very beginning.
  Add package_name to log for debugging the back gesture model.
  Add log entries for the results for the ML model for the back gesture.
  Use ML model for the Back Gesture in EdgeBackGestureHandler.
parents 9649d433 0ed68db0

cmds/statsd/src/atoms.proto

+3 −0
Original line number Diff line number Diff line
@@ -3038,6 +3038,9 @@ message BackGesture { 
    optional int32 end_y = 7;  // Y coordinate for ACTION_MOVE event.

    optional int32 left_boundary = 8;  // left edge width + left inset

    optional int32 right_boundary = 9;  // screen width - (right edge width + right inset)

    // The score between 0 and 1 which is the prediction output for the Back Gesture model.

    optional float ml_model_score = 10;

    optional string package_name = 11;  // The name of the top 100 most used package by all users.



    enum WindowHorizontalLocation {

        DEFAULT_LOCATION = 0;

core/java/com/android/internal/config/sysui/SystemUiDeviceConfigFlags.java

+16 −0
Original line number Diff line number Diff line
@@ -428,6 +428,22 @@ public final class SystemUiDeviceConfigFlags { 
     */

    public static final String SCREENSHOT_KEYCHORD_DELAY = "screenshot_keychord_delay";



    /**

     * (boolean) Whether to use an ML model for the Back Gesture.

     */

    public static final String USE_BACK_GESTURE_ML_MODEL = "use_back_gesture_ml_model";



    /**

     * (string) The name of the ML model for Back Gesture.

     */

    public static final String BACK_GESTURE_ML_MODEL_NAME = "back_gesture_ml_model_name";



    /**

     * (float) Threshold for Back Gesture ML model prediction.

     */

    public static final String BACK_GESTURE_ML_MODEL_THRESHOLD = "back_gesture_ml_model_threshold";





    private SystemUiDeviceConfigFlags() {

    }

}

packages/SystemUI/src/com/android/systemui/SystemUIFactory.java

+11 −0
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ package com.android.systemui; 


import android.annotation.NonNull;

import android.content.Context;

import android.content.res.AssetManager;

import android.content.res.Resources;

import android.os.Handler;

import android.os.Looper;
@@ -39,6 +40,7 @@ import com.android.systemui.screenshot.ScreenshotNotificationSmartActionsProvide 
import com.android.systemui.statusbar.NotificationListener;

import com.android.systemui.statusbar.NotificationMediaManager;

import com.android.systemui.statusbar.notification.NotificationWakeUpCoordinator;

import com.android.systemui.statusbar.phone.BackGestureTfClassifierProvider;

import com.android.systemui.statusbar.phone.DozeParameters;

import com.android.systemui.statusbar.phone.KeyguardBouncer;

import com.android.systemui.statusbar.phone.KeyguardBypassController;
@@ -182,4 +184,13 @@ public class SystemUIFactory { 
            return mContext;

        }

    }



    /**

     * Creates an instance of BackGestureTfClassifierProvider.

     * This method is overridden in vendor specific implementation of Sys UI.

     */

    public BackGestureTfClassifierProvider createBackGestureTfClassifierProvider(

            AssetManager am, String modelName) {

        return new BackGestureTfClassifierProvider();

    }

}

packages/SystemUI/src/com/android/systemui/statusbar/phone/BackGestureTfClassifierProvider.java

0 → 100644
+66 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2020 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package com.android.systemui.statusbar.phone;



import android.content.res.AssetManager;



import java.util.HashMap;

import java.util.Map;



/**

 * This class can be overridden by a vendor-specific sys UI implementation,

 * in order to provide classification models for the Back Gesture.

 */

public class BackGestureTfClassifierProvider {

    private static final String TAG = "BackGestureTfClassifierProvider";



    /**

     * Default implementation that returns an empty map.

     * This method is overridden in vendor-specific Sys UI implementation.

     *

     * @param am       An AssetManager to get the vocab file.

    */

    public Map<String, Integer> loadVocab(AssetManager am) {

        return new HashMap<String, Integer>();

    }



    /**

     * This method is overridden in vendor-specific Sys UI implementation.

     *

     * @param featuresVector   List of input features.

     *

    */

    public float predict(Object[] featuresVector) {

        return -1;

    }



    /**

     * Interpreter owns resources. This method releases the resources after

     * use to avoid memory leak.

     * This method is overridden in vendor-specific Sys UI implementation.

     *

     */

    public void release() {}



    /**

     * Returns whether to use the ML model for Back Gesture.

     * This method is overridden in vendor-specific Sys UI implementation.

     *

     */

    public boolean isActive() {

        return false;

    }

}

packages/SystemUI/src/com/android/systemui/statusbar/phone/EdgeBackGestureHandler.java

+124 −8
Original line number Diff line number Diff line
@@ -56,6 +56,7 @@ import com.android.internal.config.sysui.SystemUiDeviceConfigFlags; 
import com.android.internal.policy.GestureNavigationSettingsObserver;

import com.android.systemui.Dependency;

import com.android.systemui.R;

import com.android.systemui.SystemUIFactory;

import com.android.systemui.broadcast.BroadcastDispatcher;

import com.android.systemui.bubbles.BubbleController;

import com.android.systemui.model.SysUiState;
@@ -76,6 +77,7 @@ import com.android.systemui.tracing.nano.SystemUiTraceProto; 
import java.io.PrintWriter;

import java.util.ArrayList;

import java.util.List;

import java.util.Map;

import java.util.concurrent.Executor;



/**
@@ -117,8 +119,33 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa 
        public void onTaskStackChanged() {

            mGestureBlockingActivityRunning = isGestureBlockingActivityRunning();

        }

        @Override

        public void onTaskCreated(int taskId, ComponentName componentName) {

            if (componentName != null) {

                mPackageName = componentName.getPackageName();

            } else {

                mPackageName = "_UNKNOWN";

            }

        }

    };



    private DeviceConfig.OnPropertiesChangedListener mOnPropertiesChangedListener =

            new DeviceConfig.OnPropertiesChangedListener() {

                @Override

                public void onPropertiesChanged(DeviceConfig.Properties properties) {

                    if (DeviceConfig.NAMESPACE_SYSTEMUI.equals(properties.getNamespace())

                            && (properties.getKeyset().contains(

                                    SystemUiDeviceConfigFlags.BACK_GESTURE_ML_MODEL_THRESHOLD)

                            || properties.getKeyset().contains(

                                    SystemUiDeviceConfigFlags.USE_BACK_GESTURE_ML_MODEL)

                            || properties.getKeyset().contains(

                                    SystemUiDeviceConfigFlags.BACK_GESTURE_ML_MODEL_NAME))) {

                        updateMLModelState();

                    }

                }

            };





    private final Context mContext;

    private final OverviewProxyService mOverviewProxyService;

    private final Runnable mStateChangeCallback;
@@ -173,6 +200,14 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa 
    private int mRightInset;

    private int mSysUiFlags;



    // For Tf-Lite model.

    private BackGestureTfClassifierProvider mBackGestureTfClassifierProvider;

    private Map<String, Integer> mVocab;

    private boolean mUseMLModel;

    private float mMLModelThreshold;

    private String mPackageName;

    private float mMLResults;



    private final GestureNavigationSettingsObserver mGestureNavigationSettingsObserver;



    private final NavigationEdgeBackPlugin.BackCallback mBackCallback =
@@ -230,7 +265,6 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa 
                Log.e(TAG, "Failed to add gesture blocking activities", e);

            }

        }



        mLongPressTimeout = Math.min(MAX_LONG_PRESS_TIMEOUT,

                ViewConfiguration.getLongPressTimeout());
@@ -344,6 +378,7 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa 
            mContext.getSystemService(DisplayManager.class).unregisterDisplayListener(this);

            mPluginManager.removePluginListener(this);

            ActivityManagerWrapper.getInstance().unregisterTaskStackListener(mTaskStackListener);

            DeviceConfig.removeOnPropertiesChangedListener(mOnPropertiesChangedListener);



            try {

                WindowManagerGlobal.getWindowManagerService()
@@ -359,6 +394,9 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa 
            mContext.getSystemService(DisplayManager.class).registerDisplayListener(this,

                    mContext.getMainThreadHandler());

            ActivityManagerWrapper.getInstance().registerTaskStackListener(mTaskStackListener);

            DeviceConfig.addOnPropertiesChangedListener(DeviceConfig.NAMESPACE_SYSTEMUI,

                    runnable -> (mContext.getMainThreadHandler()).post(runnable),

                    mOnPropertiesChangedListener);



            try {

                WindowManagerGlobal.getWindowManagerService()
@@ -379,6 +417,8 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa 
            mPluginManager.addPluginListener(

                    this, NavigationEdgeBackPlugin.class, /*allowMultiple=*/ false);

        }

        // Update the ML model resources.

        updateMLModelState();

    }



    @Override
@@ -431,12 +471,72 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa 
        }

    }



    private void updateMLModelState() {

        boolean newState = mIsEnabled && DeviceConfig.getBoolean(DeviceConfig.NAMESPACE_SYSTEMUI,

                SystemUiDeviceConfigFlags.USE_BACK_GESTURE_ML_MODEL, false);



        if (newState == mUseMLModel) {

            return;

        }



        if (newState) {

            String mlModelName = DeviceConfig.getString(DeviceConfig.NAMESPACE_SYSTEMUI,

                    SystemUiDeviceConfigFlags.BACK_GESTURE_ML_MODEL_NAME, "backgesture");

            mBackGestureTfClassifierProvider = SystemUIFactory.getInstance()

                    .createBackGestureTfClassifierProvider(mContext.getAssets(), mlModelName);

            mMLModelThreshold = DeviceConfig.getFloat(DeviceConfig.NAMESPACE_SYSTEMUI,

                    SystemUiDeviceConfigFlags.BACK_GESTURE_ML_MODEL_THRESHOLD, 0.9f);

            if (mBackGestureTfClassifierProvider.isActive()) {

                mVocab = mBackGestureTfClassifierProvider.loadVocab(mContext.getAssets());

                mUseMLModel = true;

                return;

            }

        }



        mUseMLModel = false;

        if (mBackGestureTfClassifierProvider != null) {

            mBackGestureTfClassifierProvider.release();

            mBackGestureTfClassifierProvider = null;

        }

    }



    private float getBackGesturePredictionsCategory(int x, int y) {

        if (!mVocab.containsKey(mPackageName)) {

            return -1;

        }



        int distanceFromEdge;

        int location;

        if (x <= mDisplaySize.x / 2.0) {

            location = 1;  // left

            distanceFromEdge = x;

        } else {

            location = 2;  // right

            distanceFromEdge = mDisplaySize.x - x;

        }



        Object[] featuresVector = {

            new long[]{(long) mDisplaySize.x},

            new long[]{(long) distanceFromEdge},

            new long[]{(long) location},

            new long[]{(long) mVocab.get(mPackageName)},

            new long[]{(long) y},

        };



        mMLResults = mBackGestureTfClassifierProvider.predict(featuresVector);

        if (mMLResults == -1) return -1;



        return mMLResults >= mMLModelThreshold ? 1 : 0;

    }



    private boolean isWithinTouchRegion(int x, int y) {

        boolean withinRange = false;

        float results = -1;



        // Disallow if we are in the bottom gesture area

        if (y >= (mDisplaySize.y - mBottomGestureHeight)) {

            return false;

        }



        // If the point is way too far (twice the margin), it is

        // not interesting to us for logging purposes, nor we

        // should process it.  Simply return false and keep
@@ -446,11 +546,15 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa 
            return false;

        }



        if (mUseMLModel &&  (results = getBackGesturePredictionsCategory(x, y)) != -1) {

            withinRange = results == 1 ? true : false;

        } else {

            // Denotes whether we should proceed with the gesture.

            // Even if it is false, we may want to log it assuming

            // it is not invalid due to exclusion.

        boolean withinRange = x <= mEdgeWidthLeft + mLeftInset

            withinRange = x <= mEdgeWidthLeft + mLeftInset

                    || x >= (mDisplaySize.x - mEdgeWidthRight - mRightInset);

        }



        // Always allow if the user is in a transient sticky immersive state

        if (mIsNavBarShownTransiently) {
@@ -493,6 +597,11 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa 
            return;

        }

        mLogGesture = false;

        String logPackageName = "";

        // Due to privacy, only top 100 most used apps by all users can be logged.

        if (mUseMLModel && mVocab.containsKey(mPackageName) && mVocab.get(mPackageName) < 100) {

            logPackageName = mPackageName;

        }

        SysUiStatsLog.write(SysUiStatsLog.BACK_GESTURE_REPORTED_REPORTED, backType,

                (int) mDownPoint.y, mIsOnLeftEdge

                        ? SysUiStatsLog.BACK_GESTURE__X_LOCATION__LEFT
@@ -500,7 +609,8 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa 
                (int) mDownPoint.x, (int) mDownPoint.y,

                (int) mEndPoint.x, (int) mEndPoint.y,

                mEdgeWidthLeft + mLeftInset,

                mDisplaySize.x - (mEdgeWidthRight + mRightInset));

                mDisplaySize.x - (mEdgeWidthRight + mRightInset),

                mUseMLModel ? mMLResults : -2, logPackageName);

    }



    private void onMotionEvent(MotionEvent ev) {
@@ -509,6 +619,7 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa 
            // Verify if this is in within the touch region and we aren't in immersive mode, and

            // either the bouncer is showing or the notification panel is hidden

            mIsOnLeftEdge = ev.getX() <= mEdgeWidthLeft + mLeftInset;

            mMLResults = 0;

            mLogGesture = false;

            mInRejectedExclusion = false;

            mAllowGesture = !mDisabledForQuickstep && mIsBackGestureAllowed
@@ -648,6 +759,11 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa 
        ActivityManager.RunningTaskInfo runningTask =

                ActivityManagerWrapper.getInstance().getRunningTask();

        ComponentName topActivity = runningTask == null ? null : runningTask.topActivity;

        if (topActivity != null) {

            mPackageName = topActivity.getPackageName();

        } else {

            mPackageName = "_UNKNOWN";

        }

        return topActivity != null && mGestureBlockingActivities.contains(topActivity);

    }