Use ML model for the Back Gesture in EdgeBackGestureHandler. am: c7035fe4fe (1db145a6) · Commits · e / os / android_frameworks_base

core/java/com/android/internal/config/sysui/SystemUiDeviceConfigFlags.java

+11 −0

Original line number	Diff line number	Diff line
		@@ -428,6 +428,17 @@ public final class SystemUiDeviceConfigFlags {
		*/
		public static final String SCREENSHOT_KEYCHORD_DELAY = "screenshot_keychord_delay";

		/**
		* (boolean) Whether to use an ML model for the Back Gesture.
		*/
		public static final String USE_BACK_GESTURE_ML_MODEL = "use_back_gesture_ml_model";

		/**
		* (float) Threshold for Back Gesture ML model prediction.
		*/
		public static final String BACK_GESTURE_ML_MODEL_THRESHOLD = "back_gesture_ml_model_threshold";


		private SystemUiDeviceConfigFlags() {
		}
		}

packages/SystemUI/src/com/android/systemui/SystemUIFactory.java

+11 −0

Original line number	Diff line number	Diff line
		@@ -18,6 +18,7 @@ package com.android.systemui;

		import android.annotation.NonNull;
		import android.content.Context;
		import android.content.res.AssetManager;
		import android.content.res.Resources;
		import android.os.Handler;
		import android.os.Looper;
		@@ -39,6 +40,7 @@ import com.android.systemui.screenshot.ScreenshotNotificationSmartActionsProvide
		import com.android.systemui.statusbar.NotificationListener;
		import com.android.systemui.statusbar.NotificationMediaManager;
		import com.android.systemui.statusbar.notification.NotificationWakeUpCoordinator;
		import com.android.systemui.statusbar.phone.BackGestureTfClassifierProvider;
		import com.android.systemui.statusbar.phone.DozeParameters;
		import com.android.systemui.statusbar.phone.KeyguardBouncer;
		import com.android.systemui.statusbar.phone.KeyguardBypassController;
		@@ -182,4 +184,13 @@ public class SystemUIFactory {
		return mContext;
		}
		}

		/**
		* Creates an instance of BackGestureTfClassifierProvider.
		* This method is overridden in vendor specific implementation of Sys UI.
		*/
		public BackGestureTfClassifierProvider createBackGestureTfClassifierProvider(
		AssetManager am) {
		return new BackGestureTfClassifierProvider();
		}
		}

packages/SystemUI/src/com/android/systemui/statusbar/phone/BackGestureTfClassifierProvider.java

0 → 100644

+66 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/
		package com.android.systemui.statusbar.phone;

		import android.content.res.AssetManager;

		import java.util.HashMap;
		import java.util.Map;

		/**
		* This class can be overridden by a vendor-specific sys UI implementation,
		* in order to provide classification models for the Back Gesture.
		*/
		public class BackGestureTfClassifierProvider {
		private static final String TAG = "BackGestureTfClassifierProvider";

		/**
		* Default implementation that returns an empty map.
		* This method is overridden in vendor-specific Sys UI implementation.
		*
		* @param am An AssetManager to get the vocab file.
		*/
		public Map<String, Integer> loadVocab(AssetManager am) {
		return new HashMap<String, Integer>();
		}

		/**
		* This method is overridden in vendor-specific Sys UI implementation.
		*
		* @param featuresVector List of input features.
		*
		*/
		public float predict(Object[] featuresVector) {
		return -1;
		}

		/**
		* Interpreter owns resources. This method releases the resources after
		* use to avoid memory leak.
		* This method is overridden in vendor-specific Sys UI implementation.
		*
		*/
		public void release() {}

		/**
		* Returns whether to use the ML model for Back Gesture.
		* This method is overridden in vendor-specific Sys UI implementation.
		*
		*/
		public boolean isActive() {
		return false;
		}
		}

packages/SystemUI/src/com/android/systemui/statusbar/phone/EdgeBackGestureHandler.java

+120 −17

Original line number	Diff line number	Diff line
		@@ -56,6 +56,7 @@ import com.android.internal.config.sysui.SystemUiDeviceConfigFlags;
		import com.android.internal.policy.GestureNavigationSettingsObserver;
		import com.android.systemui.Dependency;
		import com.android.systemui.R;
		import com.android.systemui.SystemUIFactory;
		import com.android.systemui.broadcast.BroadcastDispatcher;
		import com.android.systemui.bubbles.BubbleController;
		import com.android.systemui.model.SysUiState;
		@@ -76,6 +77,7 @@ import com.android.systemui.tracing.nano.SystemUiTraceProto;
		import java.io.PrintWriter;
		import java.util.ArrayList;
		import java.util.List;
		import java.util.Map;
		import java.util.concurrent.Executor;

		/**
		@@ -117,8 +119,31 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
		public void onTaskStackChanged() {
		mGestureBlockingActivityRunning = isGestureBlockingActivityRunning();
		}
		@Override
		public void onTaskCreated(int taskId, ComponentName componentName) {
		if (componentName != null) {
		mPackageName = componentName.getPackageName();
		} else {
		mPackageName = "_UNKNOWN";
		}
		}
		};

		private DeviceConfig.OnPropertiesChangedListener mOnPropertiesChangedListener =
		new DeviceConfig.OnPropertiesChangedListener() {
		@Override
		public void onPropertiesChanged(DeviceConfig.Properties properties) {
		if (DeviceConfig.NAMESPACE_SYSTEMUI.equals(properties.getNamespace())
		&& (properties.getKeyset().contains(
		SystemUiDeviceConfigFlags.BACK_GESTURE_ML_MODEL_THRESHOLD)
		\|\| properties.getKeyset().contains(
		SystemUiDeviceConfigFlags.USE_BACK_GESTURE_ML_MODEL))) {
		updateMLModelState();
		}
		}
		};


		private final Context mContext;
		private final OverviewProxyService mOverviewProxyService;
		private final Runnable mStateChangeCallback;
		@@ -173,6 +198,13 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
		private int mRightInset;
		private int mSysUiFlags;

		// For Tf-Lite model.
		private BackGestureTfClassifierProvider mBackGestureTfClassifierProvider;
		private Map<String, Integer> mVocab;
		private boolean mUseMLModel;
		private float mMLModelThreshold;
		private String mPackageName;

		private final GestureNavigationSettingsObserver mGestureNavigationSettingsObserver;

		private final NavigationEdgeBackPlugin.BackCallback mBackCallback =
		@@ -230,7 +262,6 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
		Log.e(TAG, "Failed to add gesture blocking activities", e);
		}
		}

		mLongPressTimeout = Math.min(MAX_LONG_PRESS_TIMEOUT,
		ViewConfiguration.getLongPressTimeout());

		@@ -344,6 +375,7 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
		mContext.getSystemService(DisplayManager.class).unregisterDisplayListener(this);
		mPluginManager.removePluginListener(this);
		ActivityManagerWrapper.getInstance().unregisterTaskStackListener(mTaskStackListener);
		DeviceConfig.removeOnPropertiesChangedListener(mOnPropertiesChangedListener);

		try {
		WindowManagerGlobal.getWindowManagerService()
		@@ -359,6 +391,9 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
		mContext.getSystemService(DisplayManager.class).registerDisplayListener(this,
		mContext.getMainThreadHandler());
		ActivityManagerWrapper.getInstance().registerTaskStackListener(mTaskStackListener);
		DeviceConfig.addOnPropertiesChangedListener(DeviceConfig.NAMESPACE_SYSTEMUI,
		runnable -> (mContext.getMainThreadHandler()).post(runnable),
		mOnPropertiesChangedListener);

		try {
		WindowManagerGlobal.getWindowManagerService()
		@@ -379,6 +414,8 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
		mPluginManager.addPluginListener(
		this, NavigationEdgeBackPlugin.class, /allowMultiple=/ false);
		}
		// Update the ML model resources.
		updateMLModelState();
		}

		@Override
		@@ -431,12 +468,73 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
		}
		}

		private void updateMLModelState() {
		boolean newState = mIsEnabled && DeviceConfig.getBoolean(DeviceConfig.NAMESPACE_SYSTEMUI,
		SystemUiDeviceConfigFlags.USE_BACK_GESTURE_ML_MODEL, false);

		if (newState == mUseMLModel) {
		return;
		}

		if (newState) {
		mBackGestureTfClassifierProvider = SystemUIFactory.getInstance()
		.createBackGestureTfClassifierProvider(mContext.getAssets());
		mMLModelThreshold = DeviceConfig.getFloat(DeviceConfig.NAMESPACE_SYSTEMUI,
		SystemUiDeviceConfigFlags.BACK_GESTURE_ML_MODEL_THRESHOLD, 0.9f);
		if (mBackGestureTfClassifierProvider.isActive()) {
		mVocab = mBackGestureTfClassifierProvider.loadVocab(mContext.getAssets());
		mUseMLModel = true;
		return;
		}
		}

		mUseMLModel = false;
		if (mBackGestureTfClassifierProvider != null) {
		mBackGestureTfClassifierProvider.release();
		mBackGestureTfClassifierProvider = null;
		}
		}

		private float getBackGesturePredictionsCategory(int x, int y) {
		if (!mVocab.containsKey(mPackageName)) {
		return -1;
		}

		int distanceFromEdge;
		int location;
		if (x <= mDisplaySize.x / 2.0) {
		location = 1; // left
		distanceFromEdge = x;
		} else {
		location = 2; // right
		distanceFromEdge = mDisplaySize.x - x;
		}

		Object[] featuresVector = {
		new long[]{(long) mDisplaySize.x},
		new long[]{(long) distanceFromEdge},
		new long[]{(long) location},
		new long[]{(long) mVocab.get(mPackageName)},
		new long[]{(long) y},
		};

		final float results = mBackGestureTfClassifierProvider.predict(featuresVector);
		if (results == -1) return -1;

		return results >= mMLModelThreshold ? 1 : 0;
		}

		private boolean isWithinTouchRegion(int x, int y) {
		boolean withinRange = false;
		float results = -1;

		if (mUseMLModel && (results = getBackGesturePredictionsCategory(x, y)) != -1) {
		withinRange = results == 1 ? true : false;
		} else {
		// Disallow if we are in the bottom gesture area
		if (y >= (mDisplaySize.y - mBottomGestureHeight)) {
		return false;
		}

		// If the point is way too far (twice the margin), it is
		// not interesting to us for logging purposes, nor we
		// should process it. Simply return false and keep
		@@ -445,12 +543,12 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
		&& x < (mDisplaySize.x - 2 * (mEdgeWidthRight + mRightInset))) {
		return false;
		}

		// Denotes whether we should proceed with the gesture.
		// Even if it is false, we may want to log it assuming
		// it is not invalid due to exclusion.
		boolean withinRange = x <= mEdgeWidthLeft + mLeftInset
		withinRange = x <= mEdgeWidthLeft + mLeftInset
		\|\| x >= (mDisplaySize.x - mEdgeWidthRight - mRightInset);
		}

		// Always allow if the user is in a transient sticky immersive state
		if (mIsNavBarShownTransiently) {
		@@ -648,6 +746,11 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
		ActivityManager.RunningTaskInfo runningTask =
		ActivityManagerWrapper.getInstance().getRunningTask();
		ComponentName topActivity = runningTask == null ? null : runningTask.topActivity;
		if (topActivity != null) {
		mPackageName = topActivity.getPackageName();
		} else {
		mPackageName = "_UNKNOWN";
		}
		return topActivity != null && mGestureBlockingActivities.contains(topActivity);
		}