Merge "Expose more SystemAPI methods on AccessibilityDisplayProxy" (77eb98c8) · Commits · e / os / android_frameworks_base

core/api/system-current.txt

+6 −0

Original line number	Diff line number	Diff line
		@@ -16120,6 +16120,12 @@ package android.view.accessibility {
		public abstract class AccessibilityDisplayProxy {
		ctor public AccessibilityDisplayProxy(int, @NonNull java.util.concurrent.Executor, @NonNull java.util.List<android.accessibilityservice.AccessibilityServiceInfo>);
		method public int getDisplayId();
		method @NonNull public final java.util.List<android.accessibilityservice.AccessibilityServiceInfo> getInstalledAndEnabledServices();
		method @NonNull public java.util.List<android.view.accessibility.AccessibilityWindowInfo> getWindows();
		method public void interrupt();
		method public void onAccessibilityEvent(@NonNull android.view.accessibility.AccessibilityEvent);
		method public void onProxyConnected();
		method public void setInstalledAndEnabledServices(@NonNull java.util.List<android.accessibilityservice.AccessibilityServiceInfo>);
		}

		public final class AccessibilityManager {

core/java/android/view/accessibility/AccessibilityDisplayProxy.java

+156 −7

Original line number	Diff line number	Diff line
		@@ -20,13 +20,18 @@ import android.accessibilityservice.AccessibilityGestureEvent;
		import android.accessibilityservice.AccessibilityService;
		import android.accessibilityservice.AccessibilityServiceInfo;
		import android.accessibilityservice.IAccessibilityServiceClient;
		import android.accessibilityservice.IAccessibilityServiceConnection;
		import android.accessibilityservice.MagnificationConfig;
		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.annotation.SystemApi;
		import android.content.ComponentName;
		import android.content.Context;
		import android.content.pm.ResolveInfo;
		import android.graphics.Region;
		import android.os.IBinder;
		import android.os.RemoteException;
		import android.util.Log;
		import android.view.KeyEvent;
		import android.view.MotionEvent;
		import android.view.inputmethod.EditorInfo;
		@@ -34,14 +39,15 @@ import android.view.inputmethod.EditorInfo;
		import com.android.internal.inputmethod.IAccessibilityInputMethodSessionCallback;
		import com.android.internal.inputmethod.RemoteAccessibilityInputConnection;

		import java.util.Collections;
		import java.util.List;
		import java.util.concurrent.Executor;

		/**
		* Allows a privileged app - an app with MANAGE_ACCESSIBILITY permission and SystemAPI access - to
		* interact with the windows in the display that this proxy represents. Proxying the default display
		* or a display that is not tracked will throw an exception. Only the real user has access to global
		* clients like SystemUI.
		* or a display that is not tracked by accessibility, such as private displays, will throw an
		* exception. Only the real user has access to global clients like SystemUI.
		*
		* <p>
		* To register and unregister a proxy, use
		@@ -49,7 +55,16 @@ import java.util.concurrent.Executor;
		* and {@link AccessibilityManager#unregisterDisplayProxy(AccessibilityDisplayProxy)}. If the app
		* that has registered the proxy dies, the system will remove the proxy.
		*
		* TODO(241429275): Complete proxy impl and add additional support (if necessary) like cache methods
		* <p>
		* Avoid using the app's main thread. Proxy methods such as {@link #getWindows} and node methods
		* like {@link AccessibilityNodeInfo#getChild(int)} will happen frequently. Node methods may also
		* wait on the displayed app's UI thread to obtain accurate screen data.
		*
		* <p>
		* To get a list of {@link AccessibilityServiceInfo}s that have populated {@link ComponentName}s and
		* {@link ResolveInfo}s, retrieve the list using {@link #getInstalledAndEnabledServices()} after
		* {@link #onProxyConnected()} has been called.
		*
		* @hide
		*/
		@SystemApi
		@@ -91,7 +106,134 @@ public abstract class AccessibilityDisplayProxy {
		}

		/**
		* An IAccessibilityServiceClient that handles interrupts and accessibility events.
		* Handles {@link android.view.accessibility.AccessibilityEvent}s.
		* <p>
		* AccessibilityEvents represent changes to the UI, or what parts of the node tree have changed.
		* AccessibilityDisplayProxy should use these to query new UI and send appropriate feedback
		* to their users.
		* <p>
		* For example, a {@link AccessibilityEvent#TYPE_WINDOWS_CHANGED} indicates a change in windows,
		* so a proxy may query {@link #getWindows} to obtain updated UI and potentially inform of a new
		* window title. Or a proxy may emit an earcon on a
		* {@link AccessibilityEvent#TYPE_VIEW_SCROLLED} event.
		*/
		public void onAccessibilityEvent(@NonNull AccessibilityEvent event) {
		// Default no-op
		}

		/**
		* Handles a successful system connection after
		* {@link AccessibilityManager#registerDisplayProxy(AccessibilityDisplayProxy)} is called.
		*
		* <p>
		* At this point, querying for UI is available and {@link AccessibilityEvent}s will begin being
		* sent. An AccessibilityDisplayProxy may instantiate core infrastructure components here.
		*/
		public void onProxyConnected() {
		// Default no-op
		}

		/**
		* Handles a request to interrupt the accessibility feedback.
		* <p>
		* AccessibilityDisplayProxy should interrupt the accessibility activity occurring on its
		* display. For example, a screen reader may interrupt speech.
		*
		* @see AccessibilityManager#interrupt()
		* @see AccessibilityService#onInterrupt()
		*/
		public void interrupt() {
		// Default no-op
		}

		/**
		* Gets the focus of the window specified by {@code windowInfo}.
		*
		* @param windowInfo the window to search
		* @param focus The focus to find. One of {@link AccessibilityNodeInfo#FOCUS_INPUT} or
		* {@link AccessibilityNodeInfo#FOCUS_ACCESSIBILITY}.
		* @return The node info of the focused view or null.
		* @hide
		* TODO(254545943): Do not expose until support for accessibility focus and/or input is in place
		*/
		@Nullable
		public AccessibilityNodeInfo findFocus(@NonNull AccessibilityWindowInfo windowInfo, int focus) {
		AccessibilityNodeInfo windowRoot = windowInfo.getRoot();
		return windowRoot != null ? windowRoot.findFocus(focus) : null;
		}

		/**
		* Gets the windows of the tracked display.
		*
		* @see AccessibilityService#getWindows()
		*/
		@NonNull
		public List<AccessibilityWindowInfo> getWindows() {
		return AccessibilityInteractionClient.getInstance().getWindowsOnDisplay(mConnectionId,
		mDisplayId);
		}

		/**
		* Sets the list of {@link AccessibilityServiceInfo}s describing the services interested in the
		* {@link AccessibilityDisplayProxy}'s display.
		*
		* <p>These represent a11y features and services that are installed and running. These should
		* not include {@link AccessibilityService}s installed on the phone.
		*
		* @param installedAndEnabledServices the list of installed and running a11y services.
		*/
		public void setInstalledAndEnabledServices(
		@NonNull List<AccessibilityServiceInfo> installedAndEnabledServices) {
		mInstalledAndEnabledServices = installedAndEnabledServices;
		sendServiceInfos();
		}

		/**
		* Sets the {@link AccessibilityServiceInfo} for this service if the latter is
		* properly set and there is an {@link IAccessibilityServiceConnection} to the
		* AccessibilityManagerService.
		*/
		private void sendServiceInfos() {
		IAccessibilityServiceConnection connection =
		AccessibilityInteractionClient.getInstance().getConnection(mConnectionId);
		if (mInstalledAndEnabledServices != null && mInstalledAndEnabledServices.size() > 0
		&& connection != null) {
		try {
		connection.setInstalledAndEnabledServices(mInstalledAndEnabledServices);
		AccessibilityInteractionClient.getInstance().clearCache(mConnectionId);
		} catch (RemoteException re) {
		Log.w(LOG_TAG, "Error while setting AccessibilityServiceInfos", re);
		re.rethrowFromSystemServer();
		}
		}
		mInstalledAndEnabledServices = null;
		}

		/**
		* Gets the list of {@link AccessibilityServiceInfo}s describing the services interested in the
		* {@link AccessibilityDisplayProxy}'s display.
		*
		* @return The {@link AccessibilityServiceInfo}s of interested services.
		* @see AccessibilityServiceInfo
		*/
		@NonNull
		public final List<AccessibilityServiceInfo> getInstalledAndEnabledServices() {
		IAccessibilityServiceConnection connection =
		AccessibilityInteractionClient.getInstance().getConnection(mConnectionId);
		if (connection != null) {
		try {
		return connection.getInstalledAndEnabledServices();
		} catch (RemoteException re) {
		Log.w(LOG_TAG, "Error while getting AccessibilityServiceInfo", re);
		re.rethrowFromSystemServer();
		}
		}
		return Collections.emptyList();
		}

		/**
		* An IAccessibilityServiceClient that handles interrupts, accessibility events, and system
		* connection.
		*/
		private class IAccessibilityServiceClientImpl extends
		AccessibilityService.IAccessibilityServiceClientWrapper {
		@@ -100,17 +242,24 @@ public abstract class AccessibilityDisplayProxy {
		super(context, executor, new AccessibilityService.Callbacks() {
		@Override
		public void onAccessibilityEvent(AccessibilityEvent event) {
		// TODO: call AccessiiblityProxy.onAccessibilityEvent
		// TODO(254545943): Remove check when event processing is done more upstream in
		// AccessibilityManagerService.
		if (event.getDisplayId() == mDisplayId) {
		AccessibilityDisplayProxy.this.onAccessibilityEvent(event);
		}
		}

		@Override
		public void onInterrupt() {
		// TODO: call AccessiiblityProxy.onInterrupt
		AccessibilityDisplayProxy.this.interrupt();
		}

		@Override
		public void onServiceConnected() {
		// TODO: send service infos and call AccessiiblityProxy.onProxyConnected
		AccessibilityDisplayProxy.this.sendServiceInfos();
		AccessibilityDisplayProxy.this.onProxyConnected();
		}

		@Override
		public void init(int connectionId, IBinder windowToken) {
		mConnectionId = connectionId;

core/java/android/view/accessibility/AccessibilityInteractionClient.java

+11 −5

Original line number	Diff line number	Diff line
		@@ -458,14 +458,20 @@ public final class AccessibilityInteractionClient
		* @return The {@link AccessibilityWindowInfo} list.
		*/
		public List<AccessibilityWindowInfo> getWindows(int connectionId) {
		return getWindowsOnDisplay(connectionId, Display.DEFAULT_DISPLAY);
		}

		/**
		* Gets the info for all windows of the specified display.
		*
		* @param connectionId The id of a connection for interacting with the system.
		* @return The {@link AccessibilityWindowInfo} list belonging to {@code displayId}.
		*/
		public List<AccessibilityWindowInfo> getWindowsOnDisplay(int connectionId, int displayId) {
		final SparseArray<List<AccessibilityWindowInfo>> windows =
		getWindowsOnAllDisplays(connectionId);
		if (windows.size() > 0) {
		return windows.valueAt(Display.DEFAULT_DISPLAY);
		}
		return Collections.emptyList();
		return windows.get(displayId, Collections.emptyList());
		}

		/**
		* Gets the info for all windows of all displays.
		*

core/tests/coretests/src/android/view/accessibility/AccessibilityManagerTest.java

+15 −1

Original line number	Diff line number	Diff line
		@@ -263,10 +263,24 @@ public class AccessibilityManagerTest {
		}

		private class MyAccessibilityProxy extends AccessibilityDisplayProxy {
		// TODO(241429275): Will override A11yProxy methods in the future.
		MyAccessibilityProxy(int displayId,
		@NonNull List<AccessibilityServiceInfo> serviceInfos) {
		super(displayId, Executors.newSingleThreadExecutor(), serviceInfos);
		}

		@Override
		public void onAccessibilityEvent(@NonNull AccessibilityEvent event) {

		}

		@Override
		public void onProxyConnected() {

		}

		@Override
		public void interrupt() {

		}
		}
		}

services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java

+26 −13

Original line number	Diff line number	Diff line
		@@ -860,7 +860,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
		Slog.i(LOG_TAG, "Added global client for pid:" + Binder.getCallingPid());
		}
		return IntPair.of(
		getClientStateLocked(userState),
		combineUserStateAndProxyState(getClientStateLocked(userState),
		mProxyManager.getStateLocked()),
		client.mLastSentRelevantEventTypes);
		} else {
		userState.mUserClients.register(callback, client);
		@@ -872,7 +873,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
		+ " and userId:" + mCurrentUserId);
		}
		return IntPair.of(
		(resolvedUserId == mCurrentUserId) ? getClientStateLocked(userState) : 0,
		(resolvedUserId == mCurrentUserId) ? combineUserStateAndProxyState(
		getClientStateLocked(userState), mProxyManager.getStateLocked())
		: 0,
		client.mLastSentRelevantEventTypes);
		}
		}
		@@ -1003,6 +1006,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
		notifyAccessibilityServicesDelayedLocked(event, false);
		notifyAccessibilityServicesDelayedLocked(event, true);
		mUiAutomationManager.sendAccessibilityEventLocked(event);
		mProxyManager.sendAccessibilityEvent(event);
		}

		private void sendAccessibilityEventToInputFilter(AccessibilityEvent event) {
		@@ -1143,9 +1147,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
		}
		List<AccessibilityServiceConnection> services =
		getUserStateLocked(resolvedUserId).mBoundServices;
		int numServices = services.size();
		int numServices = services.size() + mProxyManager.getNumProxys();
		interfacesToInterrupt = new ArrayList<>(numServices);
		for (int i = 0; i < numServices; i++) {
		for (int i = 0; i < services.size(); i++) {
		AccessibilityServiceConnection service = services.get(i);
		IBinder a11yServiceBinder = service.mService;
		IAccessibilityServiceClient a11yServiceInterface = service.mServiceInterface;
		@@ -1153,6 +1157,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
		interfacesToInterrupt.add(a11yServiceInterface);
		}
		}
		mProxyManager.addServiceInterfaces(interfacesToInterrupt);
		}
		for (int i = 0, count = interfacesToInterrupt.size(); i < count; i++) {
		try {
		@@ -1941,6 +1946,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
		mUiAutomationManager.getServiceInfo(), client)
		? mUiAutomationManager.getRelevantEventTypes()
		: 0;
		relevantEventTypes \|= mProxyManager.getRelevantEventTypes();
		return relevantEventTypes;
		}

		@@ -2178,21 +2184,25 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
		updateAccessibilityEnabledSettingLocked(userState);
		}

		void scheduleUpdateClientsIfNeeded(AccessibilityUserState userState) {
		synchronized (mLock) {
		scheduleUpdateClientsIfNeededLocked(userState);
		}
		private int combineUserStateAndProxyState(int userState, int proxyState) {
		return userState \| proxyState;
		}

		void scheduleUpdateClientsIfNeededLocked(AccessibilityUserState userState) {
		final int clientState = getClientStateLocked(userState);
		if (userState.getLastSentClientStateLocked() != clientState
		final int proxyState = mProxyManager.getStateLocked();
		if ((userState.getLastSentClientStateLocked() != clientState
		\|\| mProxyManager.getLastSentStateLocked() != proxyState)
		&& (mGlobalClients.getRegisteredCallbackCount() > 0
		\|\| userState.mUserClients.getRegisteredCallbackCount() > 0)) {
		userState.setLastSentClientStateLocked(clientState);
		mProxyManager.setLastStateLocked(proxyState);
		// Send both the user and proxy state to the app for now.
		// TODO(b/250929565): Send proxy state to proxy clients
		mMainHandler.sendMessage(obtainMessage(
		AccessibilityManagerService::sendStateToAllClients,
		this, clientState, userState.mUserId));
		this, combineUserStateAndProxyState(clientState, proxyState),
		userState.mUserId));
		}
		}

		@@ -2434,7 +2444,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
		// binding we do an update pass after each bind event, so we run this
		// code and register the callback if needed.

		boolean observingWindows = mUiAutomationManager.canRetrieveInteractiveWindowsLocked();
		boolean observingWindows = mUiAutomationManager.canRetrieveInteractiveWindowsLocked()
		\|\| mProxyManager.canRetrieveInteractiveWindowsLocked();
		List<AccessibilityServiceConnection> boundServices = userState.mBoundServices;
		final int boundServiceCount = boundServices.size();
		for (int i = 0; !observingWindows && (i < boundServiceCount); i++) {
		@@ -3657,7 +3668,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
		+ "proxy-ed");
		}

		mProxyManager.registerProxy(client, displayId);
		mProxyManager.registerProxy(client, displayId, mContext,
		sIdCounter++, mMainHandler, mSecurityPolicy, this, getTraceManager(),
		mWindowManagerService, mA11yWindowManager);
		return true;
		}