Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 111ef0b9 authored by sallyyuen's avatar sallyyuen
Browse files

Expose more SystemAPI methods on AccessibilityDisplayProxy 

These are:
- onProxyConnected (equivalent to onServiceConnected)
- onAccessibilityEvent
- onInterrupt
- getWindows
- set/getInstalledAndEnabledServices

Connect the proxy client (IAccessibilityServiceClient) to a
proxy service connection
(ProxyAbstractAccessibilityServiceConnection) in ProxyManager,
so the A11yDisplayProxy can access the UI.

AccessibilityServices run on their own thread. To avoid using the
app's main thread, the app that registers the A11yDisplayProxy
can call it on a different thread. Add a note in the proxy class.

Future TODOs (b/254545943):

The a11y state returned to A11yManager includes state
of all proxies and AccessibilityServices.
Separation will require AccessibiilityManager to go from being a
process singleton to a per-display instance.

Input and a11y focus are singletons and may
shift between a proxy display and a display on the device.
Separation will require future changes.

ProxyManager behaves more like UiAutomationManager,
and centralizes proxy management. Separation into different user
states will require future refactoring.

Bug: 241429275
Test: atest AccessibilityDisplayProxyTest,
AccessibilityManagerServiceTest, manual test app that
registers/unregisters A11yDisplayProxy

Change-Id: I830c6613c2c2f8abcc2343c5110f4e7b7494fb0e
parent 0319d1cb

core/api/system-current.txt

+6 −0
Original line number Diff line number Diff line
@@ -15928,6 +15928,12 @@ package android.view.accessibility { 
  public abstract class AccessibilityDisplayProxy {

    ctor public AccessibilityDisplayProxy(int, @NonNull java.util.concurrent.Executor, @NonNull java.util.List<android.accessibilityservice.AccessibilityServiceInfo>);

    method public int getDisplayId();

    method @NonNull public final java.util.List<android.accessibilityservice.AccessibilityServiceInfo> getInstalledAndEnabledServices();

    method @NonNull public java.util.List<android.view.accessibility.AccessibilityWindowInfo> getWindows();

    method public void interrupt();

    method public void onAccessibilityEvent(@NonNull android.view.accessibility.AccessibilityEvent);

    method public void onProxyConnected();

    method public void setInstalledAndEnabledServices(@NonNull java.util.List<android.accessibilityservice.AccessibilityServiceInfo>);

  }















  public final class AccessibilityManager {

























core/java/android/view/accessibility/AccessibilityDisplayProxy.java



+156
−7




























Original line number
Diff line number
Diff line








@@ -20,13 +20,18 @@ import android.accessibilityservice.AccessibilityGestureEvent;













import android.accessibilityservice.AccessibilityService;















import android.accessibilityservice.AccessibilityServiceInfo;















import android.accessibilityservice.IAccessibilityServiceClient;













import android.accessibilityservice.IAccessibilityServiceConnection;















import android.accessibilityservice.MagnificationConfig;















import android.annotation.NonNull;















import android.annotation.Nullable;















import android.annotation.SystemApi;













import android.content.ComponentName;















import android.content.Context;













import android.content.pm.ResolveInfo;















import android.graphics.Region;















import android.os.IBinder;













import android.os.RemoteException;













import android.util.Log;















import android.view.KeyEvent;















import android.view.MotionEvent;















import android.view.inputmethod.EditorInfo;









@@ -34,14 +39,15 @@ import android.view.inputmethod.EditorInfo;













import com.android.internal.inputmethod.IAccessibilityInputMethodSessionCallback;















import com.android.internal.inputmethod.RemoteAccessibilityInputConnection;





























import java.util.Collections;















import java.util.List;















import java.util.concurrent.Executor;































/**















 * Allows a privileged app - an app with MANAGE_ACCESSIBILITY permission and SystemAPI access - to















 * interact with the windows in the display that this proxy represents. Proxying the default display













 * or a display that is not tracked will throw an exception. Only the real user has access to global













 * clients like SystemUI.













 * or a display that is not tracked by accessibility, such as private displays, will throw an













 * exception. Only the real user has access to global clients like SystemUI.















 *















 * <p>















 * To register and unregister a proxy, use









@@ -49,7 +55,16 @@ import java.util.concurrent.Executor;













 * and {@link AccessibilityManager#unregisterDisplayProxy(AccessibilityDisplayProxy)}. If the app















 * that has registered the proxy dies, the system will remove the proxy.















 *













 * TODO(241429275): Complete proxy impl and add additional support (if necessary) like cache methods













 * <p>













 * Avoid using the app's main thread. Proxy methods such as {@link #getWindows} and node methods













 * like {@link AccessibilityNodeInfo#getChild(int)} will happen frequently. Node methods may also













 * wait on the displayed app's UI thread to obtain accurate screen data.













 *













 * <p>













 * To get a list of {@link AccessibilityServiceInfo}s that have populated {@link ComponentName}s and













 * {@link ResolveInfo}s, retrieve the list using {@link #getInstalledAndEnabledServices()} after













 * {@link #onProxyConnected()} has been called.













 *















 * @hide















 */















@SystemApi










@@ -91,7 +106,134 @@ public abstract class AccessibilityDisplayProxy {













    }































    /**













     * An IAccessibilityServiceClient that handles interrupts and accessibility events.













     * Handles {@link android.view.accessibility.AccessibilityEvent}s.













     * <p>













     * AccessibilityEvents represent changes to the UI, or what parts of the node tree have changed.













     * AccessibilityDisplayProxy should use these to query new UI and send appropriate feedback













     * to their users.













     * <p>













     * For example, a {@link AccessibilityEvent#TYPE_WINDOWS_CHANGED} indicates a change in windows,













     * so a proxy may query {@link #getWindows} to obtain updated UI and potentially inform of a new













     * window title. Or a proxy may emit an earcon on a













     * {@link AccessibilityEvent#TYPE_VIEW_SCROLLED} event.













     */













    public void onAccessibilityEvent(@NonNull AccessibilityEvent event) {













        // Default no-op













    }



























    /**













     * Handles a successful system connection after













     * {@link AccessibilityManager#registerDisplayProxy(AccessibilityDisplayProxy)} is called.













     *













     * <p>













     * At this point, querying for UI is available and {@link AccessibilityEvent}s will begin being













     * sent. An AccessibilityDisplayProxy may instantiate core infrastructure components here.













     */













    public void onProxyConnected() {













        // Default no-op













    }



























    /**













     * Handles a request to interrupt the accessibility feedback.













     * <p>













     * AccessibilityDisplayProxy should interrupt the accessibility activity occurring on its













     * display. For example, a screen reader may interrupt speech.













     *













     * @see AccessibilityManager#interrupt()













     * @see AccessibilityService#onInterrupt()













     */













    public void interrupt() {













        // Default no-op













    }



























    /**













     * Gets the focus of the window specified by {@code windowInfo}.













     *













     * @param windowInfo the window to search













     * @param focus The focus to find. One of {@link AccessibilityNodeInfo#FOCUS_INPUT} or













     * {@link AccessibilityNodeInfo#FOCUS_ACCESSIBILITY}.













     * @return The node info of the focused view or null.













     * @hide













     * TODO(254545943): Do not expose until support for accessibility focus and/or input is in place













     */













    @Nullable













    public AccessibilityNodeInfo findFocus(@NonNull AccessibilityWindowInfo windowInfo, int focus) {













        AccessibilityNodeInfo windowRoot = windowInfo.getRoot();













        return windowRoot != null ? windowRoot.findFocus(focus) : null;













    }



























    /**













     * Gets the windows of the tracked display.













     *













     * @see AccessibilityService#getWindows()













     */













    @NonNull













    public List<AccessibilityWindowInfo> getWindows() {













        return AccessibilityInteractionClient.getInstance().getWindowsOnDisplay(mConnectionId,













                mDisplayId);













    }



























    /**













     * Sets the list of {@link AccessibilityServiceInfo}s describing the services interested in the













     * {@link AccessibilityDisplayProxy}'s display.













     *













     * <p>These represent a11y features and services that are installed and running. These should













     * not include {@link AccessibilityService}s installed on the phone.













     *













     * @param installedAndEnabledServices the list of installed and running a11y services.













     */













    public void setInstalledAndEnabledServices(













            @NonNull List<AccessibilityServiceInfo> installedAndEnabledServices) {













        mInstalledAndEnabledServices = installedAndEnabledServices;













        sendServiceInfos();













    }



























    /**













     * Sets the {@link AccessibilityServiceInfo} for this service if the latter is













     * properly set and there is an {@link IAccessibilityServiceConnection} to the













     * AccessibilityManagerService.













     */













    private void sendServiceInfos() {













        IAccessibilityServiceConnection connection =













                AccessibilityInteractionClient.getInstance().getConnection(mConnectionId);













        if (mInstalledAndEnabledServices != null && mInstalledAndEnabledServices.size() > 0













                && connection != null) {













            try {













                connection.setInstalledAndEnabledServices(mInstalledAndEnabledServices);













                AccessibilityInteractionClient.getInstance().clearCache(mConnectionId);













            } catch (RemoteException re) {













                Log.w(LOG_TAG, "Error while setting AccessibilityServiceInfos", re);













                re.rethrowFromSystemServer();













            }













        }













        mInstalledAndEnabledServices = null;













    }



























    /**













     * Gets the list of {@link AccessibilityServiceInfo}s describing the services interested in the













     * {@link AccessibilityDisplayProxy}'s display.













     *













     * @return The {@link AccessibilityServiceInfo}s of interested services.













     * @see AccessibilityServiceInfo













     */













    @NonNull













    public final List<AccessibilityServiceInfo> getInstalledAndEnabledServices() {













        IAccessibilityServiceConnection connection =













                AccessibilityInteractionClient.getInstance().getConnection(mConnectionId);













        if (connection != null) {













            try {













                return connection.getInstalledAndEnabledServices();













            } catch (RemoteException re) {













                Log.w(LOG_TAG, "Error while getting AccessibilityServiceInfo", re);













                re.rethrowFromSystemServer();













            }













        }













        return Collections.emptyList();













    }



























    /**













     * An IAccessibilityServiceClient that handles interrupts, accessibility events, and system













     * connection.















     */















    private class IAccessibilityServiceClientImpl extends















            AccessibilityService.IAccessibilityServiceClientWrapper {









@@ -100,17 +242,24 @@ public abstract class AccessibilityDisplayProxy {













            super(context, executor, new AccessibilityService.Callbacks() {















                @Override















                public void onAccessibilityEvent(AccessibilityEvent event) {













                    // TODO: call AccessiiblityProxy.onAccessibilityEvent













                    // TODO(254545943): Remove check when event processing is done more upstream in













                    // AccessibilityManagerService.













                    if (event.getDisplayId() == mDisplayId) {













                        AccessibilityDisplayProxy.this.onAccessibilityEvent(event);













                    }















                }































                @Override















                public void onInterrupt() {













                    // TODO: call AccessiiblityProxy.onInterrupt













                    AccessibilityDisplayProxy.this.interrupt();















                }





























                @Override















                public void onServiceConnected() {













                    // TODO: send service infos and call AccessiiblityProxy.onProxyConnected













                    AccessibilityDisplayProxy.this.sendServiceInfos();













                    AccessibilityDisplayProxy.this.onProxyConnected();















                }





























                @Override















                public void init(int connectionId, IBinder windowToken) {















                    mConnectionId = connectionId;

































core/java/android/view/accessibility/AccessibilityInteractionClient.java



+11
−5




























Original line number
Diff line number
Diff line








@@ -447,14 +447,20 @@ public final class AccessibilityInteractionClient













     * @return The {@link AccessibilityWindowInfo} list.















     */















    public List<AccessibilityWindowInfo> getWindows(int connectionId) {













        return getWindowsOnDisplay(connectionId, Display.DEFAULT_DISPLAY);













    }



























    /**













     * Gets the info for all windows of the specified display.













     *













     * @param connectionId The id of a connection for interacting with the system.













     * @return The {@link AccessibilityWindowInfo} list belonging to {@code displayId}.













     */













    public List<AccessibilityWindowInfo> getWindowsOnDisplay(int connectionId, int displayId) {















        final SparseArray<List<AccessibilityWindowInfo>> windows =















                getWindowsOnAllDisplays(connectionId);













        if (windows.size() > 0) {













            return windows.valueAt(Display.DEFAULT_DISPLAY);













        }













        return Collections.emptyList();













        return windows.get(displayId, Collections.emptyList());















    }





























    /**















     * Gets the info for all windows of all displays.















     *

































core/tests/coretests/src/android/view/accessibility/AccessibilityManagerTest.java



+15
−1




























Original line number
Diff line number
Diff line








@@ -263,10 +263,24 @@ public class AccessibilityManagerTest {













    }































    private class MyAccessibilityProxy extends AccessibilityDisplayProxy {













        // TODO(241429275): Will override A11yProxy methods in the future.















        MyAccessibilityProxy(int displayId,















                @NonNull List<AccessibilityServiceInfo> serviceInfos) {















            super(displayId, Executors.newSingleThreadExecutor(), serviceInfos);















        }



























        @Override













        public void onAccessibilityEvent(@NonNull AccessibilityEvent event) {



























        }



























        @Override













        public void onProxyConnected() {



























        }



























        @Override













        public void interrupt() {



























        }















    }















}
























services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java



+26
−13




























Original line number
Diff line number
Diff line








@@ -859,7 +859,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub













                    Slog.i(LOG_TAG, "Added global client for pid:" + Binder.getCallingPid());















                }















                return IntPair.of(













                        getClientStateLocked(userState),













                        combineUserStateAndProxyState(getClientStateLocked(userState),













                                mProxyManager.getStateLocked()),















                        client.mLastSentRelevantEventTypes);















            } else {















                userState.mUserClients.register(callback, client);









@@ -871,7 +872,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub













                            + " and userId:" + mCurrentUserId);















                }















                return IntPair.of(













                        (resolvedUserId == mCurrentUserId) ? getClientStateLocked(userState) : 0,













                        (resolvedUserId == mCurrentUserId) ? combineUserStateAndProxyState(













                                getClientStateLocked(userState), mProxyManager.getStateLocked())













                                : 0,















                        client.mLastSentRelevantEventTypes);















            }















        }










@@ -1002,6 +1005,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub













        notifyAccessibilityServicesDelayedLocked(event, false);















        notifyAccessibilityServicesDelayedLocked(event, true);















        mUiAutomationManager.sendAccessibilityEventLocked(event);













        mProxyManager.sendAccessibilityEvent(event);















    }































    private void sendAccessibilityEventToInputFilter(AccessibilityEvent event) {










@@ -1142,9 +1146,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub













            }















            List<AccessibilityServiceConnection> services =















                    getUserStateLocked(resolvedUserId).mBoundServices;













            int numServices = services.size();













            int numServices = services.size() + mProxyManager.getNumProxys();















            interfacesToInterrupt = new ArrayList<>(numServices);













            for (int i = 0; i < numServices; i++) {













            for (int i = 0; i < services.size(); i++) {















                AccessibilityServiceConnection service = services.get(i);















                IBinder a11yServiceBinder = service.mService;















                IAccessibilityServiceClient a11yServiceInterface = service.mServiceInterface;









@@ -1152,6 +1156,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub













                    interfacesToInterrupt.add(a11yServiceInterface);















                }















            }













            mProxyManager.addServiceInterfaces(interfacesToInterrupt);















        }















        for (int i = 0, count = interfacesToInterrupt.size(); i < count; i++) {















            try {










@@ -1940,6 +1945,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub













                mUiAutomationManager.getServiceInfo(), client)















                ? mUiAutomationManager.getRelevantEventTypes()















                : 0;













        relevantEventTypes |= mProxyManager.getRelevantEventTypes();















        return relevantEventTypes;















    }


























@@ -2177,21 +2183,25 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub













        updateAccessibilityEnabledSettingLocked(userState);















    }





























    void scheduleUpdateClientsIfNeeded(AccessibilityUserState userState) {













        synchronized (mLock) {













            scheduleUpdateClientsIfNeededLocked(userState);













        }













    private int combineUserStateAndProxyState(int userState, int proxyState) {













        return userState | proxyState;















    }































    void scheduleUpdateClientsIfNeededLocked(AccessibilityUserState userState) {















        final int clientState = getClientStateLocked(userState);













        if (userState.getLastSentClientStateLocked() != clientState













        final int proxyState = mProxyManager.getStateLocked();













        if ((userState.getLastSentClientStateLocked() != clientState













                || mProxyManager.getLastSentStateLocked() != proxyState)















                && (mGlobalClients.getRegisteredCallbackCount() > 0















                || userState.mUserClients.getRegisteredCallbackCount() > 0)) {















            userState.setLastSentClientStateLocked(clientState);













            mProxyManager.setLastStateLocked(proxyState);













            // Send both the user and proxy state to the app for now.













            // TODO(b/250929565): Send proxy state to proxy clients















            mMainHandler.sendMessage(obtainMessage(















                    AccessibilityManagerService::sendStateToAllClients,













                    this, clientState, userState.mUserId));













                    this, combineUserStateAndProxyState(clientState, proxyState),













                    userState.mUserId));















        }















    }


























@@ -2433,7 +2443,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub













        // binding we do an update pass after each bind event, so we run this















        // code and register the callback if needed.





























        boolean observingWindows = mUiAutomationManager.canRetrieveInteractiveWindowsLocked();













        boolean observingWindows = mUiAutomationManager.canRetrieveInteractiveWindowsLocked()













                || mProxyManager.canRetrieveInteractiveWindowsLocked();















        List<AccessibilityServiceConnection> boundServices = userState.mBoundServices;















        final int boundServiceCount = boundServices.size();















        for (int i = 0; !observingWindows && (i < boundServiceCount); i++) {










@@ -3656,7 +3667,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub













                    + "proxy-ed");















        }





























        mProxyManager.registerProxy(client, displayId);













        mProxyManager.registerProxy(client, displayId, mContext,













                sIdCounter++, mMainHandler, mSecurityPolicy, this, getTraceManager(),













                mWindowManagerService, mA11yWindowManager);















        return true;















    }