Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 8643aa01 authored by Svetoslav Ganov's avatar Svetoslav Ganov
Browse files

Interrogation of the view hierarchy from an AccessibilityService. 

1. Views are represented as AccessibilityNodeInfos to AccessibilityServices.

2. An accessibility service receives AccessibilityEvents and can ask
   for its source and gets an AccessibilityNodeInfo which can be used
   to get its parent and children infos and so on.

3. AccessibilityNodeInfo contains some attributes and actions that
   can be performed on the source.

4. AccessibilityService can request the system to preform an action
   on the source of an AccessibilityNodeInfo.

5. ViewAncestor provides an interaction connection to the
   AccessibiltyManagerService and an accessibility service uses
   its connection to the latter to interact with screen content.

6. AccessibilityService can interact ONLY with the focused window
   and all calls are routed through the AccessibilityManagerService
   which imposes security.

7. Hidden APIs on AccessibilityService can find AccessibilityNodeInfos
   based on some criteria. These API go through the AccessibilityManagerServcie
   for security check.

8. Some actions are hidden and are exposes only to eng builds for UI testing.

Change-Id: Ie34fa4219f350eb3f4f6f9f45b24f709bd98783c
parent 21945136

Android.mk

+6 −4
Original line number Diff line number Diff line
@@ -132,6 +132,8 @@ LOCAL_SRC_FILES += \ 
	core/java/android/service/wallpaper/IWallpaperConnection.aidl \

	core/java/android/service/wallpaper/IWallpaperEngine.aidl \

	core/java/android/service/wallpaper/IWallpaperService.aidl \

	core/java/android/view/accessibility/IAccessibilityInteractionConnection.aidl\

	core/java/android/view/accessibility/IAccessibilityInteractionConnectionCallback.aidl\

	core/java/android/view/accessibility/IAccessibilityManager.aidl \

	core/java/android/view/accessibility/IAccessibilityManagerClient.aidl \

	core/java/android/view/IApplicationToken.aidl \

api/14.txt

+0 −2
Original line number Diff line number Diff line
@@ -22250,10 +22250,8 @@ package android.view.accessibility { 














  public class AccessibilityRecord {















    ctor protected AccessibilityRecord();













    method protected void clear();















    method public int getAddedCount();















    method public java.lang.CharSequence getBeforeText();













    method public boolean getBooleanProperty(int);















    method public java.lang.CharSequence getClassName();















    method public java.lang.CharSequence getContentDescription();















    method public int getCurrentItemIndex();

























api/current.txt



+66
−2




























Original line number
Diff line number
Diff line








@@ -90,6 +90,7 @@ package android {













    field public static final java.lang.String REMOVE_TASKS = "android.permission.REMOVE_TASKS";















    field public static final java.lang.String REORDER_TASKS = "android.permission.REORDER_TASKS";















    field public static final deprecated java.lang.String RESTART_PACKAGES = "android.permission.RESTART_PACKAGES";













    field public static final java.lang.String RETRIEVE_WINDOW_CONTENT = "android.permission.RETRIEVE_WINDOW_CONTENT";















    field public static final java.lang.String SEND_SMS = "android.permission.SEND_SMS";















    field public static final java.lang.String SET_ACTIVITY_WATCHER = "android.permission.SET_ACTIVITY_WATCHER";















    field public static final java.lang.String SET_ALARM = "com.android.alarm.permission.SET_ALARM";










@@ -21478,6 +21479,7 @@ package android.view {













    method protected int computeVerticalScrollExtent();















    method protected int computeVerticalScrollOffset();















    method protected int computeVerticalScrollRange();













    method public android.view.accessibility.AccessibilityNodeInfo createAccessibilityNodeInfo();















    method public void createContextMenu(android.view.ContextMenu);















    method public void destroyDrawingCache();















    method public void dispatchConfigurationChanged(android.content.res.Configuration);










@@ -21506,6 +21508,7 @@ package android.view {













    method public android.view.View findFocus();















    method public final android.view.View findViewById(int);















    method public final android.view.View findViewWithTag(java.lang.Object);













    method public void findViewsWithText(java.util.ArrayList<android.view.View>, java.lang.CharSequence);















    method protected boolean fitSystemWindows(android.graphics.Rect);















    method public android.view.View focusSearch(int);















    method public void forceLayout();










@@ -21674,6 +21677,7 @@ package android.view {













    method public boolean onGenericMotionEvent(android.view.MotionEvent);















    method public boolean onHoverEvent(android.view.MotionEvent);















    method public void onInitializeAccessibilityEvent(android.view.accessibility.AccessibilityEvent);













    method public void onInitializeAccessibilityNodeInfo(android.view.accessibility.AccessibilityNodeInfo);















    method public boolean onKeyDown(int, android.view.KeyEvent);















    method public boolean onKeyLongPress(int, android.view.KeyEvent);















    method public boolean onKeyMultiple(int, int, android.view.KeyEvent);










@@ -22567,17 +22571,21 @@ package android.view.accessibility {













    method public void appendRecord(android.view.accessibility.AccessibilityRecord);















    method public int describeContents();















    method public static java.lang.String eventTypeToString(int);













    method public int getAccessibilityWindowId();















    method public long getEventTime();















    method public int getEventType();















    method public java.lang.CharSequence getPackageName();















    method public android.view.accessibility.AccessibilityRecord getRecord(int);















    method public int getRecordCount();













    method public android.view.accessibility.AccessibilityNodeInfo getSource();















    method public void initFromParcel(android.os.Parcel);















    method public static android.view.accessibility.AccessibilityEvent obtain(int);













    method public static android.view.accessibility.AccessibilityEvent obtain(android.view.accessibility.AccessibilityEvent);















    method public static android.view.accessibility.AccessibilityEvent obtain();















    method public void setEventTime(long);















    method public void setEventType(int);















    method public void setPackageName(java.lang.CharSequence);













    method public void setSource(android.view.View);















    method public void writeToParcel(android.os.Parcel, int);















    field public static final android.os.Parcelable.Creator CREATOR;















    field public static final int INVALID_POSITION = -1; // 0xffffffff









@@ -22602,20 +22610,75 @@ package android.view.accessibility {













  }





























  public final class AccessibilityManager {













    method public boolean addAccessibilityStateChangeListener(android.view.accessibility.AccessibilityManager.AccessibilityStateChangeListener);















    method public deprecated java.util.List<android.content.pm.ServiceInfo> getAccessibilityServiceList();















    method public java.util.List<android.accessibilityservice.AccessibilityServiceInfo> getEnabledAccessibilityServiceList(int);















    method public java.util.List<android.accessibilityservice.AccessibilityServiceInfo> getInstalledAccessibilityServiceList();















    method public void interrupt();















    method public boolean isEnabled();













    method public boolean removeAccessibilityStateChangeListener(android.view.accessibility.AccessibilityManager.AccessibilityStateChangeListener);















    method public void sendAccessibilityEvent(android.view.accessibility.AccessibilityEvent);















  }



























  public static abstract interface AccessibilityManager.AccessibilityStateChangeListener {













    method public abstract void onAccessibilityStateChanged(boolean);













  }

























  public class AccessibilityNodeInfo implements android.os.Parcelable {













    method public void addAction(int);













    method public void addChild(android.view.View);













    method public int describeContents();













    method public int getAccessibilityWindowId();













    method public int getActions();













    method public void getBounds(android.graphics.Rect);













    method public android.view.accessibility.AccessibilityNodeInfo getChild(int);













    method public int getChildCount();













    method public java.lang.CharSequence getClassName();













    method public java.lang.CharSequence getContentDescription();













    method public java.lang.CharSequence getPackageName();













    method public android.view.accessibility.AccessibilityNodeInfo getParent();













    method public java.lang.CharSequence getText();













    method public boolean isCheckable();













    method public boolean isChecked();













    method public boolean isClickable();













    method public boolean isEnabled();













    method public boolean isFocusable();













    method public boolean isFocused();













    method public boolean isLongClickable();













    method public boolean isPassword();













    method public boolean isSelected();













    method public static android.view.accessibility.AccessibilityNodeInfo obtain(android.view.View);













    method public static android.view.accessibility.AccessibilityNodeInfo obtain();













    method public boolean performAction(int);













    method public void recycle();













    method public void setBounds(android.graphics.Rect);













    method public void setCheckable(boolean);













    method public void setChecked(boolean);













    method public void setClassName(java.lang.CharSequence);













    method public void setClickable(boolean);













    method public void setContentDescription(java.lang.CharSequence);













    method public void setEnabled(boolean);













    method public void setFocusable(boolean);













    method public void setFocused(boolean);













    method public void setLongClickable(boolean);













    method public void setPackageName(java.lang.CharSequence);













    method public void setParent(android.view.View);













    method public void setPassword(boolean);













    method public void setSelected(boolean);













    method public void setSource(android.view.View);













    method public void setText(java.lang.CharSequence);













    method public void writeToParcel(android.os.Parcel, int);













    field public static final int ACTION_CLEAR_FOCUS = 2; // 0x2













    field public static final int ACTION_CLEAR_SELECTION = 8; // 0x8













    field public static final int ACTION_FOCUS = 1; // 0x1













    field public static final int ACTION_SELECT = 4; // 0x4













    field public static final android.os.Parcelable.Creator CREATOR;













  }



























  public class AccessibilityRecord {















    ctor protected AccessibilityRecord();













    method protected void clear();















    method public int getAddedCount();















    method public java.lang.CharSequence getBeforeText();













    method public boolean getBooleanProperty(int);















    method public java.lang.CharSequence getClassName();















    method public java.lang.CharSequence getContentDescription();















    method public int getCurrentItemIndex();









@@ -22628,6 +22691,7 @@ package android.view.accessibility {













    method public boolean isEnabled();















    method public boolean isFullScreen();















    method public boolean isPassword();













    method public static android.view.accessibility.AccessibilityRecord obtain(android.view.accessibility.AccessibilityRecord);















    method public static android.view.accessibility.AccessibilityRecord obtain();















    method public void recycle();















    method public void setAddedCount(int);

























core/java/android/accessibilityservice/AccessibilityService.java



+56
−12




























Original line number
Diff line number
Diff line








@@ -25,6 +25,7 @@ import android.os.Message;













import android.os.RemoteException;















import android.util.Log;















import android.view.accessibility.AccessibilityEvent;













import android.view.accessibility.AccessibilityNodeInfo;































/**















 * An accessibility service runs in the background and receives callbacks by the system









@@ -51,7 +52,11 @@ import android.view.accessibility.AccessibilityEvent;













 * enabling or disabling it in the device settings. After the system binds to a service it















 * calls {@link AccessibilityService#onServiceConnected()}. This method can be















 * overriden by clients that want to perform post binding setup.













 * </p>















 * <p>













 * An accessibility service can be configured to receive specific types of accessibility events,













 * listen only to specific packages, get events from each type only once in a given time frame,













 * retrieve window content, specify a settings activity, etc.















 * </p>















 * There are two approaches for configuring an accessibility service:















 * <ul>









@@ -73,6 +78,9 @@ import android.view.accessibility.AccessibilityEvent;













 *       This approach enables setting all accessibility service properties.















 *     </strong>















 *     </p>













 *     <p>













 *       For more details refer to {@link #SERVICE_META_DATA}.













 *     </p>















 *   </li>















 *   <li>















 *     Calling {@link AccessibilityService#setServiceInfo(AccessibilityServiceInfo)}. Note









@@ -88,6 +96,9 @@ import android.view.accessibility.AccessibilityEvent;













 *       {@link AccessibilityServiceInfo#packageNames}















 *     </strong>















 *     </p>













 *     <p>













 *       For more details refer to {@link AccessibilityServiceInfo}.













 *     </p>















 *   </li>















 * </ul>















 * <p>










@@ -151,16 +162,49 @@ public abstract class AccessibilityService extends Service {













     * <code>















     *   &lt;?xml version="1.0" encoding="utf-8"?&gt;<br>















     *   &lt;accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"<br>













     *   &nbsp;&nbsp;android:eventTypes="typeViewClicked|typeViewFocused"<br>













     *   &nbsp;&nbsp;android:accessibilityEventTypes="typeViewClicked|typeViewFocused"<br>















     *   &nbsp;&nbsp;android:packageNames="foo.bar, foo.baz"<br>













     *   &nbsp;&nbsp;android:feedbackType="feedbackSpoken"<br>













     *   &nbsp;&nbsp;android:accessibilityFeedbackType="feedbackSpoken"<br>















     *   &nbsp;&nbsp;android:notificationTimeout="100"<br>













     *   &nbsp;&nbsp;android:flags="flagDefault"<br>













     *   &nbsp;&nbsp;android:accessibilityFlags="flagDefault"<br>















     *   &nbsp;&nbsp;android:settingsActivity="foo.bar.TestBackActivity"<br>















     *   &nbsp;&nbsp;. . .<br>















     *   /&gt;















     * </code>















     * </p>













     * <p>













     *  <strong>Note:</strong> A service can retrieve only the content of the active window.













     *          An active window is the source of the most recent event of type













     *          {@link AccessibilityEvent#TYPE_TOUCH_EXPLORATION_GESTURE_START},













     *          {@link AccessibilityEvent#TYPE_TOUCH_EXPLORATION_GESTURE_END},













     *          {@link AccessibilityEvent#TYPE_VIEW_CLICKED},













     *          {@link AccessibilityEvent#TYPE_VIEW_FOCUSED},













     *          {@link AccessibilityEvent#TYPE_VIEW_HOVER_ENTER},













     *          {@link AccessibilityEvent#TYPE_VIEW_HOVER_EXIT},













     *          {@link AccessibilityEvent#TYPE_VIEW_LONG_CLICKED},













     *          {@link AccessibilityEvent#TYPE_VIEW_SELECTED},













     *          {@link AccessibilityEvent#TYPE_VIEW_TEXT_CHANGED},













     *          {@link AccessibilityEvent#TYPE_WINDOW_STATE_CHANGED}.













     *          Therefore the service should:













     *          <ul>













     *            <li>













     *              Register for all event types with no notification timeout and keep track













     *              for the active window by calling













     *              {@link AccessibilityEvent#getAccessibilityWindowId()} of the last received













     *              event and compare this with the













     *              {@link AccessibilityNodeInfo#getAccessibilityWindowId()} before calling













     *              retrieval methods on the latter.













     *            </li>













     *            <li>













     *              Prepare that a retrieval method on {@link AccessibilityNodeInfo} may fail













     *              since the active window has changed and the service did not get the 













     *              accessibility event. Note that it is possible to have a retrieval method













     *              failing event adopting the strategy specified in the previous bullet













     *              because the accessibility event dispatch is asynchronous and crosses













     *              process boundaries. 













     *            </li>













     *          <ul>













     * </p>















     */















    public static final String SERVICE_META_DATA = "android.accessibilityservice";


























@@ -224,7 +268,7 @@ public abstract class AccessibilityService extends Service {





























    /**















     * Implement to return the implementation of the internal accessibility













     * service interface.  Subclasses should not override.













     * service interface.















     */















    @Override















    public final IBinder onBind(Intent intent) {

































core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl



+61
−3




























Original line number
Diff line number
Diff line








@@ -17,14 +17,72 @@













package android.accessibilityservice;































import android.accessibilityservice.AccessibilityServiceInfo;













import android.view.accessibility.AccessibilityNodeInfo;































/**













 * Interface AccessibilityManagerService#Service implements, and passes to an













 * AccessibilityService so it can dynamically configure how the system handles it.













 * Interface given to an AccessibilitySerivce to talk to the AccessibilityManagerService.















 *















 * @hide















 */













oneway interface IAccessibilityServiceConnection {













interface IAccessibilityServiceConnection {































    void setServiceInfo(in AccessibilityServiceInfo info);



























    /**













     * Finds an {@link AccessibilityNodeInfo} by accessibility id.













     * <p>













     *   <strong>













     *     It is a client responsibility to recycle the received info by













     *     calling {@link AccessibilityNodeInfo#recycle()} to avoid creating













     *     of multiple instances.













     *   </strong>













     * </p>













     *













     * @param accessibilityWindowId A unique window id.













     * @param accessibilityViewId A unique View accessibility id.













     * @return The node info.













     */













    AccessibilityNodeInfo findAccessibilityNodeInfoByAccessibilityId(int accessibilityWindowId,













        int accessibilityViewId);



























    /**













     * Finds {@link AccessibilityNodeInfo}s by View text. The match is case













     * insensitive containment.













     * <p>













     *   <strong>













     *     It is a client responsibility to recycle the received infos by













     *     calling {@link AccessibilityNodeInfo#recycle()} to avoid creating













     *     of multiple instances.













     *   </strong>













     * </p>













     *













     * @param text The searched text.













     * @return A list of node info.













     */













    List<AccessibilityNodeInfo> findAccessibilityNodeInfosByViewText(String text);



























    /**













     * Finds an {@link AccessibilityNodeInfo} by View id.













     * <p>













     *   <strong>













     *     It is a client responsibility to recycle the received info by













     *     calling {@link AccessibilityNodeInfo#recycle()} to avoid creating













     *     of multiple instances.













     *   </strong>













     * </p>













     *













     * @param id The id of the node.













     * @return The node info.













     */













    AccessibilityNodeInfo findAccessibilityNodeInfoByViewId(int viewId);



























    /**













     * Performs an accessibility action on an {@link AccessibilityNodeInfo}.













     *













     * @param accessibilityWindowId The id of the window.













     * @param accessibilityViewId The of a view in the .













     * @return Whether the action was performed.













     */













    boolean performAccessibilityAction(int accessibilityWindowId, int accessibilityViewId,













        int action);















}