Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2e6f4617 authored by ryanlwlin's avatar ryanlwlin
Browse files

Make mirror window accessible (2/2) 

To support accessibility, we added a11y actions to move
the mirror window and change its scale. We also provide
content description, state description to deliver the
information of the mirror window.

Finally, we update state desceription to send window content change
 when changing the scale.

Test: manually test with voice access
      atest WindowMagnificationControllerTest
Bug: 143852371
Change-Id: Ia27c5bee39e732609ac1d8e2c7566524e6d7c64c
parent 8c1c5111

packages/SystemUI/res/values/ids.xml

+8 −0
Original line number Diff line number Diff line
@@ -175,5 +175,13 @@ 


    <!-- Accessibility actions for PIP -->

    <item type="id" name="action_pip_resize" />



    <!-- Accessibility actions for window magnification. -->

    <item type="id" name="accessibility_action_zoom_in"/>

    <item type="id" name="accessibility_action_zoom_out"/>

    <item type="id" name="accessibility_action_move_left"/>

    <item type="id" name="accessibility_action_move_right"/>

    <item type="id" name="accessibility_action_move_up"/>

    <item type="id" name="accessibility_action_move_down"/>

</resources>

packages/SystemUI/res/values/strings.xml

+12 −0
Original line number Diff line number Diff line
@@ -2666,6 +2666,18 @@ 
    <string name="magnification_window_title">Magnification Window</string>

    <!-- Title for Magnification Controls Window [CHAR LIMIT=NONE] -->

    <string name="magnification_controls_title">Magnification Window Controls</string>

    <!-- Action in accessibility menu to zoom in content of the magnification window. [CHAR LIMIT=30] -->

    <string name="accessibility_control_zoom_in">Zoom in</string>

    <!-- Action in accessibility menu to zoom out content of the magnification window. [CHAR LIMIT=30] -->

    <string name="accessibility_control_zoom_out">Zoom out</string>

    <!-- Action in accessibility menu to move the magnification window up. [CHAR LIMIT=30] -->

    <string name="accessibility_control_move_up">Move up</string>

    <!-- Action in accessibility menu to move the magnification window down. [CHAR LIMIT=30] -->

    <string name="accessibility_control_move_down">Move down</string>

    <!-- Action in accessibility menu to move the magnification window left. [CHAR LIMIT=30] -->

    <string name="accessibility_control_move_left">Move left</string>

    <!-- Action in accessibility menu to move the magnification window right. [CHAR LIMIT=30] -->

    <string name="accessibility_control_move_right">Move right</string>



    <!-- Device Controls strings -->

    <!-- Device Controls empty state, title [CHAR LIMIT=30] -->

packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnificationController.java

+99 −1
Original line number Diff line number Diff line
@@ -30,9 +30,11 @@ import android.graphics.PointF; 
import android.graphics.Rect;

import android.graphics.RectF;

import android.graphics.Region;

import android.os.Bundle;

import android.os.Handler;

import android.os.RemoteException;

import android.util.Log;

import android.util.Range;

import android.view.Choreographer;

import android.view.Display;

import android.view.Gravity;
@@ -47,12 +49,17 @@ import android.view.SurfaceView; 
import android.view.View;

import android.view.WindowManager;

import android.view.WindowManagerGlobal;

import android.view.accessibility.AccessibilityNodeInfo;

import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction;



import com.android.internal.annotations.VisibleForTesting;

import com.android.internal.graphics.SfVsyncFrameCallbackProvider;

import com.android.systemui.R;

import com.android.systemui.shared.system.WindowManagerWrapper;



import java.text.NumberFormat;

import java.util.Locale;



/**

 * Class to handle adding and removing a window magnification.

 */
@@ -60,6 +67,11 @@ class WindowMagnificationController implements View.OnTouchListener, SurfaceHold 
        MirrorWindowControl.MirrorWindowDelegate {



    private static final String TAG = "WindowMagnificationController";

    // Delay to avoid updating state description too frequently.

    private static final int UPDATE_STATE_DESCRIPTION_DELAY_MS = 100;

    // It should be consistent with the value defined in WindowMagnificationGestureHandler.

    private static final Range<Float> A11Y_ACTION_SCALE_RANGE = new Range<>(2.0f, 8.0f);

    private static final float A11Y_CHANGE_SCALE_DIFFERENCE = 1.0f;

    private final Context mContext;

    private final Resources mResources;

    private final Handler mHandler;
@@ -95,6 +107,7 @@ class WindowMagnificationController implements View.OnTouchListener, SurfaceHold 
    private final View.OnLayoutChangeListener mMirrorViewLayoutChangeListener;

    private final View.OnLayoutChangeListener mMirrorSurfaceViewLayoutChangeListener;

    private final Runnable mMirrorViewRunnable;

    private final Runnable mUpdateStateDescriptionRunnable;

    private View mMirrorView;

    private SurfaceView mMirrorSurfaceView;

    private int mMirrorSurfaceMargin;
@@ -106,6 +119,8 @@ class WindowMagnificationController implements View.OnTouchListener, SurfaceHold 


    private final SfVsyncFrameCallbackProvider mSfVsyncFrameProvider;

    private Choreographer.FrameCallback mMirrorViewGeometryVsyncCallback;

    private Locale mLocale;

    private NumberFormat mPercentFormat;



    @Nullable

    private MirrorWindowControl mMirrorWindowControl;
@@ -164,6 +179,11 @@ class WindowMagnificationController implements View.OnTouchListener, SurfaceHold 
                        mWindowMagnifierCallback.onSourceBoundsChanged(mDisplayId, mSourceBounds);

                    }

                };

        mUpdateStateDescriptionRunnable = () -> {

            if (isWindowVisible()) {

                mMirrorView.setStateDescription(formatStateDescription(mScale));

            }

        };

    }



    private void updateDimensions() {
@@ -292,12 +312,13 @@ class WindowMagnificationController implements View.OnTouchListener, SurfaceHold 
                | View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY

                | View.SYSTEM_UI_FLAG_HIDE_NAVIGATION);

        mMirrorView.addOnLayoutChangeListener(mMirrorViewLayoutChangeListener);

        mMirrorView.setAccessibilityDelegate(new MirrorWindowA11yDelegate());



        mWm.addView(mMirrorView, params);



        SurfaceHolder holder = mMirrorSurfaceView.getHolder();

        holder.addCallback(this);

        holder.setFormat(PixelFormat.RGBA_8888);



        addDragTouchListeners();

    }
@@ -526,6 +547,7 @@ class WindowMagnificationController implements View.OnTouchListener, SurfaceHold 
        final float offsetY = Float.isNaN(centerY) ? 0

                : centerY - mMagnificationFrame.exactCenterY();

        mScale = Float.isNaN(scale) ? mScale : scale;



        setMagnificationFrameBoundary();

        updateMagnificationFramePosition((int) offsetX, (int) offsetY);

        if (!isWindowVisible()) {
@@ -546,6 +568,8 @@ class WindowMagnificationController implements View.OnTouchListener, SurfaceHold 
            return;

        }

        enableWindowMagnification(scale, Float.NaN, Float.NaN);

        mHandler.removeCallbacks(mUpdateStateDescriptionRunnable);

        mHandler.postDelayed(mUpdateStateDescriptionRunnable, UPDATE_STATE_DESCRIPTION_DELAY_MS);

    }



    /**
@@ -596,4 +620,78 @@ class WindowMagnificationController implements View.OnTouchListener, SurfaceHold 
    private boolean isWindowVisible() {

        return mMirrorView != null;

    }



    private CharSequence formatStateDescription(float scale) {

        // Cache the locale-appropriate NumberFormat.  Configuration locale is guaranteed

        // non-null, so the first time this is called we will always get the appropriate

        // NumberFormat, then never regenerate it unless the locale changes on the fly.

        final Locale curLocale = mContext.getResources().getConfiguration().getLocales().get(0);

        if (!curLocale.equals(mLocale)) {

            mLocale = curLocale;

            mPercentFormat = NumberFormat.getPercentInstance(curLocale);

        }

        return mPercentFormat.format(scale);

    }



    private class MirrorWindowA11yDelegate extends View.AccessibilityDelegate {



        @Override

        public void onInitializeAccessibilityNodeInfo(View host, AccessibilityNodeInfo info) {

            super.onInitializeAccessibilityNodeInfo(host, info);

            info.addAction(

                    new AccessibilityAction(R.id.accessibility_action_zoom_in,

                            mContext.getString(R.string.accessibility_control_zoom_in)));

            info.addAction(new AccessibilityAction(R.id.accessibility_action_zoom_out,

                    mContext.getString(R.string.accessibility_control_zoom_out)));

            info.addAction(new AccessibilityAction(R.id.accessibility_action_move_up,

                    mContext.getString(R.string.accessibility_control_move_up)));

            info.addAction(new AccessibilityAction(R.id.accessibility_action_move_down,

                    mContext.getString(R.string.accessibility_control_move_down)));

            info.addAction(new AccessibilityAction(R.id.accessibility_action_move_left,

                    mContext.getString(R.string.accessibility_control_move_left)));

            info.addAction(new AccessibilityAction(R.id.accessibility_action_move_right,

                    mContext.getString(R.string.accessibility_control_move_right)));



            info.setContentDescription(mContext.getString(R.string.magnification_window_title));

            info.setStateDescription(formatStateDescription(getScale()));

        }



        @Override

        public boolean performAccessibilityAction(View host, int action, Bundle args) {

            if (performA11yAction(action)) {

                return true;

            }

            return super.performAccessibilityAction(host, action, args);

        }



        private boolean performA11yAction(int action) {

            if (action == R.id.accessibility_action_zoom_in) {

                final float scale = mScale + A11Y_CHANGE_SCALE_DIFFERENCE;

                setScale(A11Y_ACTION_SCALE_RANGE.clamp(scale));

                return true;

            }

            if (action == R.id.accessibility_action_zoom_out) {

                final float scale = mScale - A11Y_CHANGE_SCALE_DIFFERENCE;

                setScale(A11Y_ACTION_SCALE_RANGE.clamp(scale));

                return true;

            }

            if (action == R.id.accessibility_action_move_up) {

                move(0, -mSourceBounds.height());

                return true;

            }

            if (action == R.id.accessibility_action_move_down) {

                move(0, mSourceBounds.height());

                return true;

            }

            if (action == R.id.accessibility_action_move_left) {

                move(-mSourceBounds.width(), 0);

                return true;

            }

            if (action == R.id.accessibility_action_move_right) {

                move(mSourceBounds.width(), 0);

                return true;

            }

            return false;

        }

    }

}

packages/SystemUI/tests/src/com/android/systemui/accessibility/WindowMagnificationControllerTest.java

+71 −4
Original line number Diff line number Diff line
@@ -17,10 +17,17 @@ 
package com.android.systemui.accessibility;



import static android.view.Choreographer.FrameCallback;

import static android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction;



import static org.hamcrest.Matchers.containsString;

import static org.hamcrest.Matchers.hasItems;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertNotNull;

import static org.junit.Assert.assertThat;

import static org.junit.Assert.assertTrue;

import static org.mockito.ArgumentMatchers.any;

import static org.mockito.ArgumentMatchers.anyInt;

import static org.mockito.ArgumentMatchers.anyLong;

import static org.mockito.Mockito.atLeastOnce;

import static org.mockito.Mockito.doAnswer;

import static org.mockito.Mockito.verify;
@@ -37,17 +44,20 @@ import android.view.Surface; 
import android.view.SurfaceControl;

import android.view.View;

import android.view.WindowManager;

import android.view.accessibility.AccessibilityNodeInfo;



import androidx.test.InstrumentationRegistry;

import androidx.test.filters.SmallTest;



import com.android.internal.graphics.SfVsyncFrameCallbackProvider;

import com.android.systemui.R;

import com.android.systemui.SysuiTestCase;



import org.junit.After;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.ArgumentCaptor;

import org.mockito.Mock;

import org.mockito.Mockito;

import org.mockito.MockitoAnnotations;
@@ -71,6 +81,7 @@ public class WindowMagnificationControllerTest extends SysuiTestCase { 
    private Resources mResources;

    private WindowMagnificationController mWindowMagnificationController;

    private Instrumentation mInstrumentation;

    private View mMirrorView;



    @Before

    public void setUp() {
@@ -83,11 +94,15 @@ public class WindowMagnificationControllerTest extends SysuiTestCase { 
        ).when(mWindowManager).getMaximumWindowMetrics();

        mContext.addMockSystemService(Context.WINDOW_SERVICE, mWindowManager);

        doAnswer(invocation -> {

            View view = invocation.getArgument(0);

            mMirrorView = invocation.getArgument(0);

            WindowManager.LayoutParams lp = invocation.getArgument(1);

            view.setLayoutParams(lp);

            mMirrorView.setLayoutParams(lp);

            return null;

        }).when(mWindowManager).addView(any(View.class), any(WindowManager.LayoutParams.class));

        doAnswer(invocation -> {

            mMirrorView = null;

            return null;

        }).when(mWindowManager).removeView(any(View.class));

        doAnswer(invocation -> {

            FrameCallback callback = invocation.getArgument(0);

            callback.doFrame(0);
@@ -147,14 +162,18 @@ public class WindowMagnificationControllerTest extends SysuiTestCase { 
    }



    @Test

    public void setScale_enabled_expectedValue() {

    public void setScale_enabled_expectedValueAndUpdateStateDescription() {

        mInstrumentation.runOnMainSync(

                () -> mWindowMagnificationController.enableWindowMagnification(Float.NaN, Float.NaN,

                () -> mWindowMagnificationController.enableWindowMagnification(2.0f, Float.NaN,

                        Float.NaN));



        mInstrumentation.runOnMainSync(() -> mWindowMagnificationController.setScale(3.0f));



        assertEquals(3.0f, mWindowMagnificationController.getScale(), 0);

        ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);

        verify(mHandler).postDelayed(runnableArgumentCaptor.capture(), anyLong());

        runnableArgumentCaptor.getValue().run();

        assertThat(mMirrorView.getStateDescription().toString(), containsString("300"));

    }



    @Test
@@ -227,4 +246,52 @@ public class WindowMagnificationControllerTest extends SysuiTestCase { 


        verify(mResources, atLeastOnce()).getDimensionPixelSize(anyInt());

    }



    @Test

    public void initializeA11yNode_enabled_expectedValues() {

        mInstrumentation.runOnMainSync(() -> {

            mWindowMagnificationController.enableWindowMagnification(2.5f, Float.NaN,

                    Float.NaN);

        });

        assertNotNull(mMirrorView);

        final AccessibilityNodeInfo nodeInfo = new AccessibilityNodeInfo();



        mMirrorView.onInitializeAccessibilityNodeInfo(nodeInfo);



        assertNotNull(nodeInfo.getContentDescription());

        assertThat(nodeInfo.getStateDescription().toString(), containsString("250"));

        assertThat(nodeInfo.getActionList(),

                hasItems(new AccessibilityAction(R.id.accessibility_action_zoom_in, null),

                        new AccessibilityAction(R.id.accessibility_action_zoom_out, null),

                        new AccessibilityAction(R.id.accessibility_action_move_right, null),

                        new AccessibilityAction(R.id.accessibility_action_move_left, null),

                        new AccessibilityAction(R.id.accessibility_action_move_down, null),

                        new AccessibilityAction(R.id.accessibility_action_move_up, null)));

    }



    @Test

    public void performA11yActions_visible_expectedResults() {

        mInstrumentation.runOnMainSync(() -> {

            mWindowMagnificationController.enableWindowMagnification(2.5f, Float.NaN,

                    Float.NaN);

        });

        assertNotNull(mMirrorView);



        assertTrue(

                mMirrorView.performAccessibilityAction(R.id.accessibility_action_zoom_out, null));

        // Minimum scale is 2.0.

        assertEquals(2.0f, mWindowMagnificationController.getScale(), 0f);



        assertTrue(mMirrorView.performAccessibilityAction(R.id.accessibility_action_zoom_in, null));

        assertEquals(3.0f, mWindowMagnificationController.getScale(), 0f);



        // TODO: Verify the final state when the mirror surface is visible.

        assertTrue(mMirrorView.performAccessibilityAction(R.id.accessibility_action_move_up, null));

        assertTrue(

                mMirrorView.performAccessibilityAction(R.id.accessibility_action_move_down, null));

        assertTrue(

                mMirrorView.performAccessibilityAction(R.id.accessibility_action_move_right, null));

        assertTrue(

                mMirrorView.performAccessibilityAction(R.id.accessibility_action_move_left, null));

    }

}