Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 12d223ce authored by Asmita Poddar's avatar Asmita Poddar
Browse files

Add support for mouse keys on non-QWERTY keyboard layouts 

Depending on the keyboard layout in use, the mouse keys key codes should
change such that the position of the keys stays the same.
The default is to return the key codes for the
default layout (QWERTY), otherwise return the key code of the given
location on a reference QWERTY keyboard layout.
Also, the user could change the keyboard layout from settings at any
time, which would change the key character map for that device.
Hence, the key code to enum mapping should be updated every time a change
in keyboard is detected. To ensure thread safety all access to the enum
mapping for a particular device should happen from the same thread,
i.e., the handler thread.

bug: 341799888
Test: Manual testing on multiple QWERTY and DVORAK keyboard layouts,
      atest FrameworksServicesTests:MouseKeysInterceptorTest
Flag: EXEMPT bugfix
Change-Id: I4dbea8d4def95f7a710dd3b85e46d214f05fc94f
parent b381000f

services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java

+3 −0
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ import android.annotation.MainThread; 
import android.annotation.NonNull;

import android.content.Context;

import android.graphics.Region;

import android.hardware.input.InputManager;

import android.os.Looper;

import android.os.PowerManager;

import android.os.SystemClock;
@@ -57,6 +58,7 @@ import com.android.server.policy.WindowManagerPolicy; 
import java.io.FileDescriptor;

import java.io.PrintWriter;

import java.util.ArrayList;

import java.util.Objects;

import java.util.StringJoiner;



/**
@@ -748,6 +750,7 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo 


        if ((mEnabledFeatures & FLAG_FEATURE_MOUSE_KEYS) != 0) {

            mMouseKeysInterceptor = new MouseKeysInterceptor(mAms,

                    Objects.requireNonNull(mContext.getSystemService(InputManager.class)),

                    Looper.myLooper(),

                    Display.DEFAULT_DISPLAY);

            addFirstEventHandler(Display.DEFAULT_DISPLAY, mMouseKeysInterceptor);

services/accessibility/java/com/android/server/accessibility/MouseKeysInterceptor.java

+154 −41
Original line number Diff line number Diff line
@@ -23,6 +23,7 @@ import android.annotation.Nullable; 
import android.annotation.RequiresPermission;

import android.companion.virtual.VirtualDeviceManager;

import android.companion.virtual.VirtualDeviceParams;

import android.hardware.input.InputManager;

import android.hardware.input.VirtualMouse;

import android.hardware.input.VirtualMouseButtonEvent;

import android.hardware.input.VirtualMouseConfig;
@@ -34,8 +35,11 @@ import android.os.Message; 
import android.util.Log;

import android.util.Slog;

import android.util.SparseArray;

import android.view.InputDevice;

import android.view.KeyEvent;



import androidx.annotation.VisibleForTesting;



import com.android.server.LocalServices;

import com.android.server.companion.virtual.VirtualDeviceManagerInternal;
@@ -60,7 +64,7 @@ import com.android.server.companion.virtual.VirtualDeviceManagerInternal; 
 * mouse keys of each physical keyboard will control a single (global) mouse pointer.

 */

public class MouseKeysInterceptor extends BaseEventStreamTransformation

        implements Handler.Callback {

        implements Handler.Callback, InputManager.InputDeviceListener {

    private static final String LOG_TAG = "MouseKeysInterceptor";



    // To enable these logs, run: 'adb shell setprop log.tag.MouseKeysInterceptor DEBUG'
@@ -77,10 +81,19 @@ public class MouseKeysInterceptor extends BaseEventStreamTransformation 


    private final AccessibilityManagerService mAms;

    private final Handler mHandler;

    private final InputManager mInputManager;



    /** Thread to wait for virtual mouse creation to complete */

    private final Thread mCreateVirtualMouseThread;



    /**

     * Map of device IDs to a map of key codes to their corresponding {@link MouseKeyEvent} values.

     * To ensure thread safety for the map, all access and modification of the map

     * should happen on the same thread, i.e., on the handler thread.

     */

    private final SparseArray<SparseArray<MouseKeyEvent>> mDeviceKeyCodeMap =

            new SparseArray<>();



    VirtualDeviceManager.VirtualDevice mVirtualDevice = null;



    private VirtualMouse mVirtualMouse = null;
@@ -102,6 +115,21 @@ public class MouseKeysInterceptor extends BaseEventStreamTransformation 
    /** Whether scroll toggle is on */

    private boolean mScrollToggleOn = false;



    /** The ID of the input device that is currently active */

    private int mActiveInputDeviceId = 0;



    /**

     * Enum representing different types of mouse key events, each associated with a specific

     * key code.

     *

     * <p> These events correspond to various mouse actions such as directional movements,

     * clicks, and scrolls, mapped to specific keys on the keyboard.

     * The key codes here are the QWERTY key codes, and should be accessed via

     * {@link MouseKeyEvent#getKeyCode(InputDevice)}

     * so that it is mapped to the equivalent key on the keyboard layout of the keyboard device

     * that is actually in use.

     * </p>

     */

    public enum MouseKeyEvent {

        DIAGONAL_UP_LEFT_MOVE(KeyEvent.KEYCODE_7),

        UP_MOVE_OR_SCROLL(KeyEvent.KEYCODE_8),
@@ -117,31 +145,61 @@ public class MouseKeysInterceptor extends BaseEventStreamTransformation 
        RELEASE(KeyEvent.KEYCODE_COMMA),

        SCROLL_TOGGLE(KeyEvent.KEYCODE_PERIOD);



        private final int mKeyCode;

        private final int mLocationKeyCode;

        MouseKeyEvent(int enumValue) {

            mKeyCode = enumValue;

            mLocationKeyCode = enumValue;

        }



        private static final SparseArray<MouseKeyEvent> VALUE_TO_ENUM_MAP = new SparseArray<>();

        @VisibleForTesting

        public final int getKeyCodeValue() {

            return mLocationKeyCode;

        }



        static {

            for (MouseKeyEvent type : MouseKeyEvent.values()) {

                VALUE_TO_ENUM_MAP.put(type.mKeyCode, type);

        /**

         * Get the key code associated with the given MouseKeyEvent for the given keyboard

         * input device, taking into account its layout.

         * The default is to return the keycode for the default layout (QWERTY).

         * We check if the input device has been generated using {@link InputDevice#getGeneration()}

         * to test with the default {@link MouseKeyEvent} values in the unit tests.

         */

        public int getKeyCode(InputDevice inputDevice) {

            if (inputDevice.getGeneration() == -1) {

                return mLocationKeyCode;

            }

            return inputDevice.getKeyCodeForKeyLocation(mLocationKeyCode);

        }



        public final int getKeyCodeValue() {

            return mKeyCode;

        /**

         * Convert int value of the key code to corresponding {@link MouseKeyEvent}

         * enum for a particular device ID.

         * If no matching value is found, this will return {@code null}.

         */

        @Nullable

        public static MouseKeyEvent from(int keyCode, int deviceId,

                SparseArray<SparseArray<MouseKeyEvent>> deviceKeyCodeMap) {

            SparseArray<MouseKeyEvent> keyCodeToEnumMap = deviceKeyCodeMap.get(deviceId);

            if (keyCodeToEnumMap != null) {

                return keyCodeToEnumMap.get(keyCode);

            }

            return null;

        }

    }



    /**

         * Convert int value of the key code to corresponding MouseEvent enum. If no matching

         * value is found, this will return {@code null}.

     * Create a map of key codes to their corresponding {@link MouseKeyEvent} values

     * for a specific input device.

     * The key for {@code mDeviceKeyCodeMap} is the deviceId.

     * The key for {@code keyCodeToEnumMap} is the keycode for each

     * {@link MouseKeyEvent} according to the keyboard layout of the input device.

     */

        @Nullable

        public static MouseKeyEvent from(int value) {

            return VALUE_TO_ENUM_MAP.get(value);

    public void initializeDeviceToEnumMap(InputDevice inputDevice) {

        int deviceId = inputDevice.getId();

        SparseArray<MouseKeyEvent> keyCodeToEnumMap = new SparseArray<>();

        for (MouseKeyEvent mouseKeyEventType : MouseKeyEvent.values()) {

            int keyCode = mouseKeyEventType.getKeyCode(inputDevice);

            keyCodeToEnumMap.put(keyCode, mouseKeyEventType);

        }

        mDeviceKeyCodeMap.put(deviceId, keyCodeToEnumMap);

    }



    /**
@@ -152,8 +210,10 @@ public class MouseKeysInterceptor extends BaseEventStreamTransformation 
     * @param displayId Display ID to send mouse events to

     */

    @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE)

    public MouseKeysInterceptor(AccessibilityManagerService service, Looper looper, int displayId) {

    public MouseKeysInterceptor(AccessibilityManagerService service,

            InputManager inputManager, Looper looper, int displayId) {

        mAms = service;

        mInputManager = inputManager;

        mHandler = new Handler(looper, this);

        // Create the virtual mouse on a separate thread since virtual device creation

        // should happen on an auxiliary thread, and not from the handler's thread.
@@ -163,6 +223,9 @@ public class MouseKeysInterceptor extends BaseEventStreamTransformation 
            mVirtualMouse = createVirtualMouse(displayId);

        });

        mCreateVirtualMouseThread.start();

        // Register an input device listener to watch when input devices are

        // added, removed or reconfigured.

        mInputManager.registerInputDeviceListener(this, mHandler);

    }



    /**
@@ -215,7 +278,8 @@ public class MouseKeysInterceptor extends BaseEventStreamTransformation 
     */

    @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE)

    private void performMouseScrollAction(int keyCode) {

        MouseKeyEvent mouseKeyEvent = MouseKeyEvent.from(keyCode);

        MouseKeyEvent mouseKeyEvent = MouseKeyEvent.from(

                keyCode, mActiveInputDeviceId, mDeviceKeyCodeMap);

        float y = switch (mouseKeyEvent) {

            case UP_MOVE_OR_SCROLL -> 1.0f;

            case DOWN_MOVE_OR_SCROLL -> -1.0f;
@@ -247,15 +311,18 @@ public class MouseKeysInterceptor extends BaseEventStreamTransformation 
     */

    @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE)

    private void performMouseButtonAction(int keyCode) {

        MouseKeyEvent mouseKeyEvent = MouseKeyEvent.from(keyCode);

        MouseKeyEvent mouseKeyEvent = MouseKeyEvent.from(

                keyCode, mActiveInputDeviceId, mDeviceKeyCodeMap);

        int buttonCode = switch (mouseKeyEvent) {

            case LEFT_CLICK -> VirtualMouseButtonEvent.BUTTON_PRIMARY;

            case RIGHT_CLICK -> VirtualMouseButtonEvent.BUTTON_SECONDARY;

            default -> VirtualMouseButtonEvent.BUTTON_UNKNOWN;

        };

        if (buttonCode != VirtualMouseButtonEvent.BUTTON_UNKNOWN) {

            sendVirtualMouseButtonEvent(buttonCode, VirtualMouseButtonEvent.ACTION_BUTTON_PRESS);

            sendVirtualMouseButtonEvent(buttonCode, VirtualMouseButtonEvent.ACTION_BUTTON_RELEASE);

            sendVirtualMouseButtonEvent(buttonCode,

                    VirtualMouseButtonEvent.ACTION_BUTTON_PRESS);

            sendVirtualMouseButtonEvent(buttonCode,

                    VirtualMouseButtonEvent.ACTION_BUTTON_RELEASE);

        }

        if (DEBUG) {

            if (buttonCode == VirtualMouseButtonEvent.BUTTON_UNKNOWN) {
@@ -293,7 +360,9 @@ public class MouseKeysInterceptor extends BaseEventStreamTransformation 
    private void performMousePointerAction(int keyCode) {

        float x = 0f;

        float y = 0f;

        MouseKeyEvent mouseKeyEvent = MouseKeyEvent.from(keyCode);

        MouseKeyEvent mouseKeyEvent = MouseKeyEvent.from(

                keyCode, mActiveInputDeviceId, mDeviceKeyCodeMap);



        switch (mouseKeyEvent) {

            case DIAGONAL_DOWN_LEFT_MOVE -> {

                x = -MOUSE_POINTER_MOVEMENT_STEP / sqrt(2);
@@ -339,18 +408,19 @@ public class MouseKeysInterceptor extends BaseEventStreamTransformation 
        }

    }



    private boolean isMouseKey(int keyCode) {

        return MouseKeyEvent.VALUE_TO_ENUM_MAP.contains(keyCode);

    private boolean isMouseKey(int keyCode, int deviceId) {

        SparseArray<MouseKeyEvent> keyCodeToEnumMap = mDeviceKeyCodeMap.get(deviceId);

        return keyCodeToEnumMap.contains(keyCode);

    }



    private boolean isMouseButtonKey(int keyCode) {

        return keyCode == MouseKeyEvent.LEFT_CLICK.getKeyCodeValue()

                || keyCode == MouseKeyEvent.RIGHT_CLICK.getKeyCodeValue();

    private boolean isMouseButtonKey(int keyCode, InputDevice inputDevice) {

        return keyCode == MouseKeyEvent.LEFT_CLICK.getKeyCode(inputDevice)

                || keyCode == MouseKeyEvent.RIGHT_CLICK.getKeyCode(inputDevice);

    }



    private boolean isMouseScrollKey(int keyCode) {

        return keyCode == MouseKeyEvent.UP_MOVE_OR_SCROLL.getKeyCodeValue()

                || keyCode == MouseKeyEvent.DOWN_MOVE_OR_SCROLL.getKeyCodeValue();

    private boolean isMouseScrollKey(int keyCode, InputDevice inputDevice) {

        return keyCode == MouseKeyEvent.UP_MOVE_OR_SCROLL.getKeyCode(inputDevice)

                || keyCode == MouseKeyEvent.DOWN_MOVE_OR_SCROLL.getKeyCode(inputDevice);

    }



    /**
@@ -373,7 +443,7 @@ public class MouseKeysInterceptor extends BaseEventStreamTransformation 
    }



    /**

     * Handles key events and forwards mouse key events to the virtual mouse.

     * Handles key events and forwards mouse key events to the virtual mouse on the handler thread.

     *

     * @param event The key event to handle.

     * @param policyFlags The policy flags associated with the key event.
@@ -385,31 +455,45 @@ public class MouseKeysInterceptor extends BaseEventStreamTransformation 
            mAms.getTraceManager().logTrace(LOG_TAG + ".onKeyEvent",

                    FLAGS_INPUT_FILTER, "event=" + event + ";policyFlags=" + policyFlags);

        }



        mHandler.post(() -> {

            onKeyEventInternal(event, policyFlags);

        });

    }



    @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE)

    private void onKeyEventInternal(KeyEvent event, int policyFlags) {

        boolean isDown = event.getAction() == KeyEvent.ACTION_DOWN;

        int keyCode = event.getKeyCode();

        mActiveInputDeviceId = event.getDeviceId();

        InputDevice inputDevice = mInputManager.getInputDevice(mActiveInputDeviceId);



        if (!mDeviceKeyCodeMap.contains(mActiveInputDeviceId)) {

            initializeDeviceToEnumMap(inputDevice);

        }



        if (!isMouseKey(keyCode)) {

        if (!isMouseKey(keyCode, mActiveInputDeviceId)) {

            // Pass non-mouse key events to the next handler

            super.onKeyEvent(event, policyFlags);

        } else if (isDown) {

            if (keyCode == MouseKeyEvent.SCROLL_TOGGLE.getKeyCodeValue()) {

            if (keyCode == MouseKeyEvent.SCROLL_TOGGLE.getKeyCode(inputDevice)) {

                mScrollToggleOn = !mScrollToggleOn;

                if (DEBUG) {

                    Slog.d(LOG_TAG, "Scroll toggle " + (mScrollToggleOn ? "ON" : "OFF"));

                }

            } else if (keyCode == MouseKeyEvent.HOLD.getKeyCodeValue()) {

            } else if (keyCode == MouseKeyEvent.HOLD.getKeyCode(inputDevice)) {

                sendVirtualMouseButtonEvent(

                        VirtualMouseButtonEvent.BUTTON_PRIMARY,

                        VirtualMouseButtonEvent.ACTION_BUTTON_PRESS

                );

            } else if (keyCode == MouseKeyEvent.RELEASE.getKeyCodeValue()) {

            } else if (keyCode == MouseKeyEvent.RELEASE.getKeyCode(inputDevice)) {

                sendVirtualMouseButtonEvent(

                        VirtualMouseButtonEvent.BUTTON_PRIMARY,

                        VirtualMouseButtonEvent.ACTION_BUTTON_RELEASE

                );

            } else if (isMouseButtonKey(keyCode)) {

            } else if (isMouseButtonKey(keyCode, inputDevice)) {

                performMouseButtonAction(keyCode);

            } else if (mScrollToggleOn && isMouseScrollKey(keyCode)) {

            } else if (mScrollToggleOn && isMouseScrollKey(keyCode, inputDevice)) {

                // If the scroll key is pressed down and no other key is active,

                // set it as the active key and send a message to scroll the pointer

                if (mActiveScrollKey == KEY_NOT_SET) {
@@ -439,7 +523,8 @@ public class MouseKeysInterceptor extends BaseEventStreamTransformation 
                mHandler.removeMessages(MESSAGE_SCROLL_MOUSE_POINTER);

            } else {

                Slog.i(LOG_TAG, "Dropping event with key code: '" + keyCode

                        + "', with no matching down event from deviceId = " + event.getDeviceId());

                        + "', with no matching down event from deviceId = "

                        + event.getDeviceId());

            }

        }

    }
@@ -503,12 +588,40 @@ public class MouseKeysInterceptor extends BaseEventStreamTransformation 
    @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE)

    @Override

    public void onDestroy() {

        mHandler.post(() -> {

            // Clear mouse state

            mActiveMoveKey = KEY_NOT_SET;

            mActiveScrollKey = KEY_NOT_SET;

            mLastTimeKeyActionPerformed = 0;

            mDeviceKeyCodeMap.clear();

        });



        mHandler.removeCallbacksAndMessages(null);

        mVirtualDevice.close();

    }



    @Override

    public void onInputDeviceAdded(int deviceId) {

    }



    @Override

    public void onInputDeviceRemoved(int deviceId) {

        mDeviceKeyCodeMap.remove(deviceId);

    }



    /**

     * The user can change the keyboard layout from settings at anytime, which would change

     * key character map for that device. Hence, we should use this callback to

     * update the key code to enum mapping if there is a change in the physical keyboard detected.

     *

     * @param deviceId The id of the input device that changed.

     */

    @Override

    public void onInputDeviceChanged(int deviceId) {

        InputDevice inputDevice = mInputManager.getInputDevice(deviceId);

        // Update the enum mapping only if input device that changed is a keyboard

        if (inputDevice.isFullKeyboard() && !mDeviceKeyCodeMap.contains(deviceId)) {

            initializeDeviceToEnumMap(inputDevice);

        }

    }

}

services/tests/servicestests/src/com/android/server/accessibility/MouseKeysInterceptorTest.kt

+20 −1
Original line number Diff line number Diff line
@@ -48,6 +48,7 @@ import org.mockito.MockitoAnnotations 
import java.util.LinkedList

import java.util.Queue

import android.util.ArraySet

import android.view.InputDevice



/**

 * Tests for {@link MouseKeysInterceptor}
@@ -68,6 +69,8 @@ class MouseKeysInterceptorTest { 
    }



    private lateinit var mouseKeysInterceptor: MouseKeysInterceptor

    private lateinit var inputDevice: InputDevice



    private val clock = OffsettableClock()

    private val testLooper = TestLooper { clock.now() }

    private val nextInterceptor = TrackingInterceptor()
@@ -98,6 +101,10 @@ class MouseKeysInterceptorTest { 
        testSession = InputManagerGlobal.createTestSession(iInputManager)

        mockInputManager = InputManager(context)



        inputDevice = createInputDevice(DEVICE_ID)

        Mockito.`when`(iInputManager.getInputDevice(DEVICE_ID))

                .thenReturn(inputDevice)



        Mockito.`when`(mockVirtualDeviceManagerInternal.getDeviceIdsForUid(Mockito.anyInt()))

            .thenReturn(ArraySet(setOf(DEVICE_ID)))

        LocalServices.removeServiceForTest(VirtualDeviceManagerInternal::class.java)
@@ -115,7 +122,8 @@ class MouseKeysInterceptorTest { 
        Mockito.`when`(iInputManager.inputDeviceIds).thenReturn(intArrayOf(DEVICE_ID))

        Mockito.`when`(mockAms.traceManager).thenReturn(mockTraceManager)



        mouseKeysInterceptor = MouseKeysInterceptor(mockAms, testLooper.looper, DISPLAY_ID)

        mouseKeysInterceptor = MouseKeysInterceptor(mockAms, mockInputManager,

                testLooper.looper, DISPLAY_ID)

        mouseKeysInterceptor.next = nextInterceptor

    }
@@ -281,6 +289,17 @@ class MouseKeysInterceptorTest { 
        }

    }



    private fun createInputDevice(

            deviceId: Int,

            generation: Int = -1

    ): InputDevice =

            InputDevice.Builder()

                    .setId(deviceId)

                    .setName("Device $deviceId")

                    .setDescriptor("descriptor $deviceId")

                    .setGeneration(generation)

                    .build()



    private class TrackingInterceptor : BaseEventStreamTransformation() {

        val events: Queue<KeyEvent> = LinkedList()