Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6a5f4d9d authored by Vladimir Komsiyski's avatar Vladimir Komsiyski
Browse files

Tap and swipe API in relative display space. 

When the display is created in the framework, the client doesn't
have a good signal about the display dimensions. They may have
changed since the last screenshot is taken. For max robustness,
convert from relative to pixel coordinates in the system server.

Also fixing the bug where the thread is blocked for 500ms during
a swipe. Now the swipe is scheduled in the background, no blocking
is done and the call returns immediately.

Also refactoring ComputerControlSessionImpl to make it a bit more
testable.

Bug: 428284603
Fix: 439774796
Test: atest & presubmit
Flag: android.companion.virtualdevice.flags.computer_control_access
Change-Id: I695e773363259b98430c4f77ba09d192fbfd7c3c
parent f3081eda

core/java/android/companion/virtual/computercontrol/ComputerControlSession.java

+45 −0
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ 


package android.companion.virtual.computercontrol;



import android.annotation.FloatRange;

import android.annotation.IntDef;

import android.annotation.IntRange;

import android.annotation.NonNull;
@@ -143,6 +144,50 @@ public final class ComputerControlSession implements AutoCloseable { 
        return mImageReader == null ? null : mImageReader.acquireLatestImage();

    }



    /**

     * Sends a tap event to the computer control session at the given location.

     *

     * <p>The coordinates are in relative display space, e.g. (0.5, 0.5) is the center of the

     * display.</p>

     */

    public void tap(@FloatRange(from = 0.0, to = 1.0) float x,

            @FloatRange(from = 0.0, to = 1.0) float y) {

        if (x < 0 || x > 1 || y < 0 || y > 1) {

            throw new IllegalArgumentException("Tap coordinates must be in range [0, 1]");

        }

        try {

            mSession.tap(x, y);

        } catch (RemoteException e) {

            throw e.rethrowFromSystemServer();

        }

    }



    /**

     * Sends a swipe event to the computer control session for the given coordinates.

     *

     * <p>To avoid misinterpreting the swipe as a fling, the individual touches are throttled, so

     * the entire action will take ~500ms. However, this is done in the background and this method

     * returns immediately. Any ongoing swipe will be canceled if a new swipe is requested.</p>

     *

     * <p>The coordinates are in relative display space, e.g. (0.5, 0.5) is the center of the

     * display.</p>

     */

    public void swipe(

            @FloatRange(from = 0.0, to = 1.0) float fromX,

            @FloatRange(from = 0.0, to = 1.0) float fromY,

            @FloatRange(from = 0.0, to = 1.0) float toX,

            @FloatRange(from = 0.0, to = 1.0) float toY) {

        if (fromX < 0 || fromX > 1 || fromY < 0 || fromY > 1

                || toX < 0 || toX > 1 || toY < 0 || toY > 1) {

            throw new IllegalArgumentException("Swipe coordinates must be in range [0, 1]");

        }

        try {

            mSession.swipe(fromX, fromY, toX, toY);

        } catch (RemoteException e) {

            throw e.rethrowFromSystemServer();

        }

    }



    /** Returns the ID of the single trusted virtual display for this session. */

    public int getVirtualDisplayId() {

        try {

core/java/android/companion/virtual/computercontrol/IComputerControlSession.aidl

+6 −0
Original line number Diff line number Diff line
@@ -28,6 +28,12 @@ import android.view.Surface; 
 */

interface IComputerControlSession {



    /* Injects a tap event into the trusted virtual display. */

    void tap(float x, float y);



    /* Injects a swipe event into the trusted virtual display. */

    void swipe(float fromX, float fromY, float toX, float toY);



    /** Returns the ID of the single trusted virtual display for this session. */

    int getVirtualDisplayId();

core/tests/coretests/src/android/companion/virtual/computercontrol/ComputerControlSessionTest.java

+41 −0
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ import static org.mockito.ArgumentMatchers.any; 
import static org.mockito.ArgumentMatchers.eq;

import static org.mockito.Mockito.verify;

import static org.mockito.Mockito.when;

import static org.testng.Assert.assertThrows;



import android.hardware.display.DisplayManagerGlobal;

import android.hardware.display.IDisplayManager;
@@ -126,4 +127,44 @@ public class ComputerControlSessionTest { 
        mSession.close();

        verify(mMockSession).close();

    }



    @Test

    public void tap_taps() throws RemoteException {

        mSession.tap(0.1f, 0.2f);

        verify(mMockSession).tap(eq(0.1f), eq(0.2f));

    }



    @Test

    public void tapNotInRange_throws() {

        assertThrows(IllegalArgumentException.class, () -> mSession.tap(-0.1f, 0.2f));

        assertThrows(IllegalArgumentException.class, () -> mSession.tap(1.1f, 0.2f));

        assertThrows(IllegalArgumentException.class, () -> mSession.tap(0.1f, -0.2f));

        assertThrows(IllegalArgumentException.class, () -> mSession.tap(0.1f, 1.2f));

    }



    @Test

    public void swipe_swipes() throws RemoteException {

        mSession.swipe(0.1f, 0.2f, 0.3f, 0.4f);

        verify(mMockSession).swipe(eq(0.1f), eq(0.2f), eq(0.3f), eq(0.4f));

    }



    @Test

    public void swipeNotInRange_throws() {

        assertThrows(IllegalArgumentException.class,

                () -> mSession.swipe(-0.1f, 0.2f, 0.3f, 0.4f));

        assertThrows(IllegalArgumentException.class,

                () -> mSession.swipe(1.1f, 0.2f, 0.3f, 0.4f));

        assertThrows(IllegalArgumentException.class,

                () -> mSession.swipe(0.1f, -0.2f, 0.3f, 0.4f));

        assertThrows(IllegalArgumentException.class,

                () -> mSession.swipe(0.1f, 1.2f, 0.3f, 0.4f));

        assertThrows(IllegalArgumentException.class,

                () -> mSession.swipe(0.1f, 0.2f, -0.3f, 0.4f));

        assertThrows(IllegalArgumentException.class,

                () -> mSession.swipe(0.1f, 0.2f, 1.3f, 0.4f));

        assertThrows(IllegalArgumentException.class,

                () -> mSession.swipe(0.1f, 0.2f, 0.3f, -0.4f));

        assertThrows(IllegalArgumentException.class,

                () -> mSession.swipe(0.1f, 0.2f, 0.3f, 1.4f));

    }

}

libs/computercontrol/src/com/android/extensions/computercontrol/ComputerControlSession.java

+30 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ 
package com.android.extensions.computercontrol;



import android.accessibilityservice.AccessibilityServiceInfo;

import android.annotation.FloatRange;

import android.app.ActivityOptions;

import android.companion.virtual.computercontrol.InteractiveMirrorDisplay;

import android.content.Context;
@@ -111,6 +112,35 @@ public final class ComputerControlSession implements AutoCloseable { 
        return mSession.getScreenshot();

    }



    /**

     * Sends a tap event to the computer control session at the given location.

     *

     * <p>The coordinates are in relative display space, e.g. (0.5, 0.5) is the center of the

     * display.</p>

     */

    public void tap(@FloatRange(from = 0.0, to = 1.0) float x,

            @FloatRange(from = 0.0, to = 1.0) float y) {

        mSession.tap(x, y);

    }



    /**

     * Sends a swipe event to the computer control session for the given coordinates.

     *

     * <p>To avoid misinterpreting the swipe as a fling, the individual touches are throttled, so

     * the entire action will take ~500ms. However, this is done in the background and this method

     * returns immediately. Any ongoing swipe will be canceled if a new swipe is requested.</p>

     *

     * <p>The coordinates are in relative display space, e.g. (0.5, 0.5) is the center of the

     * display.</p>

     */

    public void swipe(

            @FloatRange(from = 0.0, to = 1.0) float fromX,

            @FloatRange(from = 0.0, to = 1.0) float fromY,

            @FloatRange(from = 0.0, to = 1.0) float toX,

            @FloatRange(from = 0.0, to = 1.0) float toY) {

        mSession.swipe(fromX, fromY, toX, toY);

    }



    /**

     * Injects a {@link TouchEvent} into the computer control session.

     */

libs/computercontrol/tests/src/com/android/extensions/computercontrol/ComputerControlSessionTest.java

+12 −0
Original line number Diff line number Diff line
@@ -127,6 +127,18 @@ public class ComputerControlSessionTest { 
        verify(mIComputerControlSession, times(1)).close();

    }



    @Test

    public void tap_taps() throws Exception {

        mSession.tap(0.1f, 0.2f);

        verify(mIComputerControlSession).tap(eq(0.1f), eq(0.2f));

    }



    @Test

    public void swipe_swipes() throws Exception {

        mSession.swipe(0.1f, 0.2f, 0.3f, 0.4f);

        verify(mIComputerControlSession).swipe(eq(0.1f), eq(0.2f), eq(0.3f), eq(0.4f));

    }



    @Test

    public void sendKeyEvent_sendsKeyEvent() throws Exception {

        KeyEvent event = new KeyEvent.Builder()