Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 79476f75 authored by Asmita Poddar's avatar Asmita Poddar Committed by Biswarup Pal
Browse files

StabilityCalculator in VDM 

Implement a StabilityCalculator in VDM for consolidating
platform signals for computer control stability. Currently, it
provides a temporary solution by triggering the stability listener
after a fixed timeout on input events or application launches.
Long term, we would add more functionality here to improve the
stability detection by incorporating actual signals from the
framework.
This change adds new APIs for stability listener, and deprecates
the existing stability hints APIs.

Test: atest ComputerControlSessionTest, atest ComputerControlTests
Bug: 428957982
Flag: android.companion.virtualdevice.flags.computer_control_access
Change-Id: I41ef50cd1dd084f7d14cd7c09256d6201c8ad9e5
parent 67b4e66c

core/java/android/companion/virtual/computercontrol/ComputerControlSession.java

+62 −1
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ 


package android.companion.virtual.computercontrol;



import android.annotation.CallbackExecutor;

import android.annotation.IntDef;

import android.annotation.IntRange;

import android.annotation.NonNull;
@@ -318,6 +319,35 @@ public final class ComputerControlSession implements AutoCloseable { 
        }

    }



    /**

     * Sets a {@link StabilityListener} to be notified when the computer control session is

     * potentially stable.

     *

     * @throws IllegalStateException if a listener was previously set.

     */

    public void setStabilityListener(@NonNull @CallbackExecutor Executor executor,

            @NonNull StabilityListener listener) {

        Objects.requireNonNull(executor);

        Objects.requireNonNull(listener);

        try {

            mSession.setStabilityListener(new StabilityListenerProxy(executor, listener));

        } catch (RemoteException e) {

            throw e.rethrowFromSystemServer();

        }

    }



    /**

     * Clears any {@link StabilityListener} that was previously set using

     * {@link #setStabilityListener(Executor, StabilityListener)}.

     */

    public void clearStabilityListener() {

        try {

            mSession.setStabilityListener(null);

        } catch (RemoteException e) {

            throw e.rethrowFromSystemServer();

        }

    }



    @Override

    public void close() {

        try {
@@ -369,6 +399,20 @@ public final class ComputerControlSession implements AutoCloseable { 
        void onSessionClosed();

    }



    /**

     * Listener to be notified of signals indicating that the computer control session is

     * potentially stable.

     *

     * <p>These signals indicate that the session's display content is currently stable, such as the

     * app under automation being idle and no UI animations being under progress. These are useful

     * for tasks that should only run on a static UI, such as taking screenshots to determine the

     * next step.

     */

    public interface StabilityListener {

        /** Called when the computer control session is considered stable. */

        void onSessionStable();

    }



    /** @hide */

    public static class CallbackProxy extends IComputerControlSessionCallback.Stub {
@@ -409,4 +453,21 @@ public final class ComputerControlSession implements AutoCloseable { 
                    mExecutor.execute(() -> mCallback.onSessionClosed()));

        }

    }



    private static class StabilityListenerProxy extends IComputerControlStabilityListener.Stub {



        private final Executor mExecutor;

        private final StabilityListener mListener;



        StabilityListenerProxy(@NonNull Executor executor,

                @NonNull StabilityListener listener) {

            mExecutor = executor;

            mListener = listener;

        }



        @Override

        public void onSessionStable() {

            Binder.withCleanCallingIdentity(() -> mExecutor.execute(mListener::onSessionStable));

        }

    }

}
 
 No newline at end of file

core/java/android/companion/virtual/computercontrol/IComputerControlSession.aidl

+4 −0
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ 


package android.companion.virtual.computercontrol;



import android.companion.virtual.computercontrol.IComputerControlStabilityListener;

import android.companion.virtual.computercontrol.IInteractiveMirrorDisplay;

import android.hardware.input.VirtualKeyEvent;

import android.hardware.input.VirtualTouchEvent;
@@ -66,6 +67,9 @@ interface IComputerControlSession { 
    /** Performs computer control action on the computer control display. */

    void performAction(int actionCode);



    /** Sets a listener to be notified when the computer control session is potentially stable. */

    void setStabilityListener(in IComputerControlStabilityListener listener);



    /** Closes this session. */

    void close();

}

core/java/android/companion/virtual/computercontrol/IComputerControlStabilityListener.aidl

0 → 100644
+28 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2025 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



 package android.companion.virtual.computercontrol;



/**

 * Listener for receiving stability signals from a ComputerControlSession.

 * @hide

 */

oneway interface IComputerControlStabilityListener {

    /**

     * Called when the session's display content is considered stable.

     */

    void onSessionStable();

}

libs/computercontrol/src/com/android/extensions/computercontrol/ComputerControlSession.java

+44 −2
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ 
package com.android.extensions.computercontrol;



import android.accessibilityservice.AccessibilityServiceInfo;

import android.annotation.CallbackExecutor;

import android.annotation.IntRange;

import android.app.ActivityOptions;

import android.companion.virtual.computercontrol.ComputerControlSession.Action;
@@ -46,6 +47,7 @@ import com.android.extensions.computercontrol.input.TouchEvent; 
import java.util.ArrayList;

import java.util.List;

import java.util.Objects;

import java.util.concurrent.Executor;

import java.util.concurrent.Executors;

import java.util.concurrent.atomic.AtomicBoolean;
@@ -248,16 +250,42 @@ public final class ComputerControlSession implements AutoCloseable { 
        return new InteractiveMirror(interactiveMirrorDisplay);

    }



    /** Add a callback to be notified when the computer control session is potentially stable. */

    /**

     * Add a callback to be notified when the computer control session is potentially stable.

     * @deprecated use {@link #setStabilityListener(StabilityListener, Executor)}

     */

    public void addStabilityHintCallback(long timeoutMs, @NonNull StabilityHintCallback callback) {

        mAccessibilityProxy.registerStabilityHintCallback(timeoutMs, callback);

    }



    /** Remove a stability hint callback that was previously added. */

    /**

     * Remove a stability hint callback that was previously added.

     * @deprecated use {@link #clearStabilityListener()}

     */

    public void removeStabilityHintCallback(@NonNull StabilityHintCallback callback) {

        mAccessibilityProxy.removeStabilityHintCallback(callback);

    }



    /**

     * Sets a {@link StabilityListener} to be notified when the computer control session is

     * potentially stable.

     *

     * @throws IllegalStateException if a listener was previously set.

     */

    public void setStabilityListener(@NonNull @CallbackExecutor Executor executor,

            @NonNull StabilityListener listener) {

        Objects.requireNonNull(listener);

        mSession.setStabilityListener(executor, listener::onSessionStable);

    }



    /**

     * Clears any {@link StabilityListener} that was previously set using

     * {@link #setStabilityListener(Executor, StabilityListener)}.

     */

    public void clearStabilityListener() {

        mSession.clearStabilityListener();

    }



    /**

     * Callback used to inform the session owner that the computer control session has potentially

     * reached a stable state. This can be used as a hint to determine when an action that was
@@ -552,6 +580,20 @@ public final class ComputerControlSession implements AutoCloseable { 
        void onSessionClosed();

    }



    /**

     * Listener to be notified of signals indicating that the computer control session is

     * potentially stable.

     *

     * <p>These signals indicate that the session's display content is currently stable, such as the

     * app under automation being idle and no UI animations being under progress. These are useful

     * for tasks that should only run on a static UI, such as taking screenshots to determine the

     * next step.

     */

    public interface StabilityListener {

        /** Called when the computer control session is considered stable. */

        void onSessionStable();

    }



    @VisibleForTesting

    static class ComputerControlAccessibilityProxy extends AccessibilityDisplayProxy {

        @Nullable

libs/computercontrol/tests/src/com/android/extensions/computercontrol/ComputerControlSessionTest.java

+28 −0
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ import static org.mockito.Mockito.timeout; 
import static org.mockito.Mockito.times;

import static org.mockito.Mockito.verify;

import static org.mockito.Mockito.when;

import static org.testng.Assert.assertThrows;



import android.app.Activity;

import android.companion.virtual.computercontrol.IComputerControlSession;
@@ -59,6 +60,8 @@ import org.mockito.ArgumentCaptor; 
import org.mockito.Mock;

import org.mockito.MockitoAnnotations;



import java.util.concurrent.Executors;



@RequiresFlagsEnabled(android.companion.virtualdevice.flags.Flags.FLAG_COMPUTER_CONTROL_ACCESS)

@RunWith(AndroidJUnit4.class)

public class ComputerControlSessionTest {
@@ -74,6 +77,7 @@ public class ComputerControlSessionTest { 
    @Mock IComputerControlSession mIComputerControlSession;

    private ComputerControlSession mSession;

    private int mVirtualDisplayId;

    @Mock ComputerControlSession.StabilityListener mStabilityListener;

    @Mock ComputerControlSession.StabilityHintCallback mStabilityHintCallback;

    @Mock IAccessibilityManager mIAccessibilityManager;
@@ -234,4 +238,28 @@ public class ComputerControlSessionTest { 


        verify(mStabilityHintCallback, timeout(TIMEOUT_MS)).onStabilityHint(false);

    }



    @Test

    public void setStabilityListener_withNullListener_throwsException() {

        assertThrows(NullPointerException.class, () -> mSession.setStabilityListener(

                Executors.newSingleThreadExecutor(), null));

    }



    @Test

    public void setStabilityListener_withNullExecutor_throwsException() {

        assertThrows(NullPointerException.class,

                () -> mSession.setStabilityListener(null, mStabilityListener));

    }



    @Test

    public void setStabilityListener_setsStabilityListener() throws Exception {

        mSession.setStabilityListener(Executors.newSingleThreadExecutor(), mStabilityListener);

        verify(mIComputerControlSession).setStabilityListener(any());

    }



    @Test

    public void clearStabilityListener_clearsStabilityListener() throws Exception {

        mSession.clearStabilityListener();

        verify(mIComputerControlSession).setStabilityListener(eq(null));

    }

}