Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 28b528f7 authored by Charles Chen's avatar Charles Chen
Browse files

Add VisaulQueryDetector API

Added VisualQueryDetector to manage VisualQueryDetectionService
lifecycle and provide corresponding methods to start service
functionalities.

Bug: 261783492
Test: atest CtsVoiceInteractionTestCases
Change-Id: I2b1d37ab246b28e27f75bdaf8d9ad8f2d36d6dc7
API-Coverage-Bug: 264039061
parent e2d41760
Loading
Loading
Loading
Loading
+17 −0
Original line number Diff line number Diff line
@@ -12564,11 +12564,28 @@ package android.service.voice {
    method public void onQueryRejected() throws java.lang.IllegalStateException;
  }
  public class VisualQueryDetector {
    method public void destroy();
    method @RequiresPermission(allOf={android.Manifest.permission.CAMERA, android.Manifest.permission.RECORD_AUDIO}) public boolean startRecognition() throws android.service.voice.HotwordDetector.IllegalDetectorStateException;
    method @RequiresPermission(allOf={android.Manifest.permission.CAMERA, android.Manifest.permission.RECORD_AUDIO}) public boolean stopRecognition() throws android.service.voice.HotwordDetector.IllegalDetectorStateException;
    method public void updateState(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory) throws android.service.voice.HotwordDetector.IllegalDetectorStateException;
  }
  public static interface VisualQueryDetector.Callback {
    method public void onError();
    method public void onQueryDetected(@NonNull String);
    method public void onQueryFinished();
    method public void onQueryRejected();
    method public void onVisualQueryDetectionServiceInitialized(int);
    method public void onVisualQueryDetectionServiceRestarted();
  }
  public class VoiceInteractionService extends android.app.Service {
    method @NonNull public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, android.service.voice.AlwaysOnHotwordDetector.Callback);
    method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, @Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, android.service.voice.AlwaysOnHotwordDetector.Callback);
    method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.HotwordDetector createHotwordDetector(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, @NonNull android.service.voice.HotwordDetector.Callback);
    method @NonNull @RequiresPermission("android.permission.MANAGE_VOICE_KEYPHRASES") public final android.media.voice.KeyphraseModelManager createKeyphraseModelManager();
    method @NonNull public final android.service.voice.VisualQueryDetector createVisualQueryDetector(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, @NonNull java.util.concurrent.Executor, @NonNull android.service.voice.VisualQueryDetector.Callback);
  }
}
+20 −2
Original line number Diff line number Diff line
@@ -84,7 +84,7 @@ abstract class AbstractDetector implements HotwordDetector {
            @Nullable SharedMemory sharedMemory);

    /**
     * Detect hotword from an externally supplied stream of data.
     * Detect from an externally supplied stream of data.
     *
     * @return {@code true} if the request to start recognition succeeded
     */
@@ -114,7 +114,25 @@ abstract class AbstractDetector implements HotwordDetector {
        return true;
    }

    /** {@inheritDoc} */
    /**
     * Set configuration and pass read-only data to trusted detection service.
     *
     * @param options Application configuration data to provide to the
     *         {@link VisualQueryDetectionService} and {@link HotwordDetectionService}.
     *         PersistableBundle does not allow any remotable objects or other contents that can be
     *         used to communicate with other processes.
     * @param sharedMemory The unrestricted data blob to provide to the
     *        {@link VisualQueryDetectionService} and {@link HotwordDetectionService}. Use this to
     *         provide the hotword models data or other such data to the trusted process.
     * @throws IllegalDetectorStateException Thrown when a caller has a target SDK of
     *         Android Tiramisu or above and attempts to start a recognition when the detector is
     *         not able based on the state. Because the caller receives updates via an asynchronous
     *         callback and the state of the detector can change without caller's knowledge, a
     *         checked exception is thrown.
     * @throws IllegalStateException if this {@link HotwordDetector} wasn't specified to use a
     *         {@link HotwordDetectionService} or {@link VisualQueryDetectionService} when it was
     *         created.
     */
    @Override
    public void updateState(@Nullable PersistableBundle options,
            @Nullable SharedMemory sharedMemory) throws IllegalDetectorStateException {
+24 −7
Original line number Diff line number Diff line
@@ -35,7 +35,8 @@ import android.util.AndroidException;
import java.io.PrintWriter;

/**
 * Basic functionality for sandboxed detectors.
 * Basic functionality for sandboxed detectors. This interface will be used by detectors that
 * manages their service lifecycle.
 *
 * @hide
 */
@@ -80,10 +81,21 @@ public interface HotwordDetector {
     */
    int DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE = 2;

    /**
     * Indicates that it is a visual query detector.
     *
     * @hide
     */
    int DETECTOR_TYPE_VISUAL_QUERY_DETECTOR = 3;

    /**
     * Starts sandboxed detection recognition.
     * <p>
     * On calling this, the system streams audio from the device microphone to this application's
     * If a {@link VisualQueryDetector} calls this method, {@link VisualQueryDetectionService
     * #onStartDetection(VisualQueryDetectionService.Callback)} will be called to start detection.
     * <p>
     * Otherwise if a {@link AlwaysOnHotwordDetector} or {@link SoftwareHotwordDetector} calls this,
     * the system streams audio from the device microphone to this application's
     * {@link HotwordDetectionService}. Audio is streamed until {@link #stopRecognition()} is
     * called.
     * <p>
@@ -192,6 +204,8 @@ public interface HotwordDetector {
                return "trusted_hotword_dsp";
            case DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE:
                return "trusted_hotword_software";
            case DETECTOR_TYPE_VISUAL_QUERY_DETECTOR:
                return "visual_query_detector";
            default:
                return Integer.toString(detectorType);
        }
@@ -244,18 +258,21 @@ public interface HotwordDetector {
        void onRejected(@NonNull HotwordRejectedResult result);

        /**
         * Called when the {@link HotwordDetectionService} is created by the system and given a
         * short amount of time to report its initialization state.
         * Called when the {@link HotwordDetectionService} or {@link VisualQueryDetectionService} is
         * created by the system and given a short amount of time to report their initialization
         * state.
         *
         * @param status Info about initialization state of {@link HotwordDetectionService}; the
         * allowed values are {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_SUCCESS},
         * @param status Info about initialization state of {@link HotwordDetectionService} or
         * {@link VisualQueryDetectionService}; allowed values are
         * {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_SUCCESS},
         * 1<->{@link SandboxedDetectionServiceBase#getMaxCustomInitializationStatus()},
         * {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_UNKNOWN}.
         */
        void onHotwordDetectionServiceInitialized(int status);

        /**
         * Called with the {@link HotwordDetectionService} is restarted.
         * Called with the {@link HotwordDetectionService} or {@link VisualQueryDetectionService} is
         * restarted.
         *
         * Clients are expected to call {@link HotwordDetector#updateState} to share the state with
         * the newly created service.
+241 −0
Original line number Diff line number Diff line
/*
 * Copyright (C) 2023 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package android.service.voice;

import static android.Manifest.permission.CAMERA;
import static android.Manifest.permission.RECORD_AUDIO;

import android.annotation.CallbackExecutor;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.annotation.SuppressLint;
import android.annotation.SystemApi;
import android.media.AudioFormat;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.SharedMemory;
import android.util.Slog;

import com.android.internal.app.IVoiceInteractionManagerService;

import java.io.PrintWriter;
import java.util.concurrent.Executor;
import java.util.function.Consumer;

/**
 * Manages VisualQueryDetectionService.
 *
 * This detector provides necessary functionalities to initialize, start, update and destroy a
 * {@link VisualQueryDetectionService}.
 *
 * @hide
 **/
@SystemApi
@SuppressLint("NotCloseable")
public class VisualQueryDetector {
    private static final String TAG = VisualQueryDetector.class.getSimpleName();
    private static final boolean DEBUG = false;

    private final Callback mCallback;
    private final Executor mExecutor;
    private final IVoiceInteractionManagerService mManagerService;
    private final VisualQueryDetectorInitializationDelegate mInitializationDelegate;

    VisualQueryDetector(
            IVoiceInteractionManagerService managerService,
            @NonNull @CallbackExecutor Executor executor,
            Callback callback) {

        mManagerService = managerService;
        mCallback = callback;
        mExecutor = executor;
        mInitializationDelegate = new VisualQueryDetectorInitializationDelegate();
    }

    /**
     * Initialize the {@link VisualQueryDetectionService} by passing configurations and read-only
     * data.
     */
    void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) {
        mInitializationDelegate.initialize(options, sharedMemory);
    }

    /**
     * Set configuration and pass read-only data to {@link VisualQueryDetectionService}.
     *
     * @see HotwordDetector#updateState(PersistableBundle, SharedMemory)
     */
    public void updateState(@Nullable PersistableBundle options,
            @Nullable SharedMemory sharedMemory) throws
            HotwordDetector.IllegalDetectorStateException {
        mInitializationDelegate.updateState(options, sharedMemory);
    }


    /**
     * On calling this method, {@link VisualQueryDetectionService
     * #onStartDetection(VisualQueryDetectionService.Callback)} will be called to start using
     * visual signals such as camera frames and microphone audio to perform detection. When user
     * attention is captured and the {@link VisualQueryDetectionService} streams queries,
     * {@link VisualQueryDetector.Callback#onQueryDetected(String)} is called to control the
     * behavior of handling {@code transcribedText}. When the query streaming is finished,
     * {@link VisualQueryDetector.Callback#onQueryFinished()} is called. If the current streamed
     * query is invalid, {@link VisualQueryDetector.Callback#onQueryRejected()} is called to abandon
     * the streamed query.
     *
     * @see HotwordDetector#startRecognition()
     */
    @RequiresPermission(allOf = {CAMERA, RECORD_AUDIO})
    public boolean startRecognition() throws HotwordDetector.IllegalDetectorStateException {
        if (DEBUG) {
            Slog.i(TAG, "#startRecognition");
        }
        // TODO(b/261783819): Call StartDetection on VisualQueryDetectionService with the system.
        return false;
    }

    /**
     * Stops visual query detection recognition.
     *
     * @see HotwordDetector#stopRecognition()
     */
    @RequiresPermission(allOf = {CAMERA, RECORD_AUDIO})
    public boolean stopRecognition() throws HotwordDetector.IllegalDetectorStateException {
        if (DEBUG) {
            Slog.i(TAG, "#stopRecognition");
        }
        // TODO(b/261783819): Call StopDetection on VisualQueryDetectionService with the system.
        return false;
    }

    /**
     * Destroy the current detector.
     *
     * @see HotwordDetector#destroy()
     */
    public void destroy() {
        if (DEBUG) {
            Slog.i(TAG, "#destroy");
        }
        mInitializationDelegate.destroy();
    }

    /** @hide */
    public void dump(String prefix, PrintWriter pw) {
        // TODO: implement this
    }

    /** @hide */
    public HotwordDetector getInitializationDelegate() {
        return mInitializationDelegate;
    }

    /** @hide */
    void registerOnDestroyListener(Consumer<AbstractDetector> onDestroyListener) {
        mInitializationDelegate.registerOnDestroyListener(onDestroyListener);
    }

    /**
     * A class that lets a VoiceInteractionService implementation interact with
     * visual query detection APIs.
     */
    public interface Callback {

        /**
         * Called when the {@link VisualQueryDetectionService} starts to stream partial queries.
         *
         * @param partialQuery The partial query in a text form being streamed.
         */
        void onQueryDetected(@NonNull String partialQuery);

        /**
         * Called when the {@link VisualQueryDetectionService} decides to abandon the streamed
         * partial queries.
         */
        void onQueryRejected();

        /**
         *  Called when the {@link VisualQueryDetectionService} finishes streaming partial queries.
         */
        void onQueryFinished();

        /**
         * Called when the {@link VisualQueryDetectionService} is created by the system and given a
         * short amount of time to report its initialization state.
         *
         * @param status Info about initialization state of {@link VisualQueryDetectionService}; the
         * allowed values are {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_SUCCESS},
         * 1<->{@link SandboxedDetectionServiceBase#getMaxCustomInitializationStatus()},
         * {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_UNKNOWN}.
         */
        void onVisualQueryDetectionServiceInitialized(int status);

         /**
         * Called with the {@link VisualQueryDetectionService} is restarted.
         *
         * Clients are expected to call {@link HotwordDetector#updateState} to share the state with
         * the newly created service.
         */
        void onVisualQueryDetectionServiceRestarted();

        /**
         * Called when the detection fails due to an error.
         */
        //TODO(b/265390855): Replace this callback with the new onError(DetectorError) design.
        void onError();
    }

    private class VisualQueryDetectorInitializationDelegate extends AbstractDetector {

        VisualQueryDetectorInitializationDelegate() {
            super(mManagerService, null);
        }

        @Override
        void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) {
            //TODO(261783492): call initAndVerify to create VisualQueryDetectionService
            // from the system server.
        }

        @Override
        public boolean stopRecognition() throws IllegalDetectorStateException {
            //No-op, we only reuse the initialization methods.
            return false;
        }

        @Override
        public boolean startRecognition() throws IllegalDetectorStateException {
            //No-op, we only reuse the initialization methods.
            return false;
        }

        @Override
        public final boolean startRecognition(
                @NonNull ParcelFileDescriptor audioStream,
                @NonNull AudioFormat audioFormat,
                @Nullable PersistableBundle options) throws IllegalDetectorStateException {
            //No-op, not supported by VisualQueryDetector as it should be trusted.
            return false;
        }

        @Override
        public boolean isUsingSandboxedDetectionService() {
            return true;
        }
    }
}
+87 −1
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@
package android.service.voice;

import android.Manifest;
import android.annotation.CallbackExecutor;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
@@ -58,7 +59,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Set;

import java.util.concurrent.Executor;
/**
 * Top-level service of the current global voice interactor, which is providing
 * support for hotwording, the back-end of a {@link android.app.VoiceInteractor}, etc.
@@ -164,6 +165,8 @@ public class VoiceInteractionService extends Service {

    IVoiceInteractionManagerService mSystemService;

    private VisualQueryDetector mActiveVisualQueryDetector;

    private final Object mLock = new Object();

    private KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo;
@@ -543,6 +546,85 @@ public class VoiceInteractionService extends Service {
        }
    }

    /**
     * Creates a {@link VisualQueryDetector} and initializes the application's
     * {@link VisualQueryDetectionService} using {@code options} and {@code sharedMemory}.
     *
     * <p>To be able to call this, you need to set android:visualQueryDetectionService in the
     * android.voice_interaction metadata file to a valid visual query detection service, and set
     * android:isolatedProcess="true" in the service's declaration. Otherwise, this throws an
     * {@link IllegalStateException}.
     *
     * <p>Using this has a noticeable impact on battery, since the microphone is kept open
     * for the lifetime of the recognition {@link VisualQueryDetector#startRecognition() session}.
     *
     * @param options Application configuration data to be provided to the
     * {@link VisualQueryDetectionService}. PersistableBundle does not allow any remotable objects
     * or other contents that can be used to communicate with other processes.
     * @param sharedMemory The unrestricted data blob to be provided to the
     * {@link VisualQueryDetectionService}. Use this to provide models or other such data to the
     * sandboxed process.
     * @param callback The callback to notify of detection events.
     * @return An instanece of {@link VisualQueryDetector}.
     * @throws UnsupportedOperationException if only single detector is supported. Multiple detector
     * is only available for apps targeting {@link Build.VERSION_CODES#TIRAMISU} and above.
     * @throws IllegalStateException when there is an existing {@link VisualQueryDetector}, or when
     * there is a non-trusted hotword detector running.
     *
     * @hide
     */
    // TODO: add MANAGE_HOTWORD_DETECTION permission to protect this API and update java doc.
    @SystemApi
    @NonNull
    public final VisualQueryDetector createVisualQueryDetector(
            @Nullable PersistableBundle options,
            @Nullable SharedMemory sharedMemory,
            @NonNull @CallbackExecutor Executor executor,
            @NonNull VisualQueryDetector.Callback callback) {
        Objects.requireNonNull(executor);
        Objects.requireNonNull(callback);

        if (mSystemService == null) {
            throw new IllegalStateException("Not available until onReady() is called");
        }
        synchronized (mLock) {
            if (!CompatChanges.isChangeEnabled(MULTIPLE_ACTIVE_HOTWORD_DETECTORS)) {
                throw new UnsupportedOperationException("VisualQueryDetector is only available if "
                        + "multiple detectors are allowed");
            } else {
                if (mActiveVisualQueryDetector != null) {
                    throw new IllegalStateException(
                                "There is already an active VisualQueryDetector. "
                                        + "It must be destroyed to create a new one.");
                }
                for (HotwordDetector detector : mActiveDetectors) {
                    if (!detector.isUsingSandboxedDetectionService()) {
                        throw new IllegalStateException(
                                "It disallows to create trusted and non-trusted detectors "
                                        + "at the same time.");
                    }
                }
            }

            VisualQueryDetector visualQueryDetector =
                    new VisualQueryDetector(mSystemService, executor, callback);
            HotwordDetector visualQueryDetectorInitializationDelegate =
                    visualQueryDetector.getInitializationDelegate();
            mActiveDetectors.add(visualQueryDetectorInitializationDelegate);

            try {
                visualQueryDetector.registerOnDestroyListener(this::onHotwordDetectorDestroyed);
                visualQueryDetector.initialize(options, sharedMemory);
            } catch (Exception e) {
                mActiveDetectors.remove(visualQueryDetectorInitializationDelegate);
                visualQueryDetector.destroy();
                throw e;
            }
            mActiveVisualQueryDetector = visualQueryDetector;
            return visualQueryDetector;
        }
    }

    /**
     * Creates an {@link KeyphraseModelManager} to use for enrolling voice models outside of the
     * pre-bundled system voice models.
@@ -598,6 +680,10 @@ public class VoiceInteractionService extends Service {

    private void onHotwordDetectorDestroyed(@NonNull HotwordDetector detector) {
        synchronized (mLock) {
            if (mActiveVisualQueryDetector!= null &&
                    detector == mActiveVisualQueryDetector.getInitializationDelegate()) {
                mActiveVisualQueryDetector = null;
            }
            mActiveDetectors.remove(detector);
            shutdownHotwordDetectionServiceIfRequiredLocked();
        }