Add VisaulQueryDetector API (28b528f7) · Commits · e / os / android_frameworks_base

core/api/system-current.txt

+17 −0

Original line number	Diff line number	Diff line
		@@ -12564,11 +12564,28 @@ package android.service.voice {
		method public void onQueryRejected() throws java.lang.IllegalStateException;
		}

		public class VisualQueryDetector {
		method public void destroy();
		method @RequiresPermission(allOf={android.Manifest.permission.CAMERA, android.Manifest.permission.RECORD_AUDIO}) public boolean startRecognition() throws android.service.voice.HotwordDetector.IllegalDetectorStateException;
		method @RequiresPermission(allOf={android.Manifest.permission.CAMERA, android.Manifest.permission.RECORD_AUDIO}) public boolean stopRecognition() throws android.service.voice.HotwordDetector.IllegalDetectorStateException;
		method public void updateState(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory) throws android.service.voice.HotwordDetector.IllegalDetectorStateException;
		}

		public static interface VisualQueryDetector.Callback {
		method public void onError();
		method public void onQueryDetected(@NonNull String);
		method public void onQueryFinished();
		method public void onQueryRejected();
		method public void onVisualQueryDetectionServiceInitialized(int);
		method public void onVisualQueryDetectionServiceRestarted();
		}

		public class VoiceInteractionService extends android.app.Service {
		method @NonNull public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, android.service.voice.AlwaysOnHotwordDetector.Callback);
		method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, @Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, android.service.voice.AlwaysOnHotwordDetector.Callback);
		method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.HotwordDetector createHotwordDetector(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, @NonNull android.service.voice.HotwordDetector.Callback);
		method @NonNull @RequiresPermission("android.permission.MANAGE_VOICE_KEYPHRASES") public final android.media.voice.KeyphraseModelManager createKeyphraseModelManager();
		method @NonNull public final android.service.voice.VisualQueryDetector createVisualQueryDetector(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, @NonNull java.util.concurrent.Executor, @NonNull android.service.voice.VisualQueryDetector.Callback);
		}

		}

core/java/android/service/voice/AbstractDetector.java

+20 −2

Original line number	Diff line number	Diff line
		@@ -84,7 +84,7 @@ abstract class AbstractDetector implements HotwordDetector {
		@Nullable SharedMemory sharedMemory);

		/**
		* Detect hotword from an externally supplied stream of data.
		* Detect from an externally supplied stream of data.
		*
		* @return {@code true} if the request to start recognition succeeded
		*/
		@@ -114,7 +114,25 @@ abstract class AbstractDetector implements HotwordDetector {
		return true;
		}

		/** {@inheritDoc} */
		/**
		* Set configuration and pass read-only data to trusted detection service.
		*
		* @param options Application configuration data to provide to the
		* {@link VisualQueryDetectionService} and {@link HotwordDetectionService}.
		* PersistableBundle does not allow any remotable objects or other contents that can be
		* used to communicate with other processes.
		* @param sharedMemory The unrestricted data blob to provide to the
		* {@link VisualQueryDetectionService} and {@link HotwordDetectionService}. Use this to
		* provide the hotword models data or other such data to the trusted process.
		* @throws IllegalDetectorStateException Thrown when a caller has a target SDK of
		* Android Tiramisu or above and attempts to start a recognition when the detector is
		* not able based on the state. Because the caller receives updates via an asynchronous
		* callback and the state of the detector can change without caller's knowledge, a
		* checked exception is thrown.
		* @throws IllegalStateException if this {@link HotwordDetector} wasn't specified to use a
		* {@link HotwordDetectionService} or {@link VisualQueryDetectionService} when it was
		* created.
		*/
		@Override
		public void updateState(@Nullable PersistableBundle options,
		@Nullable SharedMemory sharedMemory) throws IllegalDetectorStateException {

core/java/android/service/voice/HotwordDetector.java

+24 −7

Original line number	Diff line number	Diff line
		@@ -35,7 +35,8 @@ import android.util.AndroidException;
		import java.io.PrintWriter;

		/**
		* Basic functionality for sandboxed detectors.
		* Basic functionality for sandboxed detectors. This interface will be used by detectors that
		* manages their service lifecycle.
		*
		* @hide
		*/
		@@ -80,10 +81,21 @@ public interface HotwordDetector {
		*/
		int DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE = 2;

		/**
		* Indicates that it is a visual query detector.
		*
		* @hide
		*/
		int DETECTOR_TYPE_VISUAL_QUERY_DETECTOR = 3;

		/**
		* Starts sandboxed detection recognition.
		* <p>
		* On calling this, the system streams audio from the device microphone to this application's
		* If a {@link VisualQueryDetector} calls this method, {@link VisualQueryDetectionService
		* #onStartDetection(VisualQueryDetectionService.Callback)} will be called to start detection.
		* <p>
		* Otherwise if a {@link AlwaysOnHotwordDetector} or {@link SoftwareHotwordDetector} calls this,
		* the system streams audio from the device microphone to this application's
		* {@link HotwordDetectionService}. Audio is streamed until {@link #stopRecognition()} is
		* called.
		* <p>
		@@ -192,6 +204,8 @@ public interface HotwordDetector {
		return "trusted_hotword_dsp";
		case DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE:
		return "trusted_hotword_software";
		case DETECTOR_TYPE_VISUAL_QUERY_DETECTOR:
		return "visual_query_detector";
		default:
		return Integer.toString(detectorType);
		}
		@@ -244,18 +258,21 @@ public interface HotwordDetector {
		void onRejected(@NonNull HotwordRejectedResult result);

		/**
		* Called when the {@link HotwordDetectionService} is created by the system and given a
		* short amount of time to report its initialization state.
		* Called when the {@link HotwordDetectionService} or {@link VisualQueryDetectionService} is
		* created by the system and given a short amount of time to report their initialization
		* state.
		*
		* @param status Info about initialization state of {@link HotwordDetectionService}; the
		* allowed values are {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_SUCCESS},
		* @param status Info about initialization state of {@link HotwordDetectionService} or
		* {@link VisualQueryDetectionService}; allowed values are
		* {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_SUCCESS},
		* 1<->{@link SandboxedDetectionServiceBase#getMaxCustomInitializationStatus()},
		* {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_UNKNOWN}.
		*/
		void onHotwordDetectionServiceInitialized(int status);

		/**
		* Called with the {@link HotwordDetectionService} is restarted.
		* Called with the {@link HotwordDetectionService} or {@link VisualQueryDetectionService} is
		* restarted.
		*
		* Clients are expected to call {@link HotwordDetector#updateState} to share the state with
		* the newly created service.

core/java/android/service/voice/VisualQueryDetector.java

0 → 100644

+241 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2023 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package android.service.voice;

		import static android.Manifest.permission.CAMERA;
		import static android.Manifest.permission.RECORD_AUDIO;

		import android.annotation.CallbackExecutor;
		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.annotation.RequiresPermission;
		import android.annotation.SuppressLint;
		import android.annotation.SystemApi;
		import android.media.AudioFormat;
		import android.os.ParcelFileDescriptor;
		import android.os.PersistableBundle;
		import android.os.SharedMemory;
		import android.util.Slog;

		import com.android.internal.app.IVoiceInteractionManagerService;

		import java.io.PrintWriter;
		import java.util.concurrent.Executor;
		import java.util.function.Consumer;

		/**
		* Manages VisualQueryDetectionService.
		*
		* This detector provides necessary functionalities to initialize, start, update and destroy a
		* {@link VisualQueryDetectionService}.
		*
		* @hide
		**/
		@SystemApi
		@SuppressLint("NotCloseable")
		public class VisualQueryDetector {
		private static final String TAG = VisualQueryDetector.class.getSimpleName();
		private static final boolean DEBUG = false;

		private final Callback mCallback;
		private final Executor mExecutor;
		private final IVoiceInteractionManagerService mManagerService;
		private final VisualQueryDetectorInitializationDelegate mInitializationDelegate;

		VisualQueryDetector(
		IVoiceInteractionManagerService managerService,
		@NonNull @CallbackExecutor Executor executor,
		Callback callback) {

		mManagerService = managerService;
		mCallback = callback;
		mExecutor = executor;
		mInitializationDelegate = new VisualQueryDetectorInitializationDelegate();
		}

		/**
		* Initialize the {@link VisualQueryDetectionService} by passing configurations and read-only
		* data.
		*/
		void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) {
		mInitializationDelegate.initialize(options, sharedMemory);
		}

		/**
		* Set configuration and pass read-only data to {@link VisualQueryDetectionService}.
		*
		* @see HotwordDetector#updateState(PersistableBundle, SharedMemory)
		*/
		public void updateState(@Nullable PersistableBundle options,
		@Nullable SharedMemory sharedMemory) throws
		HotwordDetector.IllegalDetectorStateException {
		mInitializationDelegate.updateState(options, sharedMemory);
		}


		/**
		* On calling this method, {@link VisualQueryDetectionService
		* #onStartDetection(VisualQueryDetectionService.Callback)} will be called to start using
		* visual signals such as camera frames and microphone audio to perform detection. When user
		* attention is captured and the {@link VisualQueryDetectionService} streams queries,
		* {@link VisualQueryDetector.Callback#onQueryDetected(String)} is called to control the
		* behavior of handling {@code transcribedText}. When the query streaming is finished,
		* {@link VisualQueryDetector.Callback#onQueryFinished()} is called. If the current streamed
		* query is invalid, {@link VisualQueryDetector.Callback#onQueryRejected()} is called to abandon
		* the streamed query.
		*
		* @see HotwordDetector#startRecognition()
		*/
		@RequiresPermission(allOf = {CAMERA, RECORD_AUDIO})
		public boolean startRecognition() throws HotwordDetector.IllegalDetectorStateException {
		if (DEBUG) {
		Slog.i(TAG, "#startRecognition");
		}
		// TODO(b/261783819): Call StartDetection on VisualQueryDetectionService with the system.
		return false;
		}

		/**
		* Stops visual query detection recognition.
		*
		* @see HotwordDetector#stopRecognition()
		*/
		@RequiresPermission(allOf = {CAMERA, RECORD_AUDIO})
		public boolean stopRecognition() throws HotwordDetector.IllegalDetectorStateException {
		if (DEBUG) {
		Slog.i(TAG, "#stopRecognition");
		}
		// TODO(b/261783819): Call StopDetection on VisualQueryDetectionService with the system.
		return false;
		}

		/**
		* Destroy the current detector.
		*
		* @see HotwordDetector#destroy()
		*/
		public void destroy() {
		if (DEBUG) {
		Slog.i(TAG, "#destroy");
		}
		mInitializationDelegate.destroy();
		}

		/** @hide */
		public void dump(String prefix, PrintWriter pw) {
		// TODO: implement this
		}

		/** @hide */
		public HotwordDetector getInitializationDelegate() {
		return mInitializationDelegate;
		}

		/** @hide */
		void registerOnDestroyListener(Consumer<AbstractDetector> onDestroyListener) {
		mInitializationDelegate.registerOnDestroyListener(onDestroyListener);
		}

		/**
		* A class that lets a VoiceInteractionService implementation interact with
		* visual query detection APIs.
		*/
		public interface Callback {

		/**
		* Called when the {@link VisualQueryDetectionService} starts to stream partial queries.
		*
		* @param partialQuery The partial query in a text form being streamed.
		*/
		void onQueryDetected(@NonNull String partialQuery);

		/**
		* Called when the {@link VisualQueryDetectionService} decides to abandon the streamed
		* partial queries.
		*/
		void onQueryRejected();

		/**
		* Called when the {@link VisualQueryDetectionService} finishes streaming partial queries.
		*/
		void onQueryFinished();

		/**
		* Called when the {@link VisualQueryDetectionService} is created by the system and given a
		* short amount of time to report its initialization state.
		*
		* @param status Info about initialization state of {@link VisualQueryDetectionService}; the
		* allowed values are {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_SUCCESS},
		* 1<->{@link SandboxedDetectionServiceBase#getMaxCustomInitializationStatus()},
		* {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_UNKNOWN}.
		*/
		void onVisualQueryDetectionServiceInitialized(int status);

		/**
		* Called with the {@link VisualQueryDetectionService} is restarted.
		*
		* Clients are expected to call {@link HotwordDetector#updateState} to share the state with
		* the newly created service.
		*/
		void onVisualQueryDetectionServiceRestarted();

		/**
		* Called when the detection fails due to an error.
		*/
		//TODO(b/265390855): Replace this callback with the new onError(DetectorError) design.
		void onError();
		}

		private class VisualQueryDetectorInitializationDelegate extends AbstractDetector {

		VisualQueryDetectorInitializationDelegate() {
		super(mManagerService, null);
		}

		@Override
		void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) {
		//TODO(261783492): call initAndVerify to create VisualQueryDetectionService
		// from the system server.
		}

		@Override
		public boolean stopRecognition() throws IllegalDetectorStateException {
		//No-op, we only reuse the initialization methods.
		return false;
		}

		@Override
		public boolean startRecognition() throws IllegalDetectorStateException {
		//No-op, we only reuse the initialization methods.
		return false;
		}

		@Override
		public final boolean startRecognition(
		@NonNull ParcelFileDescriptor audioStream,
		@NonNull AudioFormat audioFormat,
		@Nullable PersistableBundle options) throws IllegalDetectorStateException {
		//No-op, not supported by VisualQueryDetector as it should be trusted.
		return false;
		}

		@Override
		public boolean isUsingSandboxedDetectionService() {
		return true;
		}
		}
		}

core/java/android/service/voice/VoiceInteractionService.java

+87 −1

Original line number	Diff line number	Diff line
		@@ -17,6 +17,7 @@
		package android.service.voice;

		import android.Manifest;
		import android.annotation.CallbackExecutor;
		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.annotation.RequiresPermission;
		@@ -58,7 +59,7 @@ import java.util.List;
		import java.util.Locale;
		import java.util.Objects;
		import java.util.Set;

		import java.util.concurrent.Executor;
		/**
		* Top-level service of the current global voice interactor, which is providing
		* support for hotwording, the back-end of a {@link android.app.VoiceInteractor}, etc.
		@@ -164,6 +165,8 @@ public class VoiceInteractionService extends Service {

		IVoiceInteractionManagerService mSystemService;

		private VisualQueryDetector mActiveVisualQueryDetector;

		private final Object mLock = new Object();

		private KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo;
		@@ -543,6 +546,85 @@ public class VoiceInteractionService extends Service {
		}
		}

		/**
		* Creates a {@link VisualQueryDetector} and initializes the application's
		* {@link VisualQueryDetectionService} using {@code options} and {@code sharedMemory}.
		*
		* <p>To be able to call this, you need to set android:visualQueryDetectionService in the
		* android.voice_interaction metadata file to a valid visual query detection service, and set
		* android:isolatedProcess="true" in the service's declaration. Otherwise, this throws an
		* {@link IllegalStateException}.
		*
		* <p>Using this has a noticeable impact on battery, since the microphone is kept open
		* for the lifetime of the recognition {@link VisualQueryDetector#startRecognition() session}.
		*
		* @param options Application configuration data to be provided to the
		* {@link VisualQueryDetectionService}. PersistableBundle does not allow any remotable objects
		* or other contents that can be used to communicate with other processes.
		* @param sharedMemory The unrestricted data blob to be provided to the
		* {@link VisualQueryDetectionService}. Use this to provide models or other such data to the
		* sandboxed process.
		* @param callback The callback to notify of detection events.
		* @return An instanece of {@link VisualQueryDetector}.
		* @throws UnsupportedOperationException if only single detector is supported. Multiple detector
		* is only available for apps targeting {@link Build.VERSION_CODES#TIRAMISU} and above.
		* @throws IllegalStateException when there is an existing {@link VisualQueryDetector}, or when
		* there is a non-trusted hotword detector running.
		*
		* @hide
		*/
		// TODO: add MANAGE_HOTWORD_DETECTION permission to protect this API and update java doc.
		@SystemApi
		@NonNull
		public final VisualQueryDetector createVisualQueryDetector(
		@Nullable PersistableBundle options,
		@Nullable SharedMemory sharedMemory,
		@NonNull @CallbackExecutor Executor executor,
		@NonNull VisualQueryDetector.Callback callback) {
		Objects.requireNonNull(executor);
		Objects.requireNonNull(callback);

		if (mSystemService == null) {
		throw new IllegalStateException("Not available until onReady() is called");
		}
		synchronized (mLock) {
		if (!CompatChanges.isChangeEnabled(MULTIPLE_ACTIVE_HOTWORD_DETECTORS)) {
		throw new UnsupportedOperationException("VisualQueryDetector is only available if "
		+ "multiple detectors are allowed");
		} else {
		if (mActiveVisualQueryDetector != null) {
		throw new IllegalStateException(
		"There is already an active VisualQueryDetector. "
		+ "It must be destroyed to create a new one.");
		}
		for (HotwordDetector detector : mActiveDetectors) {
		if (!detector.isUsingSandboxedDetectionService()) {
		throw new IllegalStateException(
		"It disallows to create trusted and non-trusted detectors "
		+ "at the same time.");
		}
		}
		}

		VisualQueryDetector visualQueryDetector =
		new VisualQueryDetector(mSystemService, executor, callback);
		HotwordDetector visualQueryDetectorInitializationDelegate =
		visualQueryDetector.getInitializationDelegate();
		mActiveDetectors.add(visualQueryDetectorInitializationDelegate);

		try {
		visualQueryDetector.registerOnDestroyListener(this::onHotwordDetectorDestroyed);
		visualQueryDetector.initialize(options, sharedMemory);
		} catch (Exception e) {
		mActiveDetectors.remove(visualQueryDetectorInitializationDelegate);
		visualQueryDetector.destroy();
		throw e;
		}
		mActiveVisualQueryDetector = visualQueryDetector;
		return visualQueryDetector;
		}
		}

		/**
		* Creates an {@link KeyphraseModelManager} to use for enrolling voice models outside of the
		* pre-bundled system voice models.
		@@ -598,6 +680,10 @@ public class VoiceInteractionService extends Service {

		private void onHotwordDetectorDestroyed(@NonNull HotwordDetector detector) {
		synchronized (mLock) {
		if (mActiveVisualQueryDetector!= null &&
		detector == mActiveVisualQueryDetector.getInitializationDelegate()) {
		mActiveVisualQueryDetector = null;
		}
		mActiveDetectors.remove(detector);
		shutdownHotwordDetectionServiceIfRequiredLocked();
		}