Merge "Add system change to enable stop/start recognition" (539486f7) · Commits · e / os / android_frameworks_base

core/java/android/service/voice/IVisualQueryDetectionVoiceInteractionCallback.aidl

0 → 100644

+48 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2023 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package android.service.voice;

		import android.media.AudioFormat;

		/**
		* Callback for returning the detected result from the VisualQueryDetectionService.
		*
		* @hide
		*/
		oneway interface IVisualQueryDetectionVoiceInteractionCallback {

		/**
		* Called when the detected query is streamed
		*/
		void onQueryDetected(in String partialQuery);

		/**
		* Called when the detected result is valid.
		*/
		void onQueryFinished();

		/**
		* Called when the detected result is invalid.
		*/
		void onQueryRejected();

		/**
		* Called when the detection fails due to an error.
		*/
		void onError();

		}

core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl

+5 −0

Original line number	Diff line number	Diff line
		@@ -27,6 +27,7 @@ import android.os.ParcelFileDescriptor;
		import android.os.PersistableBundle;
		import android.os.RemoteCallback;
		import android.os.SharedMemory;
		import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
		import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
		import android.service.voice.IVoiceInteractionService;
		import android.service.voice.IVoiceInteractionSession;
		@@ -299,6 +300,10 @@ interface IVoiceInteractionManagerService {
		*/
		void shutdownHotwordDetectionService();

		void startPerceiving(in IVisualQueryDetectionVoiceInteractionCallback callback);

		void stopPerceiving();

		void startListeningFromMic(
		in AudioFormat audioFormat,
		in IMicrophoneHotwordDetectionVoiceInteractionCallback callback);

services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java

+29 −0

Original line number	Diff line number	Diff line
		@@ -52,6 +52,7 @@ import android.service.voice.HotwordDetectionService;
		import android.service.voice.HotwordDetector;
		import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
		import android.service.voice.ISandboxedDetectionService;
		import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
		import android.service.voice.VisualQueryDetectionService;
		import android.service.voice.VoiceInteractionManagerInternal.HotwordDetectionServiceIdentity;
		import android.speech.IRecognitionServiceManager;
		@@ -290,6 +291,34 @@ final class HotwordDetectionConnection {
		session.startListeningFromMicLocked(audioFormat, callback);
		}

		/**
		* This method is only used by VisualQueryDetector.
		*/
		void startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback) {
		if (DEBUG) {
		Slog.d(TAG, "startPerceivingLocked");
		}
		final VisualQueryDetectorSession session = getVisualQueryDetectorSessionLocked();
		if (session == null) {
		return;
		}
		session.startPerceivingLocked(callback);
		}

		/**
		* This method is only used by VisaulQueryDetector.
		*/
		void stopPerceivingLocked() {
		if (DEBUG) {
		Slog.d(TAG, "stopPerceivingLocked");
		}
		final VisualQueryDetectorSession session = getVisualQueryDetectorSessionLocked();
		if (session == null) {
		return;
		}
		session.stopPerceivingLocked();
		}

		public void startListeningFromExternalSourceLocked(
		ParcelFileDescriptor audioStream,
		AudioFormat audioFormat,

services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java

+80 −0

Original line number	Diff line number	Diff line
		@@ -24,13 +24,18 @@ import android.media.permission.Identity;
		import android.os.IBinder;
		import android.os.ParcelFileDescriptor;
		import android.os.PersistableBundle;
		import android.os.RemoteException;
		import android.os.SharedMemory;
		import android.service.voice.IDetectorSessionVisualQueryDetectionCallback;
		import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
		import android.service.voice.ISandboxedDetectionService;
		import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
		import android.util.Slog;

		import com.android.internal.app.IHotwordRecognitionStatusCallback;

		import java.io.PrintWriter;
		import java.util.Objects;
		import java.util.concurrent.ScheduledExecutorService;

		/**
		@@ -44,6 +49,8 @@ import java.util.concurrent.ScheduledExecutorService;
		final class VisualQueryDetectorSession extends DetectorSession {

		private static final String TAG = "VisualQueryDetectorSession";
		private boolean mEgressingData;
		private boolean mQueryStreaming;

		//TODO(b/261783819): Determines actual functionalities, e.g., startRecognition etc.
		VisualQueryDetectorSession(
		@@ -55,6 +62,8 @@ final class VisualQueryDetectorSession extends DetectorSession {
		super(remoteService, lock, context, token, callback,
		voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService,
		logging);
		mEgressingData = false;
		mQueryStreaming = false;
		}

		@Override
		@@ -65,6 +74,77 @@ final class VisualQueryDetectorSession extends DetectorSession {
		//TODO(b/261783819): Starts detection in VisualQueryDetectionService.
		}

		@SuppressWarnings("GuardedBy")
		void startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback) {
		if (DEBUG) {
		Slog.d(TAG, "startPerceivingLocked");
		}

		IDetectorSessionVisualQueryDetectionCallback internalCallback =
		new IDetectorSessionVisualQueryDetectionCallback.Stub(){

		@Override
		public void onAttentionGained() {
		Slog.v(TAG, "BinderCallback#onAttentionGained");
		//TODO check to see if there is an active SysUI listener registered
		mEgressingData = true;
		}

		@Override
		public void onAttentionLost() {
		Slog.v(TAG, "BinderCallback#onAttentionLost");
		//TODO check to see if there is an active SysUI listener registered
		mEgressingData = false;
		}

		@Override
		public void onQueryDetected(@NonNull String partialQuery) throws RemoteException {
		Objects.requireNonNull(partialQuery);
		Slog.v(TAG, "BinderCallback#onQueryDetected");
		if (!mEgressingData) {
		Slog.v(TAG, "Query should not be egressed within the unattention state.");
		return;
		}
		mQueryStreaming = true;
		callback.onQueryDetected(partialQuery);
		Slog.i(TAG, "Egressed from visual query detection process.");
		}

		@Override
		public void onQueryFinished() throws RemoteException {
		Slog.v(TAG, "BinderCallback#onQueryFinished");
		if (!mQueryStreaming) {
		Slog.v(TAG, "Query streaming state signal FINISHED is block since there is"
		+ " no active query being streamed.");
		return;
		}
		callback.onQueryFinished();
		mQueryStreaming = false;
		}

		@Override
		public void onQueryRejected() throws RemoteException {
		Slog.v(TAG, "BinderCallback#onQueryRejected");
		if (!mQueryStreaming) {
		Slog.v(TAG, "Query streaming state signal REJECTED is block since there is"
		+ " no active query being streamed.");
		return;
		}
		callback.onQueryRejected();
		mQueryStreaming = false;
		}
		};
		mRemoteDetectionService.run(service -> service.detectWithVisualSignals(internalCallback));
		}

		@SuppressWarnings("GuardedBy")
		void stopPerceivingLocked() {
		if (DEBUG) {
		Slog.d(TAG, "stopPerceivingLocked");
		}
		mRemoteDetectionService.run(ISandboxedDetectionService::stopDetection);
		}

		@Override
		void startListeningFromExternalSourceLocked(
		ParcelFileDescriptor audioStream,

services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java

+41 −0

Original line number	Diff line number	Diff line
		@@ -72,6 +72,7 @@ import android.os.Trace;
		import android.os.UserHandle;
		import android.provider.Settings;
		import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
		import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
		import android.service.voice.IVoiceInteractionSession;
		import android.service.voice.VoiceInteractionManagerInternal;
		import android.service.voice.VoiceInteractionService;
		@@ -1313,6 +1314,46 @@ public class VoiceInteractionManagerService extends SystemService {
		}
		}

		@Override
		public void startPerceiving(
		IVisualQueryDetectionVoiceInteractionCallback callback)
		throws RemoteException {
		enforceCallingPermission(Manifest.permission.RECORD_AUDIO);
		enforceCallingPermission(Manifest.permission.CAMERA);
		synchronized (this) {
		enforceIsCurrentVoiceInteractionService();

		if (mImpl == null) {
		Slog.w(TAG, "startPerceiving without running voice interaction service");
		return;
		}
		final long caller = Binder.clearCallingIdentity();
		try {
		mImpl.startPerceivingLocked(callback);
		} finally {
		Binder.restoreCallingIdentity(caller);
		}
		}
		}

		@Override
		public void stopPerceiving() throws RemoteException {
		synchronized (this) {
		enforceIsCurrentVoiceInteractionService();

		if (mImpl == null) {
		Slog.w(TAG, "stopPerceiving without running voice interaction service");
		return;
		}
		final long caller = Binder.clearCallingIdentity();
		try {
		mImpl.stopPerceivingLocked();
		} finally {
		Binder.restoreCallingIdentity(caller);
		}
		}
		}

		@Override
		public void startListeningFromMic(
		AudioFormat audioFormat,