Merge "Integrate VQDS lifecycle into connection session" (ea43c801) · Commits · e / os / android_frameworks_base

core/java/android/service/voice/VisualQueryDetector.java

+79 −2

Original line number	Diff line number	Diff line
		@@ -25,12 +25,16 @@ import android.annotation.Nullable;
		import android.annotation.RequiresPermission;
		import android.annotation.SuppressLint;
		import android.annotation.SystemApi;
		import android.hardware.soundtrigger.SoundTrigger;
		import android.media.AudioFormat;
		import android.os.Binder;
		import android.os.ParcelFileDescriptor;
		import android.os.PersistableBundle;
		import android.os.RemoteException;
		import android.os.SharedMemory;
		import android.util.Slog;

		import com.android.internal.app.IHotwordRecognitionStatusCallback;
		import com.android.internal.app.IVoiceInteractionManagerService;

		import java.io.PrintWriter;
		@@ -208,8 +212,9 @@ public class VisualQueryDetector {

		@Override
		void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) {
		//TODO(261783492): call initAndVerify to create VisualQueryDetectionService
		// from the system server.
		initAndVerifyDetector(options, sharedMemory,
		new InitializationStateListener(mExecutor, mCallback),
		DETECTOR_TYPE_VISUAL_QUERY_DETECTOR);
		}

		@Override
		@@ -238,4 +243,76 @@ public class VisualQueryDetector {
		return true;
		}
		}

		private static class InitializationStateListener
		extends IHotwordRecognitionStatusCallback.Stub {
		private final Executor mExecutor;
		private final Callback mCallback;

		InitializationStateListener(Executor executor, Callback callback) {
		this.mExecutor = executor;
		this.mCallback = callback;
		}

		@Override
		public void onKeyphraseDetected(
		SoundTrigger.KeyphraseRecognitionEvent recognitionEvent,
		HotwordDetectedResult result) {
		if (DEBUG) {
		Slog.i(TAG, "Ignored #onKeyphraseDetected event");
		}
		}

		@Override
		public void onGenericSoundTriggerDetected(
		SoundTrigger.GenericRecognitionEvent recognitionEvent) throws RemoteException {
		if (DEBUG) {
		Slog.i(TAG, "Ignored #onGenericSoundTriggerDetected event");
		}
		}

		@Override
		public void onRejected(HotwordRejectedResult result) throws RemoteException {
		if (DEBUG) {
		Slog.i(TAG, "Ignored #onRejected event");
		}
		}

		@Override
		public void onRecognitionPaused() throws RemoteException {
		if (DEBUG) {
		Slog.i(TAG, "Ignored #onRecognitionPaused event");
		}
		}

		@Override
		public void onRecognitionResumed() throws RemoteException {
		if (DEBUG) {
		Slog.i(TAG, "Ignored #onRecognitionResumed event");
		}
		}

		@Override
		public void onStatusReported(int status) {
		Slog.v(TAG, "onStatusReported" + (DEBUG ? "(" + status + ")" : ""));
		//TODO: rename the target callback with a more general term
		Binder.withCleanCallingIdentity(() -> mExecutor.execute(
		() -> mCallback.onVisualQueryDetectionServiceInitialized(status)));

		}

		@Override
		public void onProcessRestarted() throws RemoteException {
		Slog.v(TAG, "onProcessRestarted()");
		//TODO: rename the target callback with a more general term
		Binder.withCleanCallingIdentity(() -> mExecutor.execute(
		() -> mCallback.onVisualQueryDetectionServiceRestarted()));
		}

		@Override
		public void onError(int status) throws RemoteException {
		Slog.v(TAG, "Initialization Error: (" + status + ")");
		// Do nothing
		}
		}
		}

services/voiceinteraction/java/com/android/server/voiceinteraction/DetectorSession.java

+64 −31

Original line number	Diff line number	Diff line
		@@ -205,9 +205,15 @@ abstract class DetectorSession {
		mVoiceInteractionServiceUid = voiceInteractionServiceUid;
		mVoiceInteractorIdentity = voiceInteractorIdentity;
		mAppOpsManager = mContext.getSystemService(AppOpsManager.class);
		mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, getDetectorType(),
		if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
		mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager,
		getDetectorType(),
		mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,
		mVoiceInteractorIdentity.attributionTag);
		} else {
		mHotwordAudioStreamCopier = null;
		}

		mScheduledExecutorService = scheduledExecutorService;
		mDebugHotwordLogging = logging;

		@@ -236,9 +242,12 @@ abstract class DetectorSession {
		future.complete(null);
		if (mUpdateStateAfterStartFinished.getAndSet(true)) {
		Slog.w(TAG, "call callback after timeout");
		if (getDetectorType()
		!= HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
		HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
		HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT,
		mVoiceInteractionServiceUid);
		}
		return;
		}
		Pair<Integer, Integer> statusResultPair = getInitStatusAndMetricsResult(bundle);
		@@ -246,28 +255,38 @@ abstract class DetectorSession {
		int initResultMetricsResult = statusResultPair.second;
		try {
		mCallback.onStatusReported(status);
		if (getDetectorType()
		!= HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
		HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(),
		initResultMetricsResult, mVoiceInteractionServiceUid);
		}
		} catch (RemoteException e) {
		Slog.w(TAG, "Failed to report initialization status: " + e);
		if (getDetectorType()
		!= HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
		HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
		METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION,
		mVoiceInteractionServiceUid);
		}
		}
		}
		};
		try {
		service.updateState(options, sharedMemory, statusCallback);
		if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
		HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
		HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE,
		mVoiceInteractionServiceUid);
		}
		} catch (RemoteException e) {
		// TODO: (b/181842909) Report an error to voice interactor
		Slog.w(TAG, "Failed to updateState for HotwordDetectionService", e);
		if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
		HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
		HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION,
		mVoiceInteractionServiceUid);
		}
		}
		return future.orTimeout(MAX_UPDATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
		}).whenComplete((res, err) -> {
		if (err instanceof TimeoutException) {
		@@ -277,14 +296,18 @@ abstract class DetectorSession {
		}
		try {
		mCallback.onStatusReported(INITIALIZATION_STATUS_UNKNOWN);
		if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
		HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(),
		METRICS_INIT_UNKNOWN_TIMEOUT, mVoiceInteractionServiceUid);
		}
		} catch (RemoteException e) {
		Slog.w(TAG, "Failed to report initialization status UNKNOWN", e);
		if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
		HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
		METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION,
		mVoiceInteractionServiceUid);
		}
		}
		} else if (err != null) {
		Slog.w(TAG, "Failed to update state: " + err);
		}
		@@ -315,9 +338,11 @@ abstract class DetectorSession {
		@SuppressWarnings("GuardedBy")
		void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory,
		Instant lastRestartInstant) {
		if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
		HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
		HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE,
		mVoiceInteractionServiceUid);
		}
		// Prevent doing the init late, so restart is handled equally to a clean process start.
		// TODO(b/191742511): this logic needs a test
		if (!mUpdateStateAfterStartFinished.get() && Instant.now().minus(
		@@ -399,10 +424,12 @@ abstract class DetectorSession {
		callback.onError();
		} catch (RemoteException ex) {
		Slog.w(TAG, "Failed to report onError status: " + ex);
		if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
		HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
		HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION,
		mVoiceInteractionServiceUid);
		}
		}
		} finally {
		synchronized (mLock) {
		mPerformingExternalSourceHotwordDetection = false;
		@@ -427,7 +454,8 @@ abstract class DetectorSession {
		throws RemoteException {
		synchronized (mLock) {
		mPerformingExternalSourceHotwordDetection = false;
		HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
		HotwordMetricsLogger.writeDetectorEvent(
		getDetectorType(),
		METRICS_EXTERNAL_SOURCE_REJECTED,
		mVoiceInteractionServiceUid);
		mScheduledExecutorService.schedule(
		@@ -454,7 +482,8 @@ abstract class DetectorSession {
		throws RemoteException {
		synchronized (mLock) {
		mPerformingExternalSourceHotwordDetection = false;
		HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
		HotwordMetricsLogger.writeDetectorEvent(
		getDetectorType(),
		METRICS_EXTERNAL_SOURCE_DETECTED,
		mVoiceInteractionServiceUid);
		mScheduledExecutorService.schedule(
		@@ -540,11 +569,13 @@ abstract class DetectorSession {
		mCallback.onError(status);
		} catch (RemoteException e) {
		Slog.w(TAG, "Failed to report onError status: " + e);
		if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
		HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
		HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION,
		mVoiceInteractionServiceUid);
		}
		}
		}

		/**
		* Called when the trusted process is restarted.
		@@ -679,6 +710,8 @@ abstract class DetectorSession {
		return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP;
		} else if (this instanceof SoftwareTrustedHotwordDetectorSession) {
		return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE;
		} else if (this instanceof VisualQueryDetectorSession) {
		return HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR;
		}
		Slog.v(TAG, "Unexpected detector type");
		return -1;

services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java

+160 −46

File changed.

Preview size limit exceeded, changes collapsed.

services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java

0 → 100644

+87 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2023 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.server.voiceinteraction;

		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.content.Context;
		import android.media.AudioFormat;
		import android.media.permission.Identity;
		import android.os.IBinder;
		import android.os.ParcelFileDescriptor;
		import android.os.PersistableBundle;
		import android.os.SharedMemory;
		import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
		import android.util.Slog;

		import com.android.internal.app.IHotwordRecognitionStatusCallback;

		import java.io.PrintWriter;
		import java.util.concurrent.ScheduledExecutorService;

		/**
		* A class that provides visual query detector to communicate with the {@link
		* android.service.voice.VisualQueryDetectionService}.
		*
		* This class can handle the visual query detection whose detector is created by using
		* {@link android.service.voice.VoiceInteractionService#createVisualQueryDetector(PersistableBundle
		* ,SharedMemory, HotwordDetector.Callback)}.
		*/
		final class VisualQueryDetectorSession extends DetectorSession {

		private static final String TAG = "VisualQueryDetectorSession";

		//TODO(b/261783819): Determines actual functionalities, e.g., startRecognition etc.
		VisualQueryDetectorSession(
		@NonNull HotwordDetectionConnection.ServiceConnection remoteService,
		@NonNull Object lock, @NonNull Context context, @NonNull IBinder token,
		@NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid,
		Identity voiceInteractorIdentity,
		@NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) {
		super(remoteService, lock, context, token, callback,
		voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService,
		logging);
		}

		@Override
		@SuppressWarnings("GuardedBy")
		void informRestartProcessLocked() {
		Slog.v(TAG, "informRestartProcessLocked");
		mUpdateStateAfterStartFinished.set(false);
		//TODO(b/261783819): Starts detection in VisualQueryDetectionService.
		}

		@Override
		void startListeningFromExternalSourceLocked(
		ParcelFileDescriptor audioStream,
		AudioFormat audioFormat,
		@Nullable PersistableBundle options,
		IMicrophoneHotwordDetectionVoiceInteractionCallback callback)
		throws UnsupportedOperationException {
		throw new UnsupportedOperationException("HotwordDetectionService method"
		+ " should not be called from VisualQueryDetectorSession.");
		}


		@SuppressWarnings("GuardedBy")
		public void dumpLocked(String prefix, PrintWriter pw) {
		super.dumpLocked(prefix, pw);
		pw.print(prefix);
		}
		}

services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java

+55 −7

Original line number	Diff line number	Diff line
		@@ -58,6 +58,7 @@ import android.os.RemoteException;
		import android.os.ServiceManager;
		import android.os.SharedMemory;
		import android.os.UserHandle;
		import android.service.voice.HotwordDetector;
		import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
		import android.service.voice.IVoiceInteractionService;
		import android.service.voice.IVoiceInteractionSession;
		@@ -109,6 +110,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
		final ComponentName mSessionComponentName;
		final IWindowManager mIWindowManager;
		final ComponentName mHotwordDetectionComponentName;
		final ComponentName mVisualQueryDetectionComponentName;
		boolean mBound = false;
		IVoiceInteractionService mService;
		volatile HotwordDetectionConnection mHotwordDetectionConnection;
		@@ -211,6 +213,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
		mInfo = null;
		mSessionComponentName = null;
		mHotwordDetectionComponentName = null;
		mVisualQueryDetectionComponentName = null;
		mIWindowManager = null;
		mValid = false;
		return;
		@@ -220,6 +223,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
		Slog.w(TAG, "Bad voice interaction service: " + mInfo.getParseError());
		mSessionComponentName = null;
		mHotwordDetectionComponentName = null;
		mVisualQueryDetectionComponentName = null;
		mIWindowManager = null;
		mValid = false;
		return;
		@@ -230,6 +234,9 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
		final String hotwordDetectionServiceName = mInfo.getHotwordDetectionService();
		mHotwordDetectionComponentName = hotwordDetectionServiceName != null
		? new ComponentName(service.getPackageName(), hotwordDetectionServiceName) : null;
		final String visualQueryDetectionServiceName = mInfo.getVisualQueryDetectionService();
		mVisualQueryDetectionComponentName = visualQueryDetectionServiceName != null ? new
		ComponentName(service.getPackageName(), visualQueryDetectionServiceName) : null;
		mIWindowManager = IWindowManager.Stub.asInterface(
		ServiceManager.getService(Context.WINDOW_SERVICE));
		IntentFilter filter = new IntentFilter();
		@@ -591,14 +598,11 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
		}
		}

		public void initAndVerifyDetectorLocked(
		@NonNull Identity voiceInteractorIdentity,
		@Nullable PersistableBundle options,
		private void verifyDetectorForHotwordDetectionLocked(
		@Nullable SharedMemory sharedMemory,
		@NonNull IBinder token,
		IHotwordRecognitionStatusCallback callback,
		int detectorType) {
		Slog.v(TAG, "initAndVerifyDetectorLocked");
		Slog.v(TAG, "verifyDetectorForHotwordDetectionLocked");
		int voiceInteractionServiceUid = mInfo.getServiceInfo().applicationInfo.uid;
		if (mHotwordDetectionComponentName == null) {
		Slog.w(TAG, "Hotword detection service name not found");
		@@ -649,11 +653,55 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne

		logDetectorCreateEventIfNeeded(callback, detectorType, true,
		voiceInteractionServiceUid);
		}

		private void verifyDetectorForVisualQueryDetectionLocked(@Nullable SharedMemory sharedMemory) {
		Slog.v(TAG, "verifyDetectorForVisualQueryDetectionLocked");

		if (mVisualQueryDetectionComponentName == null) {
		Slog.w(TAG, "Visual query detection service name not found");
		throw new IllegalStateException("Visual query detection service name not found");
		}
		ServiceInfo visualQueryDetectionServiceInfo = getServiceInfoLocked(
		mVisualQueryDetectionComponentName, mUser);
		if (visualQueryDetectionServiceInfo == null) {
		Slog.w(TAG, "Visual query detection service info not found");
		throw new IllegalStateException("Visual query detection service name not found");
		}
		if (!isIsolatedProcessLocked(visualQueryDetectionServiceInfo)) {
		Slog.w(TAG, "Visual query detection service not in isolated process");
		throw new IllegalStateException("Visual query detection not in isolated process");
		}
		if (sharedMemory != null && !sharedMemory.setProtect(OsConstants.PROT_READ)) {
		Slog.w(TAG, "Can't set sharedMemory to be read-only");
		throw new IllegalStateException("Can't set sharedMemory to be read-only");
		}
		}

		public void initAndVerifyDetectorLocked(
		@NonNull Identity voiceInteractorIdentity,
		@Nullable PersistableBundle options,
		@Nullable SharedMemory sharedMemory,
		@NonNull IBinder token,
		IHotwordRecognitionStatusCallback callback,
		int detectorType) {

		if (detectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
		verifyDetectorForHotwordDetectionLocked(sharedMemory, callback, detectorType);
		} else {
		verifyDetectorForVisualQueryDetectionLocked(sharedMemory);
		}

		if (mHotwordDetectionConnection == null) {
		mHotwordDetectionConnection = new HotwordDetectionConnection(mServiceStub, mContext,
		mInfo.getServiceInfo().applicationInfo.uid, voiceInteractorIdentity,
		mHotwordDetectionComponentName, mUser, /* bindInstantServiceAllowed= */ false,
		detectorType);
		mHotwordDetectionComponentName, mVisualQueryDetectionComponentName, mUser,
		/* bindInstantServiceAllowed= */ false, detectorType);
		} else if (detectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
		// TODO: Logger events should be handled in session instead. Temporary adding the
		// checking to prevent confusion so VisualQueryDetection events won't be logged if the
		// connection is instantiated by the VisualQueryDetector.
		mHotwordDetectionConnection.setDetectorType(detectorType);
		}
		mHotwordDetectionConnection.createDetectorLocked(options, sharedMemory, token, callback,
		detectorType);