Merge "Introduce the concept of software and external hotword." into sc-dev (86d7e7d2) · Commits · e / os / android_frameworks_base

core/api/system-current.txt

+21 −9

Original line number	Diff line number	Diff line
		@@ -10338,7 +10338,7 @@ package android.service.trust {

		package android.service.voice {

		public class AlwaysOnHotwordDetector {
		public class AlwaysOnHotwordDetector implements android.service.voice.HotwordDetector {
		method @Nullable public android.content.Intent createEnrollIntent();
		method @Nullable public android.content.Intent createReEnrollIntent();
		method @Nullable public android.content.Intent createUnEnrollIntent();
		@@ -10348,6 +10348,8 @@ package android.service.voice {
		method @Nullable @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public android.service.voice.AlwaysOnHotwordDetector.ModelParamRange queryParameter(int);
		method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public int setParameter(int, int);
		method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public boolean startRecognition(int);
		method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public boolean startRecognition();
		method @Nullable public boolean startRecognition(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, @Nullable android.os.PersistableBundle);
		method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public boolean stopRecognition();
		method public final void updateState(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory);
		field public static final int AUDIO_CAPABILITY_ECHO_CANCELLATION = 1; // 0x1
		@@ -10367,13 +10369,9 @@ package android.service.voice {
		field @Deprecated public static final int STATE_KEYPHRASE_UNSUPPORTED = -1; // 0xffffffff
		}

		public abstract static class AlwaysOnHotwordDetector.Callback {
		public abstract static class AlwaysOnHotwordDetector.Callback implements android.service.voice.HotwordDetector.Callback {
		ctor public AlwaysOnHotwordDetector.Callback();
		method public abstract void onAvailabilityChanged(int);
		method public abstract void onDetected(@NonNull android.service.voice.AlwaysOnHotwordDetector.EventPayload);
		method public abstract void onError();
		method public abstract void onRecognitionPaused();
		method public abstract void onRecognitionResumed();
		method public void onRejected(@Nullable android.service.voice.HotwordRejectedResult);
		}

		@@ -10418,23 +10416,36 @@ package android.service.voice {
		public abstract class HotwordDetectionService extends android.app.Service {
		ctor public HotwordDetectionService();
		method @Nullable public final android.os.IBinder onBind(@NonNull android.content.Intent);
		method public void onDetectFromDspSource(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, long, @NonNull android.service.voice.HotwordDetectionService.DspHotwordDetectionCallback);
		method public void onDetect(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, long, @NonNull android.service.voice.HotwordDetectionService.Callback);
		method public void onDetect(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, @NonNull android.service.voice.HotwordDetectionService.Callback);
		method public void onDetect(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, @Nullable android.os.PersistableBundle, @NonNull android.service.voice.HotwordDetectionService.Callback);
		method public void onUpdateState(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory);
		field public static final String SERVICE_INTERFACE = "android.service.voice.HotwordDetectionService";
		}

		public static final class HotwordDetectionService.DspHotwordDetectionCallback {
		method public void onDetected();
		public static final class HotwordDetectionService.Callback {
		method public void onDetected(@Nullable android.service.voice.HotwordDetectedResult);
		method public void onRejected(@Nullable android.service.voice.HotwordRejectedResult);
		}

		public interface HotwordDetector {
		method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public boolean startRecognition();
		method public boolean startRecognition(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, @Nullable android.os.PersistableBundle);
		method public boolean stopRecognition();
		field public static final int CONFIDENCE_LEVEL_HIGH = 3; // 0x3
		field public static final int CONFIDENCE_LEVEL_LOW = 1; // 0x1
		field public static final int CONFIDENCE_LEVEL_MEDIUM = 2; // 0x2
		field public static final int CONFIDENCE_LEVEL_NONE = 0; // 0x0
		}

		public static interface HotwordDetector.Callback {
		method public void onDetected(@NonNull android.service.voice.AlwaysOnHotwordDetector.EventPayload);
		method public void onError();
		method public void onRecognitionPaused();
		method public void onRecognitionResumed();
		method public void onRejected(@Nullable android.service.voice.HotwordRejectedResult);
		}

		public final class HotwordRejectedResult implements android.os.Parcelable {
		method public int describeContents();
		method public int getConfidenceLevel();
		@@ -10445,6 +10456,7 @@ package android.service.voice {
		public class VoiceInteractionService extends android.app.Service {
		method @NonNull public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, android.service.voice.AlwaysOnHotwordDetector.Callback);
		method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, @Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, android.service.voice.AlwaysOnHotwordDetector.Callback);
		method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.HotwordDetector createHotwordDetector(@NonNull android.media.AudioFormat, @Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, @NonNull android.service.voice.HotwordDetector.Callback);
		method @NonNull @RequiresPermission("android.permission.MANAGE_VOICE_KEYPHRASES") public final android.media.voice.KeyphraseModelManager createKeyphraseModelManager();
		}

core/java/android/service/voice/AbstractHotwordDetector.java

0 → 100644

+105 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2021 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package android.service.voice;

		import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;

		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.media.AudioFormat;
		import android.os.Handler;
		import android.os.Looper;
		import android.os.ParcelFileDescriptor;
		import android.os.PersistableBundle;
		import android.os.RemoteException;
		import android.util.Slog;

		import com.android.internal.app.IVoiceInteractionManagerService;

		/** Base implementation of {@link HotwordDetector}. */
		abstract class AbstractHotwordDetector implements HotwordDetector {
		private static final String TAG = AbstractHotwordDetector.class.getSimpleName();
		private static final boolean DEBUG = false;

		private final IVoiceInteractionManagerService mManagerService;
		private final Handler mHandler;
		private final HotwordDetector.Callback mCallback;

		AbstractHotwordDetector(
		IVoiceInteractionManagerService managerService,
		HotwordDetector.Callback callback) {
		mManagerService = managerService;
		// TODO: this needs to be supplied from above
		mHandler = new Handler(Looper.getMainLooper());
		mCallback = callback;
		}

		/**
		* Detect hotword from an externally supplied stream of data.
		*
		* @return a writeable file descriptor that clients can start writing data in the given format.
		* In order to stop detection, clients can close the given stream.
		*/
		@Nullable
		@Override
		public boolean startRecognition(
		@NonNull ParcelFileDescriptor audioStream,
		@NonNull AudioFormat audioFormat,
		@Nullable PersistableBundle options) {
		if (DEBUG) {
		Slog.i(TAG, "#recognizeHotword");
		}

		// TODO: consider closing existing session.

		try {
		mManagerService.startListeningFromExternalSource(
		audioStream,
		audioFormat,
		options,
		new BinderCallback(mHandler, mCallback));
		} catch (RemoteException e) {
		e.rethrowFromSystemServer();
		}

		return true;
		}

		private static class BinderCallback
		extends IMicrophoneHotwordDetectionVoiceInteractionCallback.Stub {
		private final Handler mHandler;
		// TODO: these need to be weak references.
		private final HotwordDetector.Callback mCallback;

		BinderCallback(Handler handler, HotwordDetector.Callback callback) {
		this.mHandler = handler;
		this.mCallback = callback;
		}

		/** TODO: onDetected */
		@Override
		public void onDetected(
		@Nullable HotwordDetectedResult hotwordDetectedResult,
		@Nullable AudioFormat audioFormat,
		@Nullable ParcelFileDescriptor audioStreamIgnored) {
		mHandler.sendMessage(obtainMessage(
		HotwordDetector.Callback::onDetected,
		mCallback,
		new AlwaysOnHotwordDetector.EventPayload(audioFormat, hotwordDetectedResult)));
		}
		}
		}

core/java/android/service/voice/AlwaysOnHotwordDetector.java

+17 −3

Original line number	Diff line number	Diff line
		@@ -68,7 +68,7 @@ import java.util.Locale;
		* mark and track it as such.
		*/
		@SystemApi
		public class AlwaysOnHotwordDetector {
		public class AlwaysOnHotwordDetector extends AbstractHotwordDetector {
		//---- States of Keyphrase availability. Return codes for onAvailabilityChanged() ----//
		/**
		* Indicates that this hotword detector is no longer valid for any recognition
		@@ -459,7 +459,7 @@ public class AlwaysOnHotwordDetector {
		/**
		* Callbacks for always-on hotword detection.
		*/
		public static abstract class Callback {
		public abstract static class Callback implements HotwordDetector.Callback {

		/**
		* Updates the availability state of the active keyphrase and locale on every keyphrase
		@@ -547,11 +547,13 @@ public class AlwaysOnHotwordDetector {
		IVoiceInteractionManagerService modelManagementService, int targetSdkVersion,
		boolean supportHotwordDetectionService, @Nullable PersistableBundle options,
		@Nullable SharedMemory sharedMemory) {
		super(modelManagementService, callback);

		mHandler = new MyHandler();
		mText = text;
		mLocale = locale;
		mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo;
		mExternalCallback = callback;
		mHandler = new MyHandler();
		mInternalCallback = new SoundTriggerListener(mHandler);
		mModelManagementService = modelManagementService;
		mTargetSdkVersion = targetSdkVersion;
		@@ -704,6 +706,17 @@ public class AlwaysOnHotwordDetector {
		}
		}

		/**
		* Starts recognition for the associated keyphrase.
		*
		* @see #startRecognition(int)
		*/
		@RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD})
		@Override
		public boolean startRecognition() {
		return startRecognition(0);
		}

		/**
		* Stops recognition for the associated keyphrase.
		* Caller must be the active voice interaction service via
		@@ -718,6 +731,7 @@ public class AlwaysOnHotwordDetector {
		* {@link VoiceInteractionService} hosting this detector has been shut down.
		*/
		@RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD})
		@Override
		public boolean stopRecognition() {
		if (DBG) Slog.d(TAG, "stopRecognition()");
		synchronized (mLock) {

core/java/android/service/voice/HotwordDetectionService.java

+132 −17

Original line number	Diff line number	Diff line
		@@ -20,6 +20,7 @@ import static com.android.internal.util.function.pooled.PooledLambda.obtainMessa

		import android.annotation.CallSuper;
		import android.annotation.DurationMillisLong;
		import android.annotation.IntDef;
		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.annotation.SdkConstant;
		@@ -36,6 +37,9 @@ import android.os.RemoteException;
		import android.os.SharedMemory;
		import android.util.Log;

		import java.lang.annotation.Documented;
		import java.lang.annotation.Retention;
		import java.lang.annotation.RetentionPolicy;
		import java.util.Locale;

		/**
		@@ -50,6 +54,24 @@ public abstract class HotwordDetectionService extends Service {
		// TODO (b/177502877): Set the Debug flag to false before shipping.
		private static final boolean DBG = true;

		/**
		* Source for the given audio stream.
		*
		* @hide
		*/
		@Documented
		@Retention(RetentionPolicy.SOURCE)
		@IntDef({
		AUDIO_SOURCE_MICROPHONE,
		AUDIO_SOURCE_EXTERNAL
		})
		@interface AudioSource {}

		/** @hide */
		public static final int AUDIO_SOURCE_MICROPHONE = 1;
		/** @hide */
		public static final int AUDIO_SOURCE_EXTERNAL = 2;

		/**
		* The {@link Intent} that must be declared as handled by the service.
		* To be supported, the service must also require the
		@@ -73,12 +95,12 @@ public abstract class HotwordDetectionService extends Service {
		if (DBG) {
		Log.d(TAG, "#detectFromDspSource");
		}
		mHandler.sendMessage(obtainMessage(HotwordDetectionService::onDetectFromDspSource,
		mHandler.sendMessage(obtainMessage(HotwordDetectionService::onDetect,
		HotwordDetectionService.this,
		audioStream,
		audioFormat,
		timeoutMillis,
		new DspHotwordDetectionCallback(callback)));
		new Callback(callback)));
		}

		@Override
		@@ -92,6 +114,40 @@ public abstract class HotwordDetectionService extends Service {
		options,
		sharedMemory));
		}

		@Override
		public void detectFromMicrophoneSource(
		ParcelFileDescriptor audioStream,
		@AudioSource int audioSource,
		AudioFormat audioFormat,
		PersistableBundle options,
		IDspHotwordDetectionCallback callback)
		throws RemoteException {
		if (DBG) {
		Log.d(TAG, "#detectFromMicrophoneSource");
		}
		switch (audioSource) {
		case AUDIO_SOURCE_MICROPHONE:
		mHandler.sendMessage(obtainMessage(
		HotwordDetectionService::onDetect,
		HotwordDetectionService.this,
		audioStream,
		audioFormat,
		new Callback(callback)));
		break;
		case AUDIO_SOURCE_EXTERNAL:
		mHandler.sendMessage(obtainMessage(
		HotwordDetectionService::onDetect,
		HotwordDetectionService.this,
		audioStream,
		audioFormat,
		options,
		new Callback(callback)));
		break;
		default:
		Log.i(TAG, "Unsupported audio source " + audioSource);
		}
		}
		};

		@CallSuper
		@@ -113,27 +169,30 @@ public abstract class HotwordDetectionService extends Service {
		}

		/**
		* Detect the audio data generated from Dsp.
		*
		* <p>Note: the clients are supposed to call {@code close} on the input stream when they are
		* done with the operation in order to free up resources.
		* Called when the device hardware (such as a DSP) detected the hotword, to request second stage
		* validation before handing over the audio to the {@link AlwaysOnHotwordDetector}.
		* <p>
		* After {@code callback} is invoked or {@code timeoutMillis} has passed, the system closes
		* {@code audioStream} and invokes the appropriate {@link AlwaysOnHotwordDetector.Callback
		* callback}.
		*
		* @param audioStream Stream containing audio bytes returned from DSP
		* @param audioFormat Format of the supplied audio
		* @param timeoutMillis Timeout in milliseconds for the operation to invoke the callback. If
		* the application fails to abide by the timeout, system will close the
		* microphone and cancel the operation.
		* @param callback Use {@link HotwordDetectionService#DspHotwordDetectionCallback} to return
		* the detected result.
		* @param callback The callback to use for responding to the detection request.
		*
		* @hide
		*/
		@SystemApi
		public void onDetectFromDspSource(
		public void onDetect(
		@NonNull ParcelFileDescriptor audioStream,
		@NonNull AudioFormat audioFormat,
		@DurationMillisLong long timeoutMillis,
		@NonNull DspHotwordDetectionCallback callback) {
		@NonNull Callback callback) {
		// TODO: Add a helpful error message.
		throw new UnsupportedOperationException();
		}

		/**
		@@ -154,38 +213,94 @@ public abstract class HotwordDetectionService extends Service {
		@SystemApi
		public void onUpdateState(@Nullable PersistableBundle options,
		@Nullable SharedMemory sharedMemory) {
		// TODO: Handle the unimplemented case by throwing?
		}

		/**
		* Called when the {@link VoiceInteractionService} requests that this service
		* {@link HotwordDetector#startRecognition() start} hotword recognition on audio coming directly
		* from the device microphone.
		* <p>
		* On such a request, the system streams mic audio to this service through {@code audioStream}.
		* Audio is streamed until {@link HotwordDetector#stopRecognition()} is called, at which point
		* the system closes {code audioStream}.
		* <p>
		* On successful detection of a hotword within {@code audioStream}, call
		* {@link Callback#onDetected(HotwordDetectedResult)}. The system continues to stream audio
		* through {@code audioStream}; {@code callback} is reusable.
		*
		* @param audioStream Stream containing audio bytes returned from a microphone
		* @param audioFormat Format of the supplied audio
		* @param callback The callback to use for responding to the detection request.
		* {@link Callback#onRejected(HotwordRejectedResult) callback.onRejected} cannot be used here.
		*/
		public void onDetect(
		@NonNull ParcelFileDescriptor audioStream,
		@NonNull AudioFormat audioFormat,
		@NonNull Callback callback) {
		// TODO: Add a helpful error message.
		throw new UnsupportedOperationException();
		}

		/**
		* Called when the {@link VoiceInteractionService} requests that this service
		* {@link HotwordDetector#startRecognition(ParcelFileDescriptor, AudioFormat,
		* PersistableBundle)} run} hotword recognition on audio coming from an external connected
		* microphone.
		* <p>
		* Upon invoking the {@code callback}, the system closes {@code audioStream} and sends the
		* detection result to the {@link HotwordDetector.Callback hotword detector}.
		*
		* @param audioStream Stream containing audio bytes returned from a microphone
		* @param audioFormat Format of the supplied audio
		* @param options Options supporting detection, such as configuration specific to the source of
		* the audio, provided through
		* {@link HotwordDetector#startRecognition(ParcelFileDescriptor, AudioFormat,
		* PersistableBundle)}.
		* @param callback The callback to use for responding to the detection request.
		*/
		public void onDetect(
		@NonNull ParcelFileDescriptor audioStream,
		@NonNull AudioFormat audioFormat,
		@Nullable PersistableBundle options,
		@NonNull Callback callback) {
		// TODO: Add a helpful error message.
		throw new UnsupportedOperationException();
		}

		/**
		* Callback for returning the detected result.
		* Callback for returning the detection result.
		*
		* @hide
		*/
		@SystemApi
		public static final class DspHotwordDetectionCallback {
		public static final class Callback {
		// TODO: need to make sure we don't store remote references, but not a high priority.
		private final IDspHotwordDetectionCallback mRemoteCallback;

		private DspHotwordDetectionCallback(IDspHotwordDetectionCallback remoteCallback) {
		private Callback(IDspHotwordDetectionCallback remoteCallback) {
		mRemoteCallback = remoteCallback;
		}

		/**
		* Called when the detected result is valid.
		*/
		public void onDetected() {
		public void onDetected(@Nullable HotwordDetectedResult hotwordDetectedResult) {
		try {
		mRemoteCallback.onDetected();
		mRemoteCallback.onDetected(hotwordDetectedResult);
		} catch (RemoteException e) {
		throw e.rethrowFromSystemServer();
		}
		}

		/**
		* Informs the {@link AlwaysOnHotwordDetector} that the keyphrase was not detected.
		* Informs the {@link HotwordDetector} that the keyphrase was not detected.
		* <p>
		* This cannot not be used when recognition is done through
		* {@link #onDetect(ParcelFileDescriptor, AudioFormat, Callback)}.
		*
		* @param result Info about the second stage detection result. This is provided to
		* the {@link AlwaysOnHotwordDetector}.
		* the {@link HotwordDetector}.
		*/
		public void onRejected(@Nullable HotwordRejectedResult result) {
		try {

core/java/android/service/voice/HotwordDetector.java

+100 −2

File changed.

Preview size limit exceeded, changes collapsed.