Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b7baaebb authored by Tom Chan's avatar Tom Chan
Browse files

Add support for wearable hotword. 

Add the ability for a wearable device connected to WearableSensingService to trigger a (second-stage) hotword detection on the HotwordDetectionService on the phone and forward the result and audio stream to the AlwaysOnHotwordDetector if detected.

Bug: 310055381
Test: Verified data flow between WearableSensingService, HotwordDetectionService, and AlwaysOnHotwordDetector on device. Also added CTS tests to test several end-to-end behaviors in HotwordDetectionServiceBasicTest.

Change-Id: I51d4ad3f80e463729c183c87962cc9205e52ba21
parent 7e6717a3

core/api/system-current.txt

+8 −0
Original line number Diff line number Diff line
@@ -3190,6 +3190,8 @@ package android.app.wearable { 
    method @RequiresPermission(android.Manifest.permission.MANAGE_WEARABLE_SENSING_SERVICE) public void provideDataStream(@NonNull android.os.ParcelFileDescriptor, @NonNull java.util.concurrent.Executor, @NonNull java.util.function.Consumer<java.lang.Integer>);

    method @FlaggedApi("android.app.wearable.enable_provide_wearable_connection_api") @RequiresPermission(android.Manifest.permission.MANAGE_WEARABLE_SENSING_SERVICE) public void provideWearableConnection(@NonNull android.os.ParcelFileDescriptor, @NonNull java.util.concurrent.Executor, @NonNull java.util.function.Consumer<java.lang.Integer>);

    method @FlaggedApi("android.app.wearable.enable_data_request_observer_api") @RequiresPermission(android.Manifest.permission.MANAGE_WEARABLE_SENSING_SERVICE) public void registerDataRequestObserver(int, @NonNull android.app.PendingIntent, @NonNull java.util.concurrent.Executor, @NonNull java.util.function.Consumer<java.lang.Integer>);

    method @FlaggedApi("android.app.wearable.enable_hotword_wearable_sensing_api") @RequiresPermission(android.Manifest.permission.MANAGE_WEARABLE_SENSING_SERVICE) public void startHotwordRecognition(@Nullable android.content.ComponentName, @NonNull java.util.concurrent.Executor, @NonNull java.util.function.Consumer<java.lang.Integer>);

    method @FlaggedApi("android.app.wearable.enable_hotword_wearable_sensing_api") @RequiresPermission(android.Manifest.permission.MANAGE_WEARABLE_SENSING_SERVICE) public void stopHotwordRecognition(@NonNull java.util.concurrent.Executor, @NonNull java.util.function.Consumer<java.lang.Integer>);

    method @FlaggedApi("android.app.wearable.enable_data_request_observer_api") @RequiresPermission(android.Manifest.permission.MANAGE_WEARABLE_SENSING_SERVICE) public void unregisterDataRequestObserver(int, @NonNull android.app.PendingIntent, @NonNull java.util.concurrent.Executor, @NonNull java.util.function.Consumer<java.lang.Integer>);

    field public static final int STATUS_ACCESS_DENIED = 5; // 0x5

    field @FlaggedApi("android.app.wearable.enable_provide_wearable_connection_api") public static final int STATUS_CHANNEL_ERROR = 7; // 0x7
@@ -13190,6 +13192,7 @@ package android.service.voice { 
    method @Nullable public android.service.voice.HotwordDetectedResult getHotwordDetectedResult();

    method @NonNull public java.util.List<android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra> getKeyphraseRecognitionExtras();

    method @Deprecated @Nullable public byte[] getTriggerAudio();

    method @FlaggedApi("android.app.wearable.enable_hotword_wearable_sensing_api") public boolean isRecognitionStopped();

    field public static final int DATA_FORMAT_RAW = 0; // 0x0

    field public static final int DATA_FORMAT_TRIGGER_AUDIO = 1; // 0x1

  }
@@ -13296,6 +13299,7 @@ package android.service.voice { 
    method public void onUpdateState(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, long, @Nullable java.util.function.IntConsumer);

    field @Deprecated public static final int INITIALIZATION_STATUS_SUCCESS = 0; // 0x0

    field @Deprecated public static final int INITIALIZATION_STATUS_UNKNOWN = 100; // 0x64

    field @FlaggedApi("android.app.wearable.enable_hotword_wearable_sensing_api") public static final String KEY_SYSTEM_WILL_CLOSE_AUDIO_STREAM_AFTER_CALLBACK = "android.service.voice.HotwordDetectionService.KEY_SYSTEM_WILL_CLOSE_AUDIO_STREAM_AFTER_CALLBACK";

    field public static final String SERVICE_INTERFACE = "android.service.voice.HotwordDetectionService";

  }










@@ -13581,7 +13585,11 @@ package android.service.wearable {













    method @BinderThread public abstract void onQueryServiceStatus(@NonNull java.util.Set<java.lang.Integer>, @NonNull String, @NonNull java.util.function.Consumer<android.service.ambientcontext.AmbientContextDetectionServiceStatus>);















    method @FlaggedApi("android.app.wearable.enable_provide_wearable_connection_api") @BinderThread public void onSecureWearableConnectionProvided(@NonNull android.os.ParcelFileDescriptor, @NonNull java.util.function.Consumer<java.lang.Integer>);















    method @BinderThread public abstract void onStartDetection(@NonNull android.app.ambientcontext.AmbientContextEventRequest, @NonNull String, @NonNull java.util.function.Consumer<android.service.ambientcontext.AmbientContextDetectionServiceStatus>, @NonNull java.util.function.Consumer<android.service.ambientcontext.AmbientContextDetectionResult>);













    method @FlaggedApi("android.app.wearable.enable_hotword_wearable_sensing_api") @BinderThread public void onStartHotwordRecognition(@NonNull java.util.function.Consumer<android.service.voice.HotwordAudioStream>, @NonNull java.util.function.Consumer<java.lang.Integer>);















    method public abstract void onStopDetection(@NonNull String);













    method @FlaggedApi("android.app.wearable.enable_hotword_wearable_sensing_api") @BinderThread public void onStopHotwordAudioStream();













    method @FlaggedApi("android.app.wearable.enable_hotword_wearable_sensing_api") @BinderThread public void onStopHotwordRecognition(@NonNull java.util.function.Consumer<java.lang.Integer>);













    method @FlaggedApi("android.app.wearable.enable_hotword_wearable_sensing_api") @BinderThread public void onValidatedByHotwordDetectionService();















    field public static final String SERVICE_INTERFACE = "android.service.wearable.WearableSensingService";















  }







































core/api/test-current.txt



+1
−0




























Original line number
Diff line number
Diff line








@@ -3164,6 +3164,7 @@ package android.service.voice {













    method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setDataFormat(int);















    method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setHalEventReceivedMillis(long);















    method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setHotwordDetectedResult(@NonNull android.service.voice.HotwordDetectedResult);













    method @FlaggedApi("android.app.wearable.enable_hotword_wearable_sensing_api") @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setIsRecognitionStopped(boolean);















    method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setKeyphraseRecognitionExtras(@NonNull java.util.List<android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra>);















  }

















































core/java/android/app/wearable/IWearableSensingManager.aidl



+5
−0




























Original line number
Diff line number
Diff line








@@ -17,6 +17,7 @@













package android.app.wearable;































import android.app.PendingIntent;













import android.content.ComponentName;















import android.os.ParcelFileDescriptor;















import android.os.PersistableBundle;















import android.os.RemoteCallback;









@@ -38,4 +39,8 @@ interface IWearableSensingManager {













     void registerDataRequestObserver(int dataType, in PendingIntent dataRequestPendingIntent, in RemoteCallback statusCallback);















     @JavaPassthrough(annotation="@android.annotation.RequiresPermission(android.Manifest.permission.MANAGE_WEARABLE_SENSING_SERVICE)")















     void unregisterDataRequestObserver(int dataType, in PendingIntent dataRequestPendingIntent, in RemoteCallback statusCallback);













     @JavaPassthrough(annotation="@android.annotation.RequiresPermission(android.Manifest.permission.MANAGE_WEARABLE_SENSING_SERVICE)")













     void startHotwordRecognition(in ComponentName targetVisComponentName, in RemoteCallback statusCallback);













     @JavaPassthrough(annotation="@android.annotation.RequiresPermission(android.Manifest.permission.MANAGE_WEARABLE_SENSING_SERVICE)")













     void stopHotwordRecognition(in RemoteCallback statusCallback);















}










 No newline at end of file
























core/java/android/app/wearable/WearableSensingManager.java



+84
−2




























Original line number
Diff line number
Diff line








@@ -28,6 +28,7 @@ import android.annotation.SystemService;













import android.app.PendingIntent;















import android.app.ambientcontext.AmbientContextEvent;















import android.companion.CompanionDeviceManager;













import android.content.ComponentName;















import android.content.Context;















import android.content.Intent;















import android.os.Binder;










@@ -92,9 +93,13 @@ public class WearableSensingManager {













    public static final int STATUS_SUCCESS = 1;































    /**













     * The value of the status code that indicates one or more of the













     * requested events are not supported.













     * The value of the status code that indicates one or more of the requested events are not













     * supported.















     */













    // TODO(b/324635656): Deprecate this status code. Update Javadoc:













    // @deprecated WearableSensingManager does not deal with events. Use {@link













    // STATUS_UNSUPPORTED_OPERATION} instead for operations not supported by the implementation of













    // {@link WearableSensingService}.















    public static final int STATUS_UNSUPPORTED = 2;































    /**










@@ -382,6 +387,83 @@ public class WearableSensingManager {













        }















    }





























    /**













     * Requests the wearable to start hotword recognition.













     *













     * <p>When this method is called, the system will attempt to provide a {@link













     * android.service.wearable.WearableHotwordAudioConsumer} to {@link WearableSensingService}.













     * After first-stage hotword is detected on a wearable, {@link WearableSensingService} should













     * send the hotword audio to the {@link android.service.wearable.WearableHotwordAudioConsumer},













     * which will forward the data to the {@link android.service.voice.HotwordDetectionService} for













     * second-stage hotword validation. If hotword is detected there, the audio data will be













     * forwarded to the {@link android.service.voice.VoiceInteractionService}.













     *













     * <p>If the {@code targetVisComponentName} provided here is not null, when {@link













     * WearableSensingService} sends hotword audio to the {@link













     * android.service.wearable.WearableHotwordAudioConsumer}, the system will check whether the













     * {@link android.service.voice.VoiceInteractionService} at that time is {@code













     * targetVisComponentName}. If not, the system will call {@link













     * WearableSensingService#onActiveHotwordAudioStopRequested()} and will not forward the audio













     * data to the current {@link android.service.voice.HotwordDetectionService} nor {@link













     * android.service.voice.VoiceInteractionService}. The system will not send a status code to













     * {@code statusConsumer} regarding the {@code targetVisComponentName} check. The caller is













     * responsible for determining whether the system's {@link













     * android.service.voice.VoiceInteractionService} is the same as {@code targetVisComponentName}.













     * The check here is just a protection against race conditions.













     *













     * <p>Calling this method again will send a new {@link













     * android.service.wearable.WearableHotwordAudioConsumer} to {@link WearableSensingService}. For













     * audio data sent to the new consumer, the system will perform the above check using the newly













     * provided {@code targetVisComponentName}. The {@link WearableSensingService} should not













     * continue to use the previous consumers after receiving a new one.













     *













     * <p>If the {@code statusConsumer} returns {@link STATUS_SUCCESS}, the caller should call













     * {@link #stopListeningForHotword(Executor, Consumer)} when it wants the wearable to stop













     * listening for hotword. If the {@code statusConsumer} returns any other status code, a failure













     * has occurred and calling {@link #stopListeningForHotword(Executor, Consumer)} is not













     * required. The system will not retry listening automatically. The caller should call this













     * method again if they want to retry.













     *













     * <p>If a failure occurred after the {@link statusConsumer} returns {@link STATUS_SUCCESS},













     * {@link statusConsumer} will be invoked again with a status code other than {@link













     * STATUS_SUCCESS}.













     *













     * @param targetVisComponentName The ComponentName of the target VoiceInteractionService.













     * @param executor Executor on which to run the consumer callback.













     * @param statusConsumer A consumer that handles the status codes.













     */













    @FlaggedApi(Flags.FLAG_ENABLE_HOTWORD_WEARABLE_SENSING_API)













    @RequiresPermission(Manifest.permission.MANAGE_WEARABLE_SENSING_SERVICE)













    public void startHotwordRecognition(













            @Nullable ComponentName targetVisComponentName,













            @NonNull @CallbackExecutor Executor executor,













            @NonNull @StatusCode Consumer<Integer> statusConsumer) {













        try {













            mService.startHotwordRecognition(













                    targetVisComponentName, createStatusCallback(executor, statusConsumer));













        } catch (RemoteException e) {













            throw e.rethrowFromSystemServer();













        }













    }



























    /**













     * Requests the wearable to stop hotword recognition.













     *













     * @param executor Executor on which to run the consumer callback.













     * @param statusConsumer A consumer that handles the status codes.













     */













    @FlaggedApi(Flags.FLAG_ENABLE_HOTWORD_WEARABLE_SENSING_API)













    @RequiresPermission(Manifest.permission.MANAGE_WEARABLE_SENSING_SERVICE)













    public void stopHotwordRecognition(













            @NonNull @CallbackExecutor Executor executor,













            @NonNull @StatusCode Consumer<Integer> statusConsumer) {













        try {













            mService.stopHotwordRecognition(createStatusCallback(executor, statusConsumer));













        } catch (RemoteException e) {













            throw e.rethrowFromSystemServer();













        }













    }





























    private static RemoteCallback createStatusCallback(















            Executor executor, Consumer<Integer> statusConsumer) {















        return new RemoteCallback(

































core/java/android/service/voice/AlwaysOnHotwordDetector.java



+63
−11




























Original line number
Diff line number
Diff line








@@ -22,6 +22,7 @@ import static android.service.voice.SoundTriggerFailure.ERROR_CODE_UNKNOWN;













import static android.service.voice.VoiceInteractionService.MULTIPLE_ACTIVE_HOTWORD_DETECTORS;































import android.annotation.ElapsedRealtimeLong;













import android.annotation.FlaggedApi;















import android.annotation.IntDef;















import android.annotation.NonNull;















import android.annotation.Nullable;










@@ -432,7 +433,10 @@ public class AlwaysOnHotwordDetector extends AbstractDetector {













        @ElapsedRealtimeLong















        private final long mHalEventReceivedMillis;





























        private EventPayload(boolean captureAvailable,













        private final boolean mIsRecognitionStopped;



























        private EventPayload(













                boolean captureAvailable,















                @Nullable AudioFormat audioFormat,















                int captureSession,















                @DataFormat int dataFormat,









@@ -440,7 +444,8 @@ public class AlwaysOnHotwordDetector extends AbstractDetector {













                @Nullable HotwordDetectedResult hotwordDetectedResult,















                @Nullable ParcelFileDescriptor audioStream,















                @NonNull List<KeyphraseRecognitionExtra> keyphraseExtras,













                @ElapsedRealtimeLong long halEventReceivedMillis) {













                @ElapsedRealtimeLong long halEventReceivedMillis,













                boolean isRecognitionStopped) {















            mCaptureAvailable = captureAvailable;















            mCaptureSession = captureSession;















            mAudioFormat = audioFormat;









@@ -450,6 +455,7 @@ public class AlwaysOnHotwordDetector extends AbstractDetector {













            mAudioStream = audioStream;















            mKephraseExtras = keyphraseExtras;















            mHalEventReceivedMillis = halEventReceivedMillis;













            mIsRecognitionStopped = isRecognitionStopped;















        }































        /**










@@ -592,6 +598,12 @@ public class AlwaysOnHotwordDetector extends AbstractDetector {













            return mHalEventReceivedMillis;















        }





























        /** Returns whether the system has stopped hotword recognition because of this detection. */













        @FlaggedApi(android.app.wearable.Flags.FLAG_ENABLE_HOTWORD_WEARABLE_SENSING_API)













        public boolean isRecognitionStopped() {













            return mIsRecognitionStopped;













        }





























        /**















         * Builder class for {@link EventPayload} objects















         *









@@ -610,6 +622,8 @@ public class AlwaysOnHotwordDetector extends AbstractDetector {













            private List<KeyphraseRecognitionExtra> mKeyphraseExtras = Collections.emptyList();















            @ElapsedRealtimeLong















            private long mHalEventReceivedMillis = -1;













            // default to true to keep prior behavior













            private boolean mIsRecognitionStopped = true;































            public Builder() {}


























@@ -745,14 +759,32 @@ public class AlwaysOnHotwordDetector extends AbstractDetector {













                return this;















            }





























            /**













             * Sets whether the system has stopped hotword recognition because of this detection.













             */













            @FlaggedApi(android.app.wearable.Flags.FLAG_ENABLE_HOTWORD_WEARABLE_SENSING_API)













            @NonNull













            public Builder setIsRecognitionStopped(boolean isRecognitionStopped) {













                mIsRecognitionStopped = isRecognitionStopped;













                return this;













            }





























            /**















             * Builds an {@link EventPayload} instance















             */















            @NonNull















            public EventPayload build() {













                return new EventPayload(mCaptureAvailable, mAudioFormat, mCaptureSession,













                        mDataFormat, mData, mHotwordDetectedResult, mAudioStream,













                        mKeyphraseExtras, mHalEventReceivedMillis);













                return new EventPayload(













                        mCaptureAvailable,













                        mAudioFormat,













                        mCaptureSession,













                        mDataFormat,













                        mData,













                        mHotwordDetectedResult,













                        mAudioStream,













                        mKeyphraseExtras,













                        mHalEventReceivedMillis,













                        mIsRecognitionStopped);















            }















        }















    }










@@ -786,14 +818,20 @@ public class AlwaysOnHotwordDetector extends AbstractDetector {





























        /**















         * Called when the keyphrase is spoken.













         * This implicitly stops listening for the keyphrase once it's detected.













         * Clients should start a recognition again once they are done handling this













         * detection.















         *













         * @param eventPayload Payload data for the detection event.













         *        This may contain the trigger audio, if requested when calling













         *        {@link AlwaysOnHotwordDetector#startRecognition(int)}.













         * <p>This implicitly stops listening for the keyphrase once it's detected. Clients should













         * start a recognition again once they are done handling this detection.













         *













         * @param eventPayload Payload data for the detection event. This may contain the trigger













         *     audio, if requested when calling {@link













         *     AlwaysOnHotwordDetector#startRecognition(int)}.















         */













        // TODO(b/324635656): Update Javadoc for 24Q3 release:













        // 1. Prepend to the first paragraph:













        //     If {@code eventPayload.isRecognitionStopped()} returns true, this...













        // 2. Append to the description for @param eventPayload:













        //     ...or if the audio comes from {@link













        //     android.service.wearable.WearableSensingService}.















        public abstract void onDetected(@NonNull EventPayload eventPayload);































        /**










@@ -1631,6 +1669,20 @@ public class AlwaysOnHotwordDetector extends AbstractDetector {













                    .sendToTarget();















        }





























        @Override













        public void onKeyphraseDetectedFromExternalSource(HotwordDetectedResult result) {













            Slog.i(TAG, "onKeyphraseDetectedFromExternalSource");













            EventPayload.Builder eventPayloadBuilder = new EventPayload.Builder();













            if (android.app.wearable.Flags.enableHotwordWearableSensingApi()) {













                eventPayloadBuilder.setIsRecognitionStopped(false);













            }













            Message.obtain(













                            mHandler,













                            MSG_HOTWORD_DETECTED,













                            eventPayloadBuilder.setHotwordDetectedResult(result).build())













                    .sendToTarget();













        }





























        @Override















        public void onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event) {















            Slog.w(TAG, "Generic sound trigger event detected at AOHD: " + event);