expose HAL keyphrase trigger extras to EventPayload (2f7b2d5f) · Commits · e / os / android_frameworks_base

core/api/system-current.txt

+10 −0

Original line number	Diff line number	Diff line
		@@ -5172,6 +5172,15 @@ package android.hardware.soundtrigger {
		field @NonNull public static final android.os.Parcelable.Creator<android.hardware.soundtrigger.SoundTrigger.Keyphrase> CREATOR;
		}

		public static final class SoundTrigger.KeyphraseRecognitionExtra implements android.os.Parcelable {
		method public int describeContents();
		method public int getCoarseConfidenceLevel();
		method public int getKeyphraseId();
		method public int getRecognitionModes();
		method public void writeToParcel(@NonNull android.os.Parcel, int);
		field @NonNull public static final android.os.Parcelable.Creator<android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra> CREATOR;
		}

		public static final class SoundTrigger.KeyphraseSoundModel extends android.hardware.soundtrigger.SoundTrigger.SoundModel implements android.os.Parcelable {
		ctor public SoundTrigger.KeyphraseSoundModel(@NonNull java.util.UUID, @NonNull java.util.UUID, @Nullable byte[], @Nullable android.hardware.soundtrigger.SoundTrigger.Keyphrase[], int);
		ctor public SoundTrigger.KeyphraseSoundModel(@NonNull java.util.UUID, @NonNull java.util.UUID, @Nullable byte[], @Nullable android.hardware.soundtrigger.SoundTrigger.Keyphrase[]);
		@@ -11758,6 +11767,7 @@ package android.service.voice {
		method @Nullable public byte[] getData();
		method public int getDataFormat();
		method @Nullable public android.service.voice.HotwordDetectedResult getHotwordDetectedResult();
		method @NonNull public java.util.List<android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra> getKeyphraseRecognitionExtras();
		method @Deprecated @Nullable public byte[] getTriggerAudio();
		field public static final int DATA_FORMAT_RAW = 0; // 0x0
		field public static final int DATA_FORMAT_TRIGGER_AUDIO = 1; // 0x1

core/api/test-current.txt

+15 −2

Original line number	Diff line number	Diff line
		@@ -1276,6 +1276,10 @@ package android.hardware.soundtrigger {
		field @NonNull public static final android.os.Parcelable.Creator<android.hardware.soundtrigger.KeyphraseMetadata> CREATOR;
		}

		public static final class SoundTrigger.KeyphraseRecognitionExtra implements android.os.Parcelable {
		ctor public SoundTrigger.KeyphraseRecognitionExtra(int, int, int);
		}

		public static final class SoundTrigger.ModelParamRange implements android.os.Parcelable {
		ctor public SoundTrigger.ModelParamRange(int, int);
		}
		@@ -2398,8 +2402,17 @@ package android.service.voice {
		method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public void triggerHardwareRecognitionEventForTest(int, int, boolean, int, int, int, boolean, @NonNull android.media.AudioFormat, @Nullable byte[]);
		}

		public static class AlwaysOnHotwordDetector.EventPayload {
		ctor public AlwaysOnHotwordDetector.EventPayload(boolean, boolean, @Nullable android.media.AudioFormat, int, @Nullable byte[], @Nullable android.service.voice.HotwordDetectedResult, @Nullable android.os.ParcelFileDescriptor);
		public static final class AlwaysOnHotwordDetector.EventPayload.Builder {
		ctor public AlwaysOnHotwordDetector.EventPayload.Builder();
		method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload build();
		method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setAudioStream(@NonNull android.os.ParcelFileDescriptor);
		method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureAudioFormat(@NonNull android.media.AudioFormat);
		method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureAvailable(boolean);
		method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureSession(int);
		method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setData(@NonNull byte[]);
		method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setDataFormat(int);
		method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setHotwordDetectedResult(@NonNull android.service.voice.HotwordDetectedResult);
		method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setKeyphraseRecognitionExtras(@NonNull java.util.List<android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra>);
		}

		public final class VisibleActivityInfo implements android.os.Parcelable {

core/java/android/hardware/soundtrigger/SoundTrigger.java

+96 −21

Original line number	Diff line number	Diff line
		@@ -61,6 +61,7 @@ import java.lang.annotation.Retention;
		import java.lang.annotation.RetentionPolicy;
		import java.util.ArrayList;
		import java.util.Arrays;
		import java.util.Collection;
		import java.util.Locale;
		import java.util.UUID;

		@@ -1578,29 +1579,56 @@ public class SoundTrigger {
		/**
		* Additional data conveyed by a {@link KeyphraseRecognitionEvent}
		* for a key phrase detection.
		*/
		public static final class KeyphraseRecognitionExtra implements Parcelable {
		/**
		* The keyphrase ID
		*
		* @hide
		*/
		public static class KeyphraseRecognitionExtra implements Parcelable {
		/** The keyphrase ID */
		@UnsupportedAppUsage
		public final int id;

		/** Recognition modes matched for this event */
		/**
		* Recognition modes matched for this event
		*
		* @hide
		*/
		@UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553)
		public final int recognitionModes;

		/** Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification
		* is not performed */
		/**
		* Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification
		* is not performed
		*
		* @hide
		*/
		@UnsupportedAppUsage
		public final int coarseConfidenceLevel;

		/** Confidence levels for all users recognized (KeyphraseRecognitionEvent) or to
		* be recognized (RecognitionConfig) */
		/**
		* Confidence levels for all users recognized (KeyphraseRecognitionEvent) or to
		* be recognized (RecognitionConfig)
		*
		* @hide
		*/
		@UnsupportedAppUsage
		@NonNull
		public final ConfidenceLevel[] confidenceLevels;


		/**
		* @hide
		*/
		@TestApi
		public KeyphraseRecognitionExtra(int id, @RecognitionModes int recognitionModes,
		int coarseConfidenceLevel) {
		this(id, recognitionModes, coarseConfidenceLevel, new ConfidenceLevel[0]);
		}

		/**
		* @hide
		*/
		@UnsupportedAppUsage
		public KeyphraseRecognitionExtra(int id, int recognitionModes, int coarseConfidenceLevel,
		@Nullable ConfidenceLevel[] confidenceLevels) {
		@@ -1611,7 +1639,47 @@ public class SoundTrigger {
		confidenceLevels != null ? confidenceLevels : new ConfidenceLevel[0];
		}

		public static final @android.annotation.NonNull Parcelable.Creator<KeyphraseRecognitionExtra> CREATOR
		/**
		* The keyphrase ID associated with this class' additional data
		*/
		public int getKeyphraseId() {
		return id;
		}

		/**
		* Recognition modes matched for this event
		*/
		@RecognitionModes
		public int getRecognitionModes() {
		return recognitionModes;
		}

		/**
		* Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification
		* is not performed
		*
		* <p>The confidence level is expressed in percent (0% -100%).
		*/
		public int getCoarseConfidenceLevel() {
		return coarseConfidenceLevel;
		}

		/**
		* Detected confidence level for users defined in a keyphrase.
		*
		* <p>The confidence level is expressed in percent (0% -100%).
		*
		* <p>The user ID is derived from the system ID
		* {@link android.os.UserHandle#getIdentifier()}.
		*
		* @hide
		*/
		@NonNull
		public Collection<ConfidenceLevel> getConfidenceLevels() {
		return Arrays.asList(confidenceLevels);
		}

		public static final @NonNull Parcelable.Creator<KeyphraseRecognitionExtra> CREATOR
		= new Parcelable.Creator<KeyphraseRecognitionExtra>() {
		public KeyphraseRecognitionExtra createFromParcel(Parcel in) {
		return KeyphraseRecognitionExtra.fromParcel(in);
		@@ -1632,7 +1700,7 @@ public class SoundTrigger {
		}

		@Override
		public void writeToParcel(Parcel dest, int flags) {
		public void writeToParcel(@NonNull Parcel dest, int flags) {
		dest.writeInt(id);
		dest.writeInt(recognitionModes);
		dest.writeInt(coarseConfidenceLevel);
		@@ -1657,21 +1725,28 @@ public class SoundTrigger {

		@Override
		public boolean equals(@Nullable Object obj) {
		if (this == obj)
		if (this == obj) {
		return true;
		if (obj == null)
		}
		if (obj == null) {
		return false;
		if (getClass() != obj.getClass())
		}
		if (getClass() != obj.getClass()) {
		return false;
		}
		KeyphraseRecognitionExtra other = (KeyphraseRecognitionExtra) obj;
		if (!Arrays.equals(confidenceLevels, other.confidenceLevels))
		if (!Arrays.equals(confidenceLevels, other.confidenceLevels)) {
		return false;
		if (id != other.id)
		}
		if (id != other.id) {
		return false;
		if (recognitionModes != other.recognitionModes)
		}
		if (recognitionModes != other.recognitionModes) {
		return false;
		if (coarseConfidenceLevel != other.coarseConfidenceLevel)
		}
		if (coarseConfidenceLevel != other.coarseConfidenceLevel) {
		return false;
		}
		return true;
		}

		@@ -1715,7 +1790,7 @@ public class SoundTrigger {
		keyphraseExtras != null ? keyphraseExtras : new KeyphraseRecognitionExtra[0];
		}

		public static final @android.annotation.NonNull Parcelable.Creator<KeyphraseRecognitionEvent> CREATOR
		public static final @NonNull Parcelable.Creator<KeyphraseRecognitionEvent> CREATOR
		= new Parcelable.Creator<KeyphraseRecognitionEvent>() {
		public KeyphraseRecognitionEvent createFromParcel(Parcel in) {
		return KeyphraseRecognitionEvent.fromParcelForKeyphrase(in);

core/java/android/service/voice/AbstractHotwordDetector.java

+4 −1

Original line number	Diff line number	Diff line
		@@ -146,7 +146,10 @@ abstract class AbstractHotwordDetector implements HotwordDetector {
		mHandler.sendMessage(obtainMessage(
		HotwordDetector.Callback::onDetected,
		mCallback,
		new AlwaysOnHotwordDetector.EventPayload(audioFormat, hotwordDetectedResult)));
		new AlwaysOnHotwordDetector.EventPayload.Builder()
		.setCaptureAudioFormat(audioFormat)
		.setHotwordDetectedResult(hotwordDetectedResult)
		.build()));
		}
		}
		}

core/java/android/service/voice/AlwaysOnHotwordDetector.java

+177 −35

Original line number	Diff line number	Diff line
		@@ -23,6 +23,7 @@ import android.annotation.IntDef;
		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.annotation.RequiresPermission;
		import android.annotation.SuppressLint;
		import android.annotation.SystemApi;
		import android.annotation.TestApi;
		import android.app.ActivityThread;
		@@ -59,6 +60,9 @@ import com.android.internal.app.IVoiceInteractionSoundTriggerSession;
		import java.io.PrintWriter;
		import java.lang.annotation.Retention;
		import java.lang.annotation.RetentionPolicy;
		import java.util.Arrays;
		import java.util.Collections;
		import java.util.List;
		import java.util.Locale;

		/**
		@@ -380,47 +384,24 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector {
		private final byte[] mData;
		private final HotwordDetectedResult mHotwordDetectedResult;
		private final ParcelFileDescriptor mAudioStream;
		private final List<KeyphraseRecognitionExtra> mKephraseExtras;

		EventPayload(boolean triggerInData, boolean captureAvailable,
		AudioFormat audioFormat, int captureSession, byte[] data) {
		this(triggerInData, captureAvailable, audioFormat, captureSession, data, null,
		null);
		}

		EventPayload(boolean triggerInData, boolean captureAvailable,
		AudioFormat audioFormat, int captureSession, byte[] data,
		HotwordDetectedResult hotwordDetectedResult) {
		this(triggerInData, captureAvailable, audioFormat, captureSession, data,
		hotwordDetectedResult, null);
		}

		EventPayload(AudioFormat audioFormat, HotwordDetectedResult hotwordDetectedResult) {
		this(false, false, audioFormat, -1, null, hotwordDetectedResult, null);
		}

		EventPayload(AudioFormat audioFormat,
		HotwordDetectedResult hotwordDetectedResult,
		ParcelFileDescriptor audioStream) {
		this(false, false, audioFormat, -1, null, hotwordDetectedResult, audioStream);
		}

		/** @hide */
		@TestApi
		public EventPayload(boolean triggerInData, boolean captureAvailable,
		@Nullable AudioFormat audioFormat, int captureSession, @Nullable byte[] data,
		private EventPayload(boolean captureAvailable,
		@Nullable AudioFormat audioFormat,
		int captureSession,
		@DataFormat int dataFormat,
		@Nullable byte[] data,
		@Nullable HotwordDetectedResult hotwordDetectedResult,
		@Nullable ParcelFileDescriptor audioStream) {
		@Nullable ParcelFileDescriptor audioStream,
		@NonNull List<KeyphraseRecognitionExtra> keyphraseExtras) {
		mCaptureAvailable = captureAvailable;
		mCaptureSession = captureSession;
		mAudioFormat = audioFormat;
		if (triggerInData) {
		mDataFormat = DATA_FORMAT_TRIGGER_AUDIO;
		} else {
		mDataFormat = DATA_FORMAT_RAW;
		}
		mDataFormat = dataFormat;
		mData = data;
		mHotwordDetectedResult = hotwordDetectedResult;
		mAudioStream = audioStream;
		mKephraseExtras = keyphraseExtras;
		}

		/**
		@@ -535,6 +516,166 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector {
		public ParcelFileDescriptor getAudioStream() {
		return mAudioStream;
		}

		/**
		* Returns the keyphrases recognized by the voice engine with additional confidence
		* information
		*
		* @return List of keyphrase extras describing additional data for each keyphrase the voice
		* engine triggered on for this event. The ordering of the list is preserved based on what
		* the ordering provided by {@link android.hardware.soundtrigger.SoundTriggerModule}.
		*/
		@NonNull
		public List<KeyphraseRecognitionExtra> getKeyphraseRecognitionExtras() {
		return mKephraseExtras;
		}

		/**
		* Builder class for {@link EventPayload} objects
		*
		* @hide
		*/
		@TestApi
		public static final class Builder {
		private boolean mCaptureAvailable = false;
		private int mCaptureSession = -1;
		private AudioFormat mAudioFormat = null;
		@DataFormat
		private int mDataFormat = DATA_FORMAT_RAW;
		private byte[] mData = null;
		private HotwordDetectedResult mHotwordDetectedResult = null;
		private ParcelFileDescriptor mAudioStream = null;
		private List<KeyphraseRecognitionExtra> mKeyphraseExtras = Collections.emptyList();

		public Builder() {}

		Builder(SoundTrigger.KeyphraseRecognitionEvent keyphraseRecognitionEvent) {
		setCaptureAvailable(keyphraseRecognitionEvent.isCaptureAvailable());
		setCaptureSession(keyphraseRecognitionEvent.getCaptureSession());
		if (keyphraseRecognitionEvent.getCaptureFormat() != null) {
		setCaptureAudioFormat(keyphraseRecognitionEvent.getCaptureFormat());
		}
		setDataFormat((keyphraseRecognitionEvent.triggerInData) ? DATA_FORMAT_TRIGGER_AUDIO
		: DATA_FORMAT_RAW);
		if (keyphraseRecognitionEvent.getData() != null) {
		setData(keyphraseRecognitionEvent.getData());
		}
		if (keyphraseRecognitionEvent.keyphraseExtras != null) {
		setKeyphraseRecognitionExtras(
		Arrays.asList(keyphraseRecognitionEvent.keyphraseExtras));
		}
		}

		/**
		* Indicates if {@code captureSession} can be used to continue capturing more audio from
		* the DSP hardware.
		*/
		@SuppressLint("MissingGetterMatchingBuilder")
		@NonNull
		public Builder setCaptureAvailable(boolean captureAvailable) {
		mCaptureAvailable = captureAvailable;
		return this;
		}

		/**
		* Sets the session ID to start a capture from the DSP.
		*/
		@SuppressLint("MissingGetterMatchingBuilder")
		@NonNull
		public Builder setCaptureSession(int captureSession) {
		mCaptureSession = captureSession;
		return this;
		}

		/**
		* Sets the format of the audio obtained using {@link #getTriggerAudio()}.
		*/
		@NonNull
		public Builder setCaptureAudioFormat(@NonNull AudioFormat audioFormat) {
		mAudioFormat = audioFormat;
		return this;
		}

		/**
		* Conveys the format of the additional data that is triggered with the keyphrase event.
		*
		* @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO
		* @see DataFormat
		*/
		@NonNull
		public Builder setDataFormat(@DataFormat int dataFormat) {
		mDataFormat = dataFormat;
		return this;
		}

		/**
		* Sets additional raw data that is triggered with the keyphrase event.
		*
		* <p>A {@link android.hardware.soundtrigger.SoundTriggerModule} may populate this
		* field with opaque data for use by system applications who know about voice
		* engine internals. Data may be null if the field is not populated by the
		* {@link android.hardware.soundtrigger.SoundTriggerModule}.
		*
		* <p>If {@link #getDataFormat()} is {@link #DATA_FORMAT_TRIGGER_AUDIO}, then the
		* entirety of this
		* buffer is expected to be of the format from {@link #getCaptureAudioFormat()}.
		*
		* @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO
		*/
		@NonNull
		public Builder setData(@NonNull byte[] data) {
		mData = data;
		return this;
		}

		/**
		* Sets {@link HotwordDetectedResult} associated with the hotword event, passed from
		* {@link HotwordDetectionService}.
		*/
		@NonNull
		public Builder setHotwordDetectedResult(
		@NonNull HotwordDetectedResult hotwordDetectedResult) {
		mHotwordDetectedResult = hotwordDetectedResult;
		return this;
		}

		/**
		* Sets a stream with bytes corresponding to the open audio stream with hotword data.
		*
		* <p>This data represents an audio stream in the format returned by
		* {@link #getCaptureAudioFormat}.
		*
		* <p>Clients are expected to start consuming the stream within 1 second of receiving
		* the
		* event.
		*/
		@NonNull
		public Builder setAudioStream(@NonNull ParcelFileDescriptor audioStream) {
		mAudioStream = audioStream;
		return this;
		}

		/**
		* Sets the keyphrases recognized by the voice engine with additional confidence
		* information
		*/
		@NonNull
		public Builder setKeyphraseRecognitionExtras(
		@NonNull List<KeyphraseRecognitionExtra> keyphraseRecognitionExtras) {
		mKeyphraseExtras = keyphraseRecognitionExtras;
		return this;
		}

		/**
		* Builds an {@link EventPayload} instance
		*/
		@NonNull
		public EventPayload build() {
		return new EventPayload(mCaptureAvailable, mAudioFormat, mCaptureSession,
		mDataFormat, mData, mHotwordDetectedResult, mAudioStream,
		mKeyphraseExtras);
		}
		}
		}

		/**
		@@ -1242,8 +1383,9 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector {
		Slog.i(TAG, "onDetected");
		}
		Message.obtain(mHandler, MSG_HOTWORD_DETECTED,
		new EventPayload(event.triggerInData, event.captureAvailable,
		event.captureFormat, event.captureSession, event.data, result))
		new EventPayload.Builder(event)
		.setHotwordDetectedResult(result)
		.build())
		.sendToTarget();
		}
		@Override