Merge "Hook in startRecogniton call" into lmp-dev (7ee0d458) · Commits · e / os / android_frameworks_base

core/java/android/hardware/soundtrigger/Keyphrase.java

+37 −3

Original line number	Diff line number	Diff line
		@@ -30,7 +30,11 @@ public class Keyphrase implements Parcelable {
		/** A hint text to display corresponding to this keyphrase, e.g. "Hello There". */
		public final String hintText;
		/** The locale of interest when using this Keyphrase. */
		public String locale;
		public final String locale;
		/** The various recognition modes supported by this keyphrase */
		public final int recognitionModeFlags;
		/** The users associated with this keyphrase */
		public final int[] users;

		public static final Parcelable.Creator<Keyphrase> CREATOR
		= new Parcelable.Creator<Keyphrase>() {
		@@ -44,13 +48,26 @@ public class Keyphrase implements Parcelable {
		};

		private static Keyphrase fromParcel(Parcel in) {
		return new Keyphrase(in.readInt(), in.readString(), in.readString());
		int id = in.readInt();
		String hintText = in.readString();
		String locale = in.readString();
		int recognitionModeFlags = in.readInt();
		int numUsers = in.readInt();
		int[] users = null;
		if (numUsers > 0) {
		users = new int[numUsers];
		in.readIntArray(users);
		}
		return new Keyphrase(id, hintText, locale, recognitionModeFlags, users);
		}

		public Keyphrase(int id, String hintText, String locale) {
		public Keyphrase(int id, String hintText, String locale, int recognitionModeFlags,
		int[] users) {
		this.id = id;
		this.hintText = hintText;
		this.locale = locale;
		this.recognitionModeFlags = recognitionModeFlags;
		this.users = users;
		}

		@Override
		@@ -58,6 +75,13 @@ public class Keyphrase implements Parcelable {
		dest.writeInt(id);
		dest.writeString(hintText);
		dest.writeString(locale);
		dest.writeInt(recognitionModeFlags);
		if (users != null) {
		dest.writeInt(users.length);
		dest.writeIntArray(users);
		} else {
		dest.writeInt(0);
		}
		}

		@Override
		@@ -98,4 +122,14 @@ public class Keyphrase implements Parcelable {
		return false;
		return true;
		}

		@Override
		public String toString() {
		return "Keyphrase[id=" + id + ", text=" + hintText + ", locale=" + locale
		+ ", recognitionModes=" + recognitionModeFlags + "]";
		}

		protected SoundTrigger.Keyphrase convertToSoundTriggerKeyphrase() {
		return new SoundTrigger.Keyphrase(id, recognitionModeFlags, locale, hintText, users);
		}
		}

core/java/android/hardware/soundtrigger/KeyphraseEnrollmentInfo.java

+2 −1

Original line number	Diff line number	Diff line
		@@ -239,7 +239,8 @@ public class KeyphraseEnrollmentInfo {
		* @param keyphrase The keyphrase that the user needs to be enrolled to.
		* @param locale The locale for which the enrollment needs to be performed.
		* This is a Java locale, for example "en_US".
		* @return true, if an enrollment client supports the given keyphrase and the given locale.
		* @return The metadata, if an enrollment client supports the given keyphrase
		* and the given locale, null otherwise.
		*/
		public KeyphraseMetadata getKeyphraseMetadata(String keyphrase, String locale) {
		if (mKeyphrases == null \|\| mKeyphrases.length == 0) {

core/java/android/hardware/soundtrigger/KeyphraseSoundModel.java

+11 −0

Original line number	Diff line number	Diff line
		@@ -65,4 +65,15 @@ public class KeyphraseSoundModel implements Parcelable {
		}
		dest.writeParcelableArray(keyphrases, 0);
		}

		public SoundTrigger.KeyphraseSoundModel convertToSoundTriggerKeyphraseSoundModel() {
		SoundTrigger.Keyphrase[] stKeyphrases = null;
		if (keyphrases != null) {
		stKeyphrases = new SoundTrigger.Keyphrase[keyphrases.length];
		for (int i = 0; i < keyphrases.length; i++) {
		stKeyphrases[i] = keyphrases[i].convertToSoundTriggerKeyphrase();
		}
		}
		return new SoundTrigger.KeyphraseSoundModel(uuid, data, stKeyphrases);
		}
		}

core/java/android/hardware/soundtrigger/SoundTriggerHelper.java

+73 −49

Original line number	Diff line number	Diff line
		@@ -17,6 +17,7 @@
		package android.hardware.soundtrigger;

		import android.hardware.soundtrigger.SoundTrigger.ModuleProperties;
		import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig;
		import android.hardware.soundtrigger.SoundTrigger.RecognitionEvent;
		import android.util.Slog;
		import android.util.SparseArray;
		@@ -56,7 +57,7 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener {
		private final ModuleProperties mModuleProperties;
		private final SoundTriggerModule mModule;

		private final SparseArray<Listener> mListeners;
		private final SparseArray<Listener> mActiveListeners;

		private int mCurrentSoundModelHandle = INVALID_SOUND_MODEL_HANDLE;

		@@ -77,7 +78,7 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener {
		public SoundTriggerHelper() {
		ArrayList <ModuleProperties> modules = new ArrayList<>();
		int status = SoundTrigger.listModules(modules);
		mListeners = new SparseArray<>(1);
		mActiveListeners = new SparseArray<>(1);
		if (status != SoundTrigger.STATUS_OK \|\| modules.size() == 0) {
		// TODO: Figure out how to handle errors in listing the modules here.
		dspInfo = null;
		@@ -93,28 +94,10 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener {
		}
		}

		/**
		* @return True, if the given {@link Keyphrase} is supported on DSP.
		*/
		public boolean isKeyphraseSupported(Keyphrase keyphrase) {
		// TODO: We also need to look into a SoundTrigger API that let's us
		// query this. For now just return true.
		return true;
		}

		/**
		* @return True, if the given {@link Keyphrase} has been enrolled.
		*/
		public boolean isKeyphraseEnrolled(Keyphrase keyphrase) {
		// TODO: Query VoiceInteractionManagerService
		// to list registered sound models.
		return false;
		}

		/**
		* @return True, if a recognition for the given {@link Keyphrase} is active.
		*/
		public boolean isKeyphraseActive(Keyphrase keyphrase) {
		public synchronized boolean isKeyphraseActive(Keyphrase keyphrase) {
		// TODO: Check if the recognition for the keyphrase is currently active.
		return false;
		}
		@@ -124,55 +107,98 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener {
		*
		* @param keyphraseId The identifier of the keyphrase for which
		* the recognition is to be started.
		* @param soundModel The sound model to use for recognition.
		* @param listener The listener for the recognition events related to the given keyphrase.
		* @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}.
		*/
		public int startRecognition(int keyphraseId, Listener listener) {
		public synchronized int startRecognition(int keyphraseId,
		SoundTrigger.KeyphraseSoundModel soundModel,
		Listener listener, RecognitionConfig recognitionConfig) {
		if (dspInfo == null \|\| mModule == null) {
		Slog.w(TAG, "Attempting startRecognition without the capability");
		return STATUS_ERROR;
		}

		if (mListeners.get(keyphraseId) != listener) {
		Listener oldListener = mActiveListeners.get(keyphraseId);
		if (oldListener != null && oldListener != listener) {
		if (mCurrentSoundModelHandle != INVALID_SOUND_MODEL_HANDLE) {
		Slog.w(TAG, "Canceling previous recognition");
		// TODO: Inspect the return codes here.
		mModule.unloadSoundModel(mCurrentSoundModelHandle);
		}
		mListeners.get(keyphraseId).onListeningStateChanged(STATE_STOPPED);
		mActiveListeners.get(keyphraseId).onListeningStateChanged(STATE_STOPPED);
		mActiveListeners.remove(keyphraseId);
		}

		int[] handle = new int[] { INVALID_SOUND_MODEL_HANDLE };
		int status = mModule.loadSoundModel(soundModel, handle);
		if (status != SoundTrigger.STATUS_OK) {
		Slog.w(TAG, "loadSoundModel call failed with " + status);
		return STATUS_ERROR;
		}
		if (handle[0] == INVALID_SOUND_MODEL_HANDLE) {
		Slog.w(TAG, "loadSoundModel call returned invalid sound model handle");
		return STATUS_ERROR;
		}

		// Start the recognition.
		status = mModule.startRecognition(handle[0], recognitionConfig);
		if (status != SoundTrigger.STATUS_OK) {
		Slog.w(TAG, "startRecognition failed with " + status);
		return STATUS_ERROR;
		}

		// Everything went well!
		mCurrentSoundModelHandle = handle[0];
		// Register the new listener. This replaces the old one.
		// There can only be a maximum of one active listener for a keyphrase
		// at any given time.
		mListeners.put(keyphraseId, listener);
		// TODO: Get the sound model for the given keyphrase here.
		// mModule.loadSoundModel(model, soundModelHandle);
		// mModule.startRecognition(soundModelHandle, data);
		// mCurrentSoundModelHandle = soundModelHandle;
		return STATUS_ERROR;
		mActiveListeners.put(keyphraseId, listener);
		return STATUS_OK;
		}

		/**
		* Stops recognition for the given {@link Keyphrase} if a recognition is currently active.
		*
		* @param keyphraseId The identifier of the keyphrase for which
		* the recognition is to be stopped.
		* @param listener The listener for the recognition events related to the given keyphrase.
		*
		* @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}.
		*/
		public int stopRecognition(int id, Listener listener) {
		public synchronized int stopRecognition(int keyphraseId, Listener listener) {
		if (dspInfo == null \|\| mModule == null) {
		Slog.w(TAG, "Attempting stopRecognition without the capability");
		return STATUS_ERROR;
		}

		if (mListeners.get(id) != listener) {
		Listener currentListener = mActiveListeners.get(keyphraseId);
		if (currentListener == null) {
		// startRecognition hasn't been called or it failed.
		Slog.w(TAG, "Attempting stopRecognition without a successful startRecognition");
		return STATUS_ERROR;
		} else if (currentListener != listener) {
		// TODO: Figure out if this should match the listener that was passed in during
		// startRecognition, or should we allow a different listener to stop the recognition,
		// in which case we don't need to pass in a listener here.
		Slog.w(TAG, "Attempting stopRecognition for another recognition");
		return STATUS_ERROR;
		} else {
		// Stop recognition if it's the current one, ignore otherwise.
		// TODO: Inspect the return codes here.
		mModule.stopRecognition(mCurrentSoundModelHandle);
		mModule.unloadSoundModel(mCurrentSoundModelHandle);
		int status = mModule.stopRecognition(mCurrentSoundModelHandle);
		if (status != SoundTrigger.STATUS_OK) {
		Slog.w(TAG, "stopRecognition call failed with " + status);
		return STATUS_ERROR;
		}
		status = mModule.unloadSoundModel(mCurrentSoundModelHandle);
		if (status != SoundTrigger.STATUS_OK) {
		Slog.w(TAG, "unloadSoundModel call failed with " + status);
		return STATUS_ERROR;
		}

		mCurrentSoundModelHandle = INVALID_SOUND_MODEL_HANDLE;
		mActiveListeners.remove(keyphraseId);
		return STATUS_OK;
		}
		}
		@@ -184,28 +210,26 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener {
		// TODO: Get the keyphrase out of the event and fire events on it.
		// For now, as a nasty workaround, we fire all events to the listener for
		// keyphrase with TEMP_KEYPHRASE_ID.

		switch (event.status) {
		case SoundTrigger.RECOGNITION_STATUS_SUCCESS:
		Listener listener = null;
		synchronized(this) {
		// TODO: The keyphrase should come from the recognition event
		// as it may be for a different keyphrase than the current one.
		if (mListeners.get(TEMP_KEYPHRASE_ID) != null) {
		mListeners.get(TEMP_KEYPHRASE_ID).onKeyphraseSpoken();
		listener = mActiveListeners.get(TEMP_KEYPHRASE_ID);
		}
		if (listener == null) {
		Slog.w(TAG, "received onRecognition event without any listener for it");
		return;
		}

		switch (event.status) {
		case SoundTrigger.RECOGNITION_STATUS_SUCCESS:
		listener.onKeyphraseSpoken();
		break;
		case SoundTrigger.RECOGNITION_STATUS_ABORT:
		// TODO: The keyphrase should come from the recognition event
		// as it may be for a different keyphrase than the current one.
		if (mListeners.get(TEMP_KEYPHRASE_ID) != null) {
		mListeners.get(TEMP_KEYPHRASE_ID).onListeningStateChanged(STATE_STOPPED);
		}
		listener.onListeningStateChanged(STATE_STOPPED);
		break;
		case SoundTrigger.RECOGNITION_STATUS_FAILURE:
		// TODO: The keyphrase should come from the recognition event
		// as it may be for a different keyphrase than the current one.
		if (mListeners.get(TEMP_KEYPHRASE_ID) != null) {
		mListeners.get(TEMP_KEYPHRASE_ID).onListeningStateChanged(STATE_STOPPED);
		}
		listener.onListeningStateChanged(STATE_STOPPED);
		break;
		}
		}

core/java/android/service/voice/AlwaysOnHotwordDetector.java

+83 −15

Original line number	Diff line number	Diff line
		@@ -20,9 +20,19 @@ import android.content.Intent;
		import android.hardware.soundtrigger.Keyphrase;
		import android.hardware.soundtrigger.KeyphraseEnrollmentInfo;
		import android.hardware.soundtrigger.KeyphraseMetadata;
		import android.hardware.soundtrigger.KeyphraseSoundModel;
		import android.hardware.soundtrigger.SoundTrigger;
		import android.hardware.soundtrigger.SoundTrigger.ConfidenceLevel;
		import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra;
		import android.hardware.soundtrigger.SoundTriggerHelper;
		import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig;
		import android.os.RemoteException;
		import android.util.Slog;

		import com.android.internal.app.IVoiceInteractionManagerService;

		import java.util.List;

		/**
		* A class that lets a VoiceInteractionService implementation interact with
		* always-on keyphrase detection APIs.
		@@ -72,11 +82,22 @@ public class AlwaysOnHotwordDetector {

		private final String mText;
		private final String mLocale;
		private final Keyphrase mKeyphrase;
		/**
		* The metadata of the Keyphrase, derived from the enrollment application.
		* This may be null if this keyphrase isn't supported by the enrollment application.
		*/
		private final KeyphraseMetadata mKeyphraseMetadata;
		/**
		* The sound model for the keyphrase, derived from the model management service
		* (IVoiceInteractionManagerService). May be null if the keyphrase isn't enrolled yet.
		*/
		private final KeyphraseSoundModel mEnrolledSoundModel;
		private final KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo;
		private final SoundTriggerHelper mSoundTriggerHelper;
		private final SoundTriggerHelper.Listener mListener;
		private final int mAvailability;
		private final IVoiceInteractionService mVoiceInteractionService;
		private final IVoiceInteractionManagerService mModelManagementService;

		private int mRecognitionState;

		@@ -103,25 +124,30 @@ public class AlwaysOnHotwordDetector {
		* @param text The keyphrase text to get the detector for.
		* @param locale The java locale for the detector.
		* @param callback A non-null Callback for receiving the recognition events.
		* @param voiceInteractionService The current voice interaction service.
		* @param modelManagementService A service that allows management of sound models.
		*
		* @hide
		*/
		public AlwaysOnHotwordDetector(String text, String locale, Callback callback,
		KeyphraseEnrollmentInfo keyphraseEnrollmentInfo,
		SoundTriggerHelper soundTriggerHelper) {
		SoundTriggerHelper soundTriggerHelper,
		IVoiceInteractionService voiceInteractionService,
		IVoiceInteractionManagerService modelManagementService) {
		mText = text;
		mLocale = locale;
		mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo;
		KeyphraseMetadata keyphraseMetadata =
		mKeyphraseEnrollmentInfo.getKeyphraseMetadata(text, locale);
		if (keyphraseMetadata != null) {
		mKeyphrase = new Keyphrase(keyphraseMetadata.id, text, locale);
		} else {
		mKeyphrase = null;
		}
		mKeyphraseMetadata = mKeyphraseEnrollmentInfo.getKeyphraseMetadata(text, locale);
		mListener = new SoundTriggerListener(callback);
		mSoundTriggerHelper = soundTriggerHelper;
		mAvailability = getAvailabilityInternal();
		mVoiceInteractionService = voiceInteractionService;
		mModelManagementService = modelManagementService;
		if (mKeyphraseMetadata != null) {
		mEnrolledSoundModel = internalGetKeyphraseSoundModel(mKeyphraseMetadata.id);
		} else {
		mEnrolledSoundModel = null;
		}
		mAvailability = internalGetAvailability();
		}

		/**
		@@ -171,7 +197,16 @@ public class AlwaysOnHotwordDetector {
		}

		mRecognitionState = RECOGNITION_REQUESTED;
		int code = mSoundTriggerHelper.startRecognition(mKeyphrase.id, mListener);
		mRecognitionState = RECOGNITION_REQUESTED;
		KeyphraseRecognitionExtra[] recognitionExtra = new KeyphraseRecognitionExtra[1];
		// TODO: Do we need to do something about the confidence level here?
		// TODO: Read the recognition mode flag from the KeyphraseMetadata.
		// TODO: Take in captureTriggerAudio as a method param here.
		recognitionExtra[0] = new KeyphraseRecognitionExtra(mKeyphraseMetadata.id,
		SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER, new ConfidenceLevel[0]);
		int code = mSoundTriggerHelper.startRecognition(mKeyphraseMetadata.id,
		mEnrolledSoundModel.convertToSoundTriggerKeyphraseSoundModel(), mListener,
		new RecognitionConfig(false, recognitionExtra, null /* additional data */));
		if (code != SoundTriggerHelper.STATUS_OK) {
		Slog.w(TAG, "startRecognition() failed with error code " + code);
		return STATUS_ERROR;
		@@ -195,7 +230,8 @@ public class AlwaysOnHotwordDetector {
		}

		mRecognitionState = RECOGNITION_NOT_REQUESTED;
		int code = mSoundTriggerHelper.stopRecognition(mKeyphrase.id, mListener);
		int code = mSoundTriggerHelper.stopRecognition(mKeyphraseMetadata.id, mListener);

		if (code != SoundTriggerHelper.STATUS_OK) {
		Slog.w(TAG, "stopRecognition() failed with error code " + code);
		return STATUS_ERROR;
		@@ -230,19 +266,51 @@ public class AlwaysOnHotwordDetector {
		return mKeyphraseEnrollmentInfo.getManageKeyphraseIntent(action, mText, mLocale);
		}

		private int getAvailabilityInternal() {
		private int internalGetAvailability() {
		// No DSP available
		if (mSoundTriggerHelper.dspInfo == null) {
		return KEYPHRASE_HARDWARE_UNAVAILABLE;
		}
		if (mKeyphrase == null \|\| !mSoundTriggerHelper.isKeyphraseSupported(mKeyphrase)) {
		// No enrollment application supports this keyphrase/locale
		if (mKeyphraseMetadata == null) {
		return KEYPHRASE_UNSUPPORTED;
		}
		if (!mSoundTriggerHelper.isKeyphraseEnrolled(mKeyphrase)) {
		// This keyphrase hasn't been enrolled.
		if (mEnrolledSoundModel == null) {
		return KEYPHRASE_UNENROLLED;
		}
		return KEYPHRASE_ENROLLED;
		}

		/**
		* @return The corresponding {@link KeyphraseSoundModel} or null if none is found.
		*/
		private KeyphraseSoundModel internalGetKeyphraseSoundModel(int keyphraseId) {
		List<KeyphraseSoundModel> soundModels;
		try {
		soundModels = mModelManagementService
		.listRegisteredKeyphraseSoundModels(mVoiceInteractionService);
		if (soundModels == null \|\| soundModels.isEmpty()) {
		Slog.i(TAG, "No available sound models for keyphrase ID: " + keyphraseId);
		return null;
		}
		for (KeyphraseSoundModel soundModel : soundModels) {
		if (soundModel.keyphrases == null) {
		continue;
		}
		for (Keyphrase keyphrase : soundModel.keyphrases) {
		// TODO: Check the user handle here to only load a model for the current user.
		if (keyphrase.id == keyphraseId) {
		return soundModel;
		}
		}
		}
		} catch (RemoteException e) {
		Slog.w(TAG, "RemoteException in listRegisteredKeyphraseSoundModels!");
		}
		return null;
		}

		/** @hide */
		static final class SoundTriggerListener implements SoundTriggerHelper.Listener {
		private final Callback mCallback;