Loading core/java/android/hardware/soundtrigger/SoundTrigger.java +64 −12 Original line number Diff line number Diff line Loading @@ -150,15 +150,16 @@ public class SoundTrigger { /** Key phrase text */ public final String text; /** Number of users this key phrase has been trained for */ public final int numUsers; /** Users this key phrase has been trained for. countains sound trigger specific user IDs * derived from system user IDs {@link android.os.UserHandle#getIdentifier()}. */ public final int[] users; public Keyphrase(int id, int recognitionModes, String locale, String text, int numUsers) { public Keyphrase(int id, int recognitionModes, String locale, String text, int[] users) { this.id = id; this.recognitionModes = recognitionModes; this.locale = locale; this.text = text; this.numUsers = numUsers; this.users = users; } } Loading Loading @@ -215,36 +216,86 @@ public class SoundTrigger { /** Delay in ms between end of model detection and start of audio available for capture. * A negative value is possible (e.g. if keyphrase is also available for capture) */ public final int captureDelayMs; /** Duration in ms of audio captured before the start of the trigger. 0 if none. */ public final int capturePreambleMs; /** Opaque data for use by system applications who know about voice engine internals, * typically during enrollment. */ public final byte[] data; RecognitionEvent(int status, int soundModelHandle, boolean captureAvailable, int captureSession, int captureDelayMs, byte[] data) { int captureSession, int captureDelayMs, int capturePreambleMs, byte[] data) { this.status = status; this.soundModelHandle = soundModelHandle; this.captureAvailable = captureAvailable; this.captureSession = captureSession; this.captureDelayMs = captureDelayMs; this.capturePreambleMs = capturePreambleMs; this.data = data; } } /** * A RecognitionConfig is provided to * {@link SoundTriggerModule#startRecognition(int, RecognitionConfig)} to configure the * recognition request. */ public static class RecognitionConfig { /** True if the DSP should capture the trigger sound and make it available for further * capture. */ public final boolean captureRequested; /** List of all keyphrases in the sound model for which recognition should be performed with * options for each keyphrase. */ public final KeyphraseRecognitionExtra keyphrases[]; /** Opaque data for use by system applications who know about voice engine internals, * typically during enrollment. */ public final byte[] data; public RecognitionConfig(boolean captureRequested, KeyphraseRecognitionExtra keyphrases[], byte[] data) { this.captureRequested = captureRequested; this.keyphrases = keyphrases; this.data = data; } } /** * Confidence level for users defined in a keyphrase. * - The confidence level is expressed in percent (0% -100%). * When used in a {@link KeyphraseRecognitionEvent} it indicates the detected confidence level * When used in a {@link RecognitionConfig} it indicates the minimum confidence level that * should trigger a recognition. * - The user ID is derived from the system ID {@link android.os.UserHandle#getIdentifier()}. */ public static class ConfidenceLevel { public final int userId; public final int confidenceLevel; public ConfidenceLevel(int userId, int confidenceLevel) { this.userId = userId; this.confidenceLevel = confidenceLevel; } } /** * Additional data conveyed by a {@link KeyphraseRecognitionEvent} * for a key phrase detection. */ public static class KeyphraseRecognitionExtra { /** Confidence level for each user defined in the key phrase in the same order as * users in the key phrase. The confidence level is expressed in percentage (0% -100%) */ public final int[] confidenceLevels; /** The keyphrse ID */ public final int id; /** Recognition modes matched for this event */ public final int recognitionModes; KeyphraseRecognitionExtra(int[] confidenceLevels, int recognitionModes) { this.confidenceLevels = confidenceLevels; /** Confidence levels for all users recognized (KeyphraseRecognitionEvent) or to * be recognized (RecognitionConfig) */ public final ConfidenceLevel[] confidenceLevels; public KeyphraseRecognitionExtra(int id, int recognitionModes, ConfidenceLevel[] confidenceLevels) { this.id = id; this.recognitionModes = recognitionModes; this.confidenceLevels = confidenceLevels; } } Loading @@ -259,9 +310,10 @@ public class SoundTrigger { public final boolean keyphraseInCapture; KeyphraseRecognitionEvent(int status, int soundModelHandle, boolean captureAvailable, int captureSession, int captureDelayMs, byte[] data, int captureSession, int captureDelayMs, int capturePreambleMs, byte[] data, boolean keyphraseInCapture, KeyphraseRecognitionExtra[] keyphraseExtras) { super(status, soundModelHandle, captureAvailable, captureSession, captureDelayMs, data); super(status, soundModelHandle, captureAvailable, captureSession, captureDelayMs, capturePreambleMs, data); this.keyphraseInCapture = keyphraseInCapture; this.keyphraseExtras = keyphraseExtras; } Loading core/java/android/hardware/soundtrigger/SoundTriggerModule.java +3 −2 Original line number Diff line number Diff line Loading @@ -94,7 +94,8 @@ public class SoundTriggerModule { * Recognition must be restarted after each callback (success or failure) received on * the {@link SoundTrigger.StatusListener}. * @param soundModelHandle The sound model handle to start listening to * @param data Opaque data for use by the implementation for this recognition * @param config contains configuration information for this recognition request: * recognition mode, keyphrases, users, minimum confidence levels... * @return - {@link SoundTrigger#STATUS_OK} in case of success * - {@link SoundTrigger#STATUS_ERROR} in case of unspecified error * - {@link SoundTrigger#STATUS_PERMISSION_DENIED} if the caller does not have Loading @@ -105,7 +106,7 @@ public class SoundTriggerModule { * service fails * - {@link SoundTrigger#STATUS_INVALID_OPERATION} if the call is out of sequence */ public native int startRecognition(int soundModelHandle, byte[] data); public native int startRecognition(int soundModelHandle, SoundTrigger.RecognitionConfig config); /** * Stop listening to all key phrases in a {@link SoundTrigger.SoundModel} Loading core/jni/android_hardware_SoundTrigger.cpp +154 −34 Original line number Diff line number Diff line Loading @@ -74,7 +74,7 @@ static struct { jfieldID recognitionModes; jfieldID locale; jfieldID text; jfieldID numUsers; jfieldID users; } gKeyphraseFields; static const char* const kKeyphraseSoundModelClassPathName = Loading @@ -84,6 +84,14 @@ static struct { jfieldID keyphrases; } gKeyphraseSoundModelFields; static const char* const kRecognitionConfigClassPathName = "android/hardware/soundtrigger/SoundTrigger$RecognitionConfig"; static jclass gRecognitionConfigClass; static struct { jfieldID captureRequested; jfieldID keyphrases; jfieldID data; } gRecognitionConfigFields; static const char* const kRecognitionEventClassPathName = "android/hardware/soundtrigger/SoundTrigger$RecognitionEvent"; Loading @@ -99,6 +107,20 @@ static const char* const kKeyphraseRecognitionExtraClassPathName = "android/hardware/soundtrigger/SoundTrigger$KeyphraseRecognitionExtra"; static jclass gKeyphraseRecognitionExtraClass; static jmethodID gKeyphraseRecognitionExtraCstor; static struct { jfieldID id; jfieldID recognitionModes; jfieldID confidenceLevels; } gKeyphraseRecognitionExtraFields; static const char* const kConfidenceLevelClassPathName = "android/hardware/soundtrigger/SoundTrigger$ConfidenceLevel"; static jclass gConfidenceLevelClass; static jmethodID gConfidenceLevelCstor; static struct { jfieldID userId; jfieldID confidenceLevel; } gConfidenceLevelFields; static Mutex gLock; Loading Loading @@ -183,34 +205,45 @@ void JNISoundTriggerCallback::onRecognitionEvent(struct sound_trigger_recognitio } for (size_t i = 0; i < phraseEvent->num_phrases; i++) { jintArray jConfidenceLevels = env->NewIntArray(phraseEvent->phrase_extras[i].num_users); jobjectArray jConfidenceLevels = env->NewObjectArray( phraseEvent->phrase_extras[i].num_levels, gConfidenceLevelClass, NULL); if (jConfidenceLevels == NULL) { return; } jint *nConfidenceLevels = env->GetIntArrayElements(jConfidenceLevels, NULL); memcpy(nConfidenceLevels, phraseEvent->phrase_extras[i].confidence_levels, phraseEvent->phrase_extras[i].num_users * sizeof(int)); env->ReleaseIntArrayElements(jConfidenceLevels, nConfidenceLevels, 0); for (size_t j = 0; j < phraseEvent->phrase_extras[i].num_levels; j++) { jobject jConfidenceLevel = env->NewObject(gConfidenceLevelClass, gConfidenceLevelCstor, phraseEvent->phrase_extras[i].levels[j].user_id, phraseEvent->phrase_extras[i].levels[j].level); env->SetObjectArrayElement(jConfidenceLevels, j, jConfidenceLevel); env->DeleteLocalRef(jConfidenceLevel); } jobject jNewExtra = env->NewObject(gKeyphraseRecognitionExtraClass, gKeyphraseRecognitionExtraCstor, jConfidenceLevels, phraseEvent->phrase_extras[i].recognition_modes); phraseEvent->phrase_extras[i].id, phraseEvent->phrase_extras[i].recognition_modes, jConfidenceLevels); if (jNewExtra == NULL) { return; } env->SetObjectArrayElement(jExtras, i, jNewExtra); env->DeleteLocalRef(jNewExtra); env->DeleteLocalRef(jConfidenceLevels); } jEvent = env->NewObject(gKeyphraseRecognitionEventClass, gKeyphraseRecognitionEventCstor, event->status, event->model, event->capture_available, event->capture_session, event->capture_delay_ms, jData, event->capture_session, event->capture_delay_ms, event->capture_preamble_ms, jData, phraseEvent->key_phrase_in_capture, jExtras); } else { jEvent = env->NewObject(gRecognitionEventClass, gRecognitionEventCstor, event->status, event->model, event->capture_available, event->capture_session, event->capture_delay_ms, jData); event->capture_session, event->capture_delay_ms, event->capture_preamble_ms, jData); } Loading Loading @@ -381,7 +414,7 @@ android_hardware_SoundTrigger_loadSoundModel(JNIEnv *env, jobject thiz, jobject jSoundModel, jintArray jHandle) { jint status = SOUNDTRIGGER_STATUS_OK; char *nData = NULL; jbyte *nData = NULL; struct sound_trigger_sound_model *nSoundModel; jbyteArray jData; sp<MemoryDealer> memoryDealer; Loading Loading @@ -437,7 +470,7 @@ android_hardware_SoundTrigger_loadSoundModel(JNIEnv *env, jobject thiz, } size = env->GetArrayLength(jData); nData = (char *)env->GetByteArrayElements(jData, NULL); nData = env->GetByteArrayElements(jData, NULL); if (jData == NULL) { status = SOUNDTRIGGER_STATUS_ERROR; goto exit; Loading Loading @@ -481,8 +514,16 @@ android_hardware_SoundTrigger_loadSoundModel(JNIEnv *env, jobject thiz, env->GetIntField(jPhrase,gKeyphraseFields.id); phraseModel->phrases[i].recognition_mode = env->GetIntField(jPhrase,gKeyphraseFields.recognitionModes); phraseModel->phrases[i].num_users = env->GetIntField(jPhrase, gKeyphraseFields.numUsers); jintArray jUsers = (jintArray)env->GetObjectField(jPhrase, gKeyphraseFields.users); phraseModel->phrases[i].num_users = env->GetArrayLength(jUsers); jint *nUsers = env->GetIntArrayElements(jUsers, NULL); memcpy(phraseModel->phrases[i].users, nUsers, phraseModel->phrases[i].num_users * sizeof(int)); env->ReleaseIntArrayElements(jUsers, nUsers, 0); env->DeleteLocalRef(jUsers); jstring jLocale = (jstring)env->GetObjectField(jPhrase, gKeyphraseFields.locale); const char *nLocale = env->GetStringUTFChars(jLocale, NULL); strncpy(phraseModel->phrases[i].locale, Loading @@ -500,6 +541,7 @@ android_hardware_SoundTrigger_loadSoundModel(JNIEnv *env, jobject thiz, env->DeleteLocalRef(jText); ALOGV("loadSoundModel phrases %d text %s locale %s", i, phraseModel->phrases[i].text, phraseModel->phrases[i].locale); env->DeleteLocalRef(jPhrase); } env->DeleteLocalRef(jPhrases); } Loading @@ -512,7 +554,7 @@ exit: env->ReleaseIntArrayElements(jHandle, nHandle, NULL); } if (nData != NULL) { env->ReleaseByteArrayElements(jData, (jbyte *)nData, NULL); env->ReleaseByteArrayElements(jData, nData, NULL); } return status; } Loading @@ -534,7 +576,7 @@ android_hardware_SoundTrigger_unloadSoundModel(JNIEnv *env, jobject thiz, static jint android_hardware_SoundTrigger_startRecognition(JNIEnv *env, jobject thiz, jint jHandle, jbyteArray jData) jint jHandle, jobject jConfig) { jint status = SOUNDTRIGGER_STATUS_OK; ALOGV("startRecognition"); Loading @@ -542,29 +584,82 @@ android_hardware_SoundTrigger_startRecognition(JNIEnv *env, jobject thiz, if (module == NULL) { return SOUNDTRIGGER_STATUS_ERROR; } if (!env->IsInstanceOf(jConfig, gRecognitionConfigClass)) { return SOUNDTRIGGER_STATUS_BAD_VALUE; } jbyteArray jData = (jbyteArray)env->GetObjectField(jConfig, gRecognitionConfigFields.data); jsize dataSize = 0; char *nData = NULL; sp<IMemory> memory; jbyte *nData = NULL; if (jData != NULL) { dataSize = env->GetArrayLength(jData); if (dataSize == 0) { return SOUNDTRIGGER_STATUS_BAD_VALUE; } nData = (char *)env->GetByteArrayElements(jData, NULL); nData = env->GetByteArrayElements(jData, NULL); if (nData == NULL) { return SOUNDTRIGGER_STATUS_ERROR; } } size_t totalSize = sizeof(struct sound_trigger_recognition_config) + dataSize; sp<MemoryDealer> memoryDealer = new MemoryDealer(dataSize, "SoundTrigge-JNI::StartRecognition"); new MemoryDealer(totalSize, "SoundTrigge-JNI::StartRecognition"); if (memoryDealer == 0) { return SOUNDTRIGGER_STATUS_ERROR; } memory = memoryDealer->allocate(dataSize); sp<IMemory> memory = memoryDealer->allocate(totalSize); if (memory == 0 || memory->pointer() == NULL) { return SOUNDTRIGGER_STATUS_ERROR; } memcpy(memory->pointer(), nData, dataSize); if (dataSize != 0) { memcpy((char *)memory->pointer() + sizeof(struct sound_trigger_recognition_config), nData, dataSize); env->ReleaseByteArrayElements(jData, nData, 0); } env->DeleteLocalRef(jData); struct sound_trigger_recognition_config *config = (struct sound_trigger_recognition_config *)memory->pointer(); config->data_size = dataSize; config->data_offset = sizeof(struct sound_trigger_recognition_config); config->capture_requested = env->GetIntField(jConfig, gRecognitionConfigFields.captureRequested); config->num_phrases = 0; jobjectArray jPhrases = (jobjectArray)env->GetObjectField(jConfig, gRecognitionConfigFields.keyphrases); if (jPhrases != NULL) { config->num_phrases = env->GetArrayLength(jPhrases); } ALOGV("startRecognition num phrases %d", config->num_phrases); for (size_t i = 0; i < config->num_phrases; i++) { jobject jPhrase = env->GetObjectArrayElement(jPhrases, i); config->phrases[i].id = env->GetIntField(jPhrase, gKeyphraseRecognitionExtraFields.id); config->phrases[i].recognition_modes = env->GetIntField(jPhrase, gKeyphraseRecognitionExtraFields.recognitionModes); config->phrases[i].num_levels = 0; jobjectArray jConfidenceLevels = (jobjectArray)env->GetObjectField(jPhrase, gKeyphraseRecognitionExtraFields.confidenceLevels); if (jConfidenceLevels != NULL) { config->phrases[i].num_levels = env->GetArrayLength(jConfidenceLevels); } ALOGV("startRecognition phrase %d num_levels %d", i, config->phrases[i].num_levels); for (size_t j = 0; j < config->phrases[i].num_levels; j++) { jobject jConfidenceLevel = env->GetObjectArrayElement(jConfidenceLevels, j); config->phrases[i].levels[j].user_id = env->GetIntField(jConfidenceLevel, gConfidenceLevelFields.userId); config->phrases[i].levels[j].level = env->GetIntField(jConfidenceLevel, gConfidenceLevelFields.confidenceLevel); env->DeleteLocalRef(jConfidenceLevel); } ALOGV("startRecognition phrases %d", i); env->DeleteLocalRef(jConfidenceLevels); env->DeleteLocalRef(jPhrase); } env->DeleteLocalRef(jPhrases); status = module->startRecognition(jHandle, memory); return status; Loading Loading @@ -608,7 +703,7 @@ static JNINativeMethod gModuleMethods[] = { "(I)I", (void *)android_hardware_SoundTrigger_unloadSoundModel}, {"startRecognition", "(I[B)I", "(ILandroid/hardware/soundtrigger/SoundTrigger$RecognitionConfig;)I", (void *)android_hardware_SoundTrigger_startRecognition}, {"stopRecognition", "(I)I", Loading Loading @@ -652,7 +747,7 @@ int register_android_hardware_SoundTrigger(JNIEnv *env) gKeyphraseFields.recognitionModes = env->GetFieldID(keyphraseClass, "recognitionModes", "I"); gKeyphraseFields.locale = env->GetFieldID(keyphraseClass, "locale", "Ljava/lang/String;"); gKeyphraseFields.text = env->GetFieldID(keyphraseClass, "text", "Ljava/lang/String;"); gKeyphraseFields.numUsers = env->GetFieldID(keyphraseClass, "numUsers", "I"); gKeyphraseFields.users = env->GetFieldID(keyphraseClass, "users", "[I"); jclass keyphraseSoundModelClass = env->FindClass(kKeyphraseSoundModelClassPathName); gKeyphraseSoundModelClass = (jclass) env->NewGlobalRef(keyphraseSoundModelClass); Loading @@ -664,18 +759,42 @@ int register_android_hardware_SoundTrigger(JNIEnv *env) jclass recognitionEventClass = env->FindClass(kRecognitionEventClassPathName); gRecognitionEventClass = (jclass) env->NewGlobalRef(recognitionEventClass); gRecognitionEventCstor = env->GetMethodID(recognitionEventClass, "<init>", "(IIZII[B)V"); "(IIZIII[B)V"); jclass keyphraseRecognitionEventClass = env->FindClass(kKeyphraseRecognitionEventClassPathName); gKeyphraseRecognitionEventClass = (jclass) env->NewGlobalRef(keyphraseRecognitionEventClass); gKeyphraseRecognitionEventCstor = env->GetMethodID(keyphraseRecognitionEventClass, "<init>", "(IIZII[BZ[Landroid/hardware/soundtrigger/SoundTrigger$KeyphraseRecognitionExtra;)V"); "(IIZIII[BZ[Landroid/hardware/soundtrigger/SoundTrigger$KeyphraseRecognitionExtra;)V"); jclass keyRecognitionConfigClass = env->FindClass(kRecognitionConfigClassPathName); gRecognitionConfigClass = (jclass) env->NewGlobalRef(keyRecognitionConfigClass); gRecognitionConfigFields.captureRequested = env->GetFieldID(keyRecognitionConfigClass, "captureRequested", "Z"); gRecognitionConfigFields.keyphrases = env->GetFieldID(keyRecognitionConfigClass, "keyphrases", "[Landroid/hardware/soundtrigger/SoundTrigger$KeyphraseRecognitionExtra;"); gRecognitionConfigFields.data = env->GetFieldID(keyRecognitionConfigClass, "data", "[B"); jclass keyphraseRecognitionExtraClass = env->FindClass(kKeyphraseRecognitionExtraClassPathName); gKeyphraseRecognitionExtraClass = (jclass) env->NewGlobalRef(keyphraseRecognitionExtraClass); gKeyphraseRecognitionExtraCstor = env->GetMethodID(keyphraseRecognitionExtraClass, "<init>", "([II)V"); "(II[Landroid/hardware/soundtrigger/SoundTrigger$ConfidenceLevel;)V"); gKeyphraseRecognitionExtraFields.id = env->GetFieldID(gKeyphraseRecognitionExtraClass, "id", "I"); gKeyphraseRecognitionExtraFields.recognitionModes = env->GetFieldID(gKeyphraseRecognitionExtraClass, "recognitionModes", "I"); gKeyphraseRecognitionExtraFields.confidenceLevels = env->GetFieldID(gKeyphraseRecognitionExtraClass, "confidenceLevels", "[Landroid/hardware/soundtrigger/SoundTrigger$ConfidenceLevel;"); jclass confidenceLevelClass = env->FindClass(kConfidenceLevelClassPathName); gConfidenceLevelClass = (jclass) env->NewGlobalRef(confidenceLevelClass); gConfidenceLevelCstor = env->GetMethodID(confidenceLevelClass, "<init>", "(II)V"); gConfidenceLevelFields.userId = env->GetFieldID(confidenceLevelClass, "userId", "I"); gConfidenceLevelFields.confidenceLevel = env->GetFieldID(confidenceLevelClass, "confidenceLevel", "I"); int status = AndroidRuntime::registerNativeMethods(env, kSoundTriggerClassPathName, gMethods, NELEM(gMethods)); Loading @@ -685,5 +804,6 @@ int register_android_hardware_SoundTrigger(JNIEnv *env) kModuleClassPathName, gModuleMethods, NELEM(gModuleMethods)); } return status; } services/voiceinteraction/java/com/android/server/voiceinteraction/DatabaseHelper.java +4 −4 Original line number Diff line number Diff line Loading @@ -46,7 +46,7 @@ public class DatabaseHelper extends SQLiteOpenHelper { public static final String KEY_RECOGNITION_MODES = "modes"; public static final String KEY_LOCALE = "locale"; public static final String KEY_HINT_TEXT = "hint_text"; public static final String KEY_NUM_USERS = "num_users"; public static final String KEY_USERS = "users"; public static final String KEY_SOUND_MODEL_ID = "sound_model_id"; } Loading @@ -62,7 +62,7 @@ public class DatabaseHelper extends SQLiteOpenHelper { + KeyphraseContract.TABLE + "(" + KeyphraseContract.KEY_ID + " INTEGER PRIMARY KEY," + KeyphraseContract.KEY_RECOGNITION_MODES + " INTEGER," + KeyphraseContract.KEY_NUM_USERS + " INTEGER," + KeyphraseContract.KEY_USERS + " INTEGER," + KeyphraseContract.KEY_SOUND_MODEL_ID + " TEXT," + KeyphraseContract.KEY_LOCALE + " TEXT," + KeyphraseContract.KEY_HINT_TEXT + " TEXT" + ")"; Loading Loading @@ -167,11 +167,11 @@ public class DatabaseHelper extends SQLiteOpenHelper { do { int id = c.getInt(c.getColumnIndex(KeyphraseContract.KEY_ID)); int modes = c.getInt(c.getColumnIndex(KeyphraseContract.KEY_RECOGNITION_MODES)); int numUsers = c.getInt(c.getColumnIndex(KeyphraseContract.KEY_NUM_USERS)); int[] users = {c.getInt(c.getColumnIndex(KeyphraseContract.KEY_USERS))}; String locale = c.getString(c.getColumnIndex(KeyphraseContract.KEY_LOCALE)); String hintText = c.getString(c.getColumnIndex(KeyphraseContract.KEY_HINT_TEXT)); keyphrases.add(new Keyphrase(id, modes, locale, hintText, numUsers)); keyphrases.add(new Keyphrase(id, modes, locale, hintText, users)); } while (c.moveToNext()); } Keyphrase[] keyphraseArr = new Keyphrase[keyphrases.size()]; Loading Loading
core/java/android/hardware/soundtrigger/SoundTrigger.java +64 −12 Original line number Diff line number Diff line Loading @@ -150,15 +150,16 @@ public class SoundTrigger { /** Key phrase text */ public final String text; /** Number of users this key phrase has been trained for */ public final int numUsers; /** Users this key phrase has been trained for. countains sound trigger specific user IDs * derived from system user IDs {@link android.os.UserHandle#getIdentifier()}. */ public final int[] users; public Keyphrase(int id, int recognitionModes, String locale, String text, int numUsers) { public Keyphrase(int id, int recognitionModes, String locale, String text, int[] users) { this.id = id; this.recognitionModes = recognitionModes; this.locale = locale; this.text = text; this.numUsers = numUsers; this.users = users; } } Loading Loading @@ -215,36 +216,86 @@ public class SoundTrigger { /** Delay in ms between end of model detection and start of audio available for capture. * A negative value is possible (e.g. if keyphrase is also available for capture) */ public final int captureDelayMs; /** Duration in ms of audio captured before the start of the trigger. 0 if none. */ public final int capturePreambleMs; /** Opaque data for use by system applications who know about voice engine internals, * typically during enrollment. */ public final byte[] data; RecognitionEvent(int status, int soundModelHandle, boolean captureAvailable, int captureSession, int captureDelayMs, byte[] data) { int captureSession, int captureDelayMs, int capturePreambleMs, byte[] data) { this.status = status; this.soundModelHandle = soundModelHandle; this.captureAvailable = captureAvailable; this.captureSession = captureSession; this.captureDelayMs = captureDelayMs; this.capturePreambleMs = capturePreambleMs; this.data = data; } } /** * A RecognitionConfig is provided to * {@link SoundTriggerModule#startRecognition(int, RecognitionConfig)} to configure the * recognition request. */ public static class RecognitionConfig { /** True if the DSP should capture the trigger sound and make it available for further * capture. */ public final boolean captureRequested; /** List of all keyphrases in the sound model for which recognition should be performed with * options for each keyphrase. */ public final KeyphraseRecognitionExtra keyphrases[]; /** Opaque data for use by system applications who know about voice engine internals, * typically during enrollment. */ public final byte[] data; public RecognitionConfig(boolean captureRequested, KeyphraseRecognitionExtra keyphrases[], byte[] data) { this.captureRequested = captureRequested; this.keyphrases = keyphrases; this.data = data; } } /** * Confidence level for users defined in a keyphrase. * - The confidence level is expressed in percent (0% -100%). * When used in a {@link KeyphraseRecognitionEvent} it indicates the detected confidence level * When used in a {@link RecognitionConfig} it indicates the minimum confidence level that * should trigger a recognition. * - The user ID is derived from the system ID {@link android.os.UserHandle#getIdentifier()}. */ public static class ConfidenceLevel { public final int userId; public final int confidenceLevel; public ConfidenceLevel(int userId, int confidenceLevel) { this.userId = userId; this.confidenceLevel = confidenceLevel; } } /** * Additional data conveyed by a {@link KeyphraseRecognitionEvent} * for a key phrase detection. */ public static class KeyphraseRecognitionExtra { /** Confidence level for each user defined in the key phrase in the same order as * users in the key phrase. The confidence level is expressed in percentage (0% -100%) */ public final int[] confidenceLevels; /** The keyphrse ID */ public final int id; /** Recognition modes matched for this event */ public final int recognitionModes; KeyphraseRecognitionExtra(int[] confidenceLevels, int recognitionModes) { this.confidenceLevels = confidenceLevels; /** Confidence levels for all users recognized (KeyphraseRecognitionEvent) or to * be recognized (RecognitionConfig) */ public final ConfidenceLevel[] confidenceLevels; public KeyphraseRecognitionExtra(int id, int recognitionModes, ConfidenceLevel[] confidenceLevels) { this.id = id; this.recognitionModes = recognitionModes; this.confidenceLevels = confidenceLevels; } } Loading @@ -259,9 +310,10 @@ public class SoundTrigger { public final boolean keyphraseInCapture; KeyphraseRecognitionEvent(int status, int soundModelHandle, boolean captureAvailable, int captureSession, int captureDelayMs, byte[] data, int captureSession, int captureDelayMs, int capturePreambleMs, byte[] data, boolean keyphraseInCapture, KeyphraseRecognitionExtra[] keyphraseExtras) { super(status, soundModelHandle, captureAvailable, captureSession, captureDelayMs, data); super(status, soundModelHandle, captureAvailable, captureSession, captureDelayMs, capturePreambleMs, data); this.keyphraseInCapture = keyphraseInCapture; this.keyphraseExtras = keyphraseExtras; } Loading
core/java/android/hardware/soundtrigger/SoundTriggerModule.java +3 −2 Original line number Diff line number Diff line Loading @@ -94,7 +94,8 @@ public class SoundTriggerModule { * Recognition must be restarted after each callback (success or failure) received on * the {@link SoundTrigger.StatusListener}. * @param soundModelHandle The sound model handle to start listening to * @param data Opaque data for use by the implementation for this recognition * @param config contains configuration information for this recognition request: * recognition mode, keyphrases, users, minimum confidence levels... * @return - {@link SoundTrigger#STATUS_OK} in case of success * - {@link SoundTrigger#STATUS_ERROR} in case of unspecified error * - {@link SoundTrigger#STATUS_PERMISSION_DENIED} if the caller does not have Loading @@ -105,7 +106,7 @@ public class SoundTriggerModule { * service fails * - {@link SoundTrigger#STATUS_INVALID_OPERATION} if the call is out of sequence */ public native int startRecognition(int soundModelHandle, byte[] data); public native int startRecognition(int soundModelHandle, SoundTrigger.RecognitionConfig config); /** * Stop listening to all key phrases in a {@link SoundTrigger.SoundModel} Loading
core/jni/android_hardware_SoundTrigger.cpp +154 −34 Original line number Diff line number Diff line Loading @@ -74,7 +74,7 @@ static struct { jfieldID recognitionModes; jfieldID locale; jfieldID text; jfieldID numUsers; jfieldID users; } gKeyphraseFields; static const char* const kKeyphraseSoundModelClassPathName = Loading @@ -84,6 +84,14 @@ static struct { jfieldID keyphrases; } gKeyphraseSoundModelFields; static const char* const kRecognitionConfigClassPathName = "android/hardware/soundtrigger/SoundTrigger$RecognitionConfig"; static jclass gRecognitionConfigClass; static struct { jfieldID captureRequested; jfieldID keyphrases; jfieldID data; } gRecognitionConfigFields; static const char* const kRecognitionEventClassPathName = "android/hardware/soundtrigger/SoundTrigger$RecognitionEvent"; Loading @@ -99,6 +107,20 @@ static const char* const kKeyphraseRecognitionExtraClassPathName = "android/hardware/soundtrigger/SoundTrigger$KeyphraseRecognitionExtra"; static jclass gKeyphraseRecognitionExtraClass; static jmethodID gKeyphraseRecognitionExtraCstor; static struct { jfieldID id; jfieldID recognitionModes; jfieldID confidenceLevels; } gKeyphraseRecognitionExtraFields; static const char* const kConfidenceLevelClassPathName = "android/hardware/soundtrigger/SoundTrigger$ConfidenceLevel"; static jclass gConfidenceLevelClass; static jmethodID gConfidenceLevelCstor; static struct { jfieldID userId; jfieldID confidenceLevel; } gConfidenceLevelFields; static Mutex gLock; Loading Loading @@ -183,34 +205,45 @@ void JNISoundTriggerCallback::onRecognitionEvent(struct sound_trigger_recognitio } for (size_t i = 0; i < phraseEvent->num_phrases; i++) { jintArray jConfidenceLevels = env->NewIntArray(phraseEvent->phrase_extras[i].num_users); jobjectArray jConfidenceLevels = env->NewObjectArray( phraseEvent->phrase_extras[i].num_levels, gConfidenceLevelClass, NULL); if (jConfidenceLevels == NULL) { return; } jint *nConfidenceLevels = env->GetIntArrayElements(jConfidenceLevels, NULL); memcpy(nConfidenceLevels, phraseEvent->phrase_extras[i].confidence_levels, phraseEvent->phrase_extras[i].num_users * sizeof(int)); env->ReleaseIntArrayElements(jConfidenceLevels, nConfidenceLevels, 0); for (size_t j = 0; j < phraseEvent->phrase_extras[i].num_levels; j++) { jobject jConfidenceLevel = env->NewObject(gConfidenceLevelClass, gConfidenceLevelCstor, phraseEvent->phrase_extras[i].levels[j].user_id, phraseEvent->phrase_extras[i].levels[j].level); env->SetObjectArrayElement(jConfidenceLevels, j, jConfidenceLevel); env->DeleteLocalRef(jConfidenceLevel); } jobject jNewExtra = env->NewObject(gKeyphraseRecognitionExtraClass, gKeyphraseRecognitionExtraCstor, jConfidenceLevels, phraseEvent->phrase_extras[i].recognition_modes); phraseEvent->phrase_extras[i].id, phraseEvent->phrase_extras[i].recognition_modes, jConfidenceLevels); if (jNewExtra == NULL) { return; } env->SetObjectArrayElement(jExtras, i, jNewExtra); env->DeleteLocalRef(jNewExtra); env->DeleteLocalRef(jConfidenceLevels); } jEvent = env->NewObject(gKeyphraseRecognitionEventClass, gKeyphraseRecognitionEventCstor, event->status, event->model, event->capture_available, event->capture_session, event->capture_delay_ms, jData, event->capture_session, event->capture_delay_ms, event->capture_preamble_ms, jData, phraseEvent->key_phrase_in_capture, jExtras); } else { jEvent = env->NewObject(gRecognitionEventClass, gRecognitionEventCstor, event->status, event->model, event->capture_available, event->capture_session, event->capture_delay_ms, jData); event->capture_session, event->capture_delay_ms, event->capture_preamble_ms, jData); } Loading Loading @@ -381,7 +414,7 @@ android_hardware_SoundTrigger_loadSoundModel(JNIEnv *env, jobject thiz, jobject jSoundModel, jintArray jHandle) { jint status = SOUNDTRIGGER_STATUS_OK; char *nData = NULL; jbyte *nData = NULL; struct sound_trigger_sound_model *nSoundModel; jbyteArray jData; sp<MemoryDealer> memoryDealer; Loading Loading @@ -437,7 +470,7 @@ android_hardware_SoundTrigger_loadSoundModel(JNIEnv *env, jobject thiz, } size = env->GetArrayLength(jData); nData = (char *)env->GetByteArrayElements(jData, NULL); nData = env->GetByteArrayElements(jData, NULL); if (jData == NULL) { status = SOUNDTRIGGER_STATUS_ERROR; goto exit; Loading Loading @@ -481,8 +514,16 @@ android_hardware_SoundTrigger_loadSoundModel(JNIEnv *env, jobject thiz, env->GetIntField(jPhrase,gKeyphraseFields.id); phraseModel->phrases[i].recognition_mode = env->GetIntField(jPhrase,gKeyphraseFields.recognitionModes); phraseModel->phrases[i].num_users = env->GetIntField(jPhrase, gKeyphraseFields.numUsers); jintArray jUsers = (jintArray)env->GetObjectField(jPhrase, gKeyphraseFields.users); phraseModel->phrases[i].num_users = env->GetArrayLength(jUsers); jint *nUsers = env->GetIntArrayElements(jUsers, NULL); memcpy(phraseModel->phrases[i].users, nUsers, phraseModel->phrases[i].num_users * sizeof(int)); env->ReleaseIntArrayElements(jUsers, nUsers, 0); env->DeleteLocalRef(jUsers); jstring jLocale = (jstring)env->GetObjectField(jPhrase, gKeyphraseFields.locale); const char *nLocale = env->GetStringUTFChars(jLocale, NULL); strncpy(phraseModel->phrases[i].locale, Loading @@ -500,6 +541,7 @@ android_hardware_SoundTrigger_loadSoundModel(JNIEnv *env, jobject thiz, env->DeleteLocalRef(jText); ALOGV("loadSoundModel phrases %d text %s locale %s", i, phraseModel->phrases[i].text, phraseModel->phrases[i].locale); env->DeleteLocalRef(jPhrase); } env->DeleteLocalRef(jPhrases); } Loading @@ -512,7 +554,7 @@ exit: env->ReleaseIntArrayElements(jHandle, nHandle, NULL); } if (nData != NULL) { env->ReleaseByteArrayElements(jData, (jbyte *)nData, NULL); env->ReleaseByteArrayElements(jData, nData, NULL); } return status; } Loading @@ -534,7 +576,7 @@ android_hardware_SoundTrigger_unloadSoundModel(JNIEnv *env, jobject thiz, static jint android_hardware_SoundTrigger_startRecognition(JNIEnv *env, jobject thiz, jint jHandle, jbyteArray jData) jint jHandle, jobject jConfig) { jint status = SOUNDTRIGGER_STATUS_OK; ALOGV("startRecognition"); Loading @@ -542,29 +584,82 @@ android_hardware_SoundTrigger_startRecognition(JNIEnv *env, jobject thiz, if (module == NULL) { return SOUNDTRIGGER_STATUS_ERROR; } if (!env->IsInstanceOf(jConfig, gRecognitionConfigClass)) { return SOUNDTRIGGER_STATUS_BAD_VALUE; } jbyteArray jData = (jbyteArray)env->GetObjectField(jConfig, gRecognitionConfigFields.data); jsize dataSize = 0; char *nData = NULL; sp<IMemory> memory; jbyte *nData = NULL; if (jData != NULL) { dataSize = env->GetArrayLength(jData); if (dataSize == 0) { return SOUNDTRIGGER_STATUS_BAD_VALUE; } nData = (char *)env->GetByteArrayElements(jData, NULL); nData = env->GetByteArrayElements(jData, NULL); if (nData == NULL) { return SOUNDTRIGGER_STATUS_ERROR; } } size_t totalSize = sizeof(struct sound_trigger_recognition_config) + dataSize; sp<MemoryDealer> memoryDealer = new MemoryDealer(dataSize, "SoundTrigge-JNI::StartRecognition"); new MemoryDealer(totalSize, "SoundTrigge-JNI::StartRecognition"); if (memoryDealer == 0) { return SOUNDTRIGGER_STATUS_ERROR; } memory = memoryDealer->allocate(dataSize); sp<IMemory> memory = memoryDealer->allocate(totalSize); if (memory == 0 || memory->pointer() == NULL) { return SOUNDTRIGGER_STATUS_ERROR; } memcpy(memory->pointer(), nData, dataSize); if (dataSize != 0) { memcpy((char *)memory->pointer() + sizeof(struct sound_trigger_recognition_config), nData, dataSize); env->ReleaseByteArrayElements(jData, nData, 0); } env->DeleteLocalRef(jData); struct sound_trigger_recognition_config *config = (struct sound_trigger_recognition_config *)memory->pointer(); config->data_size = dataSize; config->data_offset = sizeof(struct sound_trigger_recognition_config); config->capture_requested = env->GetIntField(jConfig, gRecognitionConfigFields.captureRequested); config->num_phrases = 0; jobjectArray jPhrases = (jobjectArray)env->GetObjectField(jConfig, gRecognitionConfigFields.keyphrases); if (jPhrases != NULL) { config->num_phrases = env->GetArrayLength(jPhrases); } ALOGV("startRecognition num phrases %d", config->num_phrases); for (size_t i = 0; i < config->num_phrases; i++) { jobject jPhrase = env->GetObjectArrayElement(jPhrases, i); config->phrases[i].id = env->GetIntField(jPhrase, gKeyphraseRecognitionExtraFields.id); config->phrases[i].recognition_modes = env->GetIntField(jPhrase, gKeyphraseRecognitionExtraFields.recognitionModes); config->phrases[i].num_levels = 0; jobjectArray jConfidenceLevels = (jobjectArray)env->GetObjectField(jPhrase, gKeyphraseRecognitionExtraFields.confidenceLevels); if (jConfidenceLevels != NULL) { config->phrases[i].num_levels = env->GetArrayLength(jConfidenceLevels); } ALOGV("startRecognition phrase %d num_levels %d", i, config->phrases[i].num_levels); for (size_t j = 0; j < config->phrases[i].num_levels; j++) { jobject jConfidenceLevel = env->GetObjectArrayElement(jConfidenceLevels, j); config->phrases[i].levels[j].user_id = env->GetIntField(jConfidenceLevel, gConfidenceLevelFields.userId); config->phrases[i].levels[j].level = env->GetIntField(jConfidenceLevel, gConfidenceLevelFields.confidenceLevel); env->DeleteLocalRef(jConfidenceLevel); } ALOGV("startRecognition phrases %d", i); env->DeleteLocalRef(jConfidenceLevels); env->DeleteLocalRef(jPhrase); } env->DeleteLocalRef(jPhrases); status = module->startRecognition(jHandle, memory); return status; Loading Loading @@ -608,7 +703,7 @@ static JNINativeMethod gModuleMethods[] = { "(I)I", (void *)android_hardware_SoundTrigger_unloadSoundModel}, {"startRecognition", "(I[B)I", "(ILandroid/hardware/soundtrigger/SoundTrigger$RecognitionConfig;)I", (void *)android_hardware_SoundTrigger_startRecognition}, {"stopRecognition", "(I)I", Loading Loading @@ -652,7 +747,7 @@ int register_android_hardware_SoundTrigger(JNIEnv *env) gKeyphraseFields.recognitionModes = env->GetFieldID(keyphraseClass, "recognitionModes", "I"); gKeyphraseFields.locale = env->GetFieldID(keyphraseClass, "locale", "Ljava/lang/String;"); gKeyphraseFields.text = env->GetFieldID(keyphraseClass, "text", "Ljava/lang/String;"); gKeyphraseFields.numUsers = env->GetFieldID(keyphraseClass, "numUsers", "I"); gKeyphraseFields.users = env->GetFieldID(keyphraseClass, "users", "[I"); jclass keyphraseSoundModelClass = env->FindClass(kKeyphraseSoundModelClassPathName); gKeyphraseSoundModelClass = (jclass) env->NewGlobalRef(keyphraseSoundModelClass); Loading @@ -664,18 +759,42 @@ int register_android_hardware_SoundTrigger(JNIEnv *env) jclass recognitionEventClass = env->FindClass(kRecognitionEventClassPathName); gRecognitionEventClass = (jclass) env->NewGlobalRef(recognitionEventClass); gRecognitionEventCstor = env->GetMethodID(recognitionEventClass, "<init>", "(IIZII[B)V"); "(IIZIII[B)V"); jclass keyphraseRecognitionEventClass = env->FindClass(kKeyphraseRecognitionEventClassPathName); gKeyphraseRecognitionEventClass = (jclass) env->NewGlobalRef(keyphraseRecognitionEventClass); gKeyphraseRecognitionEventCstor = env->GetMethodID(keyphraseRecognitionEventClass, "<init>", "(IIZII[BZ[Landroid/hardware/soundtrigger/SoundTrigger$KeyphraseRecognitionExtra;)V"); "(IIZIII[BZ[Landroid/hardware/soundtrigger/SoundTrigger$KeyphraseRecognitionExtra;)V"); jclass keyRecognitionConfigClass = env->FindClass(kRecognitionConfigClassPathName); gRecognitionConfigClass = (jclass) env->NewGlobalRef(keyRecognitionConfigClass); gRecognitionConfigFields.captureRequested = env->GetFieldID(keyRecognitionConfigClass, "captureRequested", "Z"); gRecognitionConfigFields.keyphrases = env->GetFieldID(keyRecognitionConfigClass, "keyphrases", "[Landroid/hardware/soundtrigger/SoundTrigger$KeyphraseRecognitionExtra;"); gRecognitionConfigFields.data = env->GetFieldID(keyRecognitionConfigClass, "data", "[B"); jclass keyphraseRecognitionExtraClass = env->FindClass(kKeyphraseRecognitionExtraClassPathName); gKeyphraseRecognitionExtraClass = (jclass) env->NewGlobalRef(keyphraseRecognitionExtraClass); gKeyphraseRecognitionExtraCstor = env->GetMethodID(keyphraseRecognitionExtraClass, "<init>", "([II)V"); "(II[Landroid/hardware/soundtrigger/SoundTrigger$ConfidenceLevel;)V"); gKeyphraseRecognitionExtraFields.id = env->GetFieldID(gKeyphraseRecognitionExtraClass, "id", "I"); gKeyphraseRecognitionExtraFields.recognitionModes = env->GetFieldID(gKeyphraseRecognitionExtraClass, "recognitionModes", "I"); gKeyphraseRecognitionExtraFields.confidenceLevels = env->GetFieldID(gKeyphraseRecognitionExtraClass, "confidenceLevels", "[Landroid/hardware/soundtrigger/SoundTrigger$ConfidenceLevel;"); jclass confidenceLevelClass = env->FindClass(kConfidenceLevelClassPathName); gConfidenceLevelClass = (jclass) env->NewGlobalRef(confidenceLevelClass); gConfidenceLevelCstor = env->GetMethodID(confidenceLevelClass, "<init>", "(II)V"); gConfidenceLevelFields.userId = env->GetFieldID(confidenceLevelClass, "userId", "I"); gConfidenceLevelFields.confidenceLevel = env->GetFieldID(confidenceLevelClass, "confidenceLevel", "I"); int status = AndroidRuntime::registerNativeMethods(env, kSoundTriggerClassPathName, gMethods, NELEM(gMethods)); Loading @@ -685,5 +804,6 @@ int register_android_hardware_SoundTrigger(JNIEnv *env) kModuleClassPathName, gModuleMethods, NELEM(gModuleMethods)); } return status; }
services/voiceinteraction/java/com/android/server/voiceinteraction/DatabaseHelper.java +4 −4 Original line number Diff line number Diff line Loading @@ -46,7 +46,7 @@ public class DatabaseHelper extends SQLiteOpenHelper { public static final String KEY_RECOGNITION_MODES = "modes"; public static final String KEY_LOCALE = "locale"; public static final String KEY_HINT_TEXT = "hint_text"; public static final String KEY_NUM_USERS = "num_users"; public static final String KEY_USERS = "users"; public static final String KEY_SOUND_MODEL_ID = "sound_model_id"; } Loading @@ -62,7 +62,7 @@ public class DatabaseHelper extends SQLiteOpenHelper { + KeyphraseContract.TABLE + "(" + KeyphraseContract.KEY_ID + " INTEGER PRIMARY KEY," + KeyphraseContract.KEY_RECOGNITION_MODES + " INTEGER," + KeyphraseContract.KEY_NUM_USERS + " INTEGER," + KeyphraseContract.KEY_USERS + " INTEGER," + KeyphraseContract.KEY_SOUND_MODEL_ID + " TEXT," + KeyphraseContract.KEY_LOCALE + " TEXT," + KeyphraseContract.KEY_HINT_TEXT + " TEXT" + ")"; Loading Loading @@ -167,11 +167,11 @@ public class DatabaseHelper extends SQLiteOpenHelper { do { int id = c.getInt(c.getColumnIndex(KeyphraseContract.KEY_ID)); int modes = c.getInt(c.getColumnIndex(KeyphraseContract.KEY_RECOGNITION_MODES)); int numUsers = c.getInt(c.getColumnIndex(KeyphraseContract.KEY_NUM_USERS)); int[] users = {c.getInt(c.getColumnIndex(KeyphraseContract.KEY_USERS))}; String locale = c.getString(c.getColumnIndex(KeyphraseContract.KEY_LOCALE)); String hintText = c.getString(c.getColumnIndex(KeyphraseContract.KEY_HINT_TEXT)); keyphrases.add(new Keyphrase(id, modes, locale, hintText, numUsers)); keyphrases.add(new Keyphrase(id, modes, locale, hintText, users)); } while (c.moveToNext()); } Keyphrase[] keyphraseArr = new Keyphrase[keyphrases.size()]; Loading