Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 452a6424 authored by Sandeep Siddhartha's avatar Sandeep Siddhartha
Browse files

Fix various bugs with model management 

- Tie the sound model and keyphrase for simplicity
  We won't support multiple keyphrases in a single model out of the box.
  The db schema will need to be changed by the OEM wishing to add multiple hotwords.
  This is because we currently have no way to test the flow and make sure that things work well with multiple keyphrases
  and also the framework only reads the metadata for one keyphrase.

- Make the delete/update operations atomic

- Make the flow of data from Enrollment -> VIMS; the large sound model doesn't cross the process boundary any other time.
  This is achieved by passing they key around, instead of the model themselves.

- Add a specific delete operation in DatabaseHelper rather than relying on emptying the keyphrases to delete.

Bug: 16555803
Bug: 16516353
Change-Id: I1e0cce137517502a669e431ca7e9f9f755598328
parent 90d7a3e9

core/java/android/service/voice/AlwaysOnHotwordDetector.java

+12 −30
Original line number Diff line number Diff line
@@ -133,10 +133,10 @@ public class AlwaysOnHotwordDetector { 
    private final Handler mHandler;



    /**

     * The sound model for the keyphrase, derived from the model management service

     * (IVoiceInteractionManagerService). May be null if the keyphrase isn't enrolled yet.

     * Indicates if there is a sound model enrolled for the keyphrase,

     * derived from the model management service (IVoiceInteractionManagerService).

     */

    private KeyphraseSoundModel mEnrolledSoundModel;

    private boolean mIsEnrolledForDetection;

    private int mAvailability = STATE_NOT_READY;



    /**
@@ -257,7 +257,7 @@ public class AlwaysOnHotwordDetector { 
        int code = STATUS_ERROR;

        try {

            code = mModelManagementService.startRecognition(mVoiceInteractionService,

                    mKeyphraseMetadata.id, mEnrolledSoundModel, mInternalCallback,

                    mKeyphraseMetadata.id, mInternalCallback,

                    new RecognitionConfig(

                            captureTriggerAudio, recognitionExtra, null /* additional data */));

        } catch (RemoteException e) {
@@ -417,14 +417,13 @@ public class AlwaysOnHotwordDetector { 
        @Override

        public Void doInBackground(Void... params) {

            int availability = internalGetInitialAvailability();

            KeyphraseSoundModel soundModel = null;

            boolean enrolled = false;

            // Fetch the sound model if the availability is one of the supported ones.

            if (availability == STATE_NOT_READY

                    || availability == STATE_KEYPHRASE_UNENROLLED

                    || availability == STATE_KEYPHRASE_ENROLLED) {

                soundModel =

                        internalGetKeyphraseSoundModel(mKeyphraseMetadata.id);

                if (soundModel == null) {

                enrolled = internalGetIsEnrolled(mKeyphraseMetadata.id);

                if (!enrolled) {

                    availability = STATE_KEYPHRASE_UNENROLLED;

                } else {

                    availability = STATE_KEYPHRASE_ENROLLED;
@@ -436,8 +435,8 @@ public class AlwaysOnHotwordDetector { 
                    Slog.d(TAG, "Hotword availability changed from " + mAvailability

                            + " -> " + availability);

                }

                mIsEnrolledForDetection = enrolled;

                mAvailability = availability;

                mEnrolledSoundModel = soundModel;

                notifyStateChangedLocked();

            }

            return null;
@@ -475,31 +474,14 @@ public class AlwaysOnHotwordDetector { 
        /**

         * @return The corresponding {@link KeyphraseSoundModel} or null if none is found.

         */

        private KeyphraseSoundModel internalGetKeyphraseSoundModel(int keyphraseId) {

            List<KeyphraseSoundModel> soundModels;

        private boolean internalGetIsEnrolled(int keyphraseId) {

            try {

                soundModels = mModelManagementService

                        .listRegisteredKeyphraseSoundModels(mVoiceInteractionService);

                if (soundModels == null || soundModels.isEmpty()) {

                    Slog.i(TAG, "No available sound models for keyphrase ID: " + keyphraseId);

                    return null;

                }

                for (int i = 0; i < soundModels.size(); i++) {

                    KeyphraseSoundModel soundModel = soundModels.get(i);

                    if (soundModel.keyphrases == null || soundModel.keyphrases.length == 0) {

                        continue;

                    }

                    for (int j = 0; i < soundModel.keyphrases.length; j++) {

                        Keyphrase keyphrase = soundModel.keyphrases[j];

                        if (keyphrase.id == keyphraseId) {

                            return soundModel;

                        }

                    }

                }

                return mModelManagementService.isEnrolledForKeyphrase(

                        mVoiceInteractionService, keyphraseId);

            } catch (RemoteException e) {

                Slog.w(TAG, "RemoteException in listRegisteredKeyphraseSoundModels!");

            }

            return null;

            return false;

        }

    }

}

core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl

+11 −6
Original line number Diff line number Diff line
@@ -33,18 +33,23 @@ interface IVoiceInteractionManagerService { 
    void finish(IBinder token);



    /**

     * Lists the registered Sound models for keyphrase detection.

     * Lists the registered Sound model for keyphrase detection.

     * May be null if no matching sound models exist.

     *

     * @param service The current voice interaction service.

     */

    List<SoundTrigger.KeyphraseSoundModel> listRegisteredKeyphraseSoundModels(

            in IVoiceInteractionService service);

    SoundTrigger.KeyphraseSoundModel getKeyphraseSoundModel(int keyphraseId);

    /**

     * Updates the given keyphrase sound model. Adds the model if it doesn't exist currently.

     */

    int updateKeyphraseSoundModel(in SoundTrigger.KeyphraseSoundModel model);

    /**

     * Deletes the given keyphrase sound model.

     */

    int deleteKeyphraseSoundModel(int keyphraseId);



    /**

     * Indicates if there's a keyphrase sound model available for the given keyphrase ID.

     */

    boolean isEnrolledForKeyphrase(IVoiceInteractionService service, int keyphraseId);

    /**

     * Gets the properties of the DSP hardware on this device, null if not present.

     */
@@ -53,7 +58,7 @@ interface IVoiceInteractionManagerService { 
     * Starts a recognition for the given keyphrase.

     */

    int startRecognition(in IVoiceInteractionService service, int keyphraseId,

            in SoundTrigger.KeyphraseSoundModel soundModel, in IRecognitionStatusCallback callback,

            in IRecognitionStatusCallback callback,

            in SoundTrigger.RecognitionConfig recognitionConfig);

    /**

     * Stops a recognition for the given keyphrase.

services/voiceinteraction/java/com/android/server/voiceinteraction/DatabaseHelper.java

+101 −145
Original line number Diff line number Diff line
@@ -28,8 +28,6 @@ import android.os.UserManager; 
import android.text.TextUtils;

import android.util.Slog;



import java.util.ArrayList;

import java.util.List;

import java.util.UUID;



/**
@@ -39,43 +37,35 @@ import java.util.UUID; 
 */

public class DatabaseHelper extends SQLiteOpenHelper {

    static final String TAG = "SoundModelDBHelper";

    static final boolean DBG = false;

    // TODO: Set to false.

    static final boolean DBG = true;



    private static final String NAME = "sound_model.db";

    private static final int VERSION = 2;



    public static interface KeyphraseContract {

        public static final String TABLE = "keyphrase";

        public static final String KEY_ID = "_id";

        public static final String KEY_RECOGNITION_MODES = "modes";

        public static final String KEY_LOCALE = "locale";

        public static final String KEY_HINT_TEXT = "hint_text";

        public static final String KEY_USERS = "users";

        public static final String KEY_SOUND_MODEL_ID = "sound_model_id";

    }

    private static final int VERSION = 3;



    public static interface SoundModelContract {

        public static final String TABLE = "sound_model";

        public static final String KEY_ID = "_id";

        public static final String KEY_KEYPHRASE_ID = "keyphrase_id";

        public static final String KEY_MODEL_UUID = "model_uuid";

        public static final String KEY_TYPE = "type";

        public static final String KEY_DATA = "data";

        public static final String KEY_RECOGNITION_MODES = "recognition_modes";

        public static final String KEY_LOCALE = "locale";

        public static final String KEY_HINT_TEXT = "hint_text";

        public static final String KEY_USERS = "users";

    }



    // Table Create Statements

    private static final String CREATE_TABLE_KEYPRHASES = "CREATE TABLE "

            + KeyphraseContract.TABLE + "("

            + KeyphraseContract.KEY_ID + " INTEGER PRIMARY KEY,"

            + KeyphraseContract.KEY_RECOGNITION_MODES + " INTEGER,"

            + KeyphraseContract.KEY_USERS + " TEXT,"

            + KeyphraseContract.KEY_SOUND_MODEL_ID + " TEXT,"

            + KeyphraseContract.KEY_LOCALE + " TEXT,"

            + KeyphraseContract.KEY_HINT_TEXT + " TEXT" + ")";



    // Table Create Statement

    private static final String CREATE_TABLE_SOUND_MODEL = "CREATE TABLE "

            + SoundModelContract.TABLE + "("

            + SoundModelContract.KEY_ID + " TEXT PRIMARY KEY,"

            + SoundModelContract.KEY_KEYPHRASE_ID + " INTEGER PRIMARY KEY,"

            + SoundModelContract.KEY_MODEL_UUID + " TEXT,"

            + SoundModelContract.KEY_TYPE + " INTEGER,"

            + SoundModelContract.KEY_DATA + " BLOB" + ")";

            + SoundModelContract.KEY_DATA + " BLOB,"

            + SoundModelContract.KEY_RECOGNITION_MODES + " INTEGER,"

            + SoundModelContract.KEY_LOCALE + " TEXT,"

            + SoundModelContract.KEY_HINT_TEXT + " TEXT,"

            + SoundModelContract.KEY_USERS + " TEXT" + ")";



    private final UserManager mUserManager;
@@ -87,57 +77,44 @@ public class DatabaseHelper extends SQLiteOpenHelper { 
    @Override

    public void onCreate(SQLiteDatabase db) {

        // creating required tables

        db.execSQL(CREATE_TABLE_KEYPRHASES);

        db.execSQL(CREATE_TABLE_SOUND_MODEL);

    }



    @Override

    public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {

        // TODO: For now, drop older tables and recreate new ones.

        db.execSQL("DROP TABLE IF EXISTS " + KeyphraseContract.TABLE);

        db.execSQL("DROP TABLE IF EXISTS " + SoundModelContract.TABLE);

        onCreate(db);

    }



    public boolean addOrUpdateKeyphraseSoundModel(KeyphraseSoundModel soundModel) {

    /**

     * Updates the given keyphrase model, adds it, if it doesn't already exist.

     *

     * TODO: We only support one keyphrase currently.

     */

    public boolean updateKeyphraseSoundModel(KeyphraseSoundModel soundModel) {

        synchronized(this) {

            SQLiteDatabase db = getWritableDatabase();

            ContentValues values = new ContentValues();

            // Generate a random ID for the model.

            values.put(SoundModelContract.KEY_ID, soundModel.uuid.toString());

            values.put(SoundModelContract.KEY_DATA, soundModel.data);

            values.put(SoundModelContract.KEY_MODEL_UUID, soundModel.uuid.toString());

            values.put(SoundModelContract.KEY_TYPE, SoundTrigger.SoundModel.TYPE_KEYPHRASE);

            values.put(SoundModelContract.KEY_DATA, soundModel.data);



            boolean status = true;

            if (db.insertWithOnConflict(SoundModelContract.TABLE, null, values,

                    SQLiteDatabase.CONFLICT_REPLACE) != -1) {

                for (Keyphrase keyphrase : soundModel.keyphrases) {

                    status &= addOrUpdateKeyphraseLocked(db, soundModel.uuid, keyphrase);

                }

            if (soundModel.keyphrases != null && soundModel.keyphrases.length == 1) {

                values.put(SoundModelContract.KEY_KEYPHRASE_ID, soundModel.keyphrases[0].id);

                values.put(SoundModelContract.KEY_RECOGNITION_MODES,

                        soundModel.keyphrases[0].recognitionModes);

                values.put(SoundModelContract.KEY_USERS,

                        getCommaSeparatedString(soundModel.keyphrases[0].users));

                values.put(SoundModelContract.KEY_LOCALE, soundModel.keyphrases[0].locale);

                values.put(SoundModelContract.KEY_HINT_TEXT, soundModel.keyphrases[0].text);

                try {

                    return db.insertWithOnConflict(SoundModelContract.TABLE, null, values,

                            SQLiteDatabase.CONFLICT_REPLACE) != -1;

                } finally {

                    db.close();

                return status;

            } else {

                Slog.w(TAG, "Failed to persist sound model to database");

                db.close();

                return false;

            }

                }

            }



    private boolean addOrUpdateKeyphraseLocked(

            SQLiteDatabase db, UUID modelId, Keyphrase keyphrase) {

        ContentValues values = new ContentValues();

        values.put(KeyphraseContract.KEY_ID, keyphrase.id);

        values.put(KeyphraseContract.KEY_RECOGNITION_MODES, keyphrase.recognitionModes);

        values.put(KeyphraseContract.KEY_SOUND_MODEL_ID, modelId.toString());

        values.put(KeyphraseContract.KEY_HINT_TEXT, keyphrase.text);

        values.put(KeyphraseContract.KEY_LOCALE, keyphrase.locale);

        values.put(KeyphraseContract.KEY_USERS, getCommaSeparatedString(keyphrase.users));

        if (db.insertWithOnConflict(

                KeyphraseContract.TABLE, null, values, SQLiteDatabase.CONFLICT_REPLACE) != -1) {

            return true;

        } else {

            Slog.w(TAG, "Failed to persist keyphrase to database");

            return false;

        }

    }
@@ -145,87 +122,62 @@ public class DatabaseHelper extends SQLiteOpenHelper { 
    /**

     * Deletes the sound model and associated keyphrases.

     */

    public boolean deleteKeyphraseSoundModel(UUID uuid) {

    public boolean deleteKeyphraseSoundModel(int keyphraseId) {

        synchronized(this) {

            SQLiteDatabase db = getWritableDatabase();

            String modelId = uuid.toString();

            String soundModelClause = SoundModelContract.KEY_ID + "=" + modelId;

            boolean status = true;

            if (db.delete(SoundModelContract.TABLE, soundModelClause, null) == 0) {

                Slog.w(TAG, "No sound models deleted from the database");

                status = false;

            }

            String keyphraseClause = KeyphraseContract.KEY_SOUND_MODEL_ID + "=" + modelId;

            if (db.delete(KeyphraseContract.TABLE, keyphraseClause, null) == 0) {

                Slog.w(TAG, "No keyphrases deleted from the database");

                status = false;

            }

            String soundModelClause = SoundModelContract.KEY_KEYPHRASE_ID + "=" + keyphraseId;



            try {

                return db.delete(SoundModelContract.TABLE, soundModelClause, null) != 0;

            } finally {

                db.close();

            return status;

            }

        }

    }



    /**

     * Lists all the keyphrase sound models currently registered with the system.

     * Returns a matching {@link KeyphraseSoundModel} for the keyphrase ID.

     * Returns null if a match isn't found.

     *

     * TODO: We only support one keyphrase currently.

     */

    public List<KeyphraseSoundModel> getKephraseSoundModels() {

    public KeyphraseSoundModel getKeyphraseSoundModel(int keyphraseId) {

        synchronized(this) {

            List<KeyphraseSoundModel> models = new ArrayList<>();

            String selectQuery = "SELECT  * FROM " + SoundModelContract.TABLE;

            // Find the corresponding sound model ID for the keyphrase.

            String selectQuery = "SELECT  * FROM " + SoundModelContract.TABLE

                    + " WHERE " + SoundModelContract.KEY_KEYPHRASE_ID + " = '" + keyphraseId + "'";

            SQLiteDatabase db = getReadableDatabase();

            Cursor c = db.rawQuery(selectQuery, null);



            // looping through all rows and adding to list

            try {

                if (c.moveToFirst()) {

                do {

                    int type = c.getInt(c.getColumnIndex(SoundModelContract.KEY_TYPE));

                    if (type != SoundTrigger.SoundModel.TYPE_KEYPHRASE) {

                        // Ignore non-keyphrase sound models.

                        continue;

                        Slog.w(TAG, "No KeyphraseSoundModel available for the given keyphrase");

                        return null;

                    }

                    String id = c.getString(c.getColumnIndex(SoundModelContract.KEY_ID));

                    byte[] data = c.getBlob(c.getColumnIndex(SoundModelContract.KEY_DATA));

                    // Get all the keyphrases for this this sound model.

                    // Validate the sound model.

                    if (id == null) {



                    String modelUuid = c.getString(

                            c.getColumnIndex(SoundModelContract.KEY_MODEL_UUID));

                    if (modelUuid == null) {

                        Slog.w(TAG, "Ignoring sound model since it doesn't specify an ID");

                        continue;

                    }

                    KeyphraseSoundModel model = new KeyphraseSoundModel(

                            UUID.fromString(id), data, getKeyphrasesForSoundModelLocked(db, id));

                    if (DBG) {

                        Slog.d(TAG, "Adding model: " + model);

                    }

                    models.add(model);

                } while (c.moveToNext());

            }

            c.close();

            db.close();

            return models;

        }

                        return null;

                    }



    private Keyphrase[] getKeyphrasesForSoundModelLocked(SQLiteDatabase db, String modelId) {

        List<Keyphrase> keyphrases = new ArrayList<>();

        String selectQuery = "SELECT  * FROM " + KeyphraseContract.TABLE

                + " WHERE " + KeyphraseContract.KEY_SOUND_MODEL_ID + " = '" + modelId + "'";

        Cursor c = db.rawQuery(selectQuery, null);



        // looping through all rows and adding to list

        if (c.moveToFirst()) {

            do {

                int id = c.getInt(c.getColumnIndex(KeyphraseContract.KEY_ID));

                int modes = c.getInt(c.getColumnIndex(KeyphraseContract.KEY_RECOGNITION_MODES));

                    byte[] data = c.getBlob(c.getColumnIndex(SoundModelContract.KEY_DATA));

                    int recognitionModes = c.getInt(

                            c.getColumnIndex(SoundModelContract.KEY_RECOGNITION_MODES));

                    int[] users = getArrayForCommaSeparatedString(

                        c.getString(c.getColumnIndex(KeyphraseContract.KEY_USERS)));

                String locale = c.getString(c.getColumnIndex(KeyphraseContract.KEY_LOCALE));

                String hintText = c.getString(c.getColumnIndex(KeyphraseContract.KEY_HINT_TEXT));

                            c.getString(c.getColumnIndex(SoundModelContract.KEY_USERS)));

                    String locale = c.getString(c.getColumnIndex(SoundModelContract.KEY_LOCALE));

                    String text = c.getString(

                            c.getColumnIndex(SoundModelContract.KEY_HINT_TEXT));



                    // Only add keyphrases meant for the current user.

                    if (users == null) {

                        // No users present in the keyphrase.

                        Slog.w(TAG, "Ignoring keyphrase since it doesn't specify users");

                    continue;

                        return null;

                    }

                    boolean isAvailableForCurrentUser = false;

                    int currentUser = mUserManager.getUserHandle();
@@ -237,18 +189,22 @@ public class DatabaseHelper extends SQLiteOpenHelper { 
                    }

                    if (!isAvailableForCurrentUser) {

                        Slog.w(TAG, "Ignoring keyphrase since it's not for the current user");

                    continue;

                        return null;

                    }



                keyphrases.add(new Keyphrase(id, modes, locale, hintText, users));

            } while (c.moveToNext());

                    Keyphrase[] keyphrases = new Keyphrase[1];

                    keyphrases[0] = new Keyphrase(

                            keyphraseId, recognitionModes, locale, text, users);

                    return new KeyphraseSoundModel(UUID.fromString(modelUuid), data, keyphrases);

                }

        Keyphrase[] keyphraseArr = new Keyphrase[keyphrases.size()];

        keyphrases.toArray(keyphraseArr);

                Slog.w(TAG, "No SoundModel available for the given keyphrase");

            } finally {

                c.close();

        return keyphraseArr;

                db.close();

            }

            return null;

        }

    }





    private static String getCommaSeparatedString(int[] users) {

        if (users == null || users.length == 0) {

services/voiceinteraction/java/com/android/server/voiceinteraction/SoundTriggerHelper.java

+2 −1
Original line number Diff line number Diff line
@@ -38,7 +38,8 @@ import java.util.ArrayList; 
 */

public class SoundTriggerHelper implements SoundTrigger.StatusListener {

    static final String TAG = "SoundTriggerHelper";

    static final boolean DBG = false;

    // TODO: Set to false.

    static final boolean DBG = true;

    // TODO: Remove this.

    static final int TEMP_KEYPHRASE_ID = 100;

services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java

+76 −41

File changed.

Preview size limit exceeded, changes collapsed.