Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 110f569b authored by Sandeep Siddhartha's avatar Sandeep Siddhartha
Browse files

Fix synchronization issues in AlwaysOnHotwordDetector 

- Remove unnecessary recognition status from AlwaysOnHotwordDetector

- Remove unnecessary recognition started callback from IRecognitionStatusCallback

- Fix a bug around the fact that we weren't picking up enrollment at runtime because
we were storing the availability at instantiation time.

- Handle 0-length arrays in SoundTrigger classes while parceling/unparceling

- Fix issue in SoundTrigger helper where we were not comparing binders for start/stop calls

- Unload the previous model when starting a new recognition

- Add more debug logging

Change-Id: Icc56d7f3dd1ffa49a8cfeea49080e3ab4d342c32
parent 8ca1fdc1

api/current.txt

+0 −7
Original line number Diff line number Diff line
@@ -27222,7 +27222,6 @@ package android.service.voice { 
  public class AlwaysOnHotwordDetector {

    method public int getAvailability();

    method public android.content.Intent getManageIntent(int);

    method public int getRecognitionStatus();

    method public int getSupportedRecognitionModes();

    method public int startRecognition(int);

    method public int stopRecognition();
@@ -27237,18 +27236,12 @@ package android.service.voice { 
    field public static final int RECOGNITION_FLAG_NONE = 0; // 0x0

    field public static final int RECOGNITION_MODE_USER_IDENTIFICATION = 2; // 0x2

    field public static final int RECOGNITION_MODE_VOICE_TRIGGER = 1; // 0x1

    field public static final int RECOGNITION_STATUS_ACTIVE = 16; // 0x10

    field public static final int RECOGNITION_STATUS_DISABLED_TEMPORARILY = 8; // 0x8

    field public static final int RECOGNITION_STATUS_NOT_AVAILABLE = 1; // 0x1

    field public static final int RECOGNITION_STATUS_NOT_REQUESTED = 2; // 0x2

    field public static final int RECOGNITION_STATUS_REQUESTED = 4; // 0x4

    field public static final int STATUS_ERROR = -2147483648; // 0x80000000

    field public static final int STATUS_OK = 0; // 0x0

  }















  public static abstract interface AlwaysOnHotwordDetector.Callback {















    method public abstract void onDetected(byte[]);













    method public abstract void onDetectionStarted();















    method public abstract void onDetectionStopped();















  }







































core/java/android/hardware/soundtrigger/IRecognitionStatusCallback.aidl



+0
−4




























Original line number
Diff line number
Diff line








@@ -28,10 +28,6 @@ oneway interface IRecognitionStatusCallback {













     *        This *MUST* not exceed 100K.















     */















    void onDetected(in byte[] data);













    /**













     * Called when the detection for the associated keyphrase starts.













     */













    void onDetectionStarted();















    /**















     * Called when the detection for the associated keyphrase stops.















     */

































core/java/android/hardware/soundtrigger/SoundTrigger.java



+18
−6




























Original line number
Diff line number
Diff line








@@ -247,7 +247,7 @@ public class SoundTrigger {













            String text = in.readString();















            int[] users = null;















            int numUsers = in.readInt();













            if (numUsers > 0) {













            if (numUsers >= 0) {















                users = new int[numUsers];















                in.readIntArray(users);















            }









@@ -264,7 +264,7 @@ public class SoundTrigger {













                dest.writeInt(users.length);















                dest.writeIntArray(users);















            } else {













                dest.writeInt(0);













                dest.writeInt(-1);















            }















        }


























@@ -349,7 +349,7 @@ public class SoundTrigger {













            UUID uuid = UUID.fromString(in.readString());















            byte[] data = null;















            int dataLength = in.readInt();













            if (dataLength > 0) {













            if (dataLength >= 0) {















                data = new byte[dataLength];















                in.readByteArray(data);















            }









@@ -369,10 +369,16 @@ public class SoundTrigger {













                dest.writeInt(data.length);















                dest.writeByteArray(data);















            } else {













                dest.writeInt(0);













                dest.writeInt(-1);















            }















            dest.writeTypedArray(keyphrases, 0);















        }



























        @Override













        public String toString() {













            return "KeyphraseSoundModel [keyphrases=" + Arrays.toString(keyphrases) + ", uuid="













                    + uuid + ", type=" + type + ", data? " + (data != null) + "]";













        }















    }































    /**










@@ -471,7 +477,7 @@ public class SoundTrigger {













                    in.createTypedArray(KeyphraseRecognitionExtra.CREATOR);















            byte[] data = null;















            int dataLength = in.readInt();













            if (dataLength > 0) {













            if (dataLength >= 0) {















                data = new byte[dataLength];















                in.readByteArray(data);















            }









@@ -486,7 +492,7 @@ public class SoundTrigger {













                dest.writeInt(data.length);















                dest.writeByteArray(data);















            } else {













                dest.writeInt(0);













                dest.writeInt(-1);















            }















        }

























@@ -494,6 +500,12 @@ public class SoundTrigger {













        public int describeContents() {















            return 0;















        }



























        @Override













        public String toString() {













            return "RecognitionConfig [captureRequested=" + captureRequested + ", keyphrases="













                    + Arrays.toString(keyphrases) + ", data? " + (data != null) + "]";













        }















    }































    /**

































core/java/android/service/voice/AlwaysOnHotwordDetector.java



+66
−63




























Original line number
Diff line number
Diff line








@@ -27,6 +27,8 @@ import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra;













import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel;















import android.hardware.soundtrigger.SoundTrigger.ModuleProperties;















import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig;













import android.os.Handler;













import android.os.Message;















import android.os.RemoteException;















import android.util.Slog;


























@@ -72,18 +74,6 @@ public class AlwaysOnHotwordDetector {













    public static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR;















    public static final int STATUS_OK = SoundTrigger.STATUS_OK;





























    //---- Keyphrase recognition status ----//













    /** Indicates that recognition is not available. */













    public static final int RECOGNITION_STATUS_NOT_AVAILABLE = 0x01;













    /** Indicates that recognition has not been requested. */













    public static final int RECOGNITION_STATUS_NOT_REQUESTED = 0x02;













    /** Indicates that recognition has been requested. */













    public static final int RECOGNITION_STATUS_REQUESTED = 0x04;













    /** Indicates that recognition has been temporarily disabled. */













    public static final int RECOGNITION_STATUS_DISABLED_TEMPORARILY = 0x08;













    /** Indicates that recognition is currently active . */













    public static final int RECOGNITION_STATUS_ACTIVE = 0x10;





























    //-- Flags for startRecogntion    ----//















    /** Empty flag for {@link #startRecognition(int)}. */















    public static final int RECOGNITION_FLAG_NONE = 0;









@@ -96,15 +86,22 @@ public class AlwaysOnHotwordDetector {













    //---- Recognition mode flags ----//















    // Must be kept in sync with the related attribute defined as searchKeyphraseRecognitionFlags.





























    /** Simple recognition of the key phrase. Returned by {@link #getRecognitionStatus()} */













    /**













     * Simple recognition of the key phrase. Returned by {@link #getSupportedRecognitionModes()}













     */















    public static final int RECOGNITION_MODE_VOICE_TRIGGER















            = SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER;













    /** Trigger only if one user is identified. Returned by {@link #getRecognitionStatus()} */













    /**













     * Trigger only if one user is identified. Returned by {@link #getSupportedRecognitionModes()}













     */















    public static final int RECOGNITION_MODE_USER_IDENTIFICATION















            = SoundTrigger.RECOGNITION_MODE_USER_IDENTIFICATION;































    static final String TAG = "AlwaysOnHotwordDetector";





























    private static final int MSG_HOTWORD_DETECTED = 1;













    private static final int MSG_DETECTION_STOPPED = 2;





























    private final String mText;















    private final String mLocale;















    /**









@@ -118,12 +115,11 @@ public class AlwaysOnHotwordDetector {













     */















    private final KeyphraseSoundModel mEnrolledSoundModel;















    private final KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo;













    private final int mAvailability;















    private final IVoiceInteractionService mVoiceInteractionService;















    private final IVoiceInteractionManagerService mModelManagementService;















    private final SoundTriggerListener mInternalCallback;



























    private int mRecognitionState;













    private final Callback mExternalCallback;













    private final boolean mDisabled;































    /**















     * Callbacks for always-on hotword detection.









@@ -136,10 +132,6 @@ public class AlwaysOnHotwordDetector {













         *        {@link AlwaysOnHotwordDetector#startRecognition(int)}.















         */















        void onDetected(byte[] data);













        /**













         * Called when the detection for the associated keyphrase starts.













         */













        void onDetectionStarted();















        /**















         * Called when the detection for the associated keyphrase stops.















         */









@@ -163,7 +155,8 @@ public class AlwaysOnHotwordDetector {













        mLocale = locale;















        mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo;















        mKeyphraseMetadata = mKeyphraseEnrollmentInfo.getKeyphraseMetadata(text, locale);













        mInternalCallback = new SoundTriggerListener(callback);













        mExternalCallback = callback;













        mInternalCallback = new SoundTriggerListener(new MyHandler());















        mVoiceInteractionService = voiceInteractionService;















        mModelManagementService = modelManagementService;















        if (mKeyphraseMetadata != null) {









@@ -171,7 +164,9 @@ public class AlwaysOnHotwordDetector {













        } else {















            mEnrolledSoundModel = null;















        }













        mAvailability = internalGetAvailability();













        int initialAvailability = internalGetAvailabilityLocked();













        mDisabled = (initialAvailability == KEYPHRASE_HARDWARE_UNAVAILABLE)













                || (initialAvailability == KEYPHRASE_UNSUPPORTED);















    }































    /**









@@ -186,7 +181,9 @@ public class AlwaysOnHotwordDetector {













     *         {@link #KEYPHRASE_ENROLLED}.















     */















    public int getAvailability() {













        return mAvailability;













        synchronized (this) {













            return internalGetAvailabilityLocked();













        }















    }































    /**









@@ -197,8 +194,13 @@ public class AlwaysOnHotwordDetector {













     *         before calling this method to avoid this exception.















     */















    public int getSupportedRecognitionModes() {













        if (mAvailability == KEYPHRASE_HARDWARE_UNAVAILABLE













                || mAvailability == KEYPHRASE_UNSUPPORTED) {













        synchronized (this) {













            return getSupportedRecognitionModesLocked();













        }













    }



























    private int getSupportedRecognitionModesLocked() {













        if (mDisabled) {















            throw new UnsupportedOperationException(















                    "Getting supported recognition modes for the keyphrase is not supported");















        }









@@ -206,17 +208,6 @@ public class AlwaysOnHotwordDetector {













        return mKeyphraseMetadata.recognitionModeFlags;















    }





























    /**













     * Gets the status of the recognition.













     * @return A flag comprised of {@link #RECOGNITION_STATUS_NOT_AVAILABLE},













     *         {@link #RECOGNITION_STATUS_NOT_REQUESTED}, {@link #RECOGNITION_STATUS_REQUESTED},













     *         {@link #RECOGNITION_STATUS_DISABLED_TEMPORARILY} and













     *         {@link #RECOGNITION_STATUS_ACTIVE}.













     */













    public int getRecognitionStatus() {













        return mRecognitionState;













    }





























    /**















     * Starts recognition for the associated keyphrase.















     *









@@ -229,16 +220,19 @@ public class AlwaysOnHotwordDetector {













     *         before calling this method to avoid this exception.















     */















    public int startRecognition(int recognitionFlags) {













        if (mAvailability != KEYPHRASE_ENROLLED













                || (mRecognitionState&RECOGNITION_STATUS_NOT_AVAILABLE) != 0) {













        synchronized (this) {













            return startRecognitionLocked(recognitionFlags);













        }













    }



























    private int startRecognitionLocked(int recognitionFlags) {













        if (internalGetAvailabilityLocked() != KEYPHRASE_ENROLLED) {















            throw new UnsupportedOperationException(















                    "Recognition for the given keyphrase is not supported");















        }





























        mRecognitionState &= RECOGNITION_STATUS_REQUESTED;















        KeyphraseRecognitionExtra[] recognitionExtra = new KeyphraseRecognitionExtra[1];















        // TODO: Do we need to do something about the confidence level here?













        // TODO: Take in captureTriggerAudio as a method param here.















        recognitionExtra[0] = new KeyphraseRecognitionExtra(mKeyphraseMetadata.id,















                mKeyphraseMetadata.recognitionModeFlags, new ConfidenceLevel[0]);















        boolean captureTriggerAudio =










@@ -267,12 +261,17 @@ public class AlwaysOnHotwordDetector {













     *         before calling this method to avoid this exception.















     */















    public int stopRecognition() {













        if (mAvailability != KEYPHRASE_ENROLLED) {













        synchronized (this) {













            return stopRecognitionLocked();













        }













    }



























    private synchronized int stopRecognitionLocked() {













        if (internalGetAvailabilityLocked() != KEYPHRASE_ENROLLED) {















            throw new UnsupportedOperationException(















                    "Recognition for the given keyphrase is not supported");















        }





























        mRecognitionState &= ~RECOGNITION_STATUS_NOT_REQUESTED;















        int code = STATUS_ERROR;















        try {















            code = mModelManagementService.stopRecognition(









@@ -299,8 +298,7 @@ public class AlwaysOnHotwordDetector {













     *         before calling this method to avoid this exception.















     */















    public Intent getManageIntent(int action) {













        if (mAvailability == KEYPHRASE_HARDWARE_UNAVAILABLE













                || mAvailability == KEYPHRASE_UNSUPPORTED) {













        if (mDisabled) {















            throw new UnsupportedOperationException(















                    "Managing the given keyphrase is not supported");















        }









@@ -313,7 +311,7 @@ public class AlwaysOnHotwordDetector {













        return mKeyphraseEnrollmentInfo.getManageKeyphraseIntent(action, mText, mLocale);















    }





























    private int internalGetAvailability() {













    private int internalGetAvailabilityLocked() {















        ModuleProperties dspModuleProperties = null;















        try {















            dspModuleProperties =









@@ -323,21 +321,16 @@ public class AlwaysOnHotwordDetector {













        }















        // No DSP available















        if (dspModuleProperties == null) {













            mRecognitionState = RECOGNITION_STATUS_NOT_AVAILABLE;















            return KEYPHRASE_HARDWARE_UNAVAILABLE;















        }















        // No enrollment application supports this keyphrase/locale















        if (mKeyphraseMetadata == null) {













            mRecognitionState = RECOGNITION_STATUS_NOT_AVAILABLE;















            return KEYPHRASE_UNSUPPORTED;















        }















        // This keyphrase hasn't been enrolled.















        if (mEnrolledSoundModel == null) {













            mRecognitionState = RECOGNITION_STATUS_NOT_AVAILABLE;















            return KEYPHRASE_UNENROLLED;















        }













        // Mark recognition as available













        mRecognitionState &= ~RECOGNITION_STATUS_NOT_AVAILABLE;















        return KEYPHRASE_ENROLLED;















    }

























@@ -358,7 +351,6 @@ public class AlwaysOnHotwordDetector {













                    continue;















                }















                for (Keyphrase keyphrase : soundModel.keyphrases) {













                    // TODO: Check the user handle here to only load a model for the current user.















                    if (keyphrase.id == keyphraseId) {















                        return soundModel;















                    }









@@ -372,28 +364,39 @@ public class AlwaysOnHotwordDetector {





























    /** @hide */















    static final class SoundTriggerListener extends IRecognitionStatusCallback.Stub {













        private final Callback mCallback;













        private final Handler mHandler;





























        public SoundTriggerListener(Callback callback) {













            this.mCallback = callback;













        public SoundTriggerListener(Handler handler) {













            mHandler = handler;















        }































        @Override















        public void onDetected(byte[] data) {













            Slog.i(TAG, "onKeyphraseSpoken");













            mCallback.onDetected(data);













            Slog.i(TAG, "onDetected");













            Message message = Message.obtain(mHandler, MSG_HOTWORD_DETECTED);













            message.obj = data;













            message.sendToTarget();















        }































        @Override













        public void onDetectionStarted() {













            // TODO: Set the RECOGNITION_STATUS_ACTIVE flag here.













            mCallback.onDetectionStarted();













        public void onDetectionStopped() {













            Slog.i(TAG, "onDetectionStopped");













            mHandler.sendEmptyMessage(MSG_DETECTION_STOPPED);













        }















    }





























    class MyHandler extends Handler {















        @Override













        public void onDetectionStopped() {













            // TODO: Unset the RECOGNITION_STATUS_ACTIVE flag here.













            mCallback.onDetectionStopped();













        public void handleMessage(Message msg) {













            switch (msg.what) {













                case MSG_HOTWORD_DETECTED:













                    mExternalCallback.onDetected((byte[]) msg.obj);













                    break;













                case MSG_DETECTION_STOPPED:













                    mExternalCallback.onDetectionStopped();













                default:













                    super.handleMessage(msg);













            }















        }















    }















}
























services/voiceinteraction/java/com/android/server/voiceinteraction/DatabaseHelper.java



+35
−2




























Original line number
Diff line number
Diff line








@@ -24,6 +24,7 @@ import android.database.sqlite.SQLiteOpenHelper;













import android.hardware.soundtrigger.SoundTrigger;















import android.hardware.soundtrigger.SoundTrigger.Keyphrase;















import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel;













import android.os.UserManager;















import android.text.TextUtils;















import android.util.Slog;

























@@ -38,6 +39,7 @@ import java.util.UUID;













 */















public class DatabaseHelper extends SQLiteOpenHelper {















    static final String TAG = "SoundModelDBHelper";













    static final boolean DBG = false;































    private static final String NAME = "sound_model.db";















    private static final int VERSION = 2;










@@ -75,8 +77,11 @@ public class DatabaseHelper extends SQLiteOpenHelper {













            + SoundModelContract.KEY_TYPE + " INTEGER,"















            + SoundModelContract.KEY_DATA + " BLOB" + ")";





























    private final UserManager mUserManager;





























    public DatabaseHelper(Context context) {















        super(context, NAME, null, VERSION);













        mUserManager = (UserManager) context.getSystemService(Context.USER_SERVICE);















    }































    @Override










@@ -175,8 +180,17 @@ public class DatabaseHelper extends SQLiteOpenHelper {













                String id = c.getString(c.getColumnIndex(SoundModelContract.KEY_ID));















                byte[] data = c.getBlob(c.getColumnIndex(SoundModelContract.KEY_DATA));















                // Get all the keyphrases for this this sound model.













                models.add(new KeyphraseSoundModel(













                        UUID.fromString(id), data, getKeyphrasesForSoundModel(db, id)));













                // Validate the sound model.













                if (id == null) {













                    Slog.w(TAG, "Ignoring sound model since it doesn't specify an ID");













                    continue;













                }













                KeyphraseSoundModel model = new KeyphraseSoundModel(













                        UUID.fromString(id), data, getKeyphrasesForSoundModel(db, id));













                if (DBG) {













                    Slog.d(TAG, "Adding model: " + model);













                }













                models.add(model);















            } while (c.moveToNext());















        }















        c.close();









@@ -200,6 +214,25 @@ public class DatabaseHelper extends SQLiteOpenHelper {













                String locale = c.getString(c.getColumnIndex(KeyphraseContract.KEY_LOCALE));















                String hintText = c.getString(c.getColumnIndex(KeyphraseContract.KEY_HINT_TEXT));





























                // Only add keyphrases meant for the current user.













                if (users == null) {













                    // No users present in the keyphrase.













                    Slog.w(TAG, "Ignoring keyphrase since it doesn't specify users");













                    continue;













                }













                boolean isAvailableForCurrentUser = false;













                int currentUser = mUserManager.getUserHandle();













                for (int user : users) {













                    if (currentUser == user) {













                        isAvailableForCurrentUser = true;













                        break;













                    }













                }













                if (!isAvailableForCurrentUser) {













                    Slog.w(TAG, "Ignoring keyphrase since it's not for the current user");













                    continue;













                }





























                keyphrases.add(new Keyphrase(id, modes, locale, hintText, users));















            } while (c.moveToNext());















        }