Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 96e34f05 authored by lpeter's avatar lpeter
Browse files

Move the methods into Dsp session and Software session 

Move the methods of HotwordDetectorSession into
DspTrustedHotwordDetectorSession and
SoftwareTrustedHotwordDetectorSession

Test: atest CtsVoiceInteractionTestCases
Bug: 241041976
Change-Id: I53139507697ba86f04158896ef95095993f23f2c
parent 15660161

services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java

+211 −0
Original line number Diff line number Diff line
@@ -16,20 +16,61 @@ 


package com.android.server.voiceinteraction;



import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART;



import android.annotation.NonNull;

import android.content.Context;

import android.hardware.soundtrigger.SoundTrigger;

import android.media.permission.Identity;

import android.os.PersistableBundle;

import android.os.RemoteException;

import android.os.SharedMemory;

import android.service.voice.AlwaysOnHotwordDetector;

import android.service.voice.HotwordDetectedResult;

import android.service.voice.HotwordDetectionService;

import android.service.voice.HotwordDetector;

import android.service.voice.HotwordRejectedResult;

import android.service.voice.IDspHotwordDetectionCallback;

import android.util.Slog;



import com.android.internal.annotations.GuardedBy;

import com.android.internal.app.IHotwordRecognitionStatusCallback;



import java.io.IOException;

import java.io.PrintWriter;

import java.util.Locale;

import java.util.concurrent.ScheduledExecutorService;

import java.util.concurrent.ScheduledFuture;

import java.util.concurrent.TimeUnit;

import java.util.concurrent.atomic.AtomicBoolean;



/**

 * A class that provides Dsp trusted hotword detector to communicate with the {@link

 * HotwordDetectionService}.

 *

 * This class can handle the hotword detection which detector is created by using

 * {@link android.service.voice.VoiceInteractionService#createAlwaysOnHotwordDetector(String,

 * Locale, PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)}.

 */

final class DspTrustedHotwordDetectorSession extends HotwordDetectorSession {

    private static final String TAG = "DspTrustedHotwordDetectorSession";



    // The validation timeout value is 3 seconds for onDetect of DSP trigger event.

    private static final long VALIDATION_TIMEOUT_MILLIS = 3000;

    // Write the onDetect timeout metric when it takes more time than MAX_VALIDATION_TIMEOUT_MILLIS.

    private static final long MAX_VALIDATION_TIMEOUT_MILLIS = 4000;



    @GuardedBy("mLock")

    private ScheduledFuture<?> mCancellationKeyPhraseDetectionFuture;



    @GuardedBy("mLock")

    private boolean mValidatingDspTrigger = false;



    DspTrustedHotwordDetectorSession(

            @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService,
@@ -40,4 +81,174 @@ final class DspTrustedHotwordDetectorSession extends HotwordDetectorSession { 
        super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid,

                voiceInteractorIdentity, scheduledExecutorService, logging);

    }



    @SuppressWarnings("GuardedBy")

    void detectFromDspSourceLocked(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent,

            IHotwordRecognitionStatusCallback externalCallback) {

        if (DEBUG) {

            Slog.d(TAG, "detectFromDspSourceLocked");

        }



        AtomicBoolean timeoutDetected = new AtomicBoolean(false);

        // TODO: consider making this a non-anonymous class.

        IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {

            @Override

            public void onDetected(HotwordDetectedResult result) throws RemoteException {

                if (DEBUG) {

                    Slog.d(TAG, "onDetected");

                }

                synchronized (mLock) {

                    if (mCancellationKeyPhraseDetectionFuture != null) {

                        mCancellationKeyPhraseDetectionFuture.cancel(true);

                    }

                    if (timeoutDetected.get()) {

                        return;

                    }

                    HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                            HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                            HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED);

                    if (!mValidatingDspTrigger) {

                        Slog.i(TAG, "Ignoring #onDetected due to a process restart");

                        HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                                HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                                METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK);

                        return;

                    }

                    mValidatingDspTrigger = false;

                    try {

                        enforcePermissionsForDataDelivery();

                        enforceExtraKeyphraseIdNotLeaked(result, recognitionEvent);

                    } catch (SecurityException e) {

                        Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e);

                        HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                                HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                                METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION);

                        externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION);

                        return;

                    }

                    saveProximityValueToBundle(result);

                    HotwordDetectedResult newResult;

                    try {

                        newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);

                    } catch (IOException e) {

                        externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR);

                        return;

                    }

                    externalCallback.onKeyphraseDetected(recognitionEvent, newResult);

                    Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult)

                            + " bits from hotword trusted process");

                    if (mDebugHotwordLogging) {

                        Slog.i(TAG, "Egressed detected result: " + newResult);

                    }

                }

            }



            @Override

            public void onRejected(HotwordRejectedResult result) throws RemoteException {

                if (DEBUG) {

                    Slog.d(TAG, "onRejected");

                }

                synchronized (mLock) {

                    if (mCancellationKeyPhraseDetectionFuture != null) {

                        mCancellationKeyPhraseDetectionFuture.cancel(true);

                    }

                    if (timeoutDetected.get()) {

                        return;

                    }

                    HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                            HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                            HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED);

                    if (!mValidatingDspTrigger) {

                        Slog.i(TAG, "Ignoring #onRejected due to a process restart");

                        HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                                HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                                METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK);

                        return;

                    }

                    mValidatingDspTrigger = false;

                    externalCallback.onRejected(result);

                    if (mDebugHotwordLogging && result != null) {

                        Slog.i(TAG, "Egressed rejected result: " + result);

                    }

                }

            }

        };



        mValidatingDspTrigger = true;

        mRemoteHotwordDetectionService.run(service -> {

            // We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke

            // the callback before timeout value. In order to reduce the latency impact between

            // server side and client side, we need to use another timeout value

            // MAX_VALIDATION_TIMEOUT_MILLIS to monitor it.

            mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule(

                    () -> {

                        // TODO: avoid allocate every time

                        timeoutDetected.set(true);

                        Slog.w(TAG, "Timed out on #detectFromDspSource");

                        HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                                HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                                HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT);

                        try {

                            externalCallback.onError(CALLBACK_DETECT_TIMEOUT);

                        } catch (RemoteException e) {

                            Slog.w(TAG, "Failed to report onError status: ", e);

                            HotwordMetricsLogger.writeDetectorEvent(

                                    HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                                    HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION,

                                    mVoiceInteractionServiceUid);

                        }

                    },

                    MAX_VALIDATION_TIMEOUT_MILLIS,

                    TimeUnit.MILLISECONDS);

            service.detectFromDspSource(

                    recognitionEvent,

                    recognitionEvent.getCaptureFormat(),

                    VALIDATION_TIMEOUT_MILLIS,

                    internalCallback);

        });

    }



    @Override

    @SuppressWarnings("GuardedBy")

    void informRestartProcessLocked() {

        // TODO(b/244598068): Check HotwordAudioStreamManager first

        Slog.v(TAG, "informRestartProcessLocked");

        if (mValidatingDspTrigger) {

            // We're restarting the process while it's processing a DSP trigger, so report a

            // rejection. This also allows the Interactor to startRecognition again

            try {

                mCallback.onRejected(new HotwordRejectedResult.Builder().build());

                HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                        HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                        HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART);

            } catch (RemoteException e) {

                Slog.w(TAG, "Failed to call #rejected");

                HotwordMetricsLogger.writeDetectorEvent(

                        HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                        HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION,

                        mVoiceInteractionServiceUid);

            }

            mValidatingDspTrigger = false;

        }

        mUpdateStateAfterStartFinished.set(false);



        try {

            mCallback.onProcessRestarted();

        } catch (RemoteException e) {

            Slog.w(TAG, "Failed to communicate #onProcessRestarted", e);

            HotwordMetricsLogger.writeDetectorEvent(

                    HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                    HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION,

                    mVoiceInteractionServiceUid);

        }



        mPerformingExternalSourceHotwordDetection = false;

        closeExternalAudioStreamLocked("process restarted");

    }



    @SuppressWarnings("GuardedBy")

    public void dumpLocked(String prefix, PrintWriter pw) {

        super.dumpLocked(prefix, pw);

        pw.print(prefix); pw.print("mValidatingDspTrigger="); pw.println(mValidatingDspTrigger);

    }

}

services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java

+5 −3
Original line number Diff line number Diff line
@@ -244,7 +244,8 @@ final class HotwordDetectionConnection { 
                Slog.d(TAG, "It is not a software detector");

                return;

            }

            mHotwordDetectorSession.startListeningFromMicLocked(audioFormat, callback);

            ((SoftwareTrustedHotwordDetectorSession) mHotwordDetectorSession)

                    .startListeningFromMicLocked(audioFormat, callback);

        }

    }
@@ -274,7 +275,7 @@ final class HotwordDetectionConnection { 
                Slog.d(TAG, "It is not a software detector");

                return;

            }

            mHotwordDetectorSession.stopListeningLocked();

            ((SoftwareTrustedHotwordDetectorSession) mHotwordDetectorSession).stopListeningLocked();

        }

    }
@@ -297,7 +298,8 @@ final class HotwordDetectionConnection { 
                Slog.d(TAG, "It is not a Dsp detector");

                return;

            }

            mHotwordDetectorSession.detectFromDspSourceLocked(recognitionEvent, externalCallback);

            ((DspTrustedHotwordDetectorSession) mHotwordDetectorSession).detectFromDspSourceLocked(

                    recognitionEvent, externalCallback);

        }

    }

services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java

+123 −396

File changed.

Preview size limit exceeded, changes collapsed.

services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java

+171 −0
Original line number Diff line number Diff line
@@ -16,20 +16,50 @@ 


package com.android.server.voiceinteraction;



import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_MICROPHONE;



import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED;



import android.annotation.NonNull;

import android.content.Context;

import android.media.AudioFormat;

import android.media.permission.Identity;

import android.os.PersistableBundle;

import android.os.RemoteException;

import android.os.SharedMemory;

import android.service.voice.HotwordDetectedResult;

import android.service.voice.HotwordDetectionService;

import android.service.voice.HotwordDetector;

import android.service.voice.HotwordRejectedResult;

import android.service.voice.IDspHotwordDetectionCallback;

import android.service.voice.IHotwordDetectionService;

import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;

import android.util.Slog;



import com.android.internal.annotations.GuardedBy;

import com.android.internal.app.IHotwordRecognitionStatusCallback;



import java.io.IOException;

import java.io.PrintWriter;

import java.util.concurrent.ScheduledExecutorService;



/**

 * A class that provides software trusted hotword detector to communicate with the {@link

 * HotwordDetectionService}.

 *

 * This class can handle the hotword detection which detector is created by using

 * {@link android.service.voice.VoiceInteractionService#createHotwordDetector(PersistableBundle,

 * SharedMemory, HotwordDetector.Callback)}.

 */

final class SoftwareTrustedHotwordDetectorSession extends HotwordDetectorSession {

    private static final String TAG = "SoftwareTrustedHotwordDetectorSession";



    private IMicrophoneHotwordDetectionVoiceInteractionCallback mSoftwareCallback;

    @GuardedBy("mLock")

    private boolean mPerformingSoftwareHotwordDetection;



    SoftwareTrustedHotwordDetectorSession(

            @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService,
@@ -40,4 +70,145 @@ final class SoftwareTrustedHotwordDetectorSession extends HotwordDetectorSession 
        super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid,

                voiceInteractorIdentity, scheduledExecutorService, logging);

    }



    @SuppressWarnings("GuardedBy")

    void startListeningFromMicLocked(

            AudioFormat audioFormat,

            IMicrophoneHotwordDetectionVoiceInteractionCallback callback) {

        if (DEBUG) {

            Slog.d(TAG, "startListeningFromMicLocked");

        }

        mSoftwareCallback = callback;



        if (mPerformingSoftwareHotwordDetection) {

            Slog.i(TAG, "Hotword validation is already in progress, ignoring.");

            return;

        }

        mPerformingSoftwareHotwordDetection = true;



        startListeningFromMicLocked();

    }



    @SuppressWarnings("GuardedBy")

    private void startListeningFromMicLocked() {

        // TODO: consider making this a non-anonymous class.

        IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {

            @Override

            public void onDetected(HotwordDetectedResult result) throws RemoteException {

                if (DEBUG) {

                    Slog.d(TAG, "onDetected");

                }

                synchronized (mLock) {

                    HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                            HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,

                            HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED);

                    if (!mPerformingSoftwareHotwordDetection) {

                        Slog.i(TAG, "Hotword detection has already completed");

                        HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                                HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,

                                METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK);

                        return;

                    }

                    mPerformingSoftwareHotwordDetection = false;

                    try {

                        enforcePermissionsForDataDelivery();

                    } catch (SecurityException e) {

                        HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                                HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,

                                METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION);

                        mSoftwareCallback.onError();

                        return;

                    }

                    saveProximityValueToBundle(result);

                    HotwordDetectedResult newResult;

                    try {

                        newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);

                    } catch (IOException e) {

                        // TODO: Write event

                        mSoftwareCallback.onError();

                        return;

                    }

                    mSoftwareCallback.onDetected(newResult, null, null);

                    Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult)

                            + " bits from hotword trusted process");

                    if (mDebugHotwordLogging) {

                        Slog.i(TAG, "Egressed detected result: " + newResult);

                    }

                }

            }



            @Override

            public void onRejected(HotwordRejectedResult result) throws RemoteException {

                if (DEBUG) {

                    Slog.wtf(TAG, "onRejected");

                }

                HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                        HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,

                        HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED);

                // onRejected isn't allowed here, and we are not expecting it.

            }

        };



        mRemoteHotwordDetectionService.run(

                service -> service.detectFromMicrophoneSource(

                        null,

                        AUDIO_SOURCE_MICROPHONE,

                        null,

                        null,

                        internalCallback));

        HotwordMetricsLogger.writeDetectorEvent(

                HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,

                HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION,

                mVoiceInteractionServiceUid);

    }



    @SuppressWarnings("GuardedBy")

    void stopListeningLocked() {

        if (DEBUG) {

            Slog.d(TAG, "stopListeningLocked");

        }

        if (!mPerformingSoftwareHotwordDetection) {

            Slog.i(TAG, "Hotword detection is not running");

            return;

        }

        mPerformingSoftwareHotwordDetection = false;



        mRemoteHotwordDetectionService.run(IHotwordDetectionService::stopDetection);



        closeExternalAudioStreamLocked("stopping requested");

    }



    @Override

    @SuppressWarnings("GuardedBy")

    void informRestartProcessLocked() {

        // TODO(b/244598068): Check HotwordAudioStreamManager first

        Slog.v(TAG, "informRestartProcessLocked");

        mUpdateStateAfterStartFinished.set(false);



        try {

            mCallback.onProcessRestarted();

        } catch (RemoteException e) {

            Slog.w(TAG, "Failed to communicate #onProcessRestarted", e);

            HotwordMetricsLogger.writeDetectorEvent(

                    HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,

                    HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION,

                    mVoiceInteractionServiceUid);

        }



        // Restart listening from microphone if the hotword process has been restarted.

        if (mPerformingSoftwareHotwordDetection) {

            Slog.i(TAG, "Process restarted: calling startRecognition() again");

            startListeningFromMicLocked();

        }



        mPerformingExternalSourceHotwordDetection = false;

        closeExternalAudioStreamLocked("process restarted");

    }



    @SuppressWarnings("GuardedBy")

    public void dumpLocked(String prefix, PrintWriter pw) {

        super.dumpLocked(prefix, pw);

        pw.print(prefix); pw.print("mPerformingSoftwareHotwordDetection=");

        pw.println(mPerformingSoftwareHotwordDetection);

    }

}