Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6e5e37a9 authored by TreeHugger Robot's avatar TreeHugger Robot Committed by Android (Google) Code Review
Browse files

Merge changes from topic "refactor_hotword" 

* changes:
  Move the methods into Dsp session and Software session
  Use Dsp session and Software session in HotwordDetectionConnection
  Refactor the logic of HotwordDetectionConnection
parents dda9c255 96e34f05

services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java

0 → 100644
+254 −0
Original line number Original line Diff line number Diff line 
/*

 * Copyright (C) 2022 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package com.android.server.voiceinteraction;



import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART;



import android.annotation.NonNull;

import android.content.Context;

import android.hardware.soundtrigger.SoundTrigger;

import android.media.permission.Identity;

import android.os.PersistableBundle;

import android.os.RemoteException;

import android.os.SharedMemory;

import android.service.voice.AlwaysOnHotwordDetector;

import android.service.voice.HotwordDetectedResult;

import android.service.voice.HotwordDetectionService;

import android.service.voice.HotwordDetector;

import android.service.voice.HotwordRejectedResult;

import android.service.voice.IDspHotwordDetectionCallback;

import android.util.Slog;



import com.android.internal.annotations.GuardedBy;

import com.android.internal.app.IHotwordRecognitionStatusCallback;



import java.io.IOException;

import java.io.PrintWriter;

import java.util.Locale;

import java.util.concurrent.ScheduledExecutorService;

import java.util.concurrent.ScheduledFuture;

import java.util.concurrent.TimeUnit;

import java.util.concurrent.atomic.AtomicBoolean;



/**

 * A class that provides Dsp trusted hotword detector to communicate with the {@link

 * HotwordDetectionService}.

 *

 * This class can handle the hotword detection which detector is created by using

 * {@link android.service.voice.VoiceInteractionService#createAlwaysOnHotwordDetector(String,

 * Locale, PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)}.

 */

final class DspTrustedHotwordDetectorSession extends HotwordDetectorSession {

    private static final String TAG = "DspTrustedHotwordDetectorSession";



    // The validation timeout value is 3 seconds for onDetect of DSP trigger event.

    private static final long VALIDATION_TIMEOUT_MILLIS = 3000;

    // Write the onDetect timeout metric when it takes more time than MAX_VALIDATION_TIMEOUT_MILLIS.

    private static final long MAX_VALIDATION_TIMEOUT_MILLIS = 4000;



    @GuardedBy("mLock")

    private ScheduledFuture<?> mCancellationKeyPhraseDetectionFuture;



    @GuardedBy("mLock")

    private boolean mValidatingDspTrigger = false;



    DspTrustedHotwordDetectorSession(

            @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService,

            @NonNull Object lock, @NonNull Context context,

            @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid,

            Identity voiceInteractorIdentity,

            @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) {

        super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid,

                voiceInteractorIdentity, scheduledExecutorService, logging);

    }



    @SuppressWarnings("GuardedBy")

    void detectFromDspSourceLocked(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent,

            IHotwordRecognitionStatusCallback externalCallback) {

        if (DEBUG) {

            Slog.d(TAG, "detectFromDspSourceLocked");

        }



        AtomicBoolean timeoutDetected = new AtomicBoolean(false);

        // TODO: consider making this a non-anonymous class.

        IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {

            @Override

            public void onDetected(HotwordDetectedResult result) throws RemoteException {

                if (DEBUG) {

                    Slog.d(TAG, "onDetected");

                }

                synchronized (mLock) {

                    if (mCancellationKeyPhraseDetectionFuture != null) {

                        mCancellationKeyPhraseDetectionFuture.cancel(true);

                    }

                    if (timeoutDetected.get()) {

                        return;

                    }

                    HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                            HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                            HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED);

                    if (!mValidatingDspTrigger) {

                        Slog.i(TAG, "Ignoring #onDetected due to a process restart");

                        HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                                HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                                METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK);

                        return;

                    }

                    mValidatingDspTrigger = false;

                    try {

                        enforcePermissionsForDataDelivery();

                        enforceExtraKeyphraseIdNotLeaked(result, recognitionEvent);

                    } catch (SecurityException e) {

                        Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e);

                        HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                                HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                                METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION);

                        externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION);

                        return;

                    }

                    saveProximityValueToBundle(result);

                    HotwordDetectedResult newResult;

                    try {

                        newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);

                    } catch (IOException e) {

                        externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR);

                        return;

                    }

                    externalCallback.onKeyphraseDetected(recognitionEvent, newResult);

                    Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult)

                            + " bits from hotword trusted process");

                    if (mDebugHotwordLogging) {

                        Slog.i(TAG, "Egressed detected result: " + newResult);

                    }

                }

            }



            @Override

            public void onRejected(HotwordRejectedResult result) throws RemoteException {

                if (DEBUG) {

                    Slog.d(TAG, "onRejected");

                }

                synchronized (mLock) {

                    if (mCancellationKeyPhraseDetectionFuture != null) {

                        mCancellationKeyPhraseDetectionFuture.cancel(true);

                    }

                    if (timeoutDetected.get()) {

                        return;

                    }

                    HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                            HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                            HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED);

                    if (!mValidatingDspTrigger) {

                        Slog.i(TAG, "Ignoring #onRejected due to a process restart");

                        HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                                HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                                METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK);

                        return;

                    }

                    mValidatingDspTrigger = false;

                    externalCallback.onRejected(result);

                    if (mDebugHotwordLogging && result != null) {

                        Slog.i(TAG, "Egressed rejected result: " + result);

                    }

                }

            }

        };



        mValidatingDspTrigger = true;

        mRemoteHotwordDetectionService.run(service -> {

            // We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke

            // the callback before timeout value. In order to reduce the latency impact between

            // server side and client side, we need to use another timeout value

            // MAX_VALIDATION_TIMEOUT_MILLIS to monitor it.

            mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule(

                    () -> {

                        // TODO: avoid allocate every time

                        timeoutDetected.set(true);

                        Slog.w(TAG, "Timed out on #detectFromDspSource");

                        HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                                HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                                HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT);

                        try {

                            externalCallback.onError(CALLBACK_DETECT_TIMEOUT);

                        } catch (RemoteException e) {

                            Slog.w(TAG, "Failed to report onError status: ", e);

                            HotwordMetricsLogger.writeDetectorEvent(

                                    HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                                    HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION,

                                    mVoiceInteractionServiceUid);

                        }

                    },

                    MAX_VALIDATION_TIMEOUT_MILLIS,

                    TimeUnit.MILLISECONDS);

            service.detectFromDspSource(

                    recognitionEvent,

                    recognitionEvent.getCaptureFormat(),

                    VALIDATION_TIMEOUT_MILLIS,

                    internalCallback);

        });

    }



    @Override

    @SuppressWarnings("GuardedBy")

    void informRestartProcessLocked() {

        // TODO(b/244598068): Check HotwordAudioStreamManager first

        Slog.v(TAG, "informRestartProcessLocked");

        if (mValidatingDspTrigger) {

            // We're restarting the process while it's processing a DSP trigger, so report a

            // rejection. This also allows the Interactor to startRecognition again

            try {

                mCallback.onRejected(new HotwordRejectedResult.Builder().build());

                HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                        HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                        HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART);

            } catch (RemoteException e) {

                Slog.w(TAG, "Failed to call #rejected");

                HotwordMetricsLogger.writeDetectorEvent(

                        HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                        HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION,

                        mVoiceInteractionServiceUid);

            }

            mValidatingDspTrigger = false;

        }

        mUpdateStateAfterStartFinished.set(false);



        try {

            mCallback.onProcessRestarted();

        } catch (RemoteException e) {

            Slog.w(TAG, "Failed to communicate #onProcessRestarted", e);

            HotwordMetricsLogger.writeDetectorEvent(

                    HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,

                    HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION,

                    mVoiceInteractionServiceUid);

        }



        mPerformingExternalSourceHotwordDetection = false;

        closeExternalAudioStreamLocked("process restarted");

    }



    @SuppressWarnings("GuardedBy")

    public void dumpLocked(String prefix, PrintWriter pw) {

        super.dumpLocked(prefix, pw);

        pw.print(prefix); pw.print("mValidatingDspTrigger="); pw.println(mValidatingDspTrigger);

    }

}

services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java

+73 −769

File changed.

Preview size limit exceeded, changes collapsed.

services/voiceinteraction/java/com/android/server/voiceinteraction/TrustedHotwordDetectorSession.java→services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java

+666 −0

File changed and moved.

Preview size limit exceeded, changes collapsed.

services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java

0 → 100644
+214 −0
Original line number Original line Diff line number Diff line 
/*

 * Copyright (C) 2022 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package com.android.server.voiceinteraction;



import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_MICROPHONE;



import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED;



import android.annotation.NonNull;

import android.content.Context;

import android.media.AudioFormat;

import android.media.permission.Identity;

import android.os.PersistableBundle;

import android.os.RemoteException;

import android.os.SharedMemory;

import android.service.voice.HotwordDetectedResult;

import android.service.voice.HotwordDetectionService;

import android.service.voice.HotwordDetector;

import android.service.voice.HotwordRejectedResult;

import android.service.voice.IDspHotwordDetectionCallback;

import android.service.voice.IHotwordDetectionService;

import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;

import android.util.Slog;



import com.android.internal.annotations.GuardedBy;

import com.android.internal.app.IHotwordRecognitionStatusCallback;



import java.io.IOException;

import java.io.PrintWriter;

import java.util.concurrent.ScheduledExecutorService;



/**

 * A class that provides software trusted hotword detector to communicate with the {@link

 * HotwordDetectionService}.

 *

 * This class can handle the hotword detection which detector is created by using

 * {@link android.service.voice.VoiceInteractionService#createHotwordDetector(PersistableBundle,

 * SharedMemory, HotwordDetector.Callback)}.

 */

final class SoftwareTrustedHotwordDetectorSession extends HotwordDetectorSession {

    private static final String TAG = "SoftwareTrustedHotwordDetectorSession";



    private IMicrophoneHotwordDetectionVoiceInteractionCallback mSoftwareCallback;

    @GuardedBy("mLock")

    private boolean mPerformingSoftwareHotwordDetection;



    SoftwareTrustedHotwordDetectorSession(

            @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService,

            @NonNull Object lock, @NonNull Context context,

            @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid,

            Identity voiceInteractorIdentity,

            @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) {

        super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid,

                voiceInteractorIdentity, scheduledExecutorService, logging);

    }



    @SuppressWarnings("GuardedBy")

    void startListeningFromMicLocked(

            AudioFormat audioFormat,

            IMicrophoneHotwordDetectionVoiceInteractionCallback callback) {

        if (DEBUG) {

            Slog.d(TAG, "startListeningFromMicLocked");

        }

        mSoftwareCallback = callback;



        if (mPerformingSoftwareHotwordDetection) {

            Slog.i(TAG, "Hotword validation is already in progress, ignoring.");

            return;

        }

        mPerformingSoftwareHotwordDetection = true;



        startListeningFromMicLocked();

    }



    @SuppressWarnings("GuardedBy")

    private void startListeningFromMicLocked() {

        // TODO: consider making this a non-anonymous class.

        IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {

            @Override

            public void onDetected(HotwordDetectedResult result) throws RemoteException {

                if (DEBUG) {

                    Slog.d(TAG, "onDetected");

                }

                synchronized (mLock) {

                    HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                            HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,

                            HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED);

                    if (!mPerformingSoftwareHotwordDetection) {

                        Slog.i(TAG, "Hotword detection has already completed");

                        HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                                HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,

                                METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK);

                        return;

                    }

                    mPerformingSoftwareHotwordDetection = false;

                    try {

                        enforcePermissionsForDataDelivery();

                    } catch (SecurityException e) {

                        HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                                HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,

                                METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION);

                        mSoftwareCallback.onError();

                        return;

                    }

                    saveProximityValueToBundle(result);

                    HotwordDetectedResult newResult;

                    try {

                        newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);

                    } catch (IOException e) {

                        // TODO: Write event

                        mSoftwareCallback.onError();

                        return;

                    }

                    mSoftwareCallback.onDetected(newResult, null, null);

                    Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult)

                            + " bits from hotword trusted process");

                    if (mDebugHotwordLogging) {

                        Slog.i(TAG, "Egressed detected result: " + newResult);

                    }

                }

            }



            @Override

            public void onRejected(HotwordRejectedResult result) throws RemoteException {

                if (DEBUG) {

                    Slog.wtf(TAG, "onRejected");

                }

                HotwordMetricsLogger.writeKeyphraseTriggerEvent(

                        HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,

                        HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED);

                // onRejected isn't allowed here, and we are not expecting it.

            }

        };



        mRemoteHotwordDetectionService.run(

                service -> service.detectFromMicrophoneSource(

                        null,

                        AUDIO_SOURCE_MICROPHONE,

                        null,

                        null,

                        internalCallback));

        HotwordMetricsLogger.writeDetectorEvent(

                HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,

                HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION,

                mVoiceInteractionServiceUid);

    }



    @SuppressWarnings("GuardedBy")

    void stopListeningLocked() {

        if (DEBUG) {

            Slog.d(TAG, "stopListeningLocked");

        }

        if (!mPerformingSoftwareHotwordDetection) {

            Slog.i(TAG, "Hotword detection is not running");

            return;

        }

        mPerformingSoftwareHotwordDetection = false;



        mRemoteHotwordDetectionService.run(IHotwordDetectionService::stopDetection);



        closeExternalAudioStreamLocked("stopping requested");

    }



    @Override

    @SuppressWarnings("GuardedBy")

    void informRestartProcessLocked() {

        // TODO(b/244598068): Check HotwordAudioStreamManager first

        Slog.v(TAG, "informRestartProcessLocked");

        mUpdateStateAfterStartFinished.set(false);



        try {

            mCallback.onProcessRestarted();

        } catch (RemoteException e) {

            Slog.w(TAG, "Failed to communicate #onProcessRestarted", e);

            HotwordMetricsLogger.writeDetectorEvent(

                    HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,

                    HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION,

                    mVoiceInteractionServiceUid);

        }



        // Restart listening from microphone if the hotword process has been restarted.

        if (mPerformingSoftwareHotwordDetection) {

            Slog.i(TAG, "Process restarted: calling startRecognition() again");

            startListeningFromMicLocked();

        }



        mPerformingExternalSourceHotwordDetection = false;

        closeExternalAudioStreamLocked("process restarted");

    }



    @SuppressWarnings("GuardedBy")

    public void dumpLocked(String prefix, PrintWriter pw) {

        super.dumpLocked(prefix, pw);

        pw.print(prefix); pw.print("mPerformingSoftwareHotwordDetection=");

        pw.println(mPerformingSoftwareHotwordDetection);

    }

}