Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4ee9f703 authored by Mark Punzalan's avatar Mark Punzalan Committed by Android (Google) Code Review
Browse files

Merge changes from topic "audio_egress_metric" into tm-qpr-dev 

* changes:
  Hotword: Add metrics for Audio Egress
  Rename HotwordAudioStreamManager and add Javadoc
  Remove dependency on Identity in HotwordAudioStreamManager
  Allow setting buffer length for HotwordAudioStreamManager
parents be2f4b96 48c42fda

core/java/android/service/voice/HotwordAudioStream.java

+15 −0
Original line number Diff line number Diff line
@@ -36,6 +36,21 @@ import java.util.Objects; 
 */

public final class HotwordAudioStream implements Parcelable {



    /**

     * Key for int value to be read from {@link #getMetadata()}. The value is read by the system and

     * is the length (in bytes) of the byte buffers created to copy bytes in the

     * {@link #getAudioStreamParcelFileDescriptor()} written by the {@link HotwordDetectionService}.

     * The buffer length should be chosen such that no additional latency is introduced. Typically,

     * this should be <em>at least</em> the size of byte chunks written by the

     * {@link HotwordDetectionService}.

     *

     * <p>If no value specified in the metadata for the buffer length, or if the value is less than

     * 1, or if it is greater than 65,536, or if it is not an int, the default value of 2,560 will

     * be used.</p>

     */

    public static final String KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES =

            "android.service.voice.key.AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES";



    /**

     * The {@link AudioFormat} of the audio stream.

     */

services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordAudioStreamManager.java→services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordAudioStreamCopier.java

+115 −42
Original line number Diff line number Diff line
@@ -17,16 +17,23 @@ 
package com.android.server.voiceinteraction;



import static android.app.AppOpsManager.MODE_ALLOWED;

import static android.service.voice.HotwordAudioStream.KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES;



import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__AUDIO_EGRESS_CLOSE_ERROR_FROM_SYSTEM;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__AUDIO_EGRESS_EMPTY_AUDIO_STREAM_LIST;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__AUDIO_EGRESS_END;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__AUDIO_EGRESS_ILLEGAL_COPY_BUFFER_SIZE;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__AUDIO_EGRESS_INTERRUPTED_EXCEPTION;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__AUDIO_EGRESS_NO_PERMISSION;

import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__AUDIO_EGRESS_START;

import static com.android.server.voiceinteraction.HotwordDetectionConnection.DEBUG;



import android.annotation.NonNull;

import android.app.AppOpsManager;

import android.media.permission.Identity;

import android.os.ParcelFileDescriptor;

import android.os.PersistableBundle;

import android.service.voice.HotwordAudioStream;

import android.service.voice.HotwordDetectedResult;

import android.util.Pair;

import android.util.Slog;



import java.io.IOException;
@@ -39,21 +46,40 @@ import java.util.concurrent.Callable; 
import java.util.concurrent.ExecutorService;

import java.util.concurrent.Executors;



final class HotwordAudioStreamManager {

/**

 * Copies the audio streams in {@link HotwordDetectedResult}s. This allows the system to manage the

 * lifetime of the {@link ParcelFileDescriptor}s and ensures that the flow of data is in the right

 * direction from the {@link android.service.voice.HotwordDetectionService} to the client (i.e., the

 * voice interactor).

 *

 * @hide

 */

final class HotwordAudioStreamCopier {



    private static final String TAG = "HotwordAudioStreamManager";

    private static final String TAG = "HotwordAudioStreamCopier";

    private static final String OP_MESSAGE = "Streaming hotword audio to VoiceInteractionService";

    private static final String TASK_ID_PREFIX = "HotwordDetectedResult@";

    private static final String THREAD_NAME_PREFIX = "Copy-";

    private static final int DEFAULT_COPY_BUFFER_LENGTH_BYTES = 2_560;



    // Corresponds to the OS pipe capacity in bytes

    private static final int MAX_COPY_BUFFER_LENGTH_BYTES = 65_536;



    private final AppOpsManager mAppOpsManager;

    private final Identity mVoiceInteractorIdentity;

    private final int mDetectorType;

    private final int mVoiceInteractorUid;

    private final String mVoiceInteractorPackageName;

    private final String mVoiceInteractorAttributionTag;

    private final ExecutorService mExecutorService = Executors.newCachedThreadPool();



    HotwordAudioStreamManager(@NonNull AppOpsManager appOpsManager,

            @NonNull Identity voiceInteractorIdentity) {

    HotwordAudioStreamCopier(@NonNull AppOpsManager appOpsManager, int detectorType,

            int voiceInteractorUid, @NonNull String voiceInteractorPackageName,

            @NonNull String voiceInteractorAttributionTag) {

        mAppOpsManager = appOpsManager;

        mVoiceInteractorIdentity = voiceInteractorIdentity;

        mDetectorType = detectorType;

        mVoiceInteractorUid = voiceInteractorUid;

        mVoiceInteractorPackageName = voiceInteractorPackageName;

        mVoiceInteractorAttributionTag = voiceInteractorAttributionTag;

    }



    /**
@@ -61,7 +87,7 @@ final class HotwordAudioStreamManager { 
     * <p>

     * The returned {@link HotwordDetectedResult} is identical the one that was passed in, except

     * that the {@link ParcelFileDescriptor}s within {@link HotwordDetectedResult#getAudioStreams()}

     * are replaced with descriptors from pipes managed by {@link HotwordAudioStreamManager}. The

     * are replaced with descriptors from pipes managed by {@link HotwordAudioStreamCopier}. The

     * returned value should be passed on to the client (i.e., the voice interactor).

     * </p>

     *
@@ -72,12 +98,14 @@ final class HotwordAudioStreamManager { 
            throws IOException {

        List<HotwordAudioStream> audioStreams = result.getAudioStreams();

        if (audioStreams.isEmpty()) {

            HotwordMetricsLogger.writeDetectorEvent(mDetectorType,

                    HOTWORD_DETECTOR_EVENTS__EVENT__AUDIO_EGRESS_EMPTY_AUDIO_STREAM_LIST,

                    mVoiceInteractorUid);

            return result;

        }



        List<HotwordAudioStream> newAudioStreams = new ArrayList<>(audioStreams.size());

        List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> sourcesAndSinks = new ArrayList<>(

                audioStreams.size());

        List<CopyTaskInfo> copyTaskInfos = new ArrayList<>(audioStreams.size());

        for (HotwordAudioStream audioStream : audioStreams) {

            ParcelFileDescriptor[] clientPipe = ParcelFileDescriptor.createReliablePipe();

            ParcelFileDescriptor clientAudioSource = clientPipe[0];
@@ -87,74 +115,114 @@ final class HotwordAudioStreamManager { 
                            clientAudioSource).build();

            newAudioStreams.add(newAudioStream);



            int copyBufferLength = DEFAULT_COPY_BUFFER_LENGTH_BYTES;

            PersistableBundle metadata = audioStream.getMetadata();

            if (metadata.containsKey(KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES)) {

                copyBufferLength = metadata.getInt(KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES, -1);

                if (copyBufferLength < 1 || copyBufferLength > MAX_COPY_BUFFER_LENGTH_BYTES) {

                    HotwordMetricsLogger.writeDetectorEvent(mDetectorType,

                            HOTWORD_DETECTOR_EVENTS__EVENT__AUDIO_EGRESS_ILLEGAL_COPY_BUFFER_SIZE,

                            mVoiceInteractorUid);

                    Slog.w(TAG, "Attempted to set an invalid copy buffer length ("

                            + copyBufferLength + ") for: " + audioStream);

                    copyBufferLength = DEFAULT_COPY_BUFFER_LENGTH_BYTES;

                } else if (DEBUG) {

                    Slog.i(TAG, "Copy buffer length set to " + copyBufferLength + " for: "

                            + audioStream);

                }

            }



            ParcelFileDescriptor serviceAudioSource =

                    audioStream.getAudioStreamParcelFileDescriptor();

            sourcesAndSinks.add(new Pair<>(serviceAudioSource, clientAudioSink));

            copyTaskInfos.add(new CopyTaskInfo(serviceAudioSource, clientAudioSink,

                    copyBufferLength));

        }



        String resultTaskId = TASK_ID_PREFIX + System.identityHashCode(result);

        mExecutorService.execute(new HotwordDetectedResultCopyTask(resultTaskId, sourcesAndSinks));

        mExecutorService.execute(new HotwordDetectedResultCopyTask(resultTaskId, copyTaskInfos));



        return result.buildUpon().setAudioStreams(newAudioStreams).build();

    }



    private static class CopyTaskInfo {

        private final ParcelFileDescriptor mSource;

        private final ParcelFileDescriptor mSink;

        private final int mCopyBufferLength;



        CopyTaskInfo(ParcelFileDescriptor source, ParcelFileDescriptor sink, int copyBufferLength) {

            mSource = source;

            mSink = sink;

            mCopyBufferLength = copyBufferLength;

        }

    }



    private class HotwordDetectedResultCopyTask implements Runnable {

        private final String mResultTaskId;

        private final List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> mSourcesAndSinks;

        private final List<CopyTaskInfo> mCopyTaskInfos;

        private final ExecutorService mExecutorService = Executors.newCachedThreadPool();



        HotwordDetectedResultCopyTask(String resultTaskId,

                List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> sourcesAndSinks) {

        HotwordDetectedResultCopyTask(String resultTaskId, List<CopyTaskInfo> copyTaskInfos) {

            mResultTaskId = resultTaskId;

            mSourcesAndSinks = sourcesAndSinks;

            mCopyTaskInfos = copyTaskInfos;

        }



        @Override

        public void run() {

            Thread.currentThread().setName(THREAD_NAME_PREFIX + mResultTaskId);

            int size = mSourcesAndSinks.size();

            int size = mCopyTaskInfos.size();

            List<SingleAudioStreamCopyTask> tasks = new ArrayList<>(size);

            for (int i = 0; i < size; i++) {

                Pair<ParcelFileDescriptor, ParcelFileDescriptor> sourceAndSink =

                        mSourcesAndSinks.get(i);

                ParcelFileDescriptor serviceAudioSource = sourceAndSink.first;

                ParcelFileDescriptor clientAudioSink = sourceAndSink.second;

                CopyTaskInfo copyTaskInfo = mCopyTaskInfos.get(i);

                String streamTaskId = mResultTaskId + "@" + i;

                tasks.add(new SingleAudioStreamCopyTask(streamTaskId, serviceAudioSource,

                        clientAudioSink));

                tasks.add(new SingleAudioStreamCopyTask(streamTaskId, copyTaskInfo.mSource,

                        copyTaskInfo.mSink, copyTaskInfo.mCopyBufferLength, mDetectorType,

                        mVoiceInteractorUid));

            }



            if (mAppOpsManager.startOpNoThrow(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD,

                    mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,

                    mVoiceInteractorIdentity.attributionTag, OP_MESSAGE) == MODE_ALLOWED) {

                    mVoiceInteractorUid, mVoiceInteractorPackageName,

                    mVoiceInteractorAttributionTag, OP_MESSAGE) == MODE_ALLOWED) {

                try {

                    HotwordMetricsLogger.writeDetectorEvent(mDetectorType,

                            HOTWORD_DETECTOR_EVENTS__EVENT__AUDIO_EGRESS_START,

                            mVoiceInteractorUid);

                    // TODO(b/244599891): Set timeout, close after inactivity

                    mExecutorService.invokeAll(tasks);

                    HotwordMetricsLogger.writeDetectorEvent(mDetectorType,

                            HOTWORD_DETECTOR_EVENTS__EVENT__AUDIO_EGRESS_END,

                            mVoiceInteractorUid);

                } catch (InterruptedException e) {

                    HotwordMetricsLogger.writeDetectorEvent(mDetectorType,

                            HOTWORD_DETECTOR_EVENTS__EVENT__AUDIO_EGRESS_INTERRUPTED_EXCEPTION,

                            mVoiceInteractorUid);

                    Slog.e(TAG, mResultTaskId + ": Task was interrupted", e);

                    bestEffortPropagateError(e.getMessage());

                } finally {

                    mAppOpsManager.finishOp(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD,

                            mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,

                            mVoiceInteractorIdentity.attributionTag);

                            mVoiceInteractorUid, mVoiceInteractorPackageName,

                            mVoiceInteractorAttributionTag);

                }

            } else {

                HotwordMetricsLogger.writeDetectorEvent(mDetectorType,

                        HOTWORD_DETECTOR_EVENTS__EVENT__AUDIO_EGRESS_NO_PERMISSION,

                        mVoiceInteractorUid);

                bestEffortPropagateError(

                        "Failed to obtain RECORD_AUDIO_HOTWORD permission for "

                                + SoundTriggerSessionPermissionsDecorator.toString(

                                mVoiceInteractorIdentity));

                        "Failed to obtain RECORD_AUDIO_HOTWORD permission for voice interactor with"

                                + " uid=" + mVoiceInteractorUid

                                + " packageName=" + mVoiceInteractorPackageName

                                + " attributionTag=" + mVoiceInteractorAttributionTag);

            }

        }



        private void bestEffortPropagateError(@NonNull String errorMessage) {

            try {

                for (Pair<ParcelFileDescriptor, ParcelFileDescriptor> sourceAndSink :

                        mSourcesAndSinks) {

                    ParcelFileDescriptor serviceAudioSource = sourceAndSink.first;

                    ParcelFileDescriptor clientAudioSink = sourceAndSink.second;

                    serviceAudioSource.closeWithError(errorMessage);

                    clientAudioSink.closeWithError(errorMessage);

                for (CopyTaskInfo copyTaskInfo : mCopyTaskInfos) {

                    copyTaskInfo.mSource.closeWithError(errorMessage);

                    copyTaskInfo.mSink.closeWithError(errorMessage);

                }

                HotwordMetricsLogger.writeDetectorEvent(mDetectorType,

                        HOTWORD_DETECTOR_EVENTS__EVENT__AUDIO_EGRESS_CLOSE_ERROR_FROM_SYSTEM,

                        mVoiceInteractorUid);

            } catch (IOException e) {

                Slog.e(TAG, mResultTaskId + ": Failed to propagate error", e);

            }
@@ -162,18 +230,21 @@ final class HotwordAudioStreamManager { 
    }



    private static class SingleAudioStreamCopyTask implements Callable<Void> {

        // TODO: Make this buffer size customizable from updateState()

        private static final int COPY_BUFFER_LENGTH = 2_560;



        private final String mStreamTaskId;

        private final ParcelFileDescriptor mAudioSource;

        private final ParcelFileDescriptor mAudioSink;

        private final int mCopyBufferLength;

        private final int mDetectorType;

        private final int mUid;



        SingleAudioStreamCopyTask(String streamTaskId, ParcelFileDescriptor audioSource,

                ParcelFileDescriptor audioSink) {

                ParcelFileDescriptor audioSink, int copyBufferLength, int detectorType, int uid) {

            mStreamTaskId = streamTaskId;

            mAudioSource = audioSource;

            mAudioSink = audioSink;

            mCopyBufferLength = copyBufferLength;

            mDetectorType = detectorType;

            mUid = uid;

        }



        @Override
@@ -189,7 +260,7 @@ final class HotwordAudioStreamManager { 
            try {

                fis = new ParcelFileDescriptor.AutoCloseInputStream(mAudioSource);

                fos = new ParcelFileDescriptor.AutoCloseOutputStream(mAudioSink);

                byte[] buffer = new byte[COPY_BUFFER_LENGTH];

                byte[] buffer = new byte[mCopyBufferLength];

                while (true) {

                    if (Thread.interrupted()) {

                        Slog.e(TAG,
@@ -217,6 +288,8 @@ final class HotwordAudioStreamManager { 
                mAudioSource.closeWithError(e.getMessage());

                mAudioSink.closeWithError(e.getMessage());

                Slog.e(TAG, mStreamTaskId + ": Failed to copy audio stream", e);

                HotwordMetricsLogger.writeDetectorEvent(mDetectorType,

                        HOTWORD_DETECTOR_EVENTS__EVENT__AUDIO_EGRESS_CLOSE_ERROR_FROM_SYSTEM, mUid);

            } finally {

                if (fis != null) {

                    fis.close();

services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java

+7 −6
Original line number Diff line number Diff line
@@ -170,7 +170,7 @@ final class HotwordDetectionConnection { 
    private final ScheduledExecutorService mScheduledExecutorService =

            Executors.newSingleThreadScheduledExecutor();

    private final AppOpsManager mAppOpsManager;

    private final HotwordAudioStreamManager mHotwordAudioStreamManager;

    private final HotwordAudioStreamCopier mHotwordAudioStreamCopier;

    @Nullable private final ScheduledFuture<?> mCancellationTaskFuture;

    private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false);

    private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied;
@@ -232,8 +232,9 @@ final class HotwordDetectionConnection { 
        mVoiceInteractionServiceUid = voiceInteractionServiceUid;

        mVoiceInteractorIdentity = voiceInteractorIdentity;

        mAppOpsManager = mContext.getSystemService(AppOpsManager.class);

        mHotwordAudioStreamManager = new HotwordAudioStreamManager(mAppOpsManager,

                mVoiceInteractorIdentity);

        mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, detectorType,

                mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,

                mVoiceInteractorIdentity.attributionTag);

        mDetectionComponentName = serviceName;

        mUser = userId;

        mCallback = callback;
@@ -490,7 +491,7 @@ final class HotwordDetectionConnection { 
                    saveProximityValueToBundle(result);

                    HotwordDetectedResult newResult;

                    try {

                        newResult = mHotwordAudioStreamManager.startCopyingAudioStreams(result);

                        newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);

                    } catch (IOException e) {

                        // TODO: Write event

                        mSoftwareCallback.onError();
@@ -682,7 +683,7 @@ final class HotwordDetectionConnection { 
                    saveProximityValueToBundle(result);

                    HotwordDetectedResult newResult;

                    try {

                        newResult = mHotwordAudioStreamManager.startCopyingAudioStreams(result);

                        newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);

                    } catch (IOException e) {

                        // TODO: Write event

                        externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR);
@@ -1014,7 +1015,7 @@ final class HotwordDetectionConnection { 
                                    HotwordDetectedResult newResult;

                                    try {

                                        newResult =

                                                mHotwordAudioStreamManager.startCopyingAudioStreams(

                                                mHotwordAudioStreamCopier.startCopyingAudioStreams(

                                                        triggerResult);

                                    } catch (IOException e) {

                                        // TODO: Write event