Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 7d875345 authored by Mark Punzalan's avatar Mark Punzalan Committed by Android (Google) Code Review
Browse files

Merge changes I43a24b27,I1c70f95a,I28c7f21b 

* changes:
  Rename HotwordAudioStreamManager and add Javadoc
  Remove dependency on Identity in HotwordAudioStreamManager
  Allow setting buffer length for HotwordAudioStreamManager
parents 67709e07 585c555c

core/api/system-current.txt

+1 −0
Original line number Diff line number Diff line
@@ -12277,6 +12277,7 @@ package android.service.voice { 
    method @Nullable public android.media.AudioTimestamp getTimestamp();

    method public void writeToParcel(@NonNull android.os.Parcel, int);

    field @NonNull public static final android.os.Parcelable.Creator<android.service.voice.HotwordAudioStream> CREATOR;

    field public static final String KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES = "android.service.voice.key.AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES";

  }















  public static final class HotwordAudioStream.Builder {

























core/java/android/service/voice/HotwordAudioStream.java



+17
−2




























Original line number
Diff line number
Diff line








@@ -46,6 +46,21 @@ import java.util.Objects;













@SystemApi















public final class HotwordAudioStream implements Parcelable {





























    /**













     * Key for int value to be read from {@link #getMetadata()}. The value is read by the system and













     * is the length (in bytes) of the byte buffers created to copy bytes in the













     * {@link #getAudioStreamParcelFileDescriptor()} written by the {@link HotwordDetectionService}.













     * The buffer length should be chosen such that no additional latency is introduced. Typically,













     * this should be <em>at least</em> the size of byte chunks written by the













     * {@link HotwordDetectionService}.













     *













     * <p>If no value specified in the metadata for the buffer length, or if the value is less than













     * 1, or if it is greater than 65,536, or if it is not an int, the default value of 2,560 will













     * be used.</p>













     */













    public static final String KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES =













            "android.service.voice.key.AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES";





























    /**















     * The {@link AudioFormat} of the audio stream.















     */










@@ -414,10 +429,10 @@ public final class HotwordAudioStream implements Parcelable {













    }































    @DataClass.Generated(













            time = 1669184301563L,













            time = 1669916341034L,















            codegenVersion = "1.0.23",















            sourceFile = "frameworks/base/core/java/android/service/voice/HotwordAudioStream.java",













            inputSignatures = "private final @android.annotation.NonNull android.media.AudioFormat mAudioFormat\nprivate final @android.annotation.NonNull android.os.ParcelFileDescriptor mAudioStreamParcelFileDescriptor\nprivate final @android.annotation.Nullable android.media.AudioTimestamp mTimestamp\nprivate final @android.annotation.NonNull android.os.PersistableBundle mMetadata\nprivate static  android.media.AudioTimestamp defaultTimestamp()\nprivate static  android.os.PersistableBundle defaultMetadata()\npublic  android.service.voice.HotwordAudioStream.Builder buildUpon()\nclass HotwordAudioStream extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genParcelable=true, genToString=true)")













            inputSignatures = "public static final  java.lang.String KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES\nprivate final @android.annotation.NonNull android.media.AudioFormat mAudioFormat\nprivate final @android.annotation.NonNull android.os.ParcelFileDescriptor mAudioStreamParcelFileDescriptor\nprivate final @android.annotation.Nullable android.media.AudioTimestamp mTimestamp\nprivate final @android.annotation.NonNull android.os.PersistableBundle mMetadata\nprivate static  android.media.AudioTimestamp defaultTimestamp()\nprivate static  android.os.PersistableBundle defaultMetadata()\npublic  android.service.voice.HotwordAudioStream.Builder buildUpon()\nclass HotwordAudioStream extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genParcelable=true, genToString=true)")















    @Deprecated















    private void __metadata() {}

















































services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordAudioStreamManager.java→services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordAudioStreamCopier.java



+78
−42




























Original line number
Diff line number
Diff line








@@ -17,16 +17,16 @@













package com.android.server.voiceinteraction;































import static android.app.AppOpsManager.MODE_ALLOWED;













import static android.service.voice.HotwordAudioStream.KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES;































import static com.android.server.voiceinteraction.HotwordDetectionConnection.DEBUG;































import android.annotation.NonNull;















import android.app.AppOpsManager;













import android.media.permission.Identity;















import android.os.ParcelFileDescriptor;













import android.os.PersistableBundle;















import android.service.voice.HotwordAudioStream;















import android.service.voice.HotwordDetectedResult;













import android.util.Pair;















import android.util.Slog;































import java.io.IOException;









@@ -39,21 +39,38 @@ import java.util.concurrent.Callable;













import java.util.concurrent.ExecutorService;















import java.util.concurrent.Executors;





























final class HotwordAudioStreamManager {













/**













 * Copies the audio streams in {@link HotwordDetectedResult}s. This allows the system to manage the













 * lifetime of the {@link ParcelFileDescriptor}s and ensures that the flow of data is in the right













 * direction from the {@link android.service.voice.HotwordDetectionService} to the client (i.e., the













 * voice interactor).













 *













 * @hide













 */













final class HotwordAudioStreamCopier {





























    private static final String TAG = "HotwordAudioStreamManager";













    private static final String TAG = "HotwordAudioStreamCopier";















    private static final String OP_MESSAGE = "Streaming hotword audio to VoiceInteractionService";















    private static final String TASK_ID_PREFIX = "HotwordDetectedResult@";















    private static final String THREAD_NAME_PREFIX = "Copy-";













    private static final int DEFAULT_COPY_BUFFER_LENGTH_BYTES = 2_560;



























    // Corresponds to the OS pipe capacity in bytes













    private static final int MAX_COPY_BUFFER_LENGTH_BYTES = 65_536;































    private final AppOpsManager mAppOpsManager;













    private final Identity mVoiceInteractorIdentity;













    private final int mVoiceInteractorUid;













    private final String mVoiceInteractorPackageName;













    private final String mVoiceInteractorAttributionTag;















    private final ExecutorService mExecutorService = Executors.newCachedThreadPool();





























    HotwordAudioStreamManager(@NonNull AppOpsManager appOpsManager,













            @NonNull Identity voiceInteractorIdentity) {













    HotwordAudioStreamCopier(@NonNull AppOpsManager appOpsManager,













            int voiceInteractorUid, @NonNull String voiceInteractorPackageName,













            @NonNull String voiceInteractorAttributionTag) {















        mAppOpsManager = appOpsManager;













        mVoiceInteractorIdentity = voiceInteractorIdentity;













        mVoiceInteractorUid = voiceInteractorUid;













        mVoiceInteractorPackageName = voiceInteractorPackageName;













        mVoiceInteractorAttributionTag = voiceInteractorAttributionTag;















    }































    /**









@@ -61,7 +78,7 @@ final class HotwordAudioStreamManager {













     * <p>















     * The returned {@link HotwordDetectedResult} is identical the one that was passed in, except















     * that the {@link ParcelFileDescriptor}s within {@link HotwordDetectedResult#getAudioStreams()}













     * are replaced with descriptors from pipes managed by {@link HotwordAudioStreamManager}. The













     * are replaced with descriptors from pipes managed by {@link HotwordAudioStreamCopier}. The















     * returned value should be passed on to the client (i.e., the voice interactor).















     * </p>















     *









@@ -76,8 +93,7 @@ final class HotwordAudioStreamManager {













        }































        List<HotwordAudioStream> newAudioStreams = new ArrayList<>(audioStreams.size());













        List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> sourcesAndSinks = new ArrayList<>(













                audioStreams.size());













        List<CopyTaskInfo> copyTaskInfos = new ArrayList<>(audioStreams.size());















        for (HotwordAudioStream audioStream : audioStreams) {















            ParcelFileDescriptor[] clientPipe = ParcelFileDescriptor.createReliablePipe();















            ParcelFileDescriptor clientAudioSource = clientPipe[0];









@@ -87,46 +103,69 @@ final class HotwordAudioStreamManager {













                            clientAudioSource).build();















            newAudioStreams.add(newAudioStream);





























            int copyBufferLength = DEFAULT_COPY_BUFFER_LENGTH_BYTES;













            PersistableBundle metadata = audioStream.getMetadata();













            if (metadata.containsKey(KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES)) {













                copyBufferLength = metadata.getInt(KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES, -1);













                if (copyBufferLength < 1 || copyBufferLength > MAX_COPY_BUFFER_LENGTH_BYTES) {













                    Slog.w(TAG, "Attempted to set an invalid copy buffer length ("













                            + copyBufferLength + ") for: " + audioStream);













                    copyBufferLength = DEFAULT_COPY_BUFFER_LENGTH_BYTES;













                } else if (DEBUG) {













                    Slog.i(TAG, "Copy buffer length set to " + copyBufferLength + " for: "













                            + audioStream);













                }













            }





























            ParcelFileDescriptor serviceAudioSource =















                    audioStream.getAudioStreamParcelFileDescriptor();













            sourcesAndSinks.add(new Pair<>(serviceAudioSource, clientAudioSink));













            copyTaskInfos.add(new CopyTaskInfo(serviceAudioSource, clientAudioSink,













                    copyBufferLength));















        }































        String resultTaskId = TASK_ID_PREFIX + System.identityHashCode(result);













        mExecutorService.execute(new HotwordDetectedResultCopyTask(resultTaskId, sourcesAndSinks));













        mExecutorService.execute(new HotwordDetectedResultCopyTask(resultTaskId, copyTaskInfos));































        return result.buildUpon().setAudioStreams(newAudioStreams).build();















    }





























    private static class CopyTaskInfo {













        private final ParcelFileDescriptor mSource;













        private final ParcelFileDescriptor mSink;













        private final int mCopyBufferLength;



























        CopyTaskInfo(ParcelFileDescriptor source, ParcelFileDescriptor sink, int copyBufferLength) {













            mSource = source;













            mSink = sink;













            mCopyBufferLength = copyBufferLength;













        }













    }





























    private class HotwordDetectedResultCopyTask implements Runnable {















        private final String mResultTaskId;













        private final List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> mSourcesAndSinks;













        private final List<CopyTaskInfo> mCopyTaskInfos;















        private final ExecutorService mExecutorService = Executors.newCachedThreadPool();





























        HotwordDetectedResultCopyTask(String resultTaskId,













                List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> sourcesAndSinks) {













        HotwordDetectedResultCopyTask(String resultTaskId, List<CopyTaskInfo> copyTaskInfos) {















            mResultTaskId = resultTaskId;













            mSourcesAndSinks = sourcesAndSinks;













            mCopyTaskInfos = copyTaskInfos;















        }































        @Override















        public void run() {















            Thread.currentThread().setName(THREAD_NAME_PREFIX + mResultTaskId);













            int size = mSourcesAndSinks.size();













            int size = mCopyTaskInfos.size();















            List<SingleAudioStreamCopyTask> tasks = new ArrayList<>(size);















            for (int i = 0; i < size; i++) {













                Pair<ParcelFileDescriptor, ParcelFileDescriptor> sourceAndSink =













                        mSourcesAndSinks.get(i);













                ParcelFileDescriptor serviceAudioSource = sourceAndSink.first;













                ParcelFileDescriptor clientAudioSink = sourceAndSink.second;













                CopyTaskInfo copyTaskInfo = mCopyTaskInfos.get(i);















                String streamTaskId = mResultTaskId + "@" + i;













                tasks.add(new SingleAudioStreamCopyTask(streamTaskId, serviceAudioSource,













                        clientAudioSink));













                tasks.add(new SingleAudioStreamCopyTask(streamTaskId, copyTaskInfo.mSource,













                        copyTaskInfo.mSink, copyTaskInfo.mCopyBufferLength));















            }































            if (mAppOpsManager.startOpNoThrow(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD,













                    mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,













                    mVoiceInteractorIdentity.attributionTag, OP_MESSAGE) == MODE_ALLOWED) {













                    mVoiceInteractorUid, mVoiceInteractorPackageName,













                    mVoiceInteractorAttributionTag, OP_MESSAGE) == MODE_ALLOWED) {















                try {















                    // TODO(b/244599891): Set timeout, close after inactivity















                    mExecutorService.invokeAll(tasks);









@@ -135,25 +174,23 @@ final class HotwordAudioStreamManager {













                    bestEffortPropagateError(e.getMessage());















                } finally {















                    mAppOpsManager.finishOp(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD,













                            mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,













                            mVoiceInteractorIdentity.attributionTag);













                            mVoiceInteractorUid, mVoiceInteractorPackageName,













                            mVoiceInteractorAttributionTag);















                }















            } else {















                bestEffortPropagateError(













                        "Failed to obtain RECORD_AUDIO_HOTWORD permission for "













                                + SoundTriggerSessionPermissionsDecorator.toString(













                                mVoiceInteractorIdentity));













                        "Failed to obtain RECORD_AUDIO_HOTWORD permission for voice interactor with"













                                + " uid=" + mVoiceInteractorUid













                                + " packageName=" + mVoiceInteractorPackageName













                                + " attributionTag=" + mVoiceInteractorAttributionTag);















            }















        }































        private void bestEffortPropagateError(@NonNull String errorMessage) {















            try {













                for (Pair<ParcelFileDescriptor, ParcelFileDescriptor> sourceAndSink :













                        mSourcesAndSinks) {













                    ParcelFileDescriptor serviceAudioSource = sourceAndSink.first;













                    ParcelFileDescriptor clientAudioSink = sourceAndSink.second;













                    serviceAudioSource.closeWithError(errorMessage);













                    clientAudioSink.closeWithError(errorMessage);













                for (CopyTaskInfo copyTaskInfo : mCopyTaskInfos) {













                    copyTaskInfo.mSource.closeWithError(errorMessage);













                    copyTaskInfo.mSink.closeWithError(errorMessage);















                }















            } catch (IOException e) {















                Slog.e(TAG, mResultTaskId + ": Failed to propagate error", e);









@@ -162,18 +199,17 @@ final class HotwordAudioStreamManager {













    }































    private static class SingleAudioStreamCopyTask implements Callable<Void> {













        // TODO: Make this buffer size customizable from updateState()













        private static final int COPY_BUFFER_LENGTH = 2_560;





























        private final String mStreamTaskId;















        private final ParcelFileDescriptor mAudioSource;















        private final ParcelFileDescriptor mAudioSink;













        private final int mCopyBufferLength;































        SingleAudioStreamCopyTask(String streamTaskId, ParcelFileDescriptor audioSource,













                ParcelFileDescriptor audioSink) {













                ParcelFileDescriptor audioSink, int copyBufferLength) {















            mStreamTaskId = streamTaskId;















            mAudioSource = audioSource;















            mAudioSink = audioSink;













            mCopyBufferLength = copyBufferLength;















        }































        @Override









@@ -189,7 +225,7 @@ final class HotwordAudioStreamManager {













            try {















                fis = new ParcelFileDescriptor.AutoCloseInputStream(mAudioSource);















                fos = new ParcelFileDescriptor.AutoCloseOutputStream(mAudioSink);













                byte[] buffer = new byte[COPY_BUFFER_LENGTH];













                byte[] buffer = new byte[mCopyBufferLength];















                while (true) {















                    if (Thread.interrupted()) {















                        Slog.e(TAG,

































services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java



+7
−6




























Original line number
Diff line number
Diff line








@@ -183,7 +183,7 @@ final class HotwordDetectionConnection {













    private final ScheduledExecutorService mScheduledExecutorService =















            Executors.newSingleThreadScheduledExecutor();















    private final AppOpsManager mAppOpsManager;













    private final HotwordAudioStreamManager mHotwordAudioStreamManager;













    private final HotwordAudioStreamCopier mHotwordAudioStreamCopier;















    @Nullable private final ScheduledFuture<?> mCancellationTaskFuture;















    private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false);















    private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied;










@@ -245,8 +245,9 @@ final class HotwordDetectionConnection {













        mVoiceInteractionServiceUid = voiceInteractionServiceUid;















        mVoiceInteractorIdentity = voiceInteractorIdentity;















        mAppOpsManager = mContext.getSystemService(AppOpsManager.class);













        mHotwordAudioStreamManager = new HotwordAudioStreamManager(mAppOpsManager,













                mVoiceInteractorIdentity);













        mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager,













                mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,













                mVoiceInteractorIdentity.attributionTag);















        mDetectionComponentName = serviceName;















        mUser = userId;















        mCallback = callback;










@@ -506,7 +507,7 @@ final class HotwordDetectionConnection {













                    saveProximityValueToBundle(result);















                    HotwordDetectedResult newResult;















                    try {













                        newResult = mHotwordAudioStreamManager.startCopyingAudioStreams(result);













                        newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);















                    } catch (IOException e) {















                        // TODO: Write event















                        mSoftwareCallback.onError();










@@ -641,7 +642,7 @@ final class HotwordDetectionConnection {













                    saveProximityValueToBundle(result);















                    HotwordDetectedResult newResult;















                    try {













                        newResult = mHotwordAudioStreamManager.startCopyingAudioStreams(result);













                        newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);















                    } catch (IOException e) {















                        // TODO: Write event















                        externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR);










@@ -1000,7 +1001,7 @@ final class HotwordDetectionConnection {













                                    HotwordDetectedResult newResult;















                                    try {















                                        newResult =













                                                mHotwordAudioStreamManager.startCopyingAudioStreams(













                                                mHotwordAudioStreamCopier.startCopyingAudioStreams(















                                                        triggerResult);















                                    } catch (IOException e) {















                                        // TODO: Write event

































services/voiceinteraction/java/com/android/server/voiceinteraction/TrustedHotwordDetectorSession.java



+7
−6




























Original line number
Diff line number
Diff line








@@ -183,7 +183,7 @@ final class TrustedHotwordDetectorSession {













    private final ScheduledExecutorService mScheduledExecutorService =















            Executors.newSingleThreadScheduledExecutor();















    private final AppOpsManager mAppOpsManager;













    private final HotwordAudioStreamManager mHotwordAudioStreamManager;













    private final HotwordAudioStreamCopier mHotwordAudioStreamCopier;















    @Nullable private final ScheduledFuture<?> mCancellationTaskFuture;















    private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false);















    private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied;










@@ -245,8 +245,9 @@ final class TrustedHotwordDetectorSession {













        mVoiceInteractionServiceUid = voiceInteractionServiceUid;















        mVoiceInteractorIdentity = voiceInteractorIdentity;















        mAppOpsManager = mContext.getSystemService(AppOpsManager.class);













        mHotwordAudioStreamManager = new HotwordAudioStreamManager(mAppOpsManager,













                mVoiceInteractorIdentity);













        mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager,













                mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,













                mVoiceInteractorIdentity.attributionTag);















        mDetectionComponentName = serviceName;















        mUser = userId;















        mCallback = callback;










@@ -506,7 +507,7 @@ final class TrustedHotwordDetectorSession {













                    saveProximityValueToBundle(result);















                    HotwordDetectedResult newResult;















                    try {













                        newResult = mHotwordAudioStreamManager.startCopyingAudioStreams(result);













                        newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);















                    } catch (IOException e) {















                        // TODO: Write event















                        mSoftwareCallback.onError();










@@ -641,7 +642,7 @@ final class TrustedHotwordDetectorSession {













                    saveProximityValueToBundle(result);















                    HotwordDetectedResult newResult;















                    try {













                        newResult = mHotwordAudioStreamManager.startCopyingAudioStreams(result);













                        newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);















                    } catch (IOException e) {















                        // TODO: Write event















                        externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR);










@@ -1000,7 +1001,7 @@ final class TrustedHotwordDetectorSession {













                                    HotwordDetectedResult newResult;















                                    try {















                                        newResult =













                                                mHotwordAudioStreamManager.startCopyingAudioStreams(













                                                mHotwordAudioStreamCopier.startCopyingAudioStreams(















                                                        triggerResult);















                                    } catch (IOException e) {















                                        // TODO: Write event