Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 924ea417 authored by Mark Punzalan's avatar Mark Punzalan
Browse files

Copy data in HotwordAudioStream ParcelFileDescriptors 

Bug: 244595867
Test: TODO - Manual E2E test for now, CTS tests later
Change-Id: I02050cc39146e93e9dca1f3db6e0b5b894e85dd6
parent da2e04f2

core/java/android/service/voice/HotwordAudioStream.java

+12 −2
Original line number Diff line number Diff line
@@ -127,6 +127,16 @@ public final class HotwordAudioStream implements Parcelable { 
        }

    }



    /**

     * Provides an instance of {@link Builder} with state corresponding to this instance.

     * @hide

     */

    public Builder buildUpon() {

        return new Builder(mAudioFormat, mAudioStreamParcelFileDescriptor)

            .setTimestamp(mTimestamp)

            .setMetadata(mMetadata);

    }







    // Code below generated by codegen v1.0.23.
@@ -439,10 +449,10 @@ public final class HotwordAudioStream implements Parcelable { 
    }



    @DataClass.Generated(

            time = 1665976240224L,

            time = 1666342101364L,

            codegenVersion = "1.0.23",

            sourceFile = "frameworks/base/core/java/android/service/voice/HotwordAudioStream.java",

            inputSignatures = "private final @android.annotation.NonNull android.media.AudioFormat mAudioFormat\nprivate final @android.annotation.NonNull android.os.ParcelFileDescriptor mAudioStreamParcelFileDescriptor\nprivate final @android.annotation.Nullable android.media.AudioTimestamp mTimestamp\nprivate final @android.annotation.NonNull android.os.PersistableBundle mMetadata\nprivate static  android.media.AudioTimestamp defaultTimestamp()\nprivate static  android.os.PersistableBundle defaultMetadata()\nprivate  java.lang.String timestampToString()\nprivate  void parcelTimestamp(android.os.Parcel,int)\nprivate static @android.annotation.Nullable android.media.AudioTimestamp unparcelTimestamp(android.os.Parcel)\nclass HotwordAudioStream extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genParcelable=true, genToString=true)")

            inputSignatures = "private final @android.annotation.NonNull android.media.AudioFormat mAudioFormat\nprivate final @android.annotation.NonNull android.os.ParcelFileDescriptor mAudioStreamParcelFileDescriptor\nprivate final @android.annotation.Nullable android.media.AudioTimestamp mTimestamp\nprivate final @android.annotation.NonNull android.os.PersistableBundle mMetadata\nprivate static  android.media.AudioTimestamp defaultTimestamp()\nprivate static  android.os.PersistableBundle defaultMetadata()\nprivate  java.lang.String timestampToString()\nprivate  void parcelTimestamp(android.os.Parcel,int)\nprivate static @android.annotation.Nullable android.media.AudioTimestamp unparcelTimestamp(android.os.Parcel)\npublic  android.service.voice.HotwordAudioStream.Builder buildUpon()\nclass HotwordAudioStream extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genParcelable=true, genToString=true)")

    @Deprecated

    private void __metadata() {}

core/java/android/service/voice/HotwordDetectedResult.java

+21 −2
Original line number Diff line number Diff line
@@ -388,6 +388,25 @@ public final class HotwordDetectedResult implements Parcelable { 
        }

    }



    /**

     * Provides an instance of {@link Builder} with state corresponding to this instance.

     * @hide

     */

    public Builder buildUpon() {

        return new Builder()

            .setConfidenceLevel(mConfidenceLevel)

            .setMediaSyncEvent(mMediaSyncEvent)

            .setHotwordOffsetMillis(mHotwordOffsetMillis)

            .setHotwordDurationMillis(mHotwordDurationMillis)

            .setAudioChannel(mAudioChannel)

            .setHotwordDetectionPersonalized(mHotwordDetectionPersonalized)

            .setScore(mScore)

            .setPersonalizedScore(mPersonalizedScore)

            .setHotwordPhraseId(mHotwordPhraseId)

            .setAudioStreams(mAudioStreams)

            .setExtras(mExtras);

    }







    // Code below generated by codegen v1.0.23.
@@ -984,10 +1003,10 @@ public final class HotwordDetectedResult implements Parcelable { 
    }



    @DataClass.Generated(

            time = 1665995595979L,

            time = 1666342044844L,

            codegenVersion = "1.0.23",

            sourceFile = "frameworks/base/core/java/android/service/voice/HotwordDetectedResult.java",

            inputSignatures = "public static final  int CONFIDENCE_LEVEL_NONE\npublic static final  int CONFIDENCE_LEVEL_LOW\npublic static final  int CONFIDENCE_LEVEL_LOW_MEDIUM\npublic static final  int CONFIDENCE_LEVEL_MEDIUM\npublic static final  int CONFIDENCE_LEVEL_MEDIUM_HIGH\npublic static final  int CONFIDENCE_LEVEL_HIGH\npublic static final  int CONFIDENCE_LEVEL_VERY_HIGH\npublic static final  int HOTWORD_OFFSET_UNSET\npublic static final  int AUDIO_CHANNEL_UNSET\nprivate static final  int LIMIT_HOTWORD_OFFSET_MAX_VALUE\nprivate static final  int LIMIT_AUDIO_CHANNEL_MAX_VALUE\npublic static final  java.lang.String EXTRA_PROXIMITY_METERS\nprivate final @android.service.voice.HotwordDetectedResult.HotwordConfidenceLevelValue int mConfidenceLevel\nprivate @android.annotation.Nullable android.media.MediaSyncEvent mMediaSyncEvent\nprivate  int mHotwordOffsetMillis\nprivate  int mHotwordDurationMillis\nprivate  int mAudioChannel\nprivate  boolean mHotwordDetectionPersonalized\nprivate final  int mScore\nprivate final  int mPersonalizedScore\nprivate final  int mHotwordPhraseId\nprivate final @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> mAudioStreams\nprivate final @android.annotation.NonNull android.os.PersistableBundle mExtras\nprivate static  int sMaxBundleSize\nprivate static  int defaultConfidenceLevel()\nprivate static  int defaultScore()\nprivate static  int defaultPersonalizedScore()\npublic static  int getMaxScore()\nprivate static  int defaultHotwordPhraseId()\npublic static  int getMaxHotwordPhraseId()\nprivate static  java.util.List<android.service.voice.HotwordAudioStream> defaultAudioStreams()\nprivate static  android.os.PersistableBundle defaultExtras()\npublic static  int getMaxBundleSize()\npublic @android.annotation.Nullable android.media.MediaSyncEvent getMediaSyncEvent()\npublic static  int getParcelableSize(android.os.Parcelable)\npublic static  int getUsageSize(android.service.voice.HotwordDetectedResult)\nprivate static  int bitCount(long)\nprivate  void onConstructed()\npublic @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> getAudioStreams()\nclass HotwordDetectedResult extends java.lang.Object implements [android.os.Parcelable]\npublic @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)\npublic @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []")

            inputSignatures = "public static final  int CONFIDENCE_LEVEL_NONE\npublic static final  int CONFIDENCE_LEVEL_LOW\npublic static final  int CONFIDENCE_LEVEL_LOW_MEDIUM\npublic static final  int CONFIDENCE_LEVEL_MEDIUM\npublic static final  int CONFIDENCE_LEVEL_MEDIUM_HIGH\npublic static final  int CONFIDENCE_LEVEL_HIGH\npublic static final  int CONFIDENCE_LEVEL_VERY_HIGH\npublic static final  int HOTWORD_OFFSET_UNSET\npublic static final  int AUDIO_CHANNEL_UNSET\nprivate static final  int LIMIT_HOTWORD_OFFSET_MAX_VALUE\nprivate static final  int LIMIT_AUDIO_CHANNEL_MAX_VALUE\npublic static final  java.lang.String EXTRA_PROXIMITY_METERS\nprivate final @android.service.voice.HotwordDetectedResult.HotwordConfidenceLevelValue int mConfidenceLevel\nprivate @android.annotation.Nullable android.media.MediaSyncEvent mMediaSyncEvent\nprivate  int mHotwordOffsetMillis\nprivate  int mHotwordDurationMillis\nprivate  int mAudioChannel\nprivate  boolean mHotwordDetectionPersonalized\nprivate final  int mScore\nprivate final  int mPersonalizedScore\nprivate final  int mHotwordPhraseId\nprivate final @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> mAudioStreams\nprivate final @android.annotation.NonNull android.os.PersistableBundle mExtras\nprivate static  int sMaxBundleSize\nprivate static  int defaultConfidenceLevel()\nprivate static  int defaultScore()\nprivate static  int defaultPersonalizedScore()\npublic static  int getMaxScore()\nprivate static  int defaultHotwordPhraseId()\npublic static  int getMaxHotwordPhraseId()\nprivate static  java.util.List<android.service.voice.HotwordAudioStream> defaultAudioStreams()\nprivate static  android.os.PersistableBundle defaultExtras()\npublic static  int getMaxBundleSize()\npublic @android.annotation.Nullable android.media.MediaSyncEvent getMediaSyncEvent()\npublic static  int getParcelableSize(android.os.Parcelable)\npublic static  int getUsageSize(android.service.voice.HotwordDetectedResult)\nprivate static  int bitCount(long)\nprivate  void onConstructed()\npublic @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> getAudioStreams()\npublic  android.service.voice.HotwordDetectedResult.Builder buildUpon()\nclass HotwordDetectedResult extends java.lang.Object implements [android.os.Parcelable]\npublic @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)\npublic @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []")

    @Deprecated

    private void __metadata() {}

services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordAudioStreamManager.java

0 → 100644
+233 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2022 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package com.android.server.voiceinteraction;



import static android.app.AppOpsManager.MODE_ALLOWED;



import static com.android.server.voiceinteraction.HotwordDetectionConnection.DEBUG;



import android.annotation.NonNull;

import android.app.AppOpsManager;

import android.media.permission.Identity;

import android.os.ParcelFileDescriptor;

import android.service.voice.HotwordAudioStream;

import android.service.voice.HotwordDetectedResult;

import android.util.Pair;

import android.util.Slog;



import java.io.IOException;

import java.io.InputStream;

import java.io.OutputStream;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.List;

import java.util.concurrent.Callable;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.Executors;



final class HotwordAudioStreamManager {



    private static final String TAG = "HotwordAudioStreamManager";

    private static final String OP_MESSAGE = "Streaming hotword audio to VoiceInteractionService";

    private static final String TASK_ID_PREFIX = "HotwordDetectedResult@";

    private static final String THREAD_NAME_PREFIX = "Copy-";



    private final AppOpsManager mAppOpsManager;

    private final Identity mVoiceInteractorIdentity;

    private final ExecutorService mExecutorService = Executors.newCachedThreadPool();



    HotwordAudioStreamManager(@NonNull AppOpsManager appOpsManager,

            @NonNull Identity voiceInteractorIdentity) {

        mAppOpsManager = appOpsManager;

        mVoiceInteractorIdentity = voiceInteractorIdentity;

    }



    /**

     * Starts copying the audio streams in the given {@link HotwordDetectedResult}.

     * <p>

     * The returned {@link HotwordDetectedResult} is identical the one that was passed in, except

     * that the {@link ParcelFileDescriptor}s within {@link HotwordDetectedResult#getAudioStreams()}

     * are replaced with descriptors from pipes managed by {@link HotwordAudioStreamManager}. The

     * returned value should be passed on to the client (i.e., the voice interactor).

     * </p>

     *

     * @throws IOException If there was an error creating the managed pipe.

     */

    @NonNull

    public HotwordDetectedResult startCopyingAudioStreams(@NonNull HotwordDetectedResult result)

            throws IOException {

        List<HotwordAudioStream> audioStreams = result.getAudioStreams();

        if (audioStreams.isEmpty()) {

            return result;

        }



        List<HotwordAudioStream> newAudioStreams = new ArrayList<>(audioStreams.size());

        List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> sourcesAndSinks = new ArrayList<>(

                audioStreams.size());

        for (HotwordAudioStream audioStream : audioStreams) {

            ParcelFileDescriptor[] clientPipe = ParcelFileDescriptor.createReliablePipe();

            ParcelFileDescriptor clientAudioSource = clientPipe[0];

            ParcelFileDescriptor clientAudioSink = clientPipe[1];

            HotwordAudioStream newAudioStream =

                    audioStream.buildUpon().setAudioStreamParcelFileDescriptor(

                            clientAudioSource).build();

            newAudioStreams.add(newAudioStream);



            ParcelFileDescriptor serviceAudioSource =

                    audioStream.getAudioStreamParcelFileDescriptor();

            sourcesAndSinks.add(new Pair<>(serviceAudioSource, clientAudioSink));

        }



        String resultTaskId = TASK_ID_PREFIX + System.identityHashCode(result);

        mExecutorService.execute(new HotwordDetectedResultCopyTask(resultTaskId, sourcesAndSinks));



        return result.buildUpon().setAudioStreams(newAudioStreams).build();

    }



    private class HotwordDetectedResultCopyTask implements Runnable {

        private final String mResultTaskId;

        private final List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> mSourcesAndSinks;

        private final ExecutorService mExecutorService = Executors.newCachedThreadPool();



        HotwordDetectedResultCopyTask(String resultTaskId,

                List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> sourcesAndSinks) {

            mResultTaskId = resultTaskId;

            mSourcesAndSinks = sourcesAndSinks;

        }



        @Override

        public void run() {

            Thread.currentThread().setName(THREAD_NAME_PREFIX + mResultTaskId);

            int size = mSourcesAndSinks.size();

            List<SingleAudioStreamCopyTask> tasks = new ArrayList<>(size);

            for (int i = 0; i < size; i++) {

                Pair<ParcelFileDescriptor, ParcelFileDescriptor> sourceAndSink =

                        mSourcesAndSinks.get(i);

                ParcelFileDescriptor serviceAudioSource = sourceAndSink.first;

                ParcelFileDescriptor clientAudioSink = sourceAndSink.second;

                String streamTaskId = mResultTaskId + "@" + i;

                tasks.add(new SingleAudioStreamCopyTask(streamTaskId, serviceAudioSource,

                        clientAudioSink));

            }



            if (mAppOpsManager.startOpNoThrow(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD,

                    mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,

                    mVoiceInteractorIdentity.attributionTag, OP_MESSAGE) == MODE_ALLOWED) {

                try {

                    // TODO(b/244599891): Set timeout, close after inactivity

                    mExecutorService.invokeAll(tasks);

                } catch (InterruptedException e) {

                    Slog.e(TAG, mResultTaskId + ": Task was interrupted", e);

                    bestEffortPropagateError(e.getMessage());

                } finally {

                    mAppOpsManager.finishOp(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD,

                            mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,

                            mVoiceInteractorIdentity.attributionTag);

                }

            } else {

                bestEffortPropagateError(

                        "Failed to obtain RECORD_AUDIO_HOTWORD permission for "

                                + SoundTriggerSessionPermissionsDecorator.toString(

                                mVoiceInteractorIdentity));

            }

        }



        private void bestEffortPropagateError(@NonNull String errorMessage) {

            try {

                for (Pair<ParcelFileDescriptor, ParcelFileDescriptor> sourceAndSink :

                        mSourcesAndSinks) {

                    ParcelFileDescriptor serviceAudioSource = sourceAndSink.first;

                    ParcelFileDescriptor clientAudioSink = sourceAndSink.second;

                    serviceAudioSource.closeWithError(errorMessage);

                    clientAudioSink.closeWithError(errorMessage);

                }

            } catch (IOException e) {

                Slog.e(TAG, mResultTaskId + ": Failed to propagate error", e);

            }

        }

    }



    private static class SingleAudioStreamCopyTask implements Callable<Void> {

        // TODO: Make this buffer size customizable from updateState()

        private static final int COPY_BUFFER_LENGTH = 1_024;



        private final String mStreamTaskId;

        private final ParcelFileDescriptor mAudioSource;

        private final ParcelFileDescriptor mAudioSink;



        SingleAudioStreamCopyTask(String streamTaskId, ParcelFileDescriptor audioSource,

                ParcelFileDescriptor audioSink) {

            mStreamTaskId = streamTaskId;

            mAudioSource = audioSource;

            mAudioSink = audioSink;

        }



        @Override

        public Void call() throws Exception {

            Thread.currentThread().setName(THREAD_NAME_PREFIX + mStreamTaskId);



            // Note: We are intentionally NOT using try-with-resources here. If we did,

            // the ParcelFileDescriptors will be automatically closed WITHOUT errors before we go

            // into the IOException-catch block. We want to propagate the error while closing the

            // PFDs.

            InputStream fis = null;

            OutputStream fos = null;

            try {

                fis = new ParcelFileDescriptor.AutoCloseInputStream(mAudioSource);

                fos = new ParcelFileDescriptor.AutoCloseOutputStream(mAudioSink);

                byte[] buffer = new byte[COPY_BUFFER_LENGTH];

                while (true) {

                    if (Thread.interrupted()) {

                        Slog.e(TAG,

                                mStreamTaskId + ": SingleAudioStreamCopyTask task was interrupted");

                        break;

                    }



                    int bytesRead = fis.read(buffer);

                    if (bytesRead < 0) {

                        Slog.i(TAG, mStreamTaskId + ": Reached end of audio stream");

                        break;

                    }

                    if (bytesRead > 0) {

                        if (DEBUG) {

                            // TODO(b/244599440): Add proper logging

                            Slog.d(TAG, mStreamTaskId + ": Copied " + bytesRead

                                    + " bytes from audio stream. First 20 bytes=" + Arrays.toString(

                                    Arrays.copyOfRange(buffer, 0, 20)));

                        }

                        fos.write(buffer, 0, bytesRead);

                    }

                    // TODO(b/244599891): Close PFDs after inactivity

                }

            } catch (IOException e) {

                mAudioSource.closeWithError(e.getMessage());

                mAudioSink.closeWithError(e.getMessage());

                Slog.e(TAG, mStreamTaskId + ": Failed to copy audio stream", e);

            } finally {

                if (fis != null) {

                    fis.close();

                }

                if (fos != null) {

                    fos.close();

                }

            }



            return null;

        }

    }



}

services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java

+52 −24

File changed.

Preview size limit exceeded, changes collapsed.