Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 18bd9509 authored by Ivan Chiang's avatar Ivan Chiang Committed by Android (Google) Code Review
Browse files

Merge changes I4f225864,Ic95b7014 into tm-qpr-dev 

* changes:
  Include initialAudio size in metrics and logs
  Add initialAudio field to HotwordAudioStream
parents af38551f 76bb277a

core/java/android/service/voice/HotwordAudioStream.java

+141 −19
Original line number Diff line number Diff line
@@ -27,6 +27,7 @@ import android.os.ParcelFileDescriptor; 
import android.os.Parcelable;

import android.os.PersistableBundle;



import java.util.Arrays;

import java.util.Objects;



/**
@@ -59,8 +60,17 @@ public final class HotwordAudioStream implements Parcelable { 
    private final AudioFormat mAudioFormat;



    /**

     * This stream starts with the audio bytes used for hotword detection, but continues streaming

     * the audio until the stream is shutdown by the {@link HotwordDetectionService}.

     * This stream typically starts with the audio bytes used for hotword detection, but continues

     * streaming the audio (e.g., with the query) until the stream is shutdown by the

     * {@link HotwordDetectionService}. The data format is expected to match

     * {@link #getAudioFormat()}.

     *

     * <p>

     * Alternatively, the {@link HotwordDetectionService} may use {@link #getInitialAudio()}

     * to pass the start of the audio instead of streaming it here. This may prevent added latency

     * caused by the streaming buffer (see {@link #KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES}) not

     * being large enough to handle this initial chunk of audio.

     * </p>

     */

    @NonNull

    @UnsupportedAppUsage
@@ -138,6 +148,32 @@ public final class HotwordAudioStream implements Parcelable { 
        }

    }



    /**

     * The start of the audio used for hotword detection. The data format is expected to match

     * {@link #getAudioFormat()}.

     *

     * <p>

     * The {@link HotwordDetectionService} may use this instead of using

     * {@link #getAudioStreamParcelFileDescriptor()} to stream these initial bytes of audio. This

     * may prevent added latency caused by the streaming buffer (see

     * {@link #KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES}) not being large enough to handle this

     * initial chunk of audio.

     * </p>

     */

    @NonNull

    @UnsupportedAppUsage

    private final byte[] mInitialAudio;



    private static final byte[] DEFAULT_INITIAL_EMPTY_AUDIO = {};



    private static byte[] defaultInitialAudio() {

        return DEFAULT_INITIAL_EMPTY_AUDIO;

    }



    private String initialAudioToString() {

        return "length=" + mInitialAudio.length;

    }



    /**

     * Provides an instance of {@link Builder} with state corresponding to this instance.

     * @hide
@@ -145,7 +181,8 @@ public final class HotwordAudioStream implements Parcelable { 
    public Builder buildUpon() {

        return new Builder(mAudioFormat, mAudioStreamParcelFileDescriptor)

            .setTimestamp(mTimestamp)

            .setMetadata(mMetadata);

            .setMetadata(mMetadata)

            .setInitialAudio(mInitialAudio);

    }



    /* package-private */
@@ -153,7 +190,8 @@ public final class HotwordAudioStream implements Parcelable { 
            @NonNull AudioFormat audioFormat,

            @NonNull ParcelFileDescriptor audioStreamParcelFileDescriptor,

            @Nullable AudioTimestamp timestamp,

            @NonNull PersistableBundle metadata) {

            @NonNull PersistableBundle metadata,

            @NonNull byte[] initialAudio) {

        this.mAudioFormat = audioFormat;

        com.android.internal.util.AnnotationValidations.validate(

                NonNull.class, null, mAudioFormat);
@@ -164,6 +202,9 @@ public final class HotwordAudioStream implements Parcelable { 
        this.mMetadata = metadata;

        com.android.internal.util.AnnotationValidations.validate(

                NonNull.class, null, mMetadata);

        this.mInitialAudio = initialAudio;

        com.android.internal.util.AnnotationValidations.validate(

                NonNull.class, null, mInitialAudio);



        // onConstructed(); // You can define this method to get a callback

    }
@@ -178,8 +219,17 @@ public final class HotwordAudioStream implements Parcelable { 
    }



    /**

     * This stream starts with the audio bytes used for hotword detection, but continues streaming

     * the audio until the stream is shutdown by the {@link HotwordDetectionService}.

     * This stream typically starts with the audio bytes used for hotword detection, but continues

     * streaming the audio (e.g., with the query) until the stream is shutdown by the

     * {@link HotwordDetectionService}. The data format is expected to match

     * {@link #getAudioFormat()}.

     *

     * <p>

     * Alternatively, the {@link HotwordDetectionService} may use {@link #getInitialAudio()}

     * to pass the start of the audio instead of streaming it here. This may prevent added latency

     * caused by the streaming buffer (see {@link #KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES}) not

     * being large enough to handle this initial chunk of audio.

     * </p>

     */

    @UnsupportedAppUsage

    @NonNull
@@ -220,6 +270,24 @@ public final class HotwordAudioStream implements Parcelable { 
        return mMetadata;

    }



    /**

     * The start of the audio used for hotword detection. The data format is expected to match

     * {@link #getAudioFormat()}.

     *

     * <p>

     * The {@link HotwordDetectionService} may use this instead of using

     * {@link #getAudioStreamParcelFileDescriptor()} to stream these initial bytes of audio. This

     * may prevent added latency caused by the streaming buffer (see

     * {@link #KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES}) not being large enough to handle this

     * initial chunk of audio.

     * </p>

     */

    @UnsupportedAppUsage

    @NonNull

    public byte[] getInitialAudio() {

        return mInitialAudio;

    }



    @Override

    public String toString() {

        // You can override field toString logic by defining methods like:
@@ -229,7 +297,8 @@ public final class HotwordAudioStream implements Parcelable { 
                + "audioFormat = " + mAudioFormat + ", "

                + "audioStreamParcelFileDescriptor = " + mAudioStreamParcelFileDescriptor + ", "

                + "timestamp = " + timestampToString() + ", "

                + "metadata = " + mMetadata + " }";

                + "metadata = " + mMetadata + ", "

                + "initialAudio = " + initialAudioToString() + " }";

    }



    @Override
@@ -247,7 +316,8 @@ public final class HotwordAudioStream implements Parcelable { 
                && Objects.equals(mAudioStreamParcelFileDescriptor,

                that.mAudioStreamParcelFileDescriptor)

                && Objects.equals(mTimestamp, that.mTimestamp)

                && Objects.equals(mMetadata, that.mMetadata);

                && Objects.equals(mMetadata, that.mMetadata)

                && Arrays.equals(mInitialAudio, that.mInitialAudio);

    }



    @Override
@@ -260,6 +330,7 @@ public final class HotwordAudioStream implements Parcelable { 
        _hash = 31 * _hash + Objects.hashCode(mAudioStreamParcelFileDescriptor);

        _hash = 31 * _hash + Objects.hashCode(mTimestamp);

        _hash = 31 * _hash + Objects.hashCode(mMetadata);

        _hash = 31 * _hash + Arrays.hashCode(mInitialAudio);

        return _hash;

    }
@@ -275,6 +346,7 @@ public final class HotwordAudioStream implements Parcelable { 
        dest.writeTypedObject(mAudioStreamParcelFileDescriptor, flags);

        parcelTimestamp(dest, flags);

        dest.writeTypedObject(mMetadata, flags);

        dest.writeByteArray(mInitialAudio);

    }



    @Override
@@ -296,6 +368,7 @@ public final class HotwordAudioStream implements Parcelable { 
        AudioTimestamp timestamp = unparcelTimestamp(in);

        PersistableBundle metadata = (PersistableBundle) in.readTypedObject(

                PersistableBundle.CREATOR);

        byte[] initialAudio = in.createByteArray();



        this.mAudioFormat = audioFormat;

        com.android.internal.util.AnnotationValidations.validate(
@@ -307,6 +380,9 @@ public final class HotwordAudioStream implements Parcelable { 
        this.mMetadata = metadata;

        com.android.internal.util.AnnotationValidations.validate(

                NonNull.class, null, mMetadata);

        this.mInitialAudio = initialAudio;

        com.android.internal.util.AnnotationValidations.validate(

                NonNull.class, null, mInitialAudio);



        // onConstructed(); // You can define this method to get a callback

    }
@@ -339,17 +415,29 @@ public final class HotwordAudioStream implements Parcelable { 
        private AudioTimestamp mTimestamp;

        @NonNull

        private PersistableBundle mMetadata;

        @NonNull

        private byte[] mInitialAudio;



        private long mBuilderFieldsSet = 0L;



        /**

         * Creates a new Builder.

         *

         * @param audioFormat                     The {@link AudioFormat} of the audio stream.

         * @param audioStreamParcelFileDescriptor This stream starts with the audio bytes used for

         *                                        hotword detection, but continues streaming

         *                                        the audio until the stream is shutdown by the

         *                                        {@link HotwordDetectionService}.

         * @param audioFormat

         *   The {@link AudioFormat} of the audio stream.

         * @param audioStreamParcelFileDescriptor

         *   This stream typically starts with the audio bytes used for hotword detection, but

         *   continues streaming the audio (e.g., with the query) until the stream is shutdown by

         *   the {@link HotwordDetectionService}. The data format is expected to match

         *   {@link #getAudioFormat()}.

         *

         *   <p>

         *   Alternatively, the {@link HotwordDetectionService} may use {@link #getInitialAudio()}

         *   to pass the start of the audio instead of streaming it here. This may prevent added

         *   latency caused by the streaming buffer

         *   (see {@link #KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES}) not being large enough to

         *   handle this initial chunk of audio.

         *   </p>

         */

        @UnsupportedAppUsage

        public Builder(
@@ -376,9 +464,18 @@ public final class HotwordAudioStream implements Parcelable { 
        }



        /**

         * This stream starts with the audio bytes used for hotword detection, but continues

         * streaming

         * the audio until the stream is shutdown by the {@link HotwordDetectionService}.

         * This stream typically starts with the audio bytes used for hotword detection, but

         * continues streaming the audio (e.g., with the query) until the stream is shutdown by the

         * {@link HotwordDetectionService}. The data format is expected to match

         * {@link #getAudioFormat()}.

         *

         * <p>

         * Alternatively, the {@link HotwordDetectionService} may use {@link #getInitialAudio()}

         * to pass the start of the audio instead of streaming it here. This may prevent added

         * latency caused by the streaming buffer

         * (see {@link #KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES}) not being large enough to handle

         * this initial chunk of audio.

         * </p>

         */

        @UnsupportedAppUsage

        @NonNull
@@ -428,12 +525,33 @@ public final class HotwordAudioStream implements Parcelable { 
            return this;

        }



        /**

         * The start of the audio used for hotword detection. The data format is expected to match

         * {@link #getAudioFormat()}.

         *

         * <p>

         * The {@link HotwordDetectionService} may use this instead of using

         * {@link #getAudioStreamParcelFileDescriptor()} to stream these initial bytes of audio.

         * This may prevent added latency caused by the streaming buffer (see

         * {@link #KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES}) not being large enough to handle this

         * initial chunk of audio.

         * </p>

         */

        @UnsupportedAppUsage

        @NonNull

        public Builder setInitialAudio(@NonNull byte[] value) {

            checkNotUsed();

            mBuilderFieldsSet |= 0x10;

            mInitialAudio = value;

            return this;

        }



        /** Builds the instance. This builder should not be touched after calling this! */

        @UnsupportedAppUsage

        @NonNull

        public HotwordAudioStream build() {

            checkNotUsed();

            mBuilderFieldsSet |= 0x10; // Mark builder used

            mBuilderFieldsSet |= 0x20; // Mark builder used



            if ((mBuilderFieldsSet & 0x4) == 0) {

                mTimestamp = defaultTimestamp();
@@ -441,16 +559,20 @@ public final class HotwordAudioStream implements Parcelable { 
            if ((mBuilderFieldsSet & 0x8) == 0) {

                mMetadata = defaultMetadata();

            }

            if ((mBuilderFieldsSet & 0x10) == 0) {

                mInitialAudio = defaultInitialAudio();

            }

            HotwordAudioStream o = new HotwordAudioStream(

                    mAudioFormat,

                    mAudioStreamParcelFileDescriptor,

                    mTimestamp,

                    mMetadata);

                    mMetadata,

                    mInitialAudio);

            return o;

        }



        private void checkNotUsed() {

            if ((mBuilderFieldsSet & 0x10) != 0) {

            if ((mBuilderFieldsSet & 0x20) != 0) {

                throw new IllegalStateException(

                        "This Builder should not be reused. Use a new Builder instance instead");

            }

services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordAudioStreamCopier.java

+23 −10
Original line number Diff line number Diff line
@@ -109,6 +109,7 @@ final class HotwordAudioStreamCopier { 
        List<HotwordAudioStream> newAudioStreams = new ArrayList<>(audioStreams.size());

        List<CopyTaskInfo> copyTaskInfos = new ArrayList<>(audioStreams.size());

        int totalMetadataBundleSizeBytes = 0;

        int totalInitialAudioSizeBytes = 0;

        for (HotwordAudioStream audioStream : audioStreams) {

            ParcelFileDescriptor[] clientPipe = ParcelFileDescriptor.createReliablePipe();

            ParcelFileDescriptor clientAudioSource = clientPipe[0];
@@ -137,6 +138,10 @@ final class HotwordAudioStreamCopier { 
                }

            }



            // We are including the non-streamed initial audio

            // (HotwordAudioStream.getInitialAudio()) bytes in the "stream" size metrics.

            totalInitialAudioSizeBytes += audioStream.getInitialAudio().length;



            ParcelFileDescriptor serviceAudioSource =

                    audioStream.getAudioStreamParcelFileDescriptor();

            copyTaskInfos.add(new CopyTaskInfo(serviceAudioSource, clientAudioSink,
@@ -146,7 +151,7 @@ final class HotwordAudioStreamCopier { 
        String resultTaskId = TASK_ID_PREFIX + System.identityHashCode(result);

        mExecutorService.execute(

                new HotwordDetectedResultCopyTask(resultTaskId, copyTaskInfos,

                        totalMetadataBundleSizeBytes));

                        totalMetadataBundleSizeBytes, totalInitialAudioSizeBytes));



        return result.buildUpon().setAudioStreams(newAudioStreams).build();

    }
@@ -167,13 +172,15 @@ final class HotwordAudioStreamCopier { 
        private final String mResultTaskId;

        private final List<CopyTaskInfo> mCopyTaskInfos;

        private final int mTotalMetadataSizeBytes;

        private final int mTotalInitialAudioSizeBytes;

        private final ExecutorService mExecutorService = Executors.newCachedThreadPool();



        HotwordDetectedResultCopyTask(String resultTaskId, List<CopyTaskInfo> copyTaskInfos,

                int totalMetadataSizeBytes) {

                int totalMetadataSizeBytes, int totalInitialAudioSizeBytes) {

            mResultTaskId = resultTaskId;

            mCopyTaskInfos = copyTaskInfos;

            mTotalMetadataSizeBytes = totalMetadataSizeBytes;

            mTotalInitialAudioSizeBytes = totalInitialAudioSizeBytes;

        }



        @Override
@@ -195,25 +202,30 @@ final class HotwordAudioStreamCopier { 
                try {

                    HotwordMetricsLogger.writeAudioEgressEvent(mDetectorType,

                            HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__STARTED,

                            mVoiceInteractorUid, /* streamSizeBytes= */ 0, mTotalMetadataSizeBytes,

                            size);

                            mVoiceInteractorUid, mTotalInitialAudioSizeBytes,

                            mTotalMetadataSizeBytes, size);

                    // TODO(b/244599891): Set timeout, close after inactivity

                    mExecutorService.invokeAll(tasks);



                    int totalStreamSizeBytes = 0;

                    // We are including the non-streamed initial audio

                    // (HotwordAudioStream.getInitialAudio()) bytes in the "stream" size metrics.

                    int totalStreamSizeBytes = mTotalInitialAudioSizeBytes;

                    for (SingleAudioStreamCopyTask task : tasks) {

                        totalStreamSizeBytes += task.mTotalCopiedBytes;

                    }



                    Slog.i(TAG, mResultTaskId + ": Task was completed. Total bytes streamed: "

                            + totalStreamSizeBytes + ", total metadata bundle size bytes: "

                    Slog.i(TAG, mResultTaskId + ": Task was completed. Total bytes egressed: "

                            + totalStreamSizeBytes + " (including " + mTotalInitialAudioSizeBytes

                            + " bytes NOT streamed), total metadata bundle size bytes: "

                            + mTotalMetadataSizeBytes);

                    HotwordMetricsLogger.writeAudioEgressEvent(mDetectorType,

                            HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__ENDED,

                            mVoiceInteractorUid, totalStreamSizeBytes, mTotalMetadataSizeBytes,

                            size);

                } catch (InterruptedException e) {

                    int totalStreamSizeBytes = 0;

                    // We are including the non-streamed initial audio

                    // (HotwordAudioStream.getInitialAudio()) bytes in the "stream" size metrics.

                    int totalStreamSizeBytes = mTotalInitialAudioSizeBytes;

                    for (SingleAudioStreamCopyTask task : tasks) {

                        totalStreamSizeBytes += task.mTotalCopiedBytes;

                    }
@@ -222,8 +234,9 @@ final class HotwordAudioStreamCopier { 
                            HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__INTERRUPTED_EXCEPTION,

                            mVoiceInteractorUid, totalStreamSizeBytes, mTotalMetadataSizeBytes,

                            size);

                    Slog.e(TAG, mResultTaskId + ": Task was interrupted. Total bytes streamed: "

                            + totalStreamSizeBytes + ", total metadata bundle size bytes: "

                    Slog.i(TAG, mResultTaskId + ": Task was interrupted. Total bytes egressed: "

                            + totalStreamSizeBytes + " (including " + mTotalInitialAudioSizeBytes

                            + " bytes NOT streamed), total metadata bundle size bytes: "

                            + mTotalMetadataSizeBytes);

                    bestEffortPropagateError(e.getMessage());

                } finally {