Merge "Copy data in HotwordAudioStream ParcelFileDescriptors" into tm-qpr-dev (7b9c2e99) · Commits · e / os / android_frameworks_base

core/java/android/service/voice/HotwordAudioStream.java

+10 −0

Original line number	Diff line number	Diff line
		@@ -123,6 +123,16 @@ public final class HotwordAudioStream implements Parcelable {
		}
		}

		/**
		* Provides an instance of {@link Builder} with state corresponding to this instance.
		* @hide
		*/
		public Builder buildUpon() {
		return new Builder(mAudioFormat, mAudioStreamParcelFileDescriptor)
		.setTimestamp(mTimestamp)
		.setMetadata(mMetadata);
		}

		/* package-private */
		HotwordAudioStream(
		@NonNull AudioFormat audioFormat,

core/java/android/service/voice/HotwordDetectedResult.java

+21 −2

Original line number	Diff line number	Diff line
		@@ -397,6 +397,25 @@ public final class HotwordDetectedResult implements Parcelable {
		}
		}

		/**
		* Provides an instance of {@link Builder} with state corresponding to this instance.
		* @hide
		*/
		public Builder buildUpon() {
		return new Builder()
		.setConfidenceLevel(mConfidenceLevel)
		.setMediaSyncEvent(mMediaSyncEvent)
		.setHotwordOffsetMillis(mHotwordOffsetMillis)
		.setHotwordDurationMillis(mHotwordDurationMillis)
		.setAudioChannel(mAudioChannel)
		.setHotwordDetectionPersonalized(mHotwordDetectionPersonalized)
		.setScore(mScore)
		.setPersonalizedScore(mPersonalizedScore)
		.setHotwordPhraseId(mHotwordPhraseId)
		.setAudioStreams(mAudioStreams)
		.setExtras(mExtras);
		}



		// Code below generated by codegen v1.0.23.
		@@ -993,10 +1012,10 @@ public final class HotwordDetectedResult implements Parcelable {
		}

		@DataClass.Generated(
		time = 1668405106028L,
		time = 1668466781144L,
		codegenVersion = "1.0.23",
		sourceFile = "frameworks/base/core/java/android/service/voice/HotwordDetectedResult.java",
		inputSignatures = "public static final int CONFIDENCE_LEVEL_NONE\npublic static final int CONFIDENCE_LEVEL_LOW\npublic static final int CONFIDENCE_LEVEL_LOW_MEDIUM\npublic static final int CONFIDENCE_LEVEL_MEDIUM\npublic static final int CONFIDENCE_LEVEL_MEDIUM_HIGH\npublic static final int CONFIDENCE_LEVEL_HIGH\npublic static final int CONFIDENCE_LEVEL_VERY_HIGH\npublic static final int HOTWORD_OFFSET_UNSET\npublic static final int AUDIO_CHANNEL_UNSET\nprivate static final int LIMIT_HOTWORD_OFFSET_MAX_VALUE\nprivate static final int LIMIT_AUDIO_CHANNEL_MAX_VALUE\npublic static final java.lang.String EXTRA_PROXIMITY_METERS\nprivate final @android.service.voice.HotwordDetectedResult.HotwordConfidenceLevelValue int mConfidenceLevel\nprivate @android.annotation.Nullable android.media.MediaSyncEvent mMediaSyncEvent\nprivate int mHotwordOffsetMillis\nprivate int mHotwordDurationMillis\nprivate int mAudioChannel\nprivate boolean mHotwordDetectionPersonalized\nprivate final int mScore\nprivate final int mPersonalizedScore\nprivate final int mHotwordPhraseId\nprivate final @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> mAudioStreams\nprivate final @android.annotation.NonNull android.os.PersistableBundle mExtras\nprivate static int sMaxBundleSize\nprivate static int defaultConfidenceLevel()\nprivate static int defaultScore()\nprivate static int defaultPersonalizedScore()\npublic static int getMaxScore()\nprivate static int defaultHotwordPhraseId()\npublic static int getMaxHotwordPhraseId()\nprivate static java.util.List<android.service.voice.HotwordAudioStream> defaultAudioStreams()\nprivate static android.os.PersistableBundle defaultExtras()\npublic static int getMaxBundleSize()\npublic @android.annotation.Nullable android.media.MediaSyncEvent getMediaSyncEvent()\npublic static int getParcelableSize(android.os.Parcelable)\npublic static int getUsageSize(android.service.voice.HotwordDetectedResult)\nprivate static int bitCount(long)\nprivate void onConstructed()\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> getAudioStreams()\nclass HotwordDetectedResult extends java.lang.Object implements [android.os.Parcelable]\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []")
		inputSignatures = "public static final int CONFIDENCE_LEVEL_NONE\npublic static final int CONFIDENCE_LEVEL_LOW\npublic static final int CONFIDENCE_LEVEL_LOW_MEDIUM\npublic static final int CONFIDENCE_LEVEL_MEDIUM\npublic static final int CONFIDENCE_LEVEL_MEDIUM_HIGH\npublic static final int CONFIDENCE_LEVEL_HIGH\npublic static final int CONFIDENCE_LEVEL_VERY_HIGH\npublic static final int HOTWORD_OFFSET_UNSET\npublic static final int AUDIO_CHANNEL_UNSET\nprivate static final int LIMIT_HOTWORD_OFFSET_MAX_VALUE\nprivate static final int LIMIT_AUDIO_CHANNEL_MAX_VALUE\npublic static final java.lang.String EXTRA_PROXIMITY_METERS\nprivate final @android.service.voice.HotwordDetectedResult.HotwordConfidenceLevelValue int mConfidenceLevel\nprivate @android.annotation.Nullable android.media.MediaSyncEvent mMediaSyncEvent\nprivate int mHotwordOffsetMillis\nprivate int mHotwordDurationMillis\nprivate int mAudioChannel\nprivate boolean mHotwordDetectionPersonalized\nprivate final int mScore\nprivate final int mPersonalizedScore\nprivate final int mHotwordPhraseId\nprivate final @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> mAudioStreams\nprivate final @android.annotation.NonNull android.os.PersistableBundle mExtras\nprivate static int sMaxBundleSize\nprivate static int defaultConfidenceLevel()\nprivate static int defaultScore()\nprivate static int defaultPersonalizedScore()\npublic static int getMaxScore()\nprivate static int defaultHotwordPhraseId()\npublic static int getMaxHotwordPhraseId()\nprivate static java.util.List<android.service.voice.HotwordAudioStream> defaultAudioStreams()\nprivate static android.os.PersistableBundle defaultExtras()\npublic static int getMaxBundleSize()\npublic @android.annotation.Nullable android.media.MediaSyncEvent getMediaSyncEvent()\npublic static int getParcelableSize(android.os.Parcelable)\npublic static int getUsageSize(android.service.voice.HotwordDetectedResult)\nprivate static int bitCount(long)\nprivate void onConstructed()\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> getAudioStreams()\npublic android.service.voice.HotwordDetectedResult.Builder buildUpon()\nclass HotwordDetectedResult extends java.lang.Object implements [android.os.Parcelable]\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []")
		@Deprecated
		private void __metadata() {}

services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordAudioStreamManager.java

0 → 100644

+233 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2022 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.server.voiceinteraction;

		import static android.app.AppOpsManager.MODE_ALLOWED;

		import static com.android.server.voiceinteraction.HotwordDetectionConnection.DEBUG;

		import android.annotation.NonNull;
		import android.app.AppOpsManager;
		import android.media.permission.Identity;
		import android.os.ParcelFileDescriptor;
		import android.service.voice.HotwordAudioStream;
		import android.service.voice.HotwordDetectedResult;
		import android.util.Pair;
		import android.util.Slog;

		import java.io.IOException;
		import java.io.InputStream;
		import java.io.OutputStream;
		import java.util.ArrayList;
		import java.util.Arrays;
		import java.util.List;
		import java.util.concurrent.Callable;
		import java.util.concurrent.ExecutorService;
		import java.util.concurrent.Executors;

		final class HotwordAudioStreamManager {

		private static final String TAG = "HotwordAudioStreamManager";
		private static final String OP_MESSAGE = "Streaming hotword audio to VoiceInteractionService";
		private static final String TASK_ID_PREFIX = "HotwordDetectedResult@";
		private static final String THREAD_NAME_PREFIX = "Copy-";

		private final AppOpsManager mAppOpsManager;
		private final Identity mVoiceInteractorIdentity;
		private final ExecutorService mExecutorService = Executors.newCachedThreadPool();

		HotwordAudioStreamManager(@NonNull AppOpsManager appOpsManager,
		@NonNull Identity voiceInteractorIdentity) {
		mAppOpsManager = appOpsManager;
		mVoiceInteractorIdentity = voiceInteractorIdentity;
		}

		/**
		* Starts copying the audio streams in the given {@link HotwordDetectedResult}.
		* <p>
		* The returned {@link HotwordDetectedResult} is identical the one that was passed in, except
		* that the {@link ParcelFileDescriptor}s within {@link HotwordDetectedResult#getAudioStreams()}
		* are replaced with descriptors from pipes managed by {@link HotwordAudioStreamManager}. The
		* returned value should be passed on to the client (i.e., the voice interactor).
		* </p>
		*
		* @throws IOException If there was an error creating the managed pipe.
		*/
		@NonNull
		public HotwordDetectedResult startCopyingAudioStreams(@NonNull HotwordDetectedResult result)
		throws IOException {
		List<HotwordAudioStream> audioStreams = result.getAudioStreams();
		if (audioStreams.isEmpty()) {
		return result;
		}

		List<HotwordAudioStream> newAudioStreams = new ArrayList<>(audioStreams.size());
		List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> sourcesAndSinks = new ArrayList<>(
		audioStreams.size());
		for (HotwordAudioStream audioStream : audioStreams) {
		ParcelFileDescriptor[] clientPipe = ParcelFileDescriptor.createReliablePipe();
		ParcelFileDescriptor clientAudioSource = clientPipe[0];
		ParcelFileDescriptor clientAudioSink = clientPipe[1];
		HotwordAudioStream newAudioStream =
		audioStream.buildUpon().setAudioStreamParcelFileDescriptor(
		clientAudioSource).build();
		newAudioStreams.add(newAudioStream);

		ParcelFileDescriptor serviceAudioSource =
		audioStream.getAudioStreamParcelFileDescriptor();
		sourcesAndSinks.add(new Pair<>(serviceAudioSource, clientAudioSink));
		}

		String resultTaskId = TASK_ID_PREFIX + System.identityHashCode(result);
		mExecutorService.execute(new HotwordDetectedResultCopyTask(resultTaskId, sourcesAndSinks));

		return result.buildUpon().setAudioStreams(newAudioStreams).build();
		}

		private class HotwordDetectedResultCopyTask implements Runnable {
		private final String mResultTaskId;
		private final List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> mSourcesAndSinks;
		private final ExecutorService mExecutorService = Executors.newCachedThreadPool();

		HotwordDetectedResultCopyTask(String resultTaskId,
		List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> sourcesAndSinks) {
		mResultTaskId = resultTaskId;
		mSourcesAndSinks = sourcesAndSinks;
		}

		@Override
		public void run() {
		Thread.currentThread().setName(THREAD_NAME_PREFIX + mResultTaskId);
		int size = mSourcesAndSinks.size();
		List<SingleAudioStreamCopyTask> tasks = new ArrayList<>(size);
		for (int i = 0; i < size; i++) {
		Pair<ParcelFileDescriptor, ParcelFileDescriptor> sourceAndSink =
		mSourcesAndSinks.get(i);
		ParcelFileDescriptor serviceAudioSource = sourceAndSink.first;
		ParcelFileDescriptor clientAudioSink = sourceAndSink.second;
		String streamTaskId = mResultTaskId + "@" + i;
		tasks.add(new SingleAudioStreamCopyTask(streamTaskId, serviceAudioSource,
		clientAudioSink));
		}

		if (mAppOpsManager.startOpNoThrow(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD,
		mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,
		mVoiceInteractorIdentity.attributionTag, OP_MESSAGE) == MODE_ALLOWED) {
		try {
		// TODO(b/244599891): Set timeout, close after inactivity
		mExecutorService.invokeAll(tasks);
		} catch (InterruptedException e) {
		Slog.e(TAG, mResultTaskId + ": Task was interrupted", e);
		bestEffortPropagateError(e.getMessage());
		} finally {
		mAppOpsManager.finishOp(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD,
		mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,
		mVoiceInteractorIdentity.attributionTag);
		}
		} else {
		bestEffortPropagateError(
		"Failed to obtain RECORD_AUDIO_HOTWORD permission for "
		+ SoundTriggerSessionPermissionsDecorator.toString(
		mVoiceInteractorIdentity));
		}
		}

		private void bestEffortPropagateError(@NonNull String errorMessage) {
		try {
		for (Pair<ParcelFileDescriptor, ParcelFileDescriptor> sourceAndSink :
		mSourcesAndSinks) {
		ParcelFileDescriptor serviceAudioSource = sourceAndSink.first;
		ParcelFileDescriptor clientAudioSink = sourceAndSink.second;
		serviceAudioSource.closeWithError(errorMessage);
		clientAudioSink.closeWithError(errorMessage);
		}
		} catch (IOException e) {
		Slog.e(TAG, mResultTaskId + ": Failed to propagate error", e);
		}
		}
		}

		private static class SingleAudioStreamCopyTask implements Callable<Void> {
		// TODO: Make this buffer size customizable from updateState()
		private static final int COPY_BUFFER_LENGTH = 1_024;

		private final String mStreamTaskId;
		private final ParcelFileDescriptor mAudioSource;
		private final ParcelFileDescriptor mAudioSink;

		SingleAudioStreamCopyTask(String streamTaskId, ParcelFileDescriptor audioSource,
		ParcelFileDescriptor audioSink) {
		mStreamTaskId = streamTaskId;
		mAudioSource = audioSource;
		mAudioSink = audioSink;
		}

		@Override
		public Void call() throws Exception {
		Thread.currentThread().setName(THREAD_NAME_PREFIX + mStreamTaskId);

		// Note: We are intentionally NOT using try-with-resources here. If we did,
		// the ParcelFileDescriptors will be automatically closed WITHOUT errors before we go
		// into the IOException-catch block. We want to propagate the error while closing the
		// PFDs.
		InputStream fis = null;
		OutputStream fos = null;
		try {
		fis = new ParcelFileDescriptor.AutoCloseInputStream(mAudioSource);
		fos = new ParcelFileDescriptor.AutoCloseOutputStream(mAudioSink);
		byte[] buffer = new byte[COPY_BUFFER_LENGTH];
		while (true) {
		if (Thread.interrupted()) {
		Slog.e(TAG,
		mStreamTaskId + ": SingleAudioStreamCopyTask task was interrupted");
		break;
		}

		int bytesRead = fis.read(buffer);
		if (bytesRead < 0) {
		Slog.i(TAG, mStreamTaskId + ": Reached end of audio stream");
		break;
		}
		if (bytesRead > 0) {
		if (DEBUG) {
		// TODO(b/244599440): Add proper logging
		Slog.d(TAG, mStreamTaskId + ": Copied " + bytesRead
		+ " bytes from audio stream. First 20 bytes=" + Arrays.toString(
		Arrays.copyOfRange(buffer, 0, 20)));
		}
		fos.write(buffer, 0, bytesRead);
		}
		// TODO(b/244599891): Close PFDs after inactivity
		}
		} catch (IOException e) {
		mAudioSource.closeWithError(e.getMessage());
		mAudioSink.closeWithError(e.getMessage());
		Slog.e(TAG, mStreamTaskId + ": Failed to copy audio stream", e);
		} finally {
		if (fis != null) {
		fis.close();
		}
		if (fos != null) {
		fos.close();
		}
		}

		return null;
		}
		}

		}

services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java

+52 −23

Original line number	Diff line number	Diff line
		@@ -59,6 +59,7 @@ import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPH

		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.app.AppOpsManager;
		import android.attention.AttentionManagerInternal;
		import android.content.ComponentName;
		import android.content.ContentCaptureOptions;
		@@ -137,6 +138,7 @@ final class HotwordDetectionConnection {
		// The error codes are used for onError callback
		private static final int HOTWORD_DETECTION_SERVICE_DIED = -1;
		private static final int CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION = -2;
		private static final int CALLBACK_ONDETECTED_STREAM_COPY_ERROR = -4;

		// Hotword metrics
		private static final int METRICS_INIT_UNKNOWN_TIMEOUT =
		@@ -168,6 +170,8 @@ final class HotwordDetectionConnection {
		// TODO: This may need to be a Handler(looper)
		private final ScheduledExecutorService mScheduledExecutorService =
		Executors.newSingleThreadScheduledExecutor();
		private final AppOpsManager mAppOpsManager;
		private final HotwordAudioStreamManager mHotwordAudioStreamManager;
		@Nullable private final ScheduledFuture<?> mCancellationTaskFuture;
		private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false);
		private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied;
		@@ -228,6 +232,9 @@ final class HotwordDetectionConnection {
		mContext = context;
		mVoiceInteractionServiceUid = voiceInteractionServiceUid;
		mVoiceInteractorIdentity = voiceInteractorIdentity;
		mAppOpsManager = mContext.getSystemService(AppOpsManager.class);
		mHotwordAudioStreamManager = new HotwordAudioStreamManager(mAppOpsManager,
		mVoiceInteractorIdentity);
		mDetectionComponentName = serviceName;
		mUser = userId;
		mCallback = callback;
		@@ -482,13 +489,19 @@ final class HotwordDetectionConnection {
		return;
		}
		saveProximityMetersToBundle(result);
		mSoftwareCallback.onDetected(result, null, null);
		if (result != null) {
		Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(result)
		HotwordDetectedResult newResult;
		try {
		newResult = mHotwordAudioStreamManager.startCopyingAudioStreams(result);
		} catch (IOException e) {
		// TODO: Write event
		mSoftwareCallback.onError();
		return;
		}
		mSoftwareCallback.onDetected(newResult, null, null);
		Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult)
		+ " bits from hotword trusted process");
		if (mDebugHotwordLogging) {
		Slog.i(TAG, "Egressed detected result: " + result);
		}
		Slog.i(TAG, "Egressed detected result: " + newResult);
		}
		}
		}
		@@ -660,6 +673,7 @@ final class HotwordDetectionConnection {
		try {
		enforcePermissionsForDataDelivery();
		} catch (SecurityException e) {
		Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e);
		HotwordMetricsLogger.writeKeyphraseTriggerEvent(
		mDetectorType,
		METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION);
		@@ -667,13 +681,19 @@ final class HotwordDetectionConnection {
		return;
		}
		saveProximityMetersToBundle(result);
		externalCallback.onKeyphraseDetected(recognitionEvent, result);
		if (result != null) {
		Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(result)
		HotwordDetectedResult newResult;
		try {
		newResult = mHotwordAudioStreamManager.startCopyingAudioStreams(result);
		} catch (IOException e) {
		// TODO: Write event
		externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR);
		return;
		}
		externalCallback.onKeyphraseDetected(recognitionEvent, newResult);
		Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult)
		+ " bits from hotword trusted process");
		if (mDebugHotwordLogging) {
		Slog.i(TAG, "Egressed detected result: " + result);
		}
		Slog.i(TAG, "Egressed detected result: " + newResult);
		}
		}
		}
		@@ -757,6 +777,7 @@ final class HotwordDetectionConnection {
		}

		private void restartProcessLocked() {
		// TODO(b/244598068): Check HotwordAudioStreamManager first
		Slog.v(TAG, "Restarting hotword detection process");
		ServiceConnection oldConnection = mRemoteHotwordDetectionService;
		HotwordDetectionServiceIdentity previousIdentity = mIdentity;
		@@ -991,16 +1012,24 @@ final class HotwordDetectionConnection {
		callback.onError();
		return;
		}
		callback.onDetected(triggerResult, null /* audioFormat */,
		HotwordDetectedResult newResult;
		try {
		newResult =
		mHotwordAudioStreamManager.startCopyingAudioStreams(
		triggerResult);
		} catch (IOException e) {
		// TODO: Write event
		callback.onError();
		return;
		}
		callback.onDetected(newResult, null /* audioFormat */,
		null /* audioStream */);
		if (triggerResult != null) {
		Slog.i(TAG, "Egressed "
		+ HotwordDetectedResult.getUsageSize(triggerResult)
		+ HotwordDetectedResult.getUsageSize(newResult)
		+ " bits from hotword trusted process");
		if (mDebugHotwordLogging) {
		Slog.i(TAG,
		"Egressed detected result: " + triggerResult);
		}
		"Egressed detected result: " + newResult);
		}
		}
		});