Merge "Refactor audio flows so that server streams audio to the client." into sc-dev (54e2ec28) · Commits · e / os / android_frameworks_base

core/api/system-current.txt

+1 −1

Original line number	Diff line number	Diff line
		@@ -10268,7 +10268,7 @@ package android.service.voice {
		public abstract class HotwordDetectionService extends android.app.Service {
		ctor public HotwordDetectionService();
		method @Nullable public final android.os.IBinder onBind(@NonNull android.content.Intent);
		method public void onDetectFromDspSource(int, @NonNull android.service.voice.HotwordDetectionService.DspHotwordDetectionCallback);
		method public void onDetectFromDspSource(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, long, @NonNull android.service.voice.HotwordDetectionService.DspHotwordDetectionCallback);
		field public static final String SERVICE_INTERFACE = "android.service.voice.HotwordDetectionService";
		}

core/java/android/service/voice/HotwordDetectionService.java

+24 −5

Original line number	Diff line number	Diff line
		@@ -19,15 +19,18 @@ package android.service.voice;
		import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;

		import android.annotation.CallSuper;
		import android.annotation.DurationMillisLong;
		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.annotation.SdkConstant;
		import android.annotation.SystemApi;
		import android.app.Service;
		import android.content.Intent;
		import android.media.AudioFormat;
		import android.os.Handler;
		import android.os.IBinder;
		import android.os.Looper;
		import android.os.ParcelFileDescriptor;
		import android.os.RemoteException;
		import android.util.Log;

		@@ -57,14 +60,21 @@ public abstract class HotwordDetectionService extends Service {

		private final IHotwordDetectionService mInterface = new IHotwordDetectionService.Stub() {
		@Override
		public void detectFromDspSource(int sessionId, IDspHotwordDetectionCallback callback)
		public void detectFromDspSource(
		ParcelFileDescriptor audioStream,
		AudioFormat audioFormat,
		long timeoutMillis,
		IDspHotwordDetectionCallback callback)
		throws RemoteException {
		if (DBG) {
		Log.d(TAG, "#detectFromDspSource");
		}
		mHandler.sendMessage(obtainMessage(HotwordDetectionService::onDetectFromDspSource,
		HotwordDetectionService.this,
		sessionId, new DspHotwordDetectionCallback(callback)));
		audioStream,
		audioFormat,
		timeoutMillis,
		new DspHotwordDetectionCallback(callback)));
		}
		};

		@@ -89,15 +99,24 @@ public abstract class HotwordDetectionService extends Service {
		/**
		* Detect the audio data generated from Dsp.
		*
		* @param sessionId The session to use when attempting to capture more audio from the DSP
		* hardware.
		* <p>Note: the clients are supposed to call {@code close} on the input stream when they are
		* done with the operation in order to free up resources.
		*
		* @param audioStream Stream containing audio bytes returned from DSP
		* @param audioFormat Format of the supplied audio
		* @param timeoutMillis Timeout in milliseconds for the operation to invoke the callback. If
		* the application fails to abide by the timeout, system will close the
		* microphone and cancel the operation.
		* @param callback Use {@link HotwordDetectionService#DspHotwordDetectionCallback} to return
		* the detected result.
		*
		* @hide
		*/
		@SystemApi
		public void onDetectFromDspSource(int sessionId,
		public void onDetectFromDspSource(
		@NonNull ParcelFileDescriptor audioStream,
		@NonNull AudioFormat audioFormat,
		@DurationMillisLong long timeoutMillis,
		@NonNull DspHotwordDetectionCallback callback) {
		}

core/java/android/service/voice/IHotwordDetectionService.aidl

+7 −1

Original line number	Diff line number	Diff line
		@@ -16,6 +16,8 @@

		package android.service.voice;

		import android.media.AudioFormat;
		import android.os.ParcelFileDescriptor;
		import android.service.voice.IDspHotwordDetectionCallback;

		/**
		@@ -24,5 +26,9 @@ import android.service.voice.IDspHotwordDetectionCallback;
		* @hide
		*/
		oneway interface IHotwordDetectionService {
		void detectFromDspSource(int sessionId, in IDspHotwordDetectionCallback callback);
		void detectFromDspSource(
		in ParcelFileDescriptor audioStream,
		in AudioFormat audioFormat,
		long timeoutMillis,
		in IDspHotwordDetectionCallback callback);
		}

services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java

+154 −24

Original line number	Diff line number	Diff line
		@@ -22,25 +22,49 @@ import android.content.Context;
		import android.content.Intent;
		import android.hardware.soundtrigger.IRecognitionStatusCallback;
		import android.hardware.soundtrigger.SoundTrigger;
		import android.media.AudioAttributes;
		import android.media.AudioRecord;
		import android.media.MediaRecorder;
		import android.os.ParcelFileDescriptor;
		import android.os.RemoteException;
		import android.service.voice.AlwaysOnHotwordDetector;
		import android.service.voice.HotwordDetectionService;
		import android.service.voice.IDspHotwordDetectionCallback;
		import android.service.voice.IHotwordDetectionService;
		import android.util.Pair;
		import android.util.Slog;

		import com.android.internal.app.IHotwordRecognitionStatusCallback;
		import com.android.internal.infra.ServiceConnector;

		import java.io.IOException;
		import java.io.OutputStream;
		import java.io.PrintWriter;
		import java.util.concurrent.Executor;
		import java.util.concurrent.Executors;
		import java.util.concurrent.ScheduledExecutorService;
		import java.util.concurrent.ScheduledFuture;
		import java.util.concurrent.TimeUnit;

		/**
		* A class that provides the communication with the HotwordDetectionService.
		*/
		final class HotwordDetectionConnection {
		static final String TAG = "HotwordDetectionConnection";
		private static final String TAG = "HotwordDetectionConnection";
		// TODO (b/177502877): Set the Debug flag to false before shipping.
		static final boolean DEBUG = true;
		private static final boolean DEBUG = true;

		// Number of bytes per sample of audio (which is a short).
		private static final int BYTES_PER_SAMPLE = 2;
		// TODO: These constants need to be refined.
		private static final long VALIDATION_TIMEOUT_MILLIS = 3000;
		private static final long VOICE_INTERACTION_TIMEOUT_TO_OPEN_MIC_MILLIS = 2000;
		private static final int MAX_STREAMING_SECONDS = 10;

		private final Executor mAudioCopyExecutor = Executors.newCachedThreadPool();
		// TODO: This may need to be a Handler(looper)
		private final ScheduledExecutorService mScheduledExecutorService =
		Executors.newSingleThreadScheduledExecutor();

		final Object mLock;
		final ComponentName mDetectionComponentName;
		@@ -93,31 +117,75 @@ final class HotwordDetectionConnection {
		}
		}

		private void detectFromDspSource(int sessionId, IDspHotwordDetectionCallback callback) {
		private void detectFromDspSource(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent,
		IHotwordRecognitionStatusCallback externalCallback) {
		if (DEBUG) {
		Slog.d(TAG, "detectFromDspSource");
		}
		mRemoteHotwordDetectionService.run(
		service -> service.detectFromDspSource(sessionId, callback));

		AudioRecord record = createAudioRecord(recognitionEvent);

		Pair<ParcelFileDescriptor, ParcelFileDescriptor> clientPipe = createPipe();

		if (clientPipe == null) {
		// Error.
		// Need to propagate as unknown error or something?
		return;
		}
		ParcelFileDescriptor audioSink = clientPipe.second;
		ParcelFileDescriptor clientRead = clientPipe.first;

		static final class SoundTriggerCallback extends IRecognitionStatusCallback.Stub {
		private SoundTrigger.KeyphraseRecognitionEvent mRecognitionEvent;
		private final HotwordDetectionConnection mHotwordDetectionConnection;
		private final IHotwordRecognitionStatusCallback mExternalCallback;
		private final IDspHotwordDetectionCallback mInternalCallback;
		record.startRecording();

		SoundTriggerCallback(IHotwordRecognitionStatusCallback callback,
		HotwordDetectionConnection connection) {
		mHotwordDetectionConnection = connection;
		mExternalCallback = callback;
		mInternalCallback = new IDspHotwordDetectionCallback.Stub() {
		mAudioCopyExecutor.execute(() -> {
		try (OutputStream fos =
		new ParcelFileDescriptor.AutoCloseOutputStream(audioSink)) {
		byte[] buffer = new byte[1024];

		while (true) {
		int bytesRead = record.read(buffer, 0, 1024);

		if (bytesRead < 0) {
		break;
		}

		fos.write(buffer, 0, bytesRead);
		}
		} catch (IOException e) {
		Slog.w(TAG, "Failed supplying audio data to validator", e);
		}
		});

		Runnable cancellingJob = () -> {
		record.stop();
		bestEffortCloseFileDescriptor(audioSink);
		// TODO: consider calling externalCallback.onRejected(ERROR_TIMEOUT).
		};

		ScheduledFuture<?> cancelingFuture =
		mScheduledExecutorService.schedule(
		cancellingJob, VALIDATION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);

		// TODO: consider making this a non-anonymous class.
		IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {
		@Override
		public void onDetected() throws RemoteException {
		if (DEBUG) {
		Slog.d(TAG, "onDetected");
		}
		mExternalCallback.onKeyphraseDetected(mRecognitionEvent);
		bestEffortCloseFileDescriptor(audioSink);
		cancelingFuture.cancel(true);

		// Give 2 more seconds for the interactor to start consuming the mic. If it fails to
		// do so under the given time, we'll force-close the mic to make sure resources are
		// freed up.
		// TODO: consider modelling these 2 seconds in the API.
		mScheduledExecutorService.schedule(
		cancellingJob,
		VOICE_INTERACTION_TIMEOUT_TO_OPEN_MIC_MILLIS,
		TimeUnit.MILLISECONDS);

		externalCallback.onKeyphraseDetected(recognitionEvent);
		}

		@Override
		@@ -125,10 +193,30 @@ final class HotwordDetectionConnection {
		if (DEBUG) {
		Slog.d(TAG, "onRejected");
		}
		mExternalCallback.onRejected(
		cancelingFuture.cancel(true);
		externalCallback.onRejected(
		AlwaysOnHotwordDetector.HOTWORD_DETECTION_FALSE_ALERT);
		}
		};

		mRemoteHotwordDetectionService.run(
		service -> service.detectFromDspSource(
		clientRead,
		recognitionEvent.getCaptureFormat(),
		VALIDATION_TIMEOUT_MILLIS,
		internalCallback));
		bestEffortCloseFileDescriptor(clientRead);
		}

		static final class SoundTriggerCallback extends IRecognitionStatusCallback.Stub {
		private SoundTrigger.KeyphraseRecognitionEvent mRecognitionEvent;
		private final HotwordDetectionConnection mHotwordDetectionConnection;
		private final IHotwordRecognitionStatusCallback mExternalCallback;

		SoundTriggerCallback(IHotwordRecognitionStatusCallback callback,
		HotwordDetectionConnection connection) {
		mHotwordDetectionConnection = connection;
		mExternalCallback = callback;
		}

		@Override
		@@ -142,7 +230,7 @@ final class HotwordDetectionConnection {
		if (useHotwordDetectionService) {
		mRecognitionEvent = recognitionEvent;
		mHotwordDetectionConnection.detectFromDspSource(
		recognitionEvent.getCaptureSession(), mInternalCallback);
		recognitionEvent, mExternalCallback);
		} else {
		mExternalCallback.onKeyphraseDetected(recognitionEvent);
		}
		@@ -171,6 +259,48 @@ final class HotwordDetectionConnection {
		}
		}

		// TODO: figure out if we need to let the client configure some of the parameters.
		private static AudioRecord createAudioRecord(
		@NonNull SoundTrigger.KeyphraseRecognitionEvent recognitionEvent) {
		int sampleRate = recognitionEvent.getCaptureFormat().getSampleRate();
		return new AudioRecord(
		new AudioAttributes.Builder()
		.setInternalCapturePreset(MediaRecorder.AudioSource.HOTWORD).build(),
		recognitionEvent.getCaptureFormat(),
		getBufferSizeInBytes(sampleRate, MAX_STREAMING_SECONDS),
		recognitionEvent.getCaptureSession());
		}

		/**
		* Returns the number of bytes required to store {@code bufferLengthSeconds} of audio sampled at
		* {@code sampleRate} Hz, using the format returned by DSP audio capture.
		*/
		private static int getBufferSizeInBytes(int sampleRate, int bufferLengthSeconds) {
		return BYTES_PER_SAMPLE * sampleRate * bufferLengthSeconds;
		}

		private static Pair<ParcelFileDescriptor, ParcelFileDescriptor> createPipe() {
		ParcelFileDescriptor[] fileDescriptors;
		try {
		fileDescriptors = ParcelFileDescriptor.createPipe();
		} catch (IOException e) {
		Slog.e(TAG, "Failed to create audio stream pipe", e);
		return null;
		}

		return Pair.create(fileDescriptors[0], fileDescriptors[1]);
		}

		private static void bestEffortCloseFileDescriptor(ParcelFileDescriptor fd) {
		try {
		fd.close();
		} catch (IOException e) {
		if (DEBUG) {
		Slog.w(TAG, "Failed closing file descriptor", e);
		}
		}
		}

		public void dump(String prefix, PrintWriter pw) {
		pw.print(prefix); pw.print("mBound="); pw.println(mBound);
		}