Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 69cf2859 authored by yuanjiahsu's avatar yuanjiahsu Committed by Yuanjia Hsu
Browse files

Implement VirtualAudioDevice API 

Add createVirtualAudioDevice() API in VirtualDevice to return a VirtualAudioDevice object for the caller to capture audio and inject microphone to applications running on the specified virtual display.

The VirtualAudioController in service side will responsible for listening virtual display running applications change, playback and recording config change, then notify the VirtualAudioSession to update AudioRecord/AudioTrack inside the AudioCapture/AudioInjection class internally.

Bug: 201558304
CTS-Coverage-Bug: 218528439
Test: atest FrameworksCoreTests:android.companion.virtual, atest FrameworksServicesTests:com.android.server.companion.virtual, manual testing with Exo by ag/16575248
Change-Id: I74f3da67b76aaa909943fbfc8418bf22d637ca39
parent b2134e69

core/api/system-current.txt

+34 −0
Original line number Diff line number Diff line
@@ -2750,6 +2750,7 @@ package android.companion.virtual { 
    method public void addActivityListener(@NonNull android.companion.virtual.VirtualDeviceManager.ActivityListener);

    method public void addActivityListener(@NonNull android.companion.virtual.VirtualDeviceManager.ActivityListener, @NonNull java.util.concurrent.Executor);

    method @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) public void close();

    method @NonNull @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) public android.companion.virtual.audio.VirtualAudioDevice createVirtualAudioDevice(@NonNull android.hardware.display.VirtualDisplay, @Nullable java.util.concurrent.Executor, @Nullable android.companion.virtual.audio.VirtualAudioDevice.AudioConfigurationChangeCallback);

    method @Nullable public android.hardware.display.VirtualDisplay createVirtualDisplay(int, int, int, @Nullable android.view.Surface, int, @Nullable android.os.Handler, @Nullable android.hardware.display.VirtualDisplay.Callback);

    method @NonNull @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) public android.hardware.input.VirtualKeyboard createVirtualKeyboard(@NonNull android.hardware.display.VirtualDisplay, @NonNull String, int, int);

    method @NonNull @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) public android.hardware.input.VirtualMouse createVirtualMouse(@NonNull android.hardware.display.VirtualDisplay, @NonNull String, int, int);
@@ -2782,6 +2783,39 @@ package android.companion.virtual { 














}



























package android.companion.virtual.audio {

























  public final class AudioCapture {













    ctor public AudioCapture();













    method public int getRecordingState();













    method public int read(@NonNull java.nio.ByteBuffer, int);













    method public void startRecording();













    method public void stop();













  }

























  public final class AudioInjection {













    ctor public AudioInjection();













    method public int getPlayState();













    method public void play();













    method public void stop();













    method public int write(@NonNull java.nio.ByteBuffer, int, int);













  }

























  public final class VirtualAudioDevice implements java.io.Closeable {













    method @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) public void close();













    method @Nullable public android.companion.virtual.audio.AudioCapture getAudioCapture();













    method @Nullable public android.companion.virtual.audio.AudioInjection getAudioInjection();













    method @NonNull @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) public android.companion.virtual.audio.AudioCapture startAudioCapture(@NonNull android.media.AudioFormat);













    method @NonNull @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) public android.companion.virtual.audio.AudioInjection startAudioInjection(@NonNull android.media.AudioFormat);













  }

























  public static interface VirtualAudioDevice.AudioConfigurationChangeCallback {













    method public void onPlaybackConfigChanged(@NonNull java.util.List<android.media.AudioPlaybackConfiguration>);













    method public void onRecordingConfigChanged(@NonNull java.util.List<android.media.AudioRecordingConfiguration>);













  }

























}



























package android.content {





























  public class ApexEnvironment {

























core/java/android/companion/virtual/IVirtualDevice.aidl



+10
−0




























Original line number
Diff line number
Diff line








@@ -17,6 +17,7 @@













package android.companion.virtual;































import android.app.PendingIntent;













import android.companion.virtual.audio.IAudioSessionCallback;















import android.graphics.Point;















import android.graphics.PointF;















import android.hardware.input.VirtualKeyEvent;










@@ -45,6 +46,15 @@ interface IVirtualDevice {













     */















    void close();





























    /**













     * Notifies of an audio session being started.













     */













    void onAudioSessionStarting(













            int displayId,













            IAudioSessionCallback callback);



























    void onAudioSessionEnded();





























    void createVirtualKeyboard(















            int displayId,















            String inputDeviceName,

































core/java/android/companion/virtual/VirtualDeviceManager.java



+26
−0




























Original line number
Diff line number
Diff line








@@ -25,6 +25,8 @@ import android.annotation.SystemService;













import android.app.Activity;















import android.app.PendingIntent;















import android.companion.AssociationInfo;













import android.companion.virtual.audio.VirtualAudioDevice;













import android.companion.virtual.audio.VirtualAudioDevice.AudioConfigurationChangeCallback;















import android.content.ComponentName;















import android.content.Context;















import android.graphics.Point;










@@ -338,6 +340,30 @@ public final class VirtualDeviceManager {













            }















        }





























        /**













         * Creates a VirtualAudioDevice, capable of recording audio emanating from this device,













         * or injecting audio from another device.













         *













         * <p>Note: This object does not support capturing privileged playback, such as voice call













         * audio.













         *













         * @param display The target virtual display to capture from and inject into.













         * @param executor The {@link Executor} object for the thread on which to execute













         *                the callback. If <code>null</code>, the {@link Executor} associated with













         *                the main {@link Looper} will be used.













         * @param callback Interface to be notified when playback or recording configuration of













         *                applications running on virtual display is changed.













         * @return A {@link VirtualAudioDevice} instance.













         */













        @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE)













        @NonNull













        public VirtualAudioDevice createVirtualAudioDevice(













                @NonNull VirtualDisplay display,













                @Nullable Executor executor,













                @Nullable AudioConfigurationChangeCallback callback) {













            return new VirtualAudioDevice(mContext, mVirtualDevice, display, executor, callback);













        }





























        /**















         * Sets the visibility of the pointer icon for this VirtualDevice's associated displays.















         *

































core/java/android/companion/virtual/audio/AudioCapture.java
0 → 100644



+124
−0




























Original line number
Diff line number
Diff line













/*













 * Copyright (C) 2022 The Android Open Source Project













 *













 * Licensed under the Apache License, Version 2.0 (the "License");













 * you may not use this file except in compliance with the License.













 * You may obtain a copy of the License at













 *













 *      http://www.apache.org/licenses/LICENSE-2.0













 *













 * Unless required by applicable law or agreed to in writing, software













 * distributed under the License is distributed on an "AS IS" BASIS,













 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.













 * See the License for the specific language governing permissions and













 * limitations under the License.













 */



























package android.companion.virtual.audio;



























import static android.media.AudioRecord.RECORDSTATE_RECORDING;













import static android.media.AudioRecord.RECORDSTATE_STOPPED;



























import android.annotation.NonNull;













import android.annotation.Nullable;













import android.annotation.SuppressLint;













import android.annotation.SystemApi;













import android.media.AudioRecord;













import android.util.Log;



























import com.android.internal.annotations.GuardedBy;



























import java.nio.ByteBuffer;



























/**













 * Wrapper around {@link AudioRecord} that allows for the underlying {@link AudioRecord} to













 * be swapped out while recording is ongoing.













 *













 * @hide













 */













// The stop() actually doesn't release resources, so should not force implementing Closeable.













@SuppressLint("NotCloseable")













@SystemApi













public final class AudioCapture {













    private static final String TAG = "AudioCapture";



























    private final Object mLock = new Object();



























    @GuardedBy("mLock")













    @Nullable













    private AudioRecord mAudioRecord;



























    @GuardedBy("mLock")













    private int mRecordingState = RECORDSTATE_STOPPED;



























    /**













     * Sets the {@link AudioRecord} to handle audio capturing.













     * Callers may call this multiple times with different audio records to change













     * the underlying {@link AudioRecord} without stopping and re-starting recording.













     *













     * @param audioRecord The underlying {@link AudioRecord} to use for capture,













     * or null if no audio (i.e. silence) should be captured while still keeping the













     * record in a recording state.













     */













    void setAudioRecord(@Nullable AudioRecord audioRecord) {













        Log.d(TAG, "set AudioRecord with " + audioRecord);













        synchronized (mLock) {













            // Release old reference.













            if (mAudioRecord != null) {













                mAudioRecord.release();













            }













            // Sync recording state for new reference.













            if (audioRecord != null) {













                if (mRecordingState == RECORDSTATE_RECORDING













                        && audioRecord.getRecordingState() != RECORDSTATE_RECORDING) {













                    audioRecord.startRecording();













                }













                if (mRecordingState == RECORDSTATE_STOPPED













                        && audioRecord.getRecordingState() != RECORDSTATE_STOPPED) {













                    audioRecord.stop();













                }













            }













            mAudioRecord = audioRecord;













        }













    }



























    /** See {@link AudioRecord#read(ByteBuffer, int)}. */













    public int read(@NonNull ByteBuffer audioBuffer, int sizeInBytes) {













        final int sizeRead;













        synchronized (mLock) {













            if (mAudioRecord != null) {













                sizeRead = mAudioRecord.read(audioBuffer, sizeInBytes);













            } else {













                sizeRead = 0;













            }













        }













        return sizeRead;













    }



























    /** See {@link AudioRecord#startRecording()}. */













    public void startRecording() {













        synchronized (mLock) {













            mRecordingState = RECORDSTATE_RECORDING;













            if (mAudioRecord != null && mAudioRecord.getRecordingState() != RECORDSTATE_RECORDING) {













                mAudioRecord.startRecording();













            }













        }













    }



























    /** See {@link AudioRecord#stop()}. */













    public void stop() {













        synchronized (mLock) {













            mRecordingState = RECORDSTATE_STOPPED;













            if (mAudioRecord != null && mAudioRecord.getRecordingState() != RECORDSTATE_STOPPED) {













                mAudioRecord.stop();













            }













        }













    }



























    /** See {@link AudioRecord#getRecordingState()}. */













    public int getRecordingState() {













        synchronized (mLock) {













            return mRecordingState;













        }













    }













}
























core/java/android/companion/virtual/audio/AudioInjection.java
0 → 100644



+124
−0




























Original line number
Diff line number
Diff line













/*













 * Copyright (C) 2022 The Android Open Source Project













 *













 * Licensed under the Apache License, Version 2.0 (the "License");













 * you may not use this file except in compliance with the License.













 * You may obtain a copy of the License at













 *













 *      http://www.apache.org/licenses/LICENSE-2.0













 *













 * Unless required by applicable law or agreed to in writing, software













 * distributed under the License is distributed on an "AS IS" BASIS,













 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.













 * See the License for the specific language governing permissions and













 * limitations under the License.













 */



























package android.companion.virtual.audio;



























import static android.media.AudioTrack.PLAYSTATE_PLAYING;













import static android.media.AudioTrack.PLAYSTATE_STOPPED;



























import android.annotation.NonNull;













import android.annotation.Nullable;













import android.annotation.SuppressLint;













import android.annotation.SystemApi;













import android.media.AudioTrack;













import android.util.Log;



























import com.android.internal.annotations.GuardedBy;



























import java.nio.ByteBuffer;



























/**













 * Wrapper around {@link AudioTrack} that allows for the underlying {@link AudioTrack} to













 * be swapped out while playout is ongoing.













 *













 * @hide













 */













// The stop() actually doesn't release resources, so should not force implementing Closeable.













@SuppressLint("NotCloseable")













@SystemApi













public final class AudioInjection {













    private static final String TAG = "AudioInjection";



























    private final Object mLock = new Object();



























    @GuardedBy("mLock")













    @Nullable













    private AudioTrack mAudioTrack;



























    @GuardedBy("mLock")













    private int mPlayState = PLAYSTATE_STOPPED;



























    /**













     * Sets the {@link AudioTrack} to handle audio injection.













     * Callers may call this multiple times with different audio tracks to change













     * the underlying {@link AudioTrack} without stopping and re-starting injection.













     *













     * @param audioTrack The underlying {@link AudioTrack} to use for injection,













     * or null if no audio (i.e. silence) should be injected while still keeping the













     * record in a playing state.













     */













    void setAudioTrack(@Nullable AudioTrack audioTrack) {













        Log.d(TAG, "set AudioTrack with " + audioTrack);













        synchronized (mLock) {













            // Release old reference.













            if (mAudioTrack != null) {













                mAudioTrack.release();













            }













            // Sync play state for new reference.













            if (audioTrack != null) {













                if (mPlayState == PLAYSTATE_PLAYING













                        && audioTrack.getPlayState() != PLAYSTATE_PLAYING) {













                    audioTrack.play();













                }













                if (mPlayState == PLAYSTATE_STOPPED













                        && audioTrack.getPlayState() != PLAYSTATE_STOPPED) {













                    audioTrack.stop();













                }













            }













            mAudioTrack = audioTrack;













        }













    }



























    /** See {@link AudioTrack#write(ByteBuffer, int, int)}. */













    public int write(@NonNull ByteBuffer audioBuffer, int sizeInBytes, int writeMode) {













        final int sizeWrite;













        synchronized (mLock) {













            if (mAudioTrack != null) {













                sizeWrite = mAudioTrack.write(audioBuffer, sizeInBytes, writeMode);













            } else {













                sizeWrite = 0;













            }













        }













        return sizeWrite;













    }



























    /** See {@link AudioTrack#play()}. */













    public void play() {













        synchronized (mLock) {













            mPlayState = PLAYSTATE_PLAYING;













            if (mAudioTrack != null && mAudioTrack.getPlayState() != PLAYSTATE_PLAYING) {













                mAudioTrack.play();













            }













        }













    }



























    /** See {@link AudioTrack#stop()}. */













    public void stop() {













        synchronized (mLock) {













            mPlayState = PLAYSTATE_STOPPED;













            if (mAudioTrack != null && mAudioTrack.getPlayState() != PLAYSTATE_STOPPED) {













                mAudioTrack.stop();













            }













        }













    }



























    /** See {@link AudioTrack#getPlayState()}. */













    public int getPlayState() {













        synchronized (mLock) {













            return mPlayState;













        }













    }













}