Skip to content
GitLab
Commit 3ca56021 authored by XiNGRZ's avatar XiNGRZ Committed by Luca Stefani
Browse files

Recorder: Add support for playback audio 



* Refactor to use a media muxer
Signed-off-by: default avatarLuca Stefani <luca.stefani.ge1@gmail.com>
Change-Id: I734c537b7f6ad19f74551dcd4fb0f2f5e2fbbe60
parent a413186d
Hide whitespace changes
Inline Side-by-side
app/src/main/java/org/lineageos/recorder/RecorderActivity.java
View file @ 3ca56021
......@@ -358,10 +358,8 @@ public class RecorderActivity extends AppCompatActivity implements
return hasAudioPermission() && hasPhoneReaderPermission();
}
@SuppressWarnings("SameReturnValue")
private boolean hasAllScreenRecorderPermissions() {
// None for now
return true;
return hasAudioPermission();
}
private boolean checkSoundRecPermissions() {
......@@ -385,6 +383,12 @@ public class RecorderActivity extends AppCompatActivity implements
}
private boolean checkScreenRecPermissions() {
if (!hasAudioPermission()) {
final String[] permissions = new String[]{ Manifest.permission.RECORD_AUDIO };
requestPermissions(permissions, REQUEST_SCREEN_REC_PERMS);
return true;
}
if (!hasDrawOverOtherAppsPermission()) {
Intent overlayIntent = new Intent(Settings.ACTION_MANAGE_OVERLAY_PERMISSION,
Uri.parse("package:" + getPackageName()));
......
app/src/main/java/org/lineageos/recorder/screen/ScreencastService.java
View file @ 3ca56021
......@@ -27,6 +27,14 @@ import android.content.Intent;
import android.content.IntentFilter;
import android.hardware.display.DisplayManager;
import android.hardware.display.VirtualDisplay;
import android.media.AudioAttributes;
import android.media.AudioFormat;
import android.media.AudioPlaybackCaptureConfiguration;
import android.media.AudioRecord;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.media.MediaMuxer;
import android.media.MediaRecorder;
import android.media.projection.MediaProjection;
import android.media.projection.MediaProjectionManager;
......@@ -38,10 +46,12 @@ import android.os.SystemClock;
import android.text.format.DateUtils;
import android.util.DisplayMetrics;
import android.util.Log;
import android.view.Display;
import android.view.Surface;
import android.view.WindowManager;
import android.widget.Toast;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.core.app.NotificationCompat;
......@@ -53,6 +63,7 @@ import org.lineageos.recorder.utils.Utils;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
......@@ -78,24 +89,50 @@ public class ScreencastService extends Service implements MediaProviderHelper.On
private static final String ACTION_STOP_SCAN =
"org.lineageos.recorder.server.display.STOP_SCAN";
private static final int TOTAL_NUM_TRACKS = 1;
private static final int VIDEO_BIT_RATE = 6000000;
private static final int VIDEO_FRAME_RATE = 30;
private static final int AUDIO_BIT_RATE = 16;
private static final int VIDEO_I_FRAME_INTERVAL = 1;
private static final int AUDIO_BIT_RATE = 128 * 1000;
private static final int AUDIO_SAMPLE_RATE = 44100;
private static final int AUDIO_CHANNELS = AudioFormat.CHANNEL_IN_STEREO;
public static final int NOTIFICATION_ID = 61;
private boolean mUseAudio;
private long mStartTime;
private Timer mTimer;
private NotificationCompat.Builder mBuilder;
private NotificationManager mNotificationManager;
private File mPath;
private MediaProjectionManager mMediaProjectionManager;
private MediaProjection mMediaProjection;
private Surface mInputSurface;
private VirtualDisplay mVirtualDisplay;
private MediaRecorder mMediaRecorder;
private NotificationManager mNotificationManager;
private boolean mUseAudio;
private File mPath;
private AudioRecord mAudioRecordExternal;
private AudioRecord mAudioRecordPlayback;
private MediaCodec mAudioExternalEncoder;
private MediaCodec mAudioPlaybackEncoder;
private MediaCodec mVideoEncoder;
private MediaMuxer mMediaMuxer;
private int mAudioExternalTrackIndex = -1;
private int mAudioPlaybackTrackIndex = -1;
private int mVideoTrackIndex = -1;
enum MediaMuxerStatus {
IDLE,
STARTED,
STOPPING,
}
private MediaMuxerStatus mMediaMusexStatus = MediaMuxerStatus.IDLE;
private long mCurrentTimestamp = 0;
private final BroadcastReceiver mBroadcastReceiver = new BroadcastReceiver() {
@Override
public void onReceive(Context context, Intent intent) {
......@@ -240,38 +277,230 @@ public class ScreencastService extends Service implements MediaProviderHelper.On
try {
Log.d(LOGTAG, "Writing video output to: " + mPath.getAbsolutePath());
// Set up media recorder
mMediaRecorder = new MediaRecorder();
if (mUseAudio) {
mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.MIC);
}
mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE);
mMediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4);
// Set up video
DisplayMetrics metrics = new DisplayMetrics();
WindowManager wm = getSystemService(WindowManager.class);
wm.getDefaultDisplay().getRealMetrics(metrics);
Display display = wm.getDefaultDisplay();
display.getRealMetrics(metrics);
int rotation = display.getRotation();
int screenWidth = metrics.widthPixels;
int screenHeight = metrics.heightPixels;
mMediaRecorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264);
mMediaRecorder.setVideoSize(screenWidth, screenHeight);
mMediaRecorder.setVideoFrameRate(VIDEO_FRAME_RATE);
mMediaRecorder.setVideoEncodingBitRate(VIDEO_BIT_RATE);
// Set up audio
if (mUseAudio) {
mMediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AMR_NB);
mMediaRecorder.setAudioChannels(TOTAL_NUM_TRACKS);
mMediaRecorder.setAudioEncodingBitRate(AUDIO_BIT_RATE);
mMediaRecorder.setAudioSamplingRate(AUDIO_SAMPLE_RATE);
}
// Set up muxer
mMediaMuxer = new MediaMuxer(mPath.getPath(),
MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
mMediaMuxer.setOrientationHint(rotation);
mMediaRecorder.setOutputFile(mPath);
mMediaRecorder.prepare();
// Set up video
MediaFormat video = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC,
screenWidth, screenHeight);
video.setInteger(MediaFormat.KEY_COLOR_FORMAT,
MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
video.setInteger(MediaFormat.KEY_FRAME_RATE, VIDEO_FRAME_RATE);
video.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, VIDEO_I_FRAME_INTERVAL);
video.setInteger(MediaFormat.KEY_BIT_RATE, VIDEO_BIT_RATE);
mVideoEncoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC);
mVideoEncoder.configure(video, null, null,
MediaCodec.CONFIGURE_FLAG_ENCODE);
mVideoEncoder.setCallback(new MediaCodec.Callback() {
@Override
public void onInputBufferAvailable(@NonNull MediaCodec codec, int index) {
}
@Override
public void onOutputBufferAvailable(@NonNull MediaCodec codec, int index,
@NonNull MediaCodec.BufferInfo info) {
if (mMediaMusexStatus == MediaMuxerStatus.STOPPING) {
Log.d(LOGTAG, "Video ends");
mVideoTrackIndex = -1;
if (mAudioExternalTrackIndex == -1 && mAudioPlaybackTrackIndex == -1) {
finishRecording();
}
} else {
ByteBuffer buffer = codec.getOutputBuffer(index);
if (buffer != null) {
if (mMediaMusexStatus == MediaMuxerStatus.STARTED) {
mMediaMuxer.writeSampleData(mVideoTrackIndex, buffer, info);
mCurrentTimestamp = info.presentationTimeUs;
}
codec.releaseOutputBuffer(index, false);
}
}
}
@Override
public void onError(@NonNull MediaCodec codec,
@NonNull MediaCodec.CodecException e) {
}
@Override
public void onOutputFormatChanged(@NonNull MediaCodec codec,
@NonNull MediaFormat format) {
mVideoTrackIndex = mMediaMuxer.addTrack(format);
Log.d(LOGTAG, "Video track ready: " + mVideoTrackIndex);
if ((mAudioExternalTrackIndex != -1 || mAudioPlaybackTrackIndex != -1) &&
mVideoTrackIndex != -1) {
mMediaMuxer.start();
mMediaMusexStatus = MediaMuxerStatus.STARTED;
Log.d(LOGTAG, "Start output");
}
}
});
// Setup audio defaults
int bufferSizeInBytes = AudioRecord.getMinBufferSize(
AUDIO_SAMPLE_RATE, AUDIO_CHANNELS, AudioFormat.ENCODING_PCM_16BIT);
// Set up audio playback
AudioPlaybackCaptureConfiguration config =
new AudioPlaybackCaptureConfiguration.Builder(mMediaProjection)
.addMatchingUsage(AudioAttributes.USAGE_MEDIA)
.addMatchingUsage(AudioAttributes.USAGE_GAME)
.build();
mAudioRecordPlayback = new AudioRecord.Builder()
.setAudioPlaybackCaptureConfig(config)
.setAudioFormat(new AudioFormat.Builder()
.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
.setSampleRate(AUDIO_SAMPLE_RATE)
.setChannelMask(AUDIO_CHANNELS)
.build())
.setBufferSizeInBytes(bufferSizeInBytes)
.build();
MediaFormat audioPlayback = MediaFormat.createAudioFormat(
MediaFormat.MIMETYPE_AUDIO_AAC,
mAudioRecordPlayback.getSampleRate(),
mAudioRecordPlayback.getChannelCount());
audioPlayback.setInteger(MediaFormat.KEY_AAC_PROFILE,
MediaCodecInfo.CodecProfileLevel.AACObjectLC);
audioPlayback.setInteger(MediaFormat.KEY_BIT_RATE, AUDIO_BIT_RATE);
mAudioPlaybackEncoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC);
mAudioPlaybackEncoder.configure(audioPlayback, null, null,
MediaCodec.CONFIGURE_FLAG_ENCODE);
mAudioPlaybackEncoder.setCallback(new MediaCodec.Callback() {
@Override
public void onInputBufferAvailable(@NonNull MediaCodec codec, int index) {
ByteBuffer buffer = codec.getInputBuffer(index);
if (buffer != null && mAudioRecordPlayback != null) {
int read = mAudioRecordPlayback.read(buffer, buffer.capacity(),
AudioRecord.READ_NON_BLOCKING);
if (read >= 0) {
codec.queueInputBuffer(index, 0, read, 0, 0);
} else {
mAudioPlaybackTrackIndex = -1;
if (mVideoTrackIndex == -1) {
finishRecording();
}
}
}
}
@Override
public void onOutputBufferAvailable(@NonNull MediaCodec codec, int index,
@NonNull MediaCodec.BufferInfo info) {
if (mMediaMusexStatus == MediaMuxerStatus.STOPPING) {
Log.d(LOGTAG, "Audio ends");
mAudioPlaybackTrackIndex = -1;
if (mVideoTrackIndex == -1) {
finishRecording();
}
} else {
ByteBuffer buffer = codec.getOutputBuffer(index);
if (buffer != null) {
info.presentationTimeUs = mCurrentTimestamp;
if (mMediaMusexStatus == MediaMuxerStatus.STARTED) {
mMediaMuxer.writeSampleData(mAudioPlaybackTrackIndex, buffer, info);
}
codec.releaseOutputBuffer(index, false);
}
}
}
@Override
public void onError(@NonNull MediaCodec codec,
@NonNull MediaCodec.CodecException e) {
}
@Override
public void onOutputFormatChanged(@NonNull MediaCodec codec,
@NonNull MediaFormat format) {
mAudioPlaybackTrackIndex = mMediaMuxer.addTrack(format);
Log.d(LOGTAG, "Audio track ready: " + mAudioPlaybackTrackIndex);
}
});
// Set up external audio
mAudioRecordExternal = new AudioRecord.Builder()
.setAudioSource(MediaRecorder.AudioSource.MIC)
.setAudioFormat(new AudioFormat.Builder()
.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
.setSampleRate(AUDIO_SAMPLE_RATE)
.setChannelMask(AUDIO_CHANNELS)
.build())
.setBufferSizeInBytes(bufferSizeInBytes)
.build();
MediaFormat audioExternal = MediaFormat.createAudioFormat(
MediaFormat.MIMETYPE_AUDIO_AAC,
mAudioRecordExternal.getSampleRate(),
mAudioRecordExternal.getChannelCount());
audioExternal.setInteger(MediaFormat.KEY_AAC_PROFILE,
MediaCodecInfo.CodecProfileLevel.AACObjectLC);
audioExternal.setInteger(MediaFormat.KEY_BIT_RATE, AUDIO_BIT_RATE);
mAudioExternalEncoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC);
mAudioExternalEncoder.configure(audioExternal, null, null,
MediaCodec.CONFIGURE_FLAG_ENCODE);
mAudioExternalEncoder.setCallback(new MediaCodec.Callback() {
@Override
public void onInputBufferAvailable(@NonNull MediaCodec codec, int index) {
ByteBuffer buffer = codec.getInputBuffer(index);
if (buffer != null && mAudioRecordExternal != null) {
int read = mAudioRecordExternal.read(buffer, buffer.capacity(),
AudioRecord.READ_NON_BLOCKING);
if (read >= 0) {
codec.queueInputBuffer(index, 0, read, 0, 0);
} else {
mAudioExternalTrackIndex = -1;
if (mVideoTrackIndex == -1) {
finishRecording();
}
}
}
}
@Override
public void onOutputBufferAvailable(@NonNull MediaCodec codec, int index,
@NonNull MediaCodec.BufferInfo info) {
if (mMediaMusexStatus == MediaMuxerStatus.STOPPING) {
Log.d(LOGTAG, "Audio ends");
mAudioExternalTrackIndex = -1;
if (mVideoTrackIndex == -1) {
finishRecording();
}
} else {
ByteBuffer buffer = codec.getOutputBuffer(index);
if (buffer != null) {
info.presentationTimeUs = mCurrentTimestamp;
if (mMediaMusexStatus == MediaMuxerStatus.STARTED) {
mMediaMuxer.writeSampleData(mAudioExternalTrackIndex, buffer, info);
}
codec.releaseOutputBuffer(index, false);
}
}
}
@Override
public void onError(@NonNull MediaCodec codec,
@NonNull MediaCodec.CodecException e) {
}
@Override
public void onOutputFormatChanged(@NonNull MediaCodec codec,
@NonNull MediaFormat format) {
mAudioExternalTrackIndex = mMediaMuxer.addTrack(format);
Log.d(LOGTAG, "Audio track ready: " + mAudioExternalTrackIndex);
}
});
// Create surface
mInputSurface = mMediaRecorder.getSurface();
mInputSurface = mVideoEncoder.createInputSurface();
mVirtualDisplay = mMediaProjection.createVirtualDisplay(
"Recording Display",
screenWidth,
......@@ -282,7 +511,21 @@ public class ScreencastService extends Service implements MediaProviderHelper.On
null,
null);
mMediaRecorder.start();
mMediaMusexStatus = MediaMuxerStatus.IDLE;
mVideoTrackIndex = -1;
mAudioExternalTrackIndex = -1;
mAudioPlaybackTrackIndex = -1;
mVideoEncoder.start();
if (mUseAudio) {
mAudioExternalEncoder.start();
mAudioRecordExternal.startRecording();
}
mAudioPlaybackEncoder.start();
mAudioRecordPlayback.startRecording();
mCurrentTimestamp = System.currentTimeMillis() / 100;
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e);
......@@ -304,13 +547,50 @@ public class ScreencastService extends Service implements MediaProviderHelper.On
}
private void stopRecording() {
mMediaRecorder.stop();
mMediaRecorder.release();
mMediaRecorder = null;
if (mMediaMusexStatus == MediaMuxerStatus.STOPPING || mMediaProjection == null) {
return;
}
mMediaProjection.stop();
mMediaProjection = null;
mInputSurface.release();
mInputSurface = null;
mVirtualDisplay.release();
mVirtualDisplay = null;
mMediaMusexStatus = MediaMuxerStatus.STOPPING;
finishRecording();
}
private void finishRecording() {
if (mMediaMusexStatus != MediaMuxerStatus.STOPPING) {
return;
}
mMediaMusexStatus = MediaMuxerStatus.IDLE;
mAudioRecordExternal.stop();
mAudioRecordExternal.release();
mAudioRecordExternal = null;
mVideoEncoder.stop();
mVideoEncoder.release();
mVideoEncoder = null;
mAudioExternalEncoder.stop();
mAudioExternalEncoder.release();
mAudioExternalEncoder = null;
mAudioPlaybackEncoder.stop();
mAudioPlaybackEncoder.release();
mAudioPlaybackEncoder = null;
mMediaMuxer.stop();
mMediaMuxer.release();
mMediaMuxer = null;
if (mTimer != null) {
mTimer.cancel();
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment