Recorder: Add support for playback audio (3ca56021) · Commits · e / os / android_packages_apps_Recorder

app/src/main/java/org/lineageos/recorder/RecorderActivity.java

+7 −3

Original line number	Diff line number	Diff line
		@@ -358,10 +358,8 @@ public class RecorderActivity extends AppCompatActivity implements
		return hasAudioPermission() && hasPhoneReaderPermission();
		}

		@SuppressWarnings("SameReturnValue")
		private boolean hasAllScreenRecorderPermissions() {
		// None for now
		return true;
		return hasAudioPermission();
		}

		private boolean checkSoundRecPermissions() {
		@@ -385,6 +383,12 @@ public class RecorderActivity extends AppCompatActivity implements
		}

		private boolean checkScreenRecPermissions() {
		if (!hasAudioPermission()) {
		final String[] permissions = new String[]{ Manifest.permission.RECORD_AUDIO };
		requestPermissions(permissions, REQUEST_SCREEN_REC_PERMS);
		return true;
		}

		if (!hasDrawOverOtherAppsPermission()) {
		Intent overlayIntent = new Intent(Settings.ACTION_MANAGE_OVERLAY_PERMISSION,
		Uri.parse("package:" + getPackageName()));

app/src/main/java/org/lineageos/recorder/screen/ScreencastService.java

+314 −34

Original line number	Diff line number	Diff line
		@@ -27,6 +27,14 @@ import android.content.Intent;
		import android.content.IntentFilter;
		import android.hardware.display.DisplayManager;
		import android.hardware.display.VirtualDisplay;
		import android.media.AudioAttributes;
		import android.media.AudioFormat;
		import android.media.AudioPlaybackCaptureConfiguration;
		import android.media.AudioRecord;
		import android.media.MediaCodec;
		import android.media.MediaCodecInfo;
		import android.media.MediaFormat;
		import android.media.MediaMuxer;
		import android.media.MediaRecorder;
		import android.media.projection.MediaProjection;
		import android.media.projection.MediaProjectionManager;
		@@ -38,10 +46,12 @@ import android.os.SystemClock;
		import android.text.format.DateUtils;
		import android.util.DisplayMetrics;
		import android.util.Log;
		import android.view.Display;
		import android.view.Surface;
		import android.view.WindowManager;
		import android.widget.Toast;

		import androidx.annotation.NonNull;
		import androidx.annotation.Nullable;
		import androidx.core.app.NotificationCompat;

		@@ -53,6 +63,7 @@ import org.lineageos.recorder.utils.Utils;

		import java.io.File;
		import java.io.IOException;
		import java.nio.ByteBuffer;
		import java.text.SimpleDateFormat;
		import java.util.Date;
		import java.util.Locale;
		@@ -78,24 +89,50 @@ public class ScreencastService extends Service implements MediaProviderHelper.On
		private static final String ACTION_STOP_SCAN =
		"org.lineageos.recorder.server.display.STOP_SCAN";

		private static final int TOTAL_NUM_TRACKS = 1;
		private static final int VIDEO_BIT_RATE = 6000000;
		private static final int VIDEO_FRAME_RATE = 30;
		private static final int AUDIO_BIT_RATE = 16;
		private static final int VIDEO_I_FRAME_INTERVAL = 1;
		private static final int AUDIO_BIT_RATE = 128 * 1000;
		private static final int AUDIO_SAMPLE_RATE = 44100;
		private static final int AUDIO_CHANNELS = AudioFormat.CHANNEL_IN_STEREO;

		public static final int NOTIFICATION_ID = 61;

		private boolean mUseAudio;
		private long mStartTime;
		private Timer mTimer;
		private NotificationCompat.Builder mBuilder;
		private NotificationManager mNotificationManager;
		private File mPath;

		private MediaProjectionManager mMediaProjectionManager;
		private MediaProjection mMediaProjection;

		private Surface mInputSurface;
		private VirtualDisplay mVirtualDisplay;
		private MediaRecorder mMediaRecorder;
		private NotificationManager mNotificationManager;
		private boolean mUseAudio;
		private File mPath;

		private AudioRecord mAudioRecordExternal;
		private AudioRecord mAudioRecordPlayback;

		private MediaCodec mAudioExternalEncoder;
		private MediaCodec mAudioPlaybackEncoder;
		private MediaCodec mVideoEncoder;
		private MediaMuxer mMediaMuxer;

		private int mAudioExternalTrackIndex = -1;
		private int mAudioPlaybackTrackIndex = -1;
		private int mVideoTrackIndex = -1;

		enum MediaMuxerStatus {
		IDLE,
		STARTED,
		STOPPING,
		}

		private MediaMuxerStatus mMediaMusexStatus = MediaMuxerStatus.IDLE;

		private long mCurrentTimestamp = 0;

		private final BroadcastReceiver mBroadcastReceiver = new BroadcastReceiver() {
		@Override
		public void onReceive(Context context, Intent intent) {
		@@ -240,38 +277,230 @@ public class ScreencastService extends Service implements MediaProviderHelper.On
		try {
		Log.d(LOGTAG, "Writing video output to: " + mPath.getAbsolutePath());

		// Set up media recorder
		mMediaRecorder = new MediaRecorder();
		if (mUseAudio) {
		mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.MIC);
		}
		mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE);
		mMediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4);

		// Set up video
		DisplayMetrics metrics = new DisplayMetrics();
		WindowManager wm = getSystemService(WindowManager.class);
		wm.getDefaultDisplay().getRealMetrics(metrics);
		Display display = wm.getDefaultDisplay();
		display.getRealMetrics(metrics);
		int rotation = display.getRotation();
		int screenWidth = metrics.widthPixels;
		int screenHeight = metrics.heightPixels;
		mMediaRecorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264);
		mMediaRecorder.setVideoSize(screenWidth, screenHeight);
		mMediaRecorder.setVideoFrameRate(VIDEO_FRAME_RATE);
		mMediaRecorder.setVideoEncodingBitRate(VIDEO_BIT_RATE);

		// Set up audio
		if (mUseAudio) {
		mMediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AMR_NB);
		mMediaRecorder.setAudioChannels(TOTAL_NUM_TRACKS);
		mMediaRecorder.setAudioEncodingBitRate(AUDIO_BIT_RATE);
		mMediaRecorder.setAudioSamplingRate(AUDIO_SAMPLE_RATE);
		// Set up muxer
		mMediaMuxer = new MediaMuxer(mPath.getPath(),
		MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
		mMediaMuxer.setOrientationHint(rotation);

		// Set up video
		MediaFormat video = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC,
		screenWidth, screenHeight);
		video.setInteger(MediaFormat.KEY_COLOR_FORMAT,
		MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
		video.setInteger(MediaFormat.KEY_FRAME_RATE, VIDEO_FRAME_RATE);
		video.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, VIDEO_I_FRAME_INTERVAL);
		video.setInteger(MediaFormat.KEY_BIT_RATE, VIDEO_BIT_RATE);
		mVideoEncoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC);
		mVideoEncoder.configure(video, null, null,
		MediaCodec.CONFIGURE_FLAG_ENCODE);
		mVideoEncoder.setCallback(new MediaCodec.Callback() {
		@Override
		public void onInputBufferAvailable(@NonNull MediaCodec codec, int index) {
		}

		@Override
		public void onOutputBufferAvailable(@NonNull MediaCodec codec, int index,
		@NonNull MediaCodec.BufferInfo info) {
		if (mMediaMusexStatus == MediaMuxerStatus.STOPPING) {
		Log.d(LOGTAG, "Video ends");
		mVideoTrackIndex = -1;
		if (mAudioExternalTrackIndex == -1 && mAudioPlaybackTrackIndex == -1) {
		finishRecording();
		}
		} else {
		ByteBuffer buffer = codec.getOutputBuffer(index);
		if (buffer != null) {
		if (mMediaMusexStatus == MediaMuxerStatus.STARTED) {
		mMediaMuxer.writeSampleData(mVideoTrackIndex, buffer, info);
		mCurrentTimestamp = info.presentationTimeUs;
		}
		codec.releaseOutputBuffer(index, false);
		}
		}
		}

		mMediaRecorder.setOutputFile(mPath);
		mMediaRecorder.prepare();
		@Override
		public void onError(@NonNull MediaCodec codec,
		@NonNull MediaCodec.CodecException e) {
		}

		@Override
		public void onOutputFormatChanged(@NonNull MediaCodec codec,
		@NonNull MediaFormat format) {
		mVideoTrackIndex = mMediaMuxer.addTrack(format);
		Log.d(LOGTAG, "Video track ready: " + mVideoTrackIndex);
		if ((mAudioExternalTrackIndex != -1 \|\| mAudioPlaybackTrackIndex != -1) &&
		mVideoTrackIndex != -1) {
		mMediaMuxer.start();
		mMediaMusexStatus = MediaMuxerStatus.STARTED;
		Log.d(LOGTAG, "Start output");
		}
		}
		});

		// Setup audio defaults
		int bufferSizeInBytes = AudioRecord.getMinBufferSize(
		AUDIO_SAMPLE_RATE, AUDIO_CHANNELS, AudioFormat.ENCODING_PCM_16BIT);

		// Set up audio playback
		AudioPlaybackCaptureConfiguration config =
		new AudioPlaybackCaptureConfiguration.Builder(mMediaProjection)
		.addMatchingUsage(AudioAttributes.USAGE_MEDIA)
		.addMatchingUsage(AudioAttributes.USAGE_GAME)
		.build();
		mAudioRecordPlayback = new AudioRecord.Builder()
		.setAudioPlaybackCaptureConfig(config)
		.setAudioFormat(new AudioFormat.Builder()
		.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
		.setSampleRate(AUDIO_SAMPLE_RATE)
		.setChannelMask(AUDIO_CHANNELS)
		.build())
		.setBufferSizeInBytes(bufferSizeInBytes)
		.build();
		MediaFormat audioPlayback = MediaFormat.createAudioFormat(
		MediaFormat.MIMETYPE_AUDIO_AAC,
		mAudioRecordPlayback.getSampleRate(),
		mAudioRecordPlayback.getChannelCount());
		audioPlayback.setInteger(MediaFormat.KEY_AAC_PROFILE,
		MediaCodecInfo.CodecProfileLevel.AACObjectLC);
		audioPlayback.setInteger(MediaFormat.KEY_BIT_RATE, AUDIO_BIT_RATE);
		mAudioPlaybackEncoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC);
		mAudioPlaybackEncoder.configure(audioPlayback, null, null,
		MediaCodec.CONFIGURE_FLAG_ENCODE);
		mAudioPlaybackEncoder.setCallback(new MediaCodec.Callback() {
		@Override
		public void onInputBufferAvailable(@NonNull MediaCodec codec, int index) {
		ByteBuffer buffer = codec.getInputBuffer(index);
		if (buffer != null && mAudioRecordPlayback != null) {
		int read = mAudioRecordPlayback.read(buffer, buffer.capacity(),
		AudioRecord.READ_NON_BLOCKING);
		if (read >= 0) {
		codec.queueInputBuffer(index, 0, read, 0, 0);
		} else {
		mAudioPlaybackTrackIndex = -1;
		if (mVideoTrackIndex == -1) {
		finishRecording();
		}
		}
		}
		}

		@Override
		public void onOutputBufferAvailable(@NonNull MediaCodec codec, int index,
		@NonNull MediaCodec.BufferInfo info) {
		if (mMediaMusexStatus == MediaMuxerStatus.STOPPING) {
		Log.d(LOGTAG, "Audio ends");
		mAudioPlaybackTrackIndex = -1;
		if (mVideoTrackIndex == -1) {
		finishRecording();
		}
		} else {
		ByteBuffer buffer = codec.getOutputBuffer(index);
		if (buffer != null) {
		info.presentationTimeUs = mCurrentTimestamp;
		if (mMediaMusexStatus == MediaMuxerStatus.STARTED) {
		mMediaMuxer.writeSampleData(mAudioPlaybackTrackIndex, buffer, info);
		}
		codec.releaseOutputBuffer(index, false);
		}
		}
		}

		@Override
		public void onError(@NonNull MediaCodec codec,
		@NonNull MediaCodec.CodecException e) {
		}

		@Override
		public void onOutputFormatChanged(@NonNull MediaCodec codec,
		@NonNull MediaFormat format) {
		mAudioPlaybackTrackIndex = mMediaMuxer.addTrack(format);
		Log.d(LOGTAG, "Audio track ready: " + mAudioPlaybackTrackIndex);
		}
		});

		// Set up external audio
		mAudioRecordExternal = new AudioRecord.Builder()
		.setAudioSource(MediaRecorder.AudioSource.MIC)
		.setAudioFormat(new AudioFormat.Builder()
		.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
		.setSampleRate(AUDIO_SAMPLE_RATE)
		.setChannelMask(AUDIO_CHANNELS)
		.build())
		.setBufferSizeInBytes(bufferSizeInBytes)
		.build();
		MediaFormat audioExternal = MediaFormat.createAudioFormat(
		MediaFormat.MIMETYPE_AUDIO_AAC,
		mAudioRecordExternal.getSampleRate(),
		mAudioRecordExternal.getChannelCount());
		audioExternal.setInteger(MediaFormat.KEY_AAC_PROFILE,
		MediaCodecInfo.CodecProfileLevel.AACObjectLC);
		audioExternal.setInteger(MediaFormat.KEY_BIT_RATE, AUDIO_BIT_RATE);
		mAudioExternalEncoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC);
		mAudioExternalEncoder.configure(audioExternal, null, null,
		MediaCodec.CONFIGURE_FLAG_ENCODE);
		mAudioExternalEncoder.setCallback(new MediaCodec.Callback() {
		@Override
		public void onInputBufferAvailable(@NonNull MediaCodec codec, int index) {
		ByteBuffer buffer = codec.getInputBuffer(index);
		if (buffer != null && mAudioRecordExternal != null) {
		int read = mAudioRecordExternal.read(buffer, buffer.capacity(),
		AudioRecord.READ_NON_BLOCKING);
		if (read >= 0) {
		codec.queueInputBuffer(index, 0, read, 0, 0);
		} else {
		mAudioExternalTrackIndex = -1;
		if (mVideoTrackIndex == -1) {
		finishRecording();
		}
		}
		}
		}

		@Override
		public void onOutputBufferAvailable(@NonNull MediaCodec codec, int index,
		@NonNull MediaCodec.BufferInfo info) {
		if (mMediaMusexStatus == MediaMuxerStatus.STOPPING) {
		Log.d(LOGTAG, "Audio ends");
		mAudioExternalTrackIndex = -1;
		if (mVideoTrackIndex == -1) {
		finishRecording();
		}
		} else {
		ByteBuffer buffer = codec.getOutputBuffer(index);
		if (buffer != null) {
		info.presentationTimeUs = mCurrentTimestamp;
		if (mMediaMusexStatus == MediaMuxerStatus.STARTED) {
		mMediaMuxer.writeSampleData(mAudioExternalTrackIndex, buffer, info);
		}
		codec.releaseOutputBuffer(index, false);
		}
		}
		}

		@Override
		public void onError(@NonNull MediaCodec codec,
		@NonNull MediaCodec.CodecException e) {
		}

		@Override
		public void onOutputFormatChanged(@NonNull MediaCodec codec,
		@NonNull MediaFormat format) {
		mAudioExternalTrackIndex = mMediaMuxer.addTrack(format);
		Log.d(LOGTAG, "Audio track ready: " + mAudioExternalTrackIndex);
		}
		});

		// Create surface
		mInputSurface = mMediaRecorder.getSurface();
		mInputSurface = mVideoEncoder.createInputSurface();
		mVirtualDisplay = mMediaProjection.createVirtualDisplay(
		"Recording Display",
		screenWidth,
		@@ -282,7 +511,21 @@ public class ScreencastService extends Service implements MediaProviderHelper.On
		null,
		null);

		mMediaRecorder.start();
		mMediaMusexStatus = MediaMuxerStatus.IDLE;

		mVideoTrackIndex = -1;
		mAudioExternalTrackIndex = -1;
		mAudioPlaybackTrackIndex = -1;

		mVideoEncoder.start();
		if (mUseAudio) {
		mAudioExternalEncoder.start();
		mAudioRecordExternal.startRecording();
		}
		mAudioPlaybackEncoder.start();
		mAudioRecordPlayback.startRecording();

		mCurrentTimestamp = System.currentTimeMillis() / 100;
		} catch (IOException e) {
		e.printStackTrace();
		throw new RuntimeException(e);
		@@ -304,13 +547,50 @@ public class ScreencastService extends Service implements MediaProviderHelper.On
		}

		private void stopRecording() {
		mMediaRecorder.stop();
		mMediaRecorder.release();
		mMediaRecorder = null;
		if (mMediaMusexStatus == MediaMuxerStatus.STOPPING \|\| mMediaProjection == null) {
		return;
		}

		mMediaProjection.stop();
		mMediaProjection = null;

		mInputSurface.release();
		mInputSurface = null;

		mVirtualDisplay.release();
		mVirtualDisplay = null;

		mMediaMusexStatus = MediaMuxerStatus.STOPPING;

		finishRecording();
		}

		private void finishRecording() {
		if (mMediaMusexStatus != MediaMuxerStatus.STOPPING) {
		return;
		}

		mMediaMusexStatus = MediaMuxerStatus.IDLE;

		mAudioRecordExternal.stop();
		mAudioRecordExternal.release();
		mAudioRecordExternal = null;

		mVideoEncoder.stop();
		mVideoEncoder.release();
		mVideoEncoder = null;

		mAudioExternalEncoder.stop();
		mAudioExternalEncoder.release();
		mAudioExternalEncoder = null;

		mAudioPlaybackEncoder.stop();
		mAudioPlaybackEncoder.release();
		mAudioPlaybackEncoder = null;

		mMediaMuxer.stop();
		mMediaMuxer.release();
		mMediaMuxer = null;

		if (mTimer != null) {
		mTimer.cancel();