feat(scrcpy): add recording support (eaf3a7a3) · Commits · e / tools / webadb

apps/demo/src/components/scrcpy/recorder.ts

+60 −134

Original line number	Diff line number	Diff line
		import { ScrcpyVideoStreamPacket } from "@yume-chan/scrcpy";
		import {
		ScrcpyVideoStreamFramePacket,
		ScrcpyVideoStreamPacket,
		splitH264Stream,
		} from "@yume-chan/scrcpy";
		import { InspectStream } from "@yume-chan/stream-extra";
		import WebMMuxer from "webm-muxer";

		@@ -28,103 +32,13 @@ function h264ConfigurationToAvcDecoderConfigurationRecord(
		return buffer;
		}

		function h264NaluToAvcSample(buffer: Uint8Array) {
		function h264StreamToAvcSample(buffer: Uint8Array) {
		const nalUnits: Uint8Array[] = [];
		let totalLength = 0;

		// -1 means we haven't found the first start code
		let start = -1;
		let writeIndex = 0;

		// How many `0x00`s in a row we have counted
		let zeroCount = 0;

		let inEmulation = false;

		for (const byte of buffer) {
		buffer[writeIndex] = byte;
		writeIndex += 1;

		if (inEmulation) {
		if (byte > 0x03) {
		// `0x00000304` or larger are invalid
		throw new Error("Invalid data");
		}

		inEmulation = false;
		continue;
		}

		if (byte == 0x00) {
		zeroCount += 1;
		continue;
		}

		const lastZeroCount = zeroCount;
		zeroCount = 0;

		if (start === -1) {
		// 0x000001 is the start code
		// But it can be preceded by any number of zeros
		// So 2 is the minimal
		if (lastZeroCount >= 2 && byte === 0x01) {
		// Found start of first NAL unit
		writeIndex = 0;
		start = 0;
		continue;
		}

		// Not begin with start code
		throw new Error("Invalid data");
		}

		if (lastZeroCount < 2) {
		// zero or one `0x00`s are acceptable
		continue;
		}

		if (byte === 0x01) {
		// Remove all leading `0x00`s and this `0x01`
		writeIndex -= lastZeroCount + 1;

		// Found another NAL unit
		nalUnits.push(buffer.subarray(start, writeIndex));
		totalLength += 4 + writeIndex - start;

		start = writeIndex;
		continue;
		}

		if (lastZeroCount > 2) {
		// Too much `0x00`s
		throw new Error("Invalid data");
		}

		switch (byte) {
		case 0x02:
		// Didn't find why, but 7.4.1 NAL unit semantics forbids `0x000002` appearing in NAL units
		throw new Error("Invalid data");
		case 0x03:
		// `0x000003` is the "emulation_prevention_three_byte"
		// `0x00000300`, `0x00000301`, `0x00000302` and `0x00000303` represent
		// `0x000000`, `0x000001`, `0x000002` and `0x000003` respectively

		// Remove current byte
		writeIndex -= 1;

		inEmulation = true;
		break;
		default:
		// `0x000004` or larger are as-is
		break;
		}

		if (inEmulation) {
		throw new Error("Invalid data");
		}

		nalUnits.push(buffer.subarray(start, writeIndex));
		totalLength += 4 + writeIndex - start;
		for (const unit of splitH264Stream(buffer)) {
		nalUnits.push(unit);
		totalLength += unit.byteLength + 4;
		}

		const sample = new Uint8Array(totalLength);
		@@ -149,32 +63,10 @@ export class MuxerStream extends InspectStream<ScrcpyVideoStreamPacket> {
		private firstTimestamp = -1;
		private avcConfiguration: Uint8Array \| undefined;
		private configurationWritten = false;
		private keyframeWritten = false;
		private framesFromKeyframe: ScrcpyVideoStreamFramePacket[] = [];

		constructor() {
		super((packet) => {
		if (packet.type === "configuration") {
		this.width = packet.data.croppedWidth;
		this.height = packet.data.croppedHeight;
		this.avcConfiguration =
		h264ConfigurationToAvcDecoderConfigurationRecord(
		packet.sequenceParameterSet,
		packet.pictureParameterSet
		);
		this.configurationWritten = false;
		return;
		}

		if (!this.muxer) {
		return;
		}

		// if (!this.keyframeWritten && packet.keyframe !== true) {
		// return;
		// }
		// this.keyframeWritten = true;

		let timestamp = Number(packet.pts);
		private appendFrame(frame: ScrcpyVideoStreamFramePacket) {
		let timestamp = Number(frame.pts);
		if (this.firstTimestamp === -1) {
		this.firstTimestamp = timestamp;
		timestamp = 0;
		@@ -182,12 +74,12 @@ export class MuxerStream extends InspectStream<ScrcpyVideoStreamPacket> {
		timestamp -= this.firstTimestamp;
		}

		const sample = h264NaluToAvcSample(packet.data.slice());
		this.muxer.addVideoChunk(
		const sample = h264StreamToAvcSample(frame.data);
		this.muxer!.addVideoChunk(
		{
		byteLength: sample.byteLength,
		timestamp,
		type: packet.keyframe ? "key" : "delta",
		type: frame.keyframe ? "key" : "delta",
		// Not used
		duration: null,
		copyTo: (destination) => {
		@@ -206,6 +98,34 @@ export class MuxerStream extends InspectStream<ScrcpyVideoStreamPacket> {
		}
		);
		this.configurationWritten = true;
		}

		constructor() {
		super((packet) => {
		if (packet.type === "configuration") {
		this.width = packet.data.croppedWidth;
		this.height = packet.data.croppedHeight;
		this.avcConfiguration =
		h264ConfigurationToAvcDecoderConfigurationRecord(
		packet.sequenceParameterSet,
		packet.pictureParameterSet
		);
		this.configurationWritten = false;
		return;
		}

		// To ensure the first frame is a keyframe
		// save the last keyframe and the following frames
		if (packet.keyframe === true) {
		this.framesFromKeyframe.length = 0;
		}
		this.framesFromKeyframe.push(packet);

		if (!this.muxer) {
		return;
		}

		this.appendFrame(packet);
		});
		}

		@@ -221,9 +141,15 @@ export class MuxerStream extends InspectStream<ScrcpyVideoStreamPacket> {
		},
		});

		if (this.framesFromKeyframe.length > 0) {
		for (const frame of this.framesFromKeyframe) {
		this.appendFrame(frame);
		}
		}

		setTimeout(() => {
		this.stop();
		}, 5000);
		}, 10000);
		}

		stop() {

libraries/scrcpy/src/options/1_16/h264-configuration.ts

+106 −28

Original line number	Diff line number	Diff line
		@@ -52,24 +52,24 @@ class BitReader {
		}

		/**
		* Parse NAL units from H.264 Annex B formatted data.
		* Split NAL units from a H.264 Annex B stream.
		*
		* It will overwrite the input to decode the encoding.
		* If the input is still needed, make a copy before calling this method.
		* The input is not modified.
		* The returned NAL units are views of the input (no memory allocation and copy),
		* but still contains emulation prevention bytes.
		*
		* This methods returns a generator, so it can be stopped immediately
		* after the interested NAL unit is found.
		*/
		export function* iterateNalu(buffer: Uint8Array): Generator<Uint8Array> {
		export function* splitH264Stream(buffer: Uint8Array): Generator<Uint8Array> {
		// -1 means we haven't found the first start code
		let start = -1;
		let writeIndex = 0;

		// How many `0x00`s in a row we have counted
		let zeroCount = 0;

		let inEmulation = false;

		for (const byte of buffer) {
		buffer[writeIndex] = byte;
		writeIndex += 1;
		for (let i = 0; i < buffer.length; i += 1) {
		const byte = buffer[i]!;

		if (inEmulation) {
		if (byte > 0x03) {
		@@ -81,22 +81,21 @@ export function* iterateNalu(buffer: Uint8Array): Generator<Uint8Array> {
		continue;
		}

		if (byte == 0x00) {
		if (byte === 0x00) {
		zeroCount += 1;
		continue;
		}

		const lastZeroCount = zeroCount;
		const prevZeroCount = zeroCount;
		zeroCount = 0;

		if (start === -1) {
		// 0x000001 is the start code
		// But it can be preceded by any number of zeros
		// So 2 is the minimal
		if (lastZeroCount >= 2 && byte === 0x01) {
		if (prevZeroCount >= 2 && byte === 0x01) {
		// Found start of first NAL unit
		writeIndex = 0;
		start = 0;
		start = i + 1;
		continue;
		}

		@@ -104,23 +103,20 @@ export function* iterateNalu(buffer: Uint8Array): Generator<Uint8Array> {
		throw new Error("Invalid data");
		}

		if (lastZeroCount < 2) {
		if (prevZeroCount < 2) {
		// zero or one `0x00`s are acceptable
		continue;
		}

		if (byte === 0x01) {
		// Remove all leading `0x00`s and this `0x01`
		writeIndex -= lastZeroCount + 1;

		// Found another NAL unit
		yield buffer.subarray(start, writeIndex);
		yield buffer.subarray(start, i - prevZeroCount);

		start = writeIndex;
		start = i + 1;
		continue;
		}

		if (lastZeroCount > 2) {
		if (prevZeroCount > 2) {
		// Too much `0x00`s
		throw new Error("Invalid data");
		}
		@@ -133,10 +129,6 @@ export function* iterateNalu(buffer: Uint8Array): Generator<Uint8Array> {
		// `0x000003` is the "emulation_prevention_three_byte"
		// `0x00000300`, `0x00000301`, `0x00000302` and `0x00000303` represent
		// `0x000000`, `0x000001`, `0x000002` and `0x000003` respectively

		// Remove current byte
		writeIndex -= 1;

		inEmulation = true;
		break;
		default:
		@@ -149,7 +141,93 @@ export function* iterateNalu(buffer: Uint8Array): Generator<Uint8Array> {
		throw new Error("Invalid data");
		}

		yield buffer.subarray(start, writeIndex);
		yield buffer.subarray(start, buffer.length);
		}

		/**
		* Remove emulation prevention bytes from a H.264 NAL Unit.
		*
		* The input is not modified.
		* If the input doesn't contain any emulation prevention bytes,
		* the input is returned as-is.
		* Otherwise, a new `Uint8Array` is created and returned.
		*/
		export function removeH264Emulation(buffer: Uint8Array) {
		// output will be created when first emulation prevention byte is found
		let output: Uint8Array \| undefined;
		let outputOffset = 0;

		let zeroCount = 0;
		let inEmulation = false;

		for (let i = 0; i < buffer.length; i += 1) {
		const byte = buffer[i]!;

		if (output) {
		output[outputOffset] = byte;
		outputOffset += 1;
		}

		if (inEmulation) {
		if (byte > 0x03) {
		// `0x00000304` or larger are invalid
		throw new Error("Invalid data");
		}

		inEmulation = false;
		continue;
		}

		if (byte === 0x00) {
		zeroCount += 1;
		continue;
		}

		const prevZeroCount = zeroCount;
		zeroCount = 0;

		if (prevZeroCount < 2) {
		// zero or one `0x00`s are acceptable
		continue;
		}

		if (byte === 0x01) {
		// Unexpected start code
		throw new Error("Invalid data");
		}

		if (prevZeroCount > 2) {
		// Too much `0x00`s
		throw new Error("Invalid data");
		}

		switch (byte) {
		case 0x02:
		// Didn't find why, but 7.4.1 NAL unit semantics forbids `0x000002` appearing in NAL units
		throw new Error("Invalid data");
		case 0x03:
		// `0x000003` is the "emulation_prevention_three_byte"
		// `0x00000300`, `0x00000301`, `0x00000302` and `0x00000303` represent
		// `0x000000`, `0x000001`, `0x000002` and `0x000003` respectively
		inEmulation = true;

		if (!output) {
		// Create output and copy the data before the emulation prevention byte
		output = new Uint8Array(buffer.length - 1);
		output.set(buffer.subarray(0, i - prevZeroCount));
		outputOffset = i - prevZeroCount + 1;
		} else {
		// Remove the emulation prevention byte
		outputOffset -= 1;
		}
		break;
		default:
		// `0x000004` or larger are as-is
		break;
		}
		}

		return output?.subarray(0, outputOffset) ?? buffer;
		}

		// 7.3.2.1.1 Sequence parameter set data syntax
		@@ -330,7 +408,7 @@ export function parseH264Configuration(buffer: Uint8Array) {
		let sequenceParameterSet: Uint8Array \| undefined;
		let pictureParameterSet: Uint8Array \| undefined;

		for (const nalu of iterateNalu(buffer)) {
		for (const nalu of splitH264Stream(buffer)) {
		const naluType = nalu[0]! & 0x1f;
		switch (naluType) {
		case 7: // Sequence parameter set

libraries/scrcpy/src/options/1_16/options.ts

+5 −2

Original line number	Diff line number	Diff line
		@@ -19,6 +19,7 @@ import { CodecOptions } from "./codec-options.js";
		import {
		parseH264Configuration,
		parseSequenceParameterSet,
		removeH264Emulation,
		} from "./h264-configuration.js";
		import {
		ScrcpyScrollController1_16,
		@@ -213,7 +214,7 @@ export class ScrcpyOptions1_16<
		const {
		sequenceParameterSet,
		pictureParameterSet,
		} = parseH264Configuration(packet.data.slice());
		} = parseH264Configuration(packet.data);

		const {
		profile_idc: profileIndex,
		@@ -226,7 +227,9 @@ export class ScrcpyOptions1_16<
		frame_crop_right_offset,
		frame_crop_top_offset,
		frame_crop_bottom_offset,
		} = parseSequenceParameterSet(sequenceParameterSet);
		} = parseSequenceParameterSet(
		removeH264Emulation(sequenceParameterSet)
		);

		const encodedWidth =
		(pic_width_in_mbs_minus1 + 1) * 16;