feat(scrcpy): test and fix AV1 (14dbc45e) · Commits · e / tools / webadb

libraries/adb-scrcpy/src/client.ts

+41 −14

Original line number	Diff line number	Diff line
		@@ -14,6 +14,7 @@ import type {
		ScrcpyVideoStreamMetadata,
		} from "@yume-chan/scrcpy";
		import {
		Av1,
		DEFAULT_SERVER_PATH,
		ScrcpyControlMessageWriter,
		ScrcpyDeviceMessageDeserializeStream,
		@@ -344,6 +345,37 @@ export class AdbScrcpyClient {
		}
		}

		#configureH264(data: Uint8Array) {
		const { croppedWidth, croppedHeight } = h264ParseConfiguration(data);

		this.#screenWidth = croppedWidth;
		this.#screenHeight = croppedHeight;
		}

		#configureH265(data: Uint8Array) {
		const { croppedWidth, croppedHeight } = h265ParseConfiguration(data);

		this.#screenWidth = croppedWidth;
		this.#screenHeight = croppedHeight;
		}

		#configureAv1(data: Uint8Array) {
		const parser = new Av1(data);
		const sequenceHeader = parser.searchSequenceHeaderObu();
		if (!sequenceHeader) {
		return;
		}

		const { max_frame_width_minus_1, max_frame_height_minus_1 } =
		sequenceHeader;

		const width = max_frame_width_minus_1 + 1;
		const height = max_frame_height_minus_1 + 1;

		this.#screenWidth = width;
		this.#screenHeight = height;
		}

		async #createVideoStream(initialStream: ReadableStream<Uint8Array>) {
		const { stream, metadata } =
		await this.#options.parseVideoStreamMetadata(initialStream);
		@@ -355,23 +387,18 @@ export class AdbScrcpyClient {
		new InspectStream((packet) => {
		if (packet.type === "configuration") {
		switch (metadata.codec) {
		case ScrcpyVideoCodecId.H264: {
		const { croppedWidth, croppedHeight } =
		h264ParseConfiguration(packet.data);

		this.#screenWidth = croppedWidth;
		this.#screenHeight = croppedHeight;
		case ScrcpyVideoCodecId.H264:
		this.#configureH264(packet.data);
		break;
		}
		case ScrcpyVideoCodecId.H265: {
		const { croppedWidth, croppedHeight } =
		h265ParseConfiguration(packet.data);

		this.#screenWidth = croppedWidth;
		this.#screenHeight = croppedHeight;
		case ScrcpyVideoCodecId.H265:
		this.#configureH265(packet.data);
		break;
		case ScrcpyVideoCodecId.AV1:
		// AV1 configuration is in normal stream
		break;
		}
		}
		} else if (metadata.codec === ScrcpyVideoCodecId.AV1) {
		this.#configureAv1(packet.data);
		}
		}),
		),

libraries/scrcpy-decoder-webcodecs/src/index.ts

+84 −19

Original line number	Diff line number	Diff line
		@@ -12,7 +12,10 @@ import type {
		ScrcpyVideoDecoder,
		ScrcpyVideoDecoderCapability,
		} from "@yume-chan/scrcpy-decoder-tinyh264";
		import { WritableStream } from "@yume-chan/stream-extra";
		import {
		WritableStream,
		type WritableStreamDefaultController,
		} from "@yume-chan/stream-extra";

		import { BitmapFrameRenderer } from "./bitmap.js";
		import type { FrameRenderer } from "./renderer.js";
		@@ -98,6 +101,18 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
		}
		this.#currentFrameRendered = false;

		if (
		frame.displayWidth !== this.#canvas.width \|\|
		frame.displayHeight !== this.#canvas.height
		) {
		this.#canvas.width = frame.displayWidth;
		this.#canvas.height = frame.displayHeight;
		this.#sizeChanged.fire({
		width: frame.displayWidth,
		height: frame.displayHeight,
		});
		}

		// PERF: H.264 renderer may draw multiple frames in one vertical sync interval to minimize latency.
		// When multiple frames are drawn in one vertical sync interval,
		// only the last one is visible to users.
		@@ -107,16 +122,44 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
		this.#renderer.draw(frame);
		},
		error(e) {
		console.warn(
		"[@yume-chan/scrcpy-decoder-webcodecs]",
		"VideoDecoder error",
		e,
		);
		if (controller) {
		try {
		controller.error(e);
		} catch {}
		} else {
		error = e;
		}
		},
		});

		let error: Error \| undefined;
		let controller: WritableStreamDefaultController \| undefined;
		this.#writable = new WritableStream<ScrcpyMediaStreamPacket>({
		write: (packet) => {
		start: (_controller) => {
		if (error) {
		_controller.error(error);
		} else {
		controller = _controller;
		}
		},
		write: (packet, _controller) => {
		if (this.#codec === ScrcpyVideoCodecId.AV1) {
		if (packet.type === "configuration") {
		return;
		}

		this.#configureAv1(packet.data);
		this.#decoder.decode(
		new EncodedVideoChunk({
		// Treat `undefined` as `key`, otherwise won't decode.
		type: packet.keyframe === false ? "delta" : "key",
		timestamp: 0,
		data: packet.data,
		}),
		);
		return;
		}

		switch (packet.type) {
		case "configuration":
		this.#configure(packet.data);
		@@ -199,32 +242,53 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
		}

		#configureAv1(data: Uint8Array) {
		let sequenceHeader: Av1.SequenceHeaderObu \| undefined;
		const av1 = new Av1(data);
		for (const obu of av1.bitstream()) {
		if (obu.sequence_header_obu) {
		sequenceHeader = obu.sequence_header_obu;
		}
		}
		const parser = new Av1(data);
		const sequenceHeader = parser.searchSequenceHeaderObu();

		if (!sequenceHeader) {
		throw new Error("No sequence header found");
		return;
		}

		const {
		seq_profile: seqProfile,
		seq_level_idx: [seqLevelIdx = 0],
		max_frame_width_minus_1,
		max_frame_height_minus_1,
		color_config: {
		BitDepth,
		mono_chrome: monoChrome,
		subsampling_x: subsamplingX,
		subsampling_y: subsamplingY,
		chroma_sample_position: chromaSamplePosition,
		color_description_present_flag,
		},
		} = sequenceHeader;

		let colorPrimaries: Av1.ColorPrimaries;
		let transferCharacteristics: Av1.TransferCharacteristics;
		let matrixCoefficients: Av1.MatrixCoefficients;
		let colorRange: boolean;
		if (color_description_present_flag) {
		({
		color_primaries: colorPrimaries,
		transfer_characteristics: transferCharacteristics,
		matrix_coefficients: matrixCoefficients,
		color_range: colorRange,
		},
		} = sequenceHeader;
		} = sequenceHeader.color_config);
		} else {
		colorPrimaries = Av1.ColorPrimaries.Bt709;
		transferCharacteristics = Av1.TransferCharacteristics.Bt709;
		matrixCoefficients = Av1.MatrixCoefficients.Bt709;
		colorRange = false;
		}

		const width = max_frame_width_minus_1 + 1;
		const height = max_frame_height_minus_1 + 1;

		this.#canvas.width = width;
		this.#canvas.height = height;
		this.#sizeChanged.fire({ width, height });

		const codec = [
		"av01",
		seqProfile.toString(16),
		@@ -250,15 +314,16 @@ export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
		switch (this.#codec) {
		case ScrcpyVideoCodecId.H264:
		this.#configureH264(data);
		this.#config = data;
		break;
		case ScrcpyVideoCodecId.H265:
		this.#configureH265(data);
		this.#config = data;
		break;
		case ScrcpyVideoCodecId.AV1:
		this.#configureAv1(data);
		// AV1 configuration is in normal stream
		break;
		}
		this.#config = data;
		}

		#decode(packet: ScrcpyMediaStreamDataPacket) {

libraries/scrcpy/src/codec/av1.ts

+30 −8

Original line number	Diff line number	Diff line
		@@ -81,14 +81,22 @@ class BitReader {
		}

		skip(n: number) {
		if (n <= this.#bitPosition + 1) {
		this.#bytePosition += 1;
		this.#bitPosition = 7;
		this.#byte = this.#data[this.#bytePosition]!;
		return;
		}

		n -= this.#bitPosition + 1;
		this.#bytePosition += 1;

		const bytes = (n / 8) \| 0;
		if (bytes > 0) {
		this.#bytePosition += bytes;
		n -= bytes * 8;
		}

		n -= this.#bitPosition + 1;
		this.#bytePosition += 1;
		this.#bitPosition = 7 - n;
		this.#byte = this.#data[this.#bytePosition]!;
		}
		@@ -151,7 +159,7 @@ export class Av1 extends BitReader {
		return value;
		}

		*bitstream(): Generator<Av1.OpenBitstreamUnit, void, void> {
		*annexBBitstream(): Generator<Av1.OpenBitstreamUnit, void, void> {
		while (!this.ended) {
		const temporal_unit_size = this.leb128();
		yield* this.temporalUnit(temporal_unit_size);
		@@ -181,13 +189,15 @@ export class Av1 extends BitReader {

		#OperatingPointIdc = 0;

		openBitstreamUnit(sz: bigint) {
		openBitstreamUnit(sz?: bigint) {
		const obu_header = this.obuHeader();
		let obu_size: bigint;
		if (obu_header.obu_has_size_field) {
		obu_size = this.leb128();
		} else {
		} else if (sz !== undefined) {
		obu_size = sz - 1n - (obu_header.obu_extension_flag ? 1n : 0n);
		} else {
		throw new Error("obu_has_size_field must be true");
		}

		const startPosition = this.getPosition();
		@@ -227,14 +237,13 @@ export class Av1 extends BitReader {
		(startPosition[1] - currentPosition[1]);

		if (
		obu_size > 0 &&
		obu_size > 0 /* &&
		obu_header.obu_type !== Av1.ObuType.TileGroup &&
		obu_header.obu_type !== Av1.ObuType.TileList &&
		obu_header.obu_type !== Av1.ObuType.Frame
		obu_header.obu_type !== Av1.ObuType.Frame */
		) {
		this.skip(Number(obu_size) * 8 - payloadBits);
		}

		return {
		obu_header,
		obu_size,
		@@ -464,6 +473,19 @@ export class Av1 extends BitReader {
		};
		}

		searchSequenceHeaderObu() {
		while (!this.ended) {
		const obu = this.openBitstreamUnit();
		if (!obu) {
		continue;
		}
		if (obu.sequence_header_obu) {
		return obu.sequence_header_obu;
		}
		}
		return undefined;
		}

		timingInfo() {
		const num_units_in_display_tick = this.f(32);
		const time_scale = this.f(32);