feat(scrcpy): add some AV1 parsing (6cfd8c12) · Commits · e / tools / webadb

.vscode/settings.json

+3 −0

Original line number	Diff line number	Diff line
		@@ -8,6 +8,7 @@
		"autorun",
		"Backquote",
		"Bframes",
		"Bitstream",
		"bootloader",
		"brotli",
		"Callout",
		@@ -26,6 +27,7 @@
		"Embedder",
		"entrypoints",
		"fflate",
		"flac",
		"fluentui",
		"genymobile",
		"Genymobile's",
		@@ -56,6 +58,7 @@
		"streamsaver",
		"struct",
		"struct's",
		"subsampling",
		"tcpip",
		"tinyh",
		"transferables",

libraries/scrcpy-decoder-webcodecs/src/index.ts

+71 −20

Original line number	Diff line number	Diff line
		import { EventEmitter } from "@yume-chan/event";
		import type {
		ScrcpyMediaStreamDataPacket,
		ScrcpyMediaStreamPacket,
		} from "@yume-chan/scrcpy";
		import { getUint32LittleEndian } from "@yume-chan/no-data-view";
		import {
		Av1,
		ScrcpyVideoCodecId,
		h264ParseConfiguration,
		h265ParseConfiguration,
		type ScrcpyMediaStreamDataPacket,
		type ScrcpyMediaStreamPacket,
		} from "@yume-chan/scrcpy";
		import { getUint32LittleEndian } from "@yume-chan/no-data-view";
		import type {
		ScrcpyVideoDecoder,
		ScrcpyVideoDecoderCapability,
		@@ -19,11 +18,19 @@ import { BitmapFrameRenderer } from "./bitmap.js";
		import type { FrameRenderer } from "./renderer.js";
		import { WebGLFrameRenderer } from "./webgl.js";

		function toHex(value: number) {
		return value.toString(16).padStart(2, "0").toUpperCase();
		function hexDigits(value: number) {
		return value.toString(16).toUpperCase();
		}

		function hexTwoDigits(value: number) {
		return value.toString(16).toUpperCase().padStart(2, "0");
		}

		export class WebCodecsDecoder implements ScrcpyVideoDecoder {
		function decimalTwoDigits(value: number) {
		return value.toString(10).padStart(2, "0");
		}

		export class WebCodecsVideoDecoder implements ScrcpyVideoDecoder {
		static isSupported() {
		return typeof globalThis.VideoDecoder !== "undefined";
		}
		@@ -149,9 +156,9 @@ export class WebCodecsDecoder implements ScrcpyVideoDecoder {
		// ISO Base Media File Format Name Space
		const codec =
		"avc1." +
		toHex(profileIndex) +
		toHex(constraintSet) +
		toHex(levelIndex);
		hexTwoDigits(profileIndex) +
		hexTwoDigits(constraintSet) +
		hexTwoDigits(levelIndex);
		this.#decoder.configure({
		codec: codec,
		optimizeForLatency: true,
		@@ -181,16 +188,57 @@ export class WebCodecsDecoder implements ScrcpyVideoDecoder {
		"hev1",
		["", "A", "B", "C"][generalProfileSpace]! +
		generalProfileIndex.toString(),
		getUint32LittleEndian(generalProfileCompatibilitySet, 0).toString(
		16,
		),
		hexDigits(getUint32LittleEndian(generalProfileCompatibilitySet, 0)),
		(generalTierFlag ? "H" : "L") + generalLevelIndex.toString(),
		getUint32LittleEndian(generalConstraintSet, 0)
		.toString(16)
		.toUpperCase(),
		getUint32LittleEndian(generalConstraintSet, 4)
		.toString(16)
		.toUpperCase(),
		...Array.from(generalConstraintSet, hexDigits),
		].join(".");
		this.#decoder.configure({
		codec,
		optimizeForLatency: true,
		});
		}

		#configureAv1(data: Uint8Array) {
		let sequenceHeader: Av1.SequenceHeaderObu \| undefined;
		const av1 = new Av1(data);
		for (const obu of av1.bitstream()) {
		if (obu.sequence_header_obu) {
		sequenceHeader = obu.sequence_header_obu;
		}
		}
		if (!sequenceHeader) {
		throw new Error("No sequence header found");
		}

		const {
		seq_profile: seqProfile,
		seq_level_idx: [seqLevelIdx = 0],
		color_config: {
		BitDepth,
		mono_chrome: monoChrome,
		subsampling_x: subsamplingX,
		subsampling_y: subsamplingY,
		chroma_sample_position: chromaSamplePosition,
		color_primaries: colorPrimaries,
		transfer_characteristics: transferCharacteristics,
		matrix_coefficients: matrixCoefficients,
		color_range: colorRange,
		},
		} = sequenceHeader;
		const codec = [
		"av01",
		seqProfile.toString(16),
		decimalTwoDigits(seqLevelIdx) +
		(sequenceHeader.seq_tier[0] ? "H" : "M"),
		decimalTwoDigits(BitDepth),
		monoChrome ? "1" : "0",
		(subsamplingX ? "1" : "0") +
		(subsamplingY ? "1" : "0") +
		chromaSamplePosition.toString(),
		decimalTwoDigits(colorPrimaries),
		decimalTwoDigits(transferCharacteristics),
		decimalTwoDigits(matrixCoefficients),
		colorRange ? "1" : "0",
		].join(".");
		this.#decoder.configure({
		codec,
		@@ -206,6 +254,9 @@ export class WebCodecsDecoder implements ScrcpyVideoDecoder {
		case ScrcpyVideoCodecId.H265:
		this.#configureH265(data);
		break;
		case ScrcpyVideoCodecId.AV1:
		this.#configureAv1(data);
		break;
		}
		this.#config = data;
		}

libraries/scrcpy/src/codec/av1.ts

+648 −0

File changed.

Preview size limit exceeded, changes collapsed.

libraries/scrcpy/src/codec/nalu.spec.ts

+170 −56

Original line number	Diff line number	Diff line
		@@ -3,26 +3,103 @@ import { describe, expect, it } from "@jest/globals";
		import { NaluSodbBitReader } from "./nalu.js";

		describe("nalu", () => {
		describe.only("NaluSodbReader", () => {
		it("should throw error if no end bit found", () => {
		describe("NaluSodbReader", () => {
		describe("constructor", () => {
		it("should set `ended` if stream is effectively empty", () => {
		const reader = new NaluSodbBitReader(
		new Uint8Array([0b10000000]),
		);
		expect(reader).toHaveProperty("ended", true);
		});

		it("should throw error if stream is empty", () => {
		expect(
		() => new NaluSodbBitReader(new Uint8Array([0b00000000])),
		() => new NaluSodbBitReader(new Uint8Array(0)),
		).toThrowErrorMatchingInlineSnapshot(`"Stop bit not found"`);
		});

		it("should throw error if no end bit found (single byte)", () => {
		expect(
		() =>
		new NaluSodbBitReader(
		new Uint8Array([0b00000000, 0b00000000]),
		),
		() => new NaluSodbBitReader(new Uint8Array(1)),
		).toThrowErrorMatchingInlineSnapshot(`"Stop bit not found"`);
		});

		it("should throw error if read after end bit", () => {
		let reader = new NaluSodbBitReader(new Uint8Array([0b10000000]));
		it("should throw error if no end bit found (multiple bytes)", () => {
		expect(
		() => new NaluSodbBitReader(new Uint8Array(10)),
		).toThrowErrorMatchingInlineSnapshot(`"Stop bit not found"`);
		});
		});

		describe("next", () => {
		it("should read bits in Big Endian (single byte)", () => {
		const reader = new NaluSodbBitReader(
		new Uint8Array([0b10110111]),
		);
		expect(reader.next()).toBe(1);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(1);
		expect(reader.next()).toBe(1);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(1);
		expect(reader.next()).toBe(1);
		});

		it("should read bits in Big Endian (multiple bytes)", () => {
		const reader = new NaluSodbBitReader(
		new Uint8Array([0b01001000, 0b10000100, 0b00010001]),
		);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(1);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(1);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(1);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(1);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(1);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(0);
		});

		it("should throw error if read after end bit (single byte, middle)", () => {
		const reader = new NaluSodbBitReader(
		new Uint8Array([0b11111000]),
		);
		for (let i = 0; i < 4; i += 1) {
		expect(reader.next()).toBe(1);
		}
		expect(() => reader.next()).toThrowErrorMatchingInlineSnapshot(
		`"Bit index out of bounds"`,
		);
		});

		it("should throw error if read after end bit (single byte, end)", () => {
		const reader = new NaluSodbBitReader(
		new Uint8Array([0b11111111]),
		);
		for (let i = 0; i < 7; i += 1) {
		expect(reader.next()).toBe(1);
		}
		expect(() => reader.next()).toThrowErrorMatchingInlineSnapshot(
		`"Bit index out of bounds"`,
		);
		});

		reader = new NaluSodbBitReader(
		it("should throw error if read after end bit (multiple bytes, start)", () => {
		const reader = new NaluSodbBitReader(
		new Uint8Array([0b11111111, 0b10000000]),
		);
		for (let i = 0; i < 8; i += 1) {
		@@ -33,6 +110,18 @@ describe("nalu", () => {
		);
		});

		it("should throw error if read after end bit (multiple bytes, middle)", () => {
		const reader = new NaluSodbBitReader(
		new Uint8Array([0b11111111, 0b11111000]),
		);
		for (let i = 0; i < 12; i += 1) {
		expect(reader.next()).toBe(1);
		}
		expect(() => reader.next()).toThrowErrorMatchingInlineSnapshot(
		`"Bit index out of bounds"`,
		);
		});

		it("should skip emulation prevent byte", () => {
		const reader = new NaluSodbBitReader(
		new Uint8Array([0xff, 0x00, 0x00, 0x03, 0xff, 0x80]),
		@@ -64,23 +153,48 @@ describe("nalu", () => {
		expect(reader.next()).toBe(1);
		}
		});
		});
		});

		describe("skip", () => {
		it("should skip <8 bits in single byte", () => {
		const reader = new NaluSodbBitReader(new Uint8Array([0b01000011]));

		it("should read bits in Big Endian", () => {
		let reader = new NaluSodbBitReader(new Uint8Array([0b10110011]));
		reader.skip(1);
		expect(reader.next()).toBe(1);
		expect(reader.next()).toBe(0);

		reader.skip(3);
		expect(reader.next()).toBe(1);
		expect(() => reader.next()).toThrowErrorMatchingInlineSnapshot(
		`"Bit index out of bounds"`,
		);
		});

		it("should skip <8 bits in multiple bytes", () => {
		const reader = new NaluSodbBitReader(
		new Uint8Array([0b00000100, 0b00101000]),
		);

		reader.skip(5);
		expect(reader.next()).toBe(1);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(1);

		reader = new NaluSodbBitReader(new Uint8Array([0b01001100]));
		expect(reader.next()).toBe(0);
		reader.skip(3);
		expect(reader.next()).toBe(1);
		expect(reader.next()).toBe(0);
		expect(reader.next()).toBe(0);
		expect(() => reader.next()).toThrowErrorMatchingInlineSnapshot(
		`"Bit index out of bounds"`,
		);
		});

		it("should skip >8 bits without emulation prevention byte", () => {
		const reader = new NaluSodbBitReader(
		new Uint8Array([0b00000000, 0b00100001]),
		);
		reader.skip(10);
		expect(reader.next()).toBe(1);
		expect(reader.next()).toBe(0);
		});
		});
		});

libraries/scrcpy/src/codec/nalu.ts

+50 −15

Original line number	Diff line number	Diff line
		@@ -232,12 +232,15 @@ export function naluRemoveEmulation(buffer: Uint8Array) {

		export class NaluSodbBitReader {
		readonly #nalu: Uint8Array;
		// logical length is `#byteLength * 8 + (7 - #stopBitIndex)`
		readonly #byteLength: number;
		readonly #stopBitIndex: number;

		#zeroCount = 0;
		#bytePosition = -1;
		#bitPosition = -1;

		// logical position is `#bytePosition * 8 + (7 - #bitPosition)`
		#bytePosition = 0;
		#bitPosition = 7;
		#byte = 0;

		get byteLength() {
		@@ -258,8 +261,8 @@ export class NaluSodbBitReader {

		get ended() {
		return (
		this.#bytePosition === this.#byteLength &&
		this.#bitPosition === this.#stopBitIndex
		this.#bytePosition >= this.#byteLength &&
		this.#bitPosition <= this.#stopBitIndex
		);
		}

		@@ -276,7 +279,7 @@ export class NaluSodbBitReader {
		if (((byte >> j) & 1) === 1) {
		this.#byteLength = i;
		this.#stopBitIndex = j;
		this.#readByte();
		this.#loadByte();
		return;
		}
		}
		@@ -285,15 +288,20 @@ export class NaluSodbBitReader {
		throw new Error("Stop bit not found");
		}

		#readByte() {
		#loadByte() {
		this.#byte = this.#nalu[this.#bytePosition]!;

		// If the current sequence is `0x000003`, skip to the next byte.
		// `annexBSplitNalu` had validated the input, so don't need to check here.
		if (this.#zeroCount === 2 && this.#byte === 3) {
		this.#zeroCount = 0;
		this.#bytePosition += 1;
		this.#readByte();
		this.#loadByte();
		return;
		}

		// `0x00000301` becomes `0x000001`, so only the `0x03` byte needs to be skipped
		// The `0x00` bytes are still returned as-is
		if (this.#byte === 0) {
		this.#zeroCount += 1;
		} else {
		@@ -302,18 +310,19 @@ export class NaluSodbBitReader {
		}

		next() {
		if (this.#bitPosition === -1) {
		this.#bitPosition = 7;
		this.#bytePosition += 1;
		this.#readByte();
		}

		if (this.ended) {
		throw new Error("Bit index out of bounds");
		}

		const value = (this.#byte >> this.#bitPosition) & 1;

		this.#bitPosition -= 1;
		if (this.#bitPosition < 0) {
		this.#bytePosition += 1;
		this.#bitPosition = 7;
		this.#loadByte();
		}

		return value;
		}

		@@ -329,10 +338,36 @@ export class NaluSodbBitReader {
		return result;
		}

		#ensurePositionValid() {
		if (
		this.#bytePosition >= this.#byteLength &&
		this.#bitPosition < this.#stopBitIndex
		) {
		throw new Error("Bit index out of bounds");
		}
		}

		skip(length: number) {
		for (let i = 0; i < length; i += 1) {
		this.next();
		if (length <= this.#bitPosition + 1) {
		this.#bitPosition -= length;
		this.#ensurePositionValid();
		return;
		}

		length -= this.#bitPosition + 1;
		this.#bytePosition += 1;
		this.#bitPosition = 7;
		this.#loadByte();
		this.#ensurePositionValid();

		for (; length >= 8; length -= 8) {
		this.#bytePosition += 1;
		this.#loadByte();
		this.#ensurePositionValid();
		}

		this.#bitPosition = 7 - length;
		this.#ensurePositionValid();
		}

		decodeExponentialGolombNumber(): number {