feat(aoa): emulating HID keyboard via AOA protocol (d08a6891) · Commits · e / tools / webadb

apps/demo/package.json

+2 −0

Original line number	Diff line number	Diff line
		@@ -22,6 +22,7 @@
		"@yume-chan/adb-backend-ws": "workspace:^0.0.9",
		"@yume-chan/adb-credential-web": "workspace:^0.0.18",
		"@yume-chan/android-bin": "workspace:^0.0.18",
		"@yume-chan/aoa": "workspace:^0.0.18",
		"@yume-chan/async": "^2.2.0",
		"@yume-chan/b-tree": "workspace:^0.0.16",
		"@yume-chan/event": "workspace:^0.0.18",
		@@ -46,6 +47,7 @@
		"@mdx-js/loader": "^2.2.1",
		"@mdx-js/react": "^2.2.1",
		"@next/mdx": "^13.1.1",
		"@types/node": "^18.11.18",
		"@types/react": "18.0.27",
		"eslint": "^8.31.0",
		"eslint-config-next": "13.1.5",

apps/demo/src/components/scrcpy/input.ts

0 → 100644

+224 −0

Original line number	Diff line number	Diff line
		import { AoaHidDevice, HidKeyCode, HidKeyboard } from "@yume-chan/aoa";
		import { Disposable } from "@yume-chan/event";
		import {
		AdbScrcpyClient,
		AndroidKeyCode,
		AndroidKeyEventAction,
		AndroidKeyEventMeta,
		} from "@yume-chan/scrcpy";

		export interface KeyboardInjector extends Disposable {
		down(key: string): Promise<void>;
		up(key: string): Promise<void>;
		reset(): Promise<void>;
		}

		export class ScrcpyKeyboardInjector implements KeyboardInjector {
		private readonly client: AdbScrcpyClient;

		private _controlLeft = false;
		private _controlRight = false;
		private _shiftLeft = false;
		private _shiftRight = false;
		private _altLeft = false;
		private _altRight = false;
		private _metaLeft = false;
		private _metaRight = false;

		private _capsLock = false;
		private _numLock = true;

		private _keys: Set<AndroidKeyCode> = new Set();

		public constructor(client: AdbScrcpyClient) {
		this.client = client;
		}

		private setModifier(keyCode: AndroidKeyCode, value: boolean) {
		switch (keyCode) {
		case AndroidKeyCode.ControlLeft:
		this._controlLeft = value;
		break;
		case AndroidKeyCode.ControlRight:
		this._controlRight = value;
		break;
		case AndroidKeyCode.ShiftLeft:
		this._shiftLeft = value;
		break;
		case AndroidKeyCode.ShiftRight:
		this._shiftRight = value;
		break;
		case AndroidKeyCode.AltLeft:
		this._altLeft = value;
		break;
		case AndroidKeyCode.AltRight:
		this._altRight = value;
		break;
		case AndroidKeyCode.MetaLeft:
		this._metaLeft = value;
		break;
		case AndroidKeyCode.MetaRight:
		this._metaRight = value;
		break;
		case AndroidKeyCode.CapsLock:
		if (value) {
		this._capsLock = !this._capsLock;
		}
		break;
		case AndroidKeyCode.NumLock:
		if (value) {
		this._numLock = !this._numLock;
		}
		break;
		}
		}

		private getMetaState(): AndroidKeyEventMeta {
		let metaState = 0;
		if (this._altLeft) {
		metaState \|=
		AndroidKeyEventMeta.AltOn \| AndroidKeyEventMeta.AltLeftOn;
		}
		if (this._altRight) {
		metaState \|=
		AndroidKeyEventMeta.AltOn \| AndroidKeyEventMeta.AltRightOn;
		}
		if (this._shiftLeft) {
		metaState \|=
		AndroidKeyEventMeta.ShiftOn \| AndroidKeyEventMeta.ShiftLeftOn;
		}
		if (this._shiftRight) {
		metaState \|=
		AndroidKeyEventMeta.ShiftOn \| AndroidKeyEventMeta.ShiftRightOn;
		}
		if (this._controlLeft) {
		metaState \|=
		AndroidKeyEventMeta.CtrlOn \| AndroidKeyEventMeta.CtrlLeftOn;
		}
		if (this._controlRight) {
		metaState \|=
		AndroidKeyEventMeta.CtrlOn \| AndroidKeyEventMeta.CtrlRightOn;
		}
		if (this._metaLeft) {
		metaState \|=
		AndroidKeyEventMeta.MetaOn \| AndroidKeyEventMeta.MetaLeftOn;
		}
		if (this._metaRight) {
		metaState \|=
		AndroidKeyEventMeta.MetaOn \| AndroidKeyEventMeta.MetaRightOn;
		}
		if (this._capsLock) {
		metaState \|= AndroidKeyEventMeta.CapsLockOn;
		}
		if (this._numLock) {
		metaState \|= AndroidKeyEventMeta.NumLockOn;
		}
		return metaState;
		}

		public async down(key: string): Promise<void> {
		const keyCode = AndroidKeyCode[key as keyof typeof AndroidKeyCode];
		if (!keyCode) {
		return;
		}

		this.setModifier(keyCode, true);
		this._keys.add(keyCode);
		await this.client.controlMessageSerializer?.injectKeyCode({
		action: AndroidKeyEventAction.Down,
		keyCode,
		metaState: this.getMetaState(),
		repeat: 0,
		});
		}

		public async up(key: string): Promise<void> {
		const keyCode = AndroidKeyCode[key as keyof typeof AndroidKeyCode];
		if (!keyCode) {
		return;
		}

		this.setModifier(keyCode, false);
		this._keys.delete(keyCode);
		await this.client.controlMessageSerializer?.injectKeyCode({
		action: AndroidKeyEventAction.Up,
		keyCode,
		metaState: this.getMetaState(),
		repeat: 0,
		});
		}

		public async reset(): Promise<void> {
		this._controlLeft = false;
		this._controlRight = false;
		this._shiftLeft = false;
		this._shiftRight = false;
		this._altLeft = false;
		this._altRight = false;
		this._metaLeft = false;
		this._metaRight = false;
		for (const key of this._keys) {
		this.up(AndroidKeyCode[key]);
		}
		this._keys.clear();
		}

		public dispose(): void {
		// do nothing
		}
		}

		export class AoaKeyboardInjector implements KeyboardInjector {
		public static async register(
		device: USBDevice
		): Promise<AoaKeyboardInjector> {
		const keyboard = await AoaHidDevice.register(
		device,
		0,
		HidKeyboard.DESCRIPTOR
		);
		return new AoaKeyboardInjector(keyboard);
		}

		private readonly aoaKeyboard: AoaHidDevice;
		private readonly hidKeyboard = new HidKeyboard();

		public constructor(aoaKeyboard: AoaHidDevice) {
		this.aoaKeyboard = aoaKeyboard;
		}

		public async down(key: string): Promise<void> {
		const keyCode = HidKeyCode[key as keyof typeof HidKeyCode];
		if (!keyCode) {
		return;
		}

		this.hidKeyboard.down(keyCode);
		await this.aoaKeyboard.sendInputReport(
		this.hidKeyboard.serializeInputReport()
		);
		}

		public async up(key: string): Promise<void> {
		const keyCode = HidKeyCode[key as keyof typeof HidKeyCode];
		if (!keyCode) {
		return;
		}

		this.hidKeyboard.up(keyCode);
		await this.aoaKeyboard.sendInputReport(
		this.hidKeyboard.serializeInputReport()
		);
		}

		public async reset(): Promise<void> {
		this.hidKeyboard.reset();
		await this.aoaKeyboard.sendInputReport(
		this.hidKeyboard.serializeInputReport()
		);
		}

		public async dispose(): Promise<void> {
		await this.aoaKeyboard.unregister();
		}
		}

apps/demo/src/components/scrcpy/state.tsx

+17 −3

Original line number	Diff line number	Diff line
		import { ADB_SYNC_MAX_PACKET_SIZE } from "@yume-chan/adb";
		import { AdbWebUsbBackend } from "@yume-chan/adb-backend-webusb";
		import {
		AdbScrcpyClient,
		AdbScrcpyOptions1_22,
		AndroidKeyCode,
		AndroidScreenPowerMode,
		CodecOptions,
		DEFAULT_SERVER_PATH,
		@@ -25,6 +25,11 @@ import { action, autorun, makeAutoObservable, runInAction } from "mobx";
		import { GLOBAL_STATE } from "../../state";
		import { ProgressStream } from "../../utils";
		import { fetchServer } from "./fetch-server";
		import {
		AoaKeyboardInjector,
		KeyboardInjector,
		ScrcpyKeyboardInjector,
		} from "./input";
		import { MuxerStream, RECORD_STATE } from "./recorder";
		import { H264Decoder, SETTING_STATE } from "./settings";

		@@ -58,7 +63,7 @@ export class ScrcpyPageState {

		client: AdbScrcpyClient \| undefined = undefined;
		hoverHelper: ScrcpyHoverHelper \| undefined = undefined;
		pressedKeys: Set<AndroidKeyCode> = new Set();
		keyboard: KeyboardInjector \| undefined = undefined;

		async pushServer() {
		const serverBuffer = await fetchServer();
		@@ -351,6 +356,14 @@ export class ScrcpyPageState {
		this.hoverHelper = new ScrcpyHoverHelper();
		this.running = true;
		});

		if (GLOBAL_STATE.backend instanceof AdbWebUsbBackend) {
		this.keyboard = await AoaKeyboardInjector.register(
		GLOBAL_STATE.backend.device
		);
		} else {
		this.keyboard = new ScrcpyKeyboardInjector(client);
		}
		} catch (e: any) {
		GLOBAL_STATE.showErrorDialog(e);
		} finally {
		@@ -376,7 +389,8 @@ export class ScrcpyPageState {
		RECORD_STATE.recording = false;
		}

		this.pressedKeys.clear();
		this.keyboard?.dispose();
		this.keyboard = undefined;

		this.fps = "0";
		clearTimeout(this.fpsCounterIntervalId);

apps/demo/src/components/scrcpy/video-container.tsx

+0 −1

Original line number	Diff line number	Diff line
		@@ -111,7 +111,6 @@ function handlePointerLeave(e: PointerEvent<HTMLDivElement>) {

		e.preventDefault();
		e.stopPropagation();

		// Because pointer capture on pointer down, this event only happens for hovering mouse and pen.
		// Release the injected pointer, otherwise it will stuck at the last position.
		injectTouch(AndroidMotionEventAction.HoverExit, e);

apps/demo/src/pages/audio.tsx

0 → 100644

+79 −0

Original line number	Diff line number	Diff line
		import { DefaultButton, PrimaryButton } from "@fluentui/react";
		import { AdbWebUsbBackend } from "@yume-chan/adb-backend-webusb";
		import {
		aoaGetProtocol,
		aoaSetAudioMode,
		aoaStartAccessory,
		} from "@yume-chan/aoa";
		import { observer } from "mobx-react-lite";
		import { useCallback, useState } from "react";
		import { GLOBAL_STATE } from "../state";

		function AudioPage() {
		const [supported, setSupported] = useState<boolean \| undefined>(undefined);
		const handleQuerySupportClick = useCallback(async () => {
		const backend = GLOBAL_STATE.backend as AdbWebUsbBackend;
		const device = backend.device;
		const version = await aoaGetProtocol(device);
		setSupported(version >= 2);
		}, []);

		const handleEnableClick = useCallback(async () => {
		const backend = GLOBAL_STATE.backend as AdbWebUsbBackend;
		const device = backend.device;
		const version = await aoaGetProtocol(device);
		if (version < 2) {
		return;
		}
		await aoaSetAudioMode(device, 1);
		await aoaStartAccessory(device);
		}, []);
		const handleDisableClick = useCallback(async () => {
		const backend = GLOBAL_STATE.backend as AdbWebUsbBackend;
		const device = backend.device;
		const version = await aoaGetProtocol(device);
		if (version < 2) {
		return;
		}
		await aoaSetAudioMode(device, 0);
		await aoaStartAccessory(device);
		}, []);

		if (
		!GLOBAL_STATE.backend \|\|
		!(GLOBAL_STATE.backend instanceof AdbWebUsbBackend)
		) {
		return <div>Audio forward can only be used with WebUSB backend.</div>;
		}

		return (
		<div>
		<div>
		Supported:{" "}
		{supported === undefined ? "Unknown" : supported ? "Yes" : "No"}
		</div>
		<div>
		<PrimaryButton
		disabled={!GLOBAL_STATE.backend}
		onClick={handleQuerySupportClick}
		>
		Query Support
		</PrimaryButton>
		<DefaultButton
		disabled={!supported}
		onClick={handleEnableClick}
		>
		Enable
		</DefaultButton>
		<DefaultButton
		disabled={!supported}
		onClick={handleDisableClick}
		>
		Disable
		</DefaultButton>
		</div>
		</div>
		);
		}

		export default observer(AudioPage);