feat(scrcpy): improve support for scroll and hover (076d67e2) · Commits · e / tools / webadb

apps/demo/src/pages/scrcpy.tsx

+34 −20

Original line number	Diff line number	Diff line
		@@ -40,9 +40,11 @@ import {
		AndroidKeyEventAction,
		AndroidMotionEventAction,
		AndroidScreenPowerMode,
		clamp,
		CodecOptions,
		DEFAULT_SERVER_PATH,
		ScrcpyDeviceMessageType,
		ScrcpyHoverHelper,
		ScrcpyLogLevel,
		ScrcpyOptions1_25,
		ScrcpyOptionsInit1_24,
		@@ -157,18 +159,6 @@ function fetchServer(
		return cachedValue.promise;
		}

		function clamp(value: number, min: number, max: number): number {
		if (value < min) {
		return min;
		}

		if (value > max) {
		return max;
		}

		return value;
		}

		export interface H264Decoder extends Disposable {
		readonly maxProfile: AndroidCodecProfile \| undefined;
		readonly maxLevel: AndroidCodecLevel \| undefined;
		@@ -258,6 +248,7 @@ const useClasses = makeStyles({
		},
		video: {
		transformOrigin: "center center",
		touchAction: "none",
		},
		});

		@@ -369,6 +360,7 @@ class ScrcpyPageState {
		}

		client: AdbScrcpyClient \| undefined = undefined;
		hoverHelper: ScrcpyHoverHelper \| undefined = undefined;

		async pushServer() {
		const serverBuffer = await fetchServer();
		@@ -891,6 +883,7 @@ class ScrcpyPageState {
		handlePointerDown: false,
		handlePointerMove: false,
		handlePointerUp: false,
		handlePointerLeave: false,
		handleWheel: false,
		handleContextMenu: false,
		handleKeyDown: false,
		@@ -1117,6 +1110,7 @@ class ScrcpyPageState {

		runInAction(() => {
		this.client = client;
		this.hoverHelper = new ScrcpyHoverHelper();
		this.running = true;
		});
		} catch (e: any) {
		@@ -1301,37 +1295,46 @@ class ScrcpyPageState {

		const { pointerType } = e;
		let pointerId: bigint;
		let { pressure } = e;
		if (pointerType === "mouse") {
		// ScrcpyPointerId.Mouse doesn't work with Chrome browser
		// https://github.com/Genymobile/scrcpy/issues/3635
		pointerId = ScrcpyPointerId.Finger;
		pressure = pressure === 0 ? 0 : 1;
		} else {
		pointerId = BigInt(e.pointerId);
		}

		const { x, y } = this.calculatePointerPosition(e.clientX, e.clientY);
		this.client!.controlMessageSerializer!.injectTouch({

		const messages = this.hoverHelper!.process({
		action,
		pointerId,
		screenWidth: this.client!.screenWidth!,
		screenHeight: this.client!.screenHeight!,
		screenWidth: this.client.screenWidth!,
		screenHeight: this.client.screenHeight!,
		pointerX: x,
		pointerY: y,
		pressure,
		pressure: e.pressure,
		buttons: e.buttons,
		});
		for (const message of messages) {
		this.client.controlMessageSerializer!.injectTouch(message);
		}
		};

		handlePointerDown = (e: React.PointerEvent<HTMLDivElement>) => {
		this.rendererContainer!.focus();
		e.preventDefault();
		e.stopPropagation();
		e.currentTarget.setPointerCapture(e.pointerId);
		this.injectTouch(AndroidMotionEventAction.Down, e);
		};

		handlePointerMove = (e: React.PointerEvent<HTMLDivElement>) => {
		if (!this.client) {
		return;
		}

		e.preventDefault();
		e.stopPropagation();
		this.injectTouch(
		e.buttons === 0
		? AndroidMotionEventAction.HoverMove
		@@ -1341,6 +1344,16 @@ class ScrcpyPageState {
		};

		handlePointerUp = (e: React.PointerEvent<HTMLDivElement>) => {
		e.preventDefault();
		e.stopPropagation();
		this.injectTouch(AndroidMotionEventAction.Up, e);
		};

		handlePointerLeave = (e: React.PointerEvent<HTMLDivElement>) => {
		e.preventDefault();
		e.stopPropagation();
		// Prevent hover state on device from "stucking" at the last position
		this.injectTouch(AndroidMotionEventAction.HoverExit, e);
		this.injectTouch(AndroidMotionEventAction.Up, e);
		};

		@@ -1358,8 +1371,8 @@ class ScrcpyPageState {
		screenHeight: this.client!.screenHeight!,
		pointerX: x,
		pointerY: y,
		scrollX: e.deltaX / 100,
		scrollY: e.deltaY / 100,
		scrollX: -e.deltaX / 100,
		scrollY: -e.deltaY / 100,
		buttons: 0,
		});
		};
		@@ -1591,6 +1604,7 @@ const Scrcpy: NextPage = () => {
		onPointerMove={state.handlePointerMove}
		onPointerUp={state.handlePointerUp}
		onPointerCancel={state.handlePointerUp}
		onPointerLeave={state.handlePointerLeave}
		onKeyDown={state.handleKeyDown}
		onContextMenu={state.handleContextMenu}
		/>

libraries/scrcpy/src/control/hover-helper.ts

0 → 100644

+51 −0

Original line number	Diff line number	Diff line
		import {
		AndroidMotionEventAction,
		type ScrcpyInjectTouchControlMessage,
		} from "./inject-touch.js";
		import { ScrcpyControlMessageType } from "./type.js";

		/**
		* On Android, touching the screen with a finger will disable mouse cursor.
		* However, Scrcpy doesn't do that, and can inject two pointers at the same time.
		* This can cause finger events to be "ignored" because mouse is still the primary pointer.
		*
		* This helper class injects an extra `ACTION_UP` event,
		* so Scrcpy server can remove the previously hovering pointer.
		*/
		export class ScrcpyHoverHelper {
		// AFAIK, only mouse and pen can have hover state
		// and you can't have two mouses or pens.
		private lastHoverMessage: ScrcpyInjectTouchControlMessage \| undefined;

		public process(
		message: Omit<ScrcpyInjectTouchControlMessage, "type">
		): ScrcpyInjectTouchControlMessage[] {
		const result: ScrcpyInjectTouchControlMessage[] = [];

		// A different pointer appeared,
		// Cancel previously hovering pointer so Scrcpy server can free up the pointer ID.
		if (
		this.lastHoverMessage &&
		this.lastHoverMessage.pointerId !== message.pointerId
		) {
		// TODO: Inject MotionEvent.ACTION_HOVER_EXIT
		// From testing, it seems no App cares about this event.
		result.push({
		...this.lastHoverMessage,
		action: AndroidMotionEventAction.Up,
		});
		this.lastHoverMessage = undefined;
		}

		if (message.action === AndroidMotionEventAction.HoverMove) {
		// TODO: Inject MotionEvent.ACTION_HOVER_ENTER
		this.lastHoverMessage = message as ScrcpyInjectTouchControlMessage;
		}

		(message as ScrcpyInjectTouchControlMessage).type =
		ScrcpyControlMessageType.InjectTouch;
		result.push(message as ScrcpyInjectTouchControlMessage);

		return result;
		}
		}

libraries/scrcpy/src/control/index.ts

+1 −0

Original line number	Diff line number	Diff line
		export * from "./back-or-screen-on.js";
		export * from "./hover-helper.js";
		export * from "./inject-keycode.js";
		export * from "./inject-scroll.js";
		export * from "./inject-text.js";

libraries/scrcpy/src/control/inject-touch.ts

+13 −1

Original line number	Diff line number	Diff line
		@@ -31,6 +31,18 @@ export namespace ScrcpyPointerId {
		export const VirtualFinger = BigInt(-4);
		}

		export function clamp(value: number, min: number, max: number): number {
		if (value < min) {
		return min;
		}

		if (value > max) {
		return max;
		}

		return value;
		}

		const Uint16Max = (1 << 16) - 1;

		const ScrcpyFloatToUint16NumberType: NumberFieldType = {
		@@ -41,7 +53,7 @@ const ScrcpyFloatToUint16NumberType: NumberFieldType = {
		return value / Uint16Max;
		},
		serialize(dataView, offset, value, littleEndian) {
		value = value * Uint16Max;
		value = clamp(value, 0, 1) * Uint16Max;
		NumberFieldType.Uint16.serialize(dataView, offset, value, littleEndian);
		},
		};

libraries/scrcpy/src/options/1_16/scroll.ts

+4 −4

Original line number	Diff line number	Diff line
		@@ -40,18 +40,18 @@ export class ScrcpyScrollController1_16 implements ScrcpyScrollController {
		let scrollY = 0;
		if (this.accumulatedX >= 1) {
		scrollX = 1;
		this.accumulatedX -= 1;
		this.accumulatedX = 0;
		} else if (this.accumulatedX <= -1) {
		scrollX = -1;
		this.accumulatedX += 1;
		this.accumulatedX = 0;
		}

		if (this.accumulatedY >= 1) {
		scrollY = 1;
		this.accumulatedY -= 1;
		this.accumulatedY = 0;
		} else if (this.accumulatedY <= -1) {
		scrollY = -1;
		this.accumulatedY += 1;
		this.accumulatedY = 0;
		}

		if (scrollX === 0 && scrollY === 0) {