Merge "Update to ToT RemoteCompose" into main (00ff8969) · Commits · e / os / android_frameworks_base

core/java/com/android/internal/widget/remotecompose/accessibility/AndroidPlatformSemanticNodeApplier.java

0 → 100644

+171 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2024 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/
		package com.android.internal.widget.remotecompose.accessibility;

		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.graphics.Rect;
		import android.view.accessibility.AccessibilityNodeInfo;

		import com.android.internal.widget.remotecompose.core.operations.layout.Component;
		import com.android.internal.widget.remotecompose.core.semantics.AccessibilitySemantics;
		import com.android.internal.widget.remotecompose.core.semantics.AccessibleComponent;
		import com.android.internal.widget.remotecompose.core.semantics.CoreSemantics;

		import java.util.List;

		public class AndroidPlatformSemanticNodeApplier
		implements SemanticNodeApplier<AccessibilityNodeInfo, Component, AccessibilitySemantics> {

		private static final String ROLE_DESCRIPTION_KEY = "AccessibilityNodeInfo.roleDescription";

		@Override
		public void applyComponent(
		@NonNull
		RemoteComposeDocumentAccessibility<Component, AccessibilitySemantics>
		remoteComposeAccessibility,
		AccessibilityNodeInfo nodeInfo,
		Component component,
		List<AccessibilitySemantics> semantics) {
		if (component instanceof AccessibleComponent) {
		applyContentDescription(
		((AccessibleComponent) component).getContentDescriptionId(),
		nodeInfo,
		remoteComposeAccessibility);

		applyRole(((AccessibleComponent) component).getRole(), nodeInfo);
		}

		applySemantics(remoteComposeAccessibility, nodeInfo, semantics);

		float[] locationInWindow = new float[2];
		component.getLocationInWindow(locationInWindow);
		Rect bounds =
		new Rect(
		(int) locationInWindow[0],
		(int) locationInWindow[1],
		(int) (locationInWindow[0] + component.getWidth()),
		(int) (locationInWindow[1] + component.getHeight()));
		//noinspection deprecation
		nodeInfo.setBoundsInParent(bounds);
		nodeInfo.setBoundsInScreen(bounds);

		if (component instanceof AccessibleComponent) {
		applyContentDescription(
		((AccessibleComponent) component).getContentDescriptionId(),
		nodeInfo,
		remoteComposeAccessibility);

		applyText(
		((AccessibleComponent) component).getTextId(),
		nodeInfo,
		remoteComposeAccessibility);

		applyRole(((AccessibleComponent) component).getRole(), nodeInfo);
		}

		applySemantics(remoteComposeAccessibility, nodeInfo, semantics);

		if (nodeInfo.getText() == null && nodeInfo.getContentDescription() == null) {
		nodeInfo.setContentDescription("");
		}
		}

		public void applySemantics(
		RemoteComposeDocumentAccessibility<Component, AccessibilitySemantics>
		remoteComposeAccessibility,
		AccessibilityNodeInfo nodeInfo,
		List<AccessibilitySemantics> semantics) {
		for (AccessibilitySemantics semantic : semantics) {
		if (semantic.isInterestingForSemantics()) {
		if (semantic instanceof CoreSemantics) {
		applyCoreSemantics(
		remoteComposeAccessibility, nodeInfo, (CoreSemantics) semantic);
		} else if (semantic instanceof AccessibleComponent) {
		AccessibleComponent s = (AccessibleComponent) semantic;

		applyContentDescription(
		s.getContentDescriptionId(), nodeInfo, remoteComposeAccessibility);

		applyRole(s.getRole(), nodeInfo);

		applyText(s.getTextId(), nodeInfo, remoteComposeAccessibility);

		if (s.isClickable()) {
		nodeInfo.setClickable(true);
		nodeInfo.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_CLICK);
		}
		}
		}
		}
		}

		private void applyCoreSemantics(
		RemoteComposeDocumentAccessibility<Component, AccessibilitySemantics>
		remoteComposeAccessibility,
		AccessibilityNodeInfo nodeInfo,
		CoreSemantics semantics) {
		applyContentDescription(
		semantics.getContentDescriptionId(), nodeInfo, remoteComposeAccessibility);

		applyRole(semantics.getRole(), nodeInfo);

		applyText(semantics.getTextId(), nodeInfo, remoteComposeAccessibility);

		applyStateDescription(
		semantics.getStateDescriptionId(), nodeInfo, remoteComposeAccessibility);

		nodeInfo.setEnabled(semantics.mEnabled);
		}

		void applyRole(@Nullable AccessibleComponent.Role role, AccessibilityNodeInfo nodeInfo) {
		if (role != null) {
		nodeInfo.getExtras().putCharSequence(ROLE_DESCRIPTION_KEY, role.getDescription());
		}
		}

		void applyContentDescription(
		@Nullable Integer contentDescriptionId,
		AccessibilityNodeInfo nodeInfo,
		RemoteComposeDocumentAccessibility<Component, AccessibilitySemantics>
		remoteComposeAccessibility) {
		if (contentDescriptionId != null) {
		nodeInfo.setContentDescription(
		remoteComposeAccessibility.stringValue(contentDescriptionId));
		}
		}

		void applyText(
		@Nullable Integer textId,
		AccessibilityNodeInfo nodeInfo,
		RemoteComposeDocumentAccessibility<Component, AccessibilitySemantics>
		remoteComposeAccessibility) {
		if (textId != null) {
		nodeInfo.setText(remoteComposeAccessibility.stringValue(textId));
		}
		}

		void applyStateDescription(
		@Nullable Integer stateDescriptionId,
		AccessibilityNodeInfo nodeInfo,
		RemoteComposeDocumentAccessibility<Component, AccessibilitySemantics>
		remoteComposeAccessibility) {
		if (stateDescriptionId != null) {
		nodeInfo.setStateDescription(
		remoteComposeAccessibility.stringValue(stateDescriptionId));
		}
		}
		}

core/java/com/android/internal/widget/remotecompose/accessibility/CoreDocumentAccessibility.java

0 → 100644

+182 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2024 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/
		package com.android.internal.widget.remotecompose.accessibility;

		import android.annotation.Nullable;
		import android.graphics.PointF;
		import android.graphics.Rect;
		import android.os.Bundle;

		import com.android.internal.widget.remotecompose.core.CoreDocument;
		import com.android.internal.widget.remotecompose.core.operations.layout.ClickModifierOperation;
		import com.android.internal.widget.remotecompose.core.operations.layout.Component;
		import com.android.internal.widget.remotecompose.core.operations.layout.LayoutComponent;
		import com.android.internal.widget.remotecompose.core.operations.layout.RootLayoutComponent;
		import com.android.internal.widget.remotecompose.core.operations.layout.modifiers.ComponentModifiers;
		import com.android.internal.widget.remotecompose.core.operations.layout.modifiers.ModifierOperation;
		import com.android.internal.widget.remotecompose.core.semantics.AccessibilitySemantics;
		import com.android.internal.widget.remotecompose.core.semantics.AccessibleComponent;
		import com.android.internal.widget.remotecompose.core.semantics.CoreSemantics;

		import java.util.Collections;
		import java.util.List;
		import java.util.Objects;
		import java.util.stream.Collectors;
		import java.util.stream.Stream;

		/**
		* Java Player implementation of the {@link RemoteComposeDocumentAccessibility} interface. Each item
		* in the semantic tree is a {@link Component} from the remote Compose UI. Each Component can have a
		* list of modifiers that must be tagged with {@link AccessibilitySemantics} either incidentally
		* (see {@link ClickModifierOperation}) or explicitly (see {@link CoreSemantics}).
		*/
		public class CoreDocumentAccessibility
		implements RemoteComposeDocumentAccessibility<Component, AccessibilitySemantics> {
		private final CoreDocument mDocument;

		private final Rect mMissingBounds = new Rect(0, 0, 1, 1);

		public CoreDocumentAccessibility(CoreDocument document) {
		this.mDocument = document;
		}

		@Nullable
		@Override
		public Integer getComponentIdAt(PointF point) {
		return RootId;
		}

		@Override
		public @Nullable Component findComponentById(int virtualViewId) {
		RootLayoutComponent root = mDocument.getRootLayoutComponent();

		if (root == null \|\| virtualViewId == -1) {
		return root;
		}

		return componentStream(root)
		.filter(op -> op.getComponentId() == virtualViewId)
		.findFirst()
		.orElse(null);
		}

		@Override
		public List<CoreSemantics.Mode> mergeMode(Component component) {
		if (!(component instanceof LayoutComponent)) {
		return Collections.singletonList(CoreSemantics.Mode.SET);
		}

		return ((LayoutComponent) component)
		.getComponentModifiers().getList().stream()
		.filter(i -> i instanceof AccessibleComponent)
		.map(i -> ((AccessibleComponent) i).getMode())
		.filter(Objects::nonNull)
		.collect(Collectors.toList());
		}

		@Override
		public boolean performAction(Component component, int action, Bundle arguments) {
		if (action == ACTION_CLICK) {
		mDocument.performClick(component.getComponentId());
		return true;
		} else {
		return false;
		}
		}

		@Nullable
		@Override
		public String stringValue(int id) {
		Object value = mDocument.getRemoteComposeState().getFromId(id);
		return value != null ? String.valueOf(value) : null;
		}

		@Override
		public List<AccessibilitySemantics> semanticModifiersForComponent(Component component) {
		if (!(component instanceof LayoutComponent)) {
		return Collections.emptyList();
		}

		List<ModifierOperation> modifiers =
		((LayoutComponent) component).getComponentModifiers().getList();

		return modifiers.stream()
		.filter(
		it ->
		it instanceof AccessibilitySemantics
		&& ((AccessibilitySemantics) it)
		.isInterestingForSemantics())
		.map(i -> (AccessibilitySemantics) i)
		.collect(Collectors.toList());
		}

		@Override
		public List<Integer> semanticallyRelevantChildComponents(Component component) {
		return componentStream(component)
		.filter(i -> i.getComponentId() != component.getComponentId())
		.filter(CoreDocumentAccessibility::isInteresting)
		.map(Component::getComponentId)
		.collect(Collectors.toList());
		}

		static Stream<Component> componentStream(Component root) {
		return Stream.concat(
		Stream.of(root),
		root.mList.stream()
		.flatMap(
		op -> {
		if (op instanceof Component) {
		return componentStream((Component) op);
		} else {
		return Stream.empty();
		}
		}));
		}

		static Stream<ModifierOperation> modifiersStream(Component component) {
		return component.mList.stream()
		.filter(it -> it instanceof ComponentModifiers)
		.flatMap(it -> ((ComponentModifiers) it).getList().stream());
		}

		static boolean isInteresting(Component component) {
		boolean interesting =
		isContainerWithSemantics(component)
		\|\| modifiersStream(component)
		.anyMatch(CoreDocumentAccessibility::isModifierWithSemantics);

		return interesting && component.isVisible();
		}

		static boolean isModifierWithSemantics(ModifierOperation modifier) {
		return modifier instanceof AccessibilitySemantics
		&& ((AccessibilitySemantics) modifier).isInterestingForSemantics();
		}

		static boolean isContainerWithSemantics(Component component) {
		if (component instanceof AccessibilitySemantics) {
		return ((AccessibilitySemantics) component).isInterestingForSemantics();
		}

		if (!(component instanceof LayoutComponent)) {
		return false;
		}

		return ((LayoutComponent) component)
		.getComponentModifiers().getList().stream()
		.anyMatch(CoreDocumentAccessibility::isModifierWithSemantics);
		}
		}

core/java/com/android/internal/widget/remotecompose/accessibility/PlatformRemoteComposeTouchHelper.java

0 → 100644

+163 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2024 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/
		package com.android.internal.widget.remotecompose.accessibility;

		import static com.android.internal.widget.remotecompose.accessibility.RemoteComposeDocumentAccessibility.RootId;

		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.graphics.PointF;
		import android.os.Bundle;
		import android.util.IntArray;
		import android.view.View;
		import android.view.accessibility.AccessibilityEvent;
		import android.view.accessibility.AccessibilityNodeInfo;

		import com.android.internal.widget.ExploreByTouchHelper;
		import com.android.internal.widget.remotecompose.core.CoreDocument;
		import com.android.internal.widget.remotecompose.core.operations.layout.Component;
		import com.android.internal.widget.remotecompose.core.semantics.AccessibilitySemantics;
		import com.android.internal.widget.remotecompose.core.semantics.CoreSemantics.Mode;

		import java.util.HashSet;
		import java.util.List;
		import java.util.Set;
		import java.util.Stack;

		public class PlatformRemoteComposeTouchHelper<N, C, S> extends ExploreByTouchHelper {
		private final RemoteComposeDocumentAccessibility<C, S> mRemoteDocA11y;

		private final SemanticNodeApplier<AccessibilityNodeInfo, C, S> mApplier;

		public PlatformRemoteComposeTouchHelper(
		View host,
		RemoteComposeDocumentAccessibility<C, S> remoteDocA11y,
		SemanticNodeApplier<AccessibilityNodeInfo, C, S> applier) {
		super(host);
		this.mRemoteDocA11y = remoteDocA11y;
		this.mApplier = applier;
		}

		public static PlatformRemoteComposeTouchHelper<
		AccessibilityNodeInfo, Component, AccessibilitySemantics>
		forRemoteComposePlayer(View player, @NonNull CoreDocument coreDocument) {
		return new PlatformRemoteComposeTouchHelper<>(
		player,
		new CoreDocumentAccessibility(coreDocument),
		new AndroidPlatformSemanticNodeApplier());
		}

		/**
		* Gets the virtual view ID at a given location on the screen.
		*
		* <p>This method is called by the Accessibility framework to determine which virtual view, if
		* any, is located at a specific point on the screen. It uses the {@link
		* RemoteComposeDocumentAccessibility#getComponentIdAt(PointF)} method to find the ID of the
		* component at the given coordinates.
		*
		* @param x The x-coordinate of the location in pixels.
		* @param y The y-coordinate of the location in pixels.
		* @return The ID of the virtual view at the given location, or {@link #INVALID_ID} if no
		* virtual view is found at that location.
		*/
		@Override
		protected int getVirtualViewAt(float x, float y) {
		Integer root = mRemoteDocA11y.getComponentIdAt(new PointF(x, y));

		if (root == null) {
		return INVALID_ID;
		}

		return root;
		}

		/**
		* Populates a list with the visible virtual view IDs.
		*
		* <p>This method is called by the accessibility framework to retrieve the IDs of all visible
		* virtual views in the accessibility hierarchy. It traverses the hierarchy starting from the
		* root node (RootId) and adds the ID of each visible view to the provided list.
		*
		* @param virtualViewIds The list to be populated with the visible virtual view IDs.
		*/
		@Override
		protected void getVisibleVirtualViews(IntArray virtualViewIds) {
		Stack<Integer> toVisit = new Stack<>();
		Set<Integer> visited = new HashSet<>();

		toVisit.push(RootId);

		while (!toVisit.isEmpty()) {
		Integer componentId = toVisit.remove(0);

		if (visited.add(componentId)) {
		virtualViewIds.add(componentId);

		C component = mRemoteDocA11y.findComponentById(componentId);

		if (component != null) {
		boolean allSet =
		mRemoteDocA11y.mergeMode(component).stream()
		.allMatch(i -> i == Mode.SET);

		if (allSet) {
		List<Integer> childViews =
		mRemoteDocA11y.semanticallyRelevantChildComponents(component);

		toVisit.addAll(childViews);
		}
		}
		}
		}
		}

		@Override
		public void onPopulateNodeForVirtualView(
		int virtualViewId, @NonNull AccessibilityNodeInfo node) {
		C component = mRemoteDocA11y.findComponentById(virtualViewId);

		List<Mode> mode = mRemoteDocA11y.mergeMode(component);

		if (mode.contains(Mode.MERGE)) {
		List<Integer> childViews =
		mRemoteDocA11y.semanticallyRelevantChildComponents(component);

		for (Integer childView : childViews) {
		onPopulateNodeForVirtualView(childView, node);
		}
		}

		List<S> semantics = mRemoteDocA11y.semanticModifiersForComponent(component);
		mApplier.applyComponent(mRemoteDocA11y, node, component, semantics);
		}

		@Override
		protected void onPopulateEventForVirtualView(int virtualViewId, AccessibilityEvent event) {
		// TODO
		}

		@Override
		protected boolean onPerformActionForVirtualView(
		int virtualViewId, int action, @Nullable Bundle arguments) {
		C component = mRemoteDocA11y.findComponentById(virtualViewId);

		if (component != null) {
		return mRemoteDocA11y.performAction(component, action, arguments);
		} else {
		return false;
		}
		}
		}

core/java/com/android/internal/widget/remotecompose/accessibility/RemoteComposeDocumentAccessibility.java

0 → 100644

+100 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2024 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/
		package com.android.internal.widget.remotecompose.accessibility;

		import android.annotation.Nullable;
		import android.graphics.PointF;
		import android.os.Bundle;
		import android.view.View;

		import com.android.internal.widget.remotecompose.core.semantics.CoreSemantics;

		import java.util.List;

		/**
		* Interface for interacting with the accessibility features of a remote Compose UI. This interface
		* provides methods to perform actions, retrieve state, and query the accessibility tree of the
		* remote Compose UI.
		*
		* @param <C> The type of component in the remote Compose UI.
		* @param <S> The type representing semantic modifiers applied to components.
		*/
		public interface RemoteComposeDocumentAccessibility<C, S> {
		// Matches ExploreByTouchHelper.HOST_ID
		Integer RootId = View.NO_ID;

		// androidx.core.view.accessibility.AccessibilityNodeInfoCompat.ACTION_CLICK
		int ACTION_CLICK = 0x00000010;

		/**
		* Performs the specified action on the given component.
		*
		* @param component The component on which to perform the action.
		* @param action The action to perform.
		* @param arguments Optional arguments for the action.
		* @return {@code true} if the action was performed successfully, {@code false} otherwise.
		*/
		boolean performAction(C component, int action, Bundle arguments);

		/**
		* Retrieves the string value associated with the given ID.
		*
		* @param id The ID to retrieve the string value for.
		* @return The string value associated with the ID, or {@code null} if no such value exists.
		*/
		@Nullable
		String stringValue(int id);

		/**
		* Retrieves a list of child view IDs semantically contained within the given component/virtual
		* view. These may later be hidden from accessibility services by properties, but should contain
		* only possibly semantically relevant virtual views.
		*
		* @param component The component to retrieve child view IDs from, or [RootId] for the top
		* level.
		* @return A list of integer IDs representing the child views of the component.
		*/
		List<Integer> semanticallyRelevantChildComponents(C component);

		/**
		* Retrieves the semantic modifiers associated with a given component.
		*
		* @param component The component for which to retrieve semantic modifiers.
		* @return A list of semantic modifiers applicable to the component.
		*/
		List<S> semanticModifiersForComponent(C component);

		/**
		* Gets all applied merge modes of the given component. A Merge mode is one of Set, Merge or
		* Clear and describes how to apply and combine hierarchical semantics.
		*
		* @param component The component to merge the mode for.
		* @return A list of merged modes, potentially conflicting but to be resolved by the caller.
		*/
		List<CoreSemantics.Mode> mergeMode(C component);

		/**
		* Finds a component by its ID.
		*
		* @param id the ID of the component to find
		* @return the component with the given ID, or {@code null} if no such component exists
		*/
		@Nullable
		C findComponentById(int id);

		@Nullable
		Integer getComponentIdAt(PointF point);
		}

core/java/com/android/internal/widget/remotecompose/accessibility/RemoteComposeTouchHelper.java

0 → 100644

+31 −0

File added.

Preview size limit exceeded, changes collapsed.