Merge "Add a new function to export virtual assist node to ccapi" into main (5e7e47ba) · Commits · e / os / android_frameworks_base

apct-tests/perftests/contentcapture/res/layout/test_export_virtual_assist_node_activity.xml

0 → 100644

+17 −0

Original line number	Diff line number	Diff line
		<?xml version="1.0" encoding="utf-8"?>
		<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
		android:layout_width="match_parent"
		android:layout_height="match_parent"
		android:orientation="vertical">

		<TextView
		android:layout_width="wrap_content"
		android:layout_height="wrap_content"
		android:text="a standard text view"/>

		<android.view.contentcapture.MyCustomViewWithA11yProvider
		android:id="@+id/custom_view_with_a11y_provider"
		android:layout_width="100dp"
		android:layout_height="100dp"/>

		</LinearLayout>
		No newline at end of file

apct-tests/perftests/contentcapture/src/android/view/contentcapture/LoginTest.java

+37 −1

Original line number	Diff line number	Diff line
		@@ -21,12 +21,12 @@ import android.content.Intent;
		import android.os.RemoteCallback;
		import android.perftests.utils.BenchmarkState;
		import android.view.View;

		import androidx.test.filters.LargeTest;

		import com.android.compatibility.common.util.ActivitiesWatcher.ActivityWatcher;
		import com.android.perftests.contentcapture.R;

		import org.junit.Assert;
		import org.junit.Test;

		@LargeTest
		@@ -231,4 +231,40 @@ public class LoginTest extends AbstractContentCapturePerfTestCase {
		state.resumeTiming();
		}
		}

		@Test
		public void testNotifyVirtualChildrenAppearedWithCustomView() throws Throwable {
		// Arrange
		MyContentCaptureService service = enableService();
		CustomTestActivity activity = launchActivity(
		R.layout.test_export_virtual_assist_node_activity, 0);
		// TODO: Add multiple views with virtual children and check the performance.
		View hostView = activity.findViewById(R.id.custom_view_with_a11y_provider);
		hostView.setImportantForContentCapture(View.IMPORTANT_FOR_CONTENT_CAPTURE_YES);
		// Expected: 1 for a11y host node (-1), 3 for virtual children (1, 2, 3)
		// 1 for host view
		int expectedViewAppeared = 5;
		long eventTimeoutMs = 20000;
		BenchmarkState state = mPerfStatusReporter.getBenchmarkState();

		// Act
		while (state.keepRunning()) {
		state.pauseTiming();
		service.clearEvents();
		// Trigger content capture structure provision.
		sInstrumentation.runOnMainSync(() -> hostView.setVisibility(View.GONE));
		sInstrumentation.waitForIdleSync();
		state.resumeTiming();
		sInstrumentation.runOnMainSync(() -> hostView.setVisibility(View.VISIBLE));
		sInstrumentation.waitForIdleSync();
		state.pauseTiming();
		service.waitForAppearedEvents(expectedViewAppeared, eventTimeoutMs);

		// Assert
		Assert.assertEquals("Expected " + expectedViewAppeared
		+ " TYPE_VIEW_APPEARED events", expectedViewAppeared,
		service.getAppearedCount());
		state.resumeTiming();
		}
		}
		}

apct-tests/perftests/contentcapture/src/android/view/contentcapture/MyContentCaptureService.java

+65 −1

Original line number	Diff line number	Diff line
		@@ -25,9 +25,13 @@ import android.util.Pair;
		import androidx.annotation.NonNull;
		import androidx.annotation.Nullable;

		import java.util.ArrayList;
		import java.util.List;
		import java.util.Set;
		import java.util.concurrent.CountDownLatch;
		import java.util.concurrent.TimeUnit;
		import java.util.concurrent.locks.Condition;
		import java.util.concurrent.locks.ReentrantLock;

		public class MyContentCaptureService extends ContentCaptureService {

		@@ -37,7 +41,10 @@ public class MyContentCaptureService extends ContentCaptureService {
		+ MyContentCaptureService.class.getName();

		private static ServiceWatcher sServiceWatcher;

		private final ReentrantLock lock = new ReentrantLock();
		private final Condition eventsChanged = lock.newCondition();
		private final List<ContentCaptureEvent> mCapturedEvents = new ArrayList<>();
		private int appearedCount = 0;
		@NonNull
		public static ServiceWatcher setServiceWatcher() {
		if (sServiceWatcher != null) {
		@@ -114,12 +121,69 @@ public class MyContentCaptureService extends ContentCaptureService {
		public void onContentCaptureEvent(ContentCaptureSessionId sessionId,
		ContentCaptureEvent event) {
		Log.i(TAG, "onContentCaptureEventsRequest(session=" + sessionId + "): " + event);
		lock.lock();
		try {
		mCapturedEvents.add(event);
		if (event.getType() == ContentCaptureEvent.TYPE_VIEW_APPEARED) {
		appearedCount++;
		}
		eventsChanged.signalAll();
		} finally {
		lock.unlock();
		}
		if (sServiceWatcher != null
		&& event.getType() == ContentCaptureEvent.TYPE_SESSION_PAUSED) {
		sServiceWatcher.mSessionPaused.countDown();
		}
		}

		public void clearEvents() {
		lock.lock();
		try {
		mCapturedEvents.clear();
		appearedCount = 0;
		} finally {
		lock.unlock();
		}
		Log.i(TAG, "Cleared captured events");
		}

		public List<ContentCaptureEvent> getCapturedEvents() {
		lock.lock();
		try {
		return new ArrayList<>(mCapturedEvents);
		} finally {
		lock.unlock();
		}
		}

		public int getAppearedCount() {
		lock.lock();
		try {
		return appearedCount;
		} finally {
		lock.unlock();
		}
		}

		public boolean waitForAppearedEvents(
		int expectedCount, long timeoutMillis) throws InterruptedException {
		long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis);
		lock.lock();
		try {
		while (appearedCount < expectedCount) {
		long remainingNanos = deadline - System.nanoTime();
		if (remainingNanos <= 0) {
		return false;
		}
		eventsChanged.await(remainingNanos, TimeUnit.NANOSECONDS);
		}
		return true;
		} finally {
		lock.unlock();
		}
		}

		@Override
		public void onActivityEvent(ActivityEvent event) {
		Log.i(TAG, "onActivityEvent(): " + event);

apct-tests/perftests/contentcapture/src/android/view/contentcapture/MyCustomViewWithA11yProvider.java

0 → 100644

+116 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2025 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/
		package android.view.contentcapture;

		import android.content.Context;
		import android.graphics.Rect;
		import android.os.Bundle;
		import android.util.AttributeSet;
		import android.view.View;
		import android.view.accessibility.AccessibilityNodeInfo;
		import android.view.accessibility.AccessibilityNodeProvider;

		import androidx.annotation.NonNull;
		import androidx.annotation.Nullable;

		import java.util.ArrayList;
		import java.util.List;

		public class MyCustomViewWithA11yProvider extends View {
		private MyAccessibilityNodeProvider mProvider;

		public MyCustomViewWithA11yProvider(Context context) {
		super(context);
		init();
		}

		public MyCustomViewWithA11yProvider(Context context, @Nullable AttributeSet attrs) {
		super(context, attrs);
		init();
		}

		public MyCustomViewWithA11yProvider(Context context,
		@Nullable AttributeSet attrs, int defStyleAttr) {
		super(context, attrs, defStyleAttr);
		init();
		}

		@Override
		public AccessibilityNodeProvider getAccessibilityNodeProvider() {
		return mProvider;
		}
		private void init() {
		mProvider = new MyAccessibilityNodeProvider(this);
		}

		private static class MyAccessibilityNodeProvider extends AccessibilityNodeProvider {
		private static final int VIRTUAL_CHILD_ID_1 = 1;
		private static final int VIRTUAL_CHILD_ID_2 = 2;
		private static final int VIRTUAL_CHILD_ID_3 = 3;
		private final View mHostView;
		private final String mPackageName;
		MyAccessibilityNodeProvider(View hostView) {
		mHostView = hostView;
		mPackageName = hostView.getContext().getPackageName();
		}

		@Override
		public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) {
		final AccessibilityNodeInfo info;
		if (virtualViewId == AccessibilityNodeProvider.HOST_VIEW_ID) {
		info = AccessibilityNodeInfo.obtain(mHostView);
		mHostView.onInitializeAccessibilityNodeInfo(info);
		info.setPackageName(mPackageName);
		info.addChild(mHostView, VIRTUAL_CHILD_ID_1);
		info.addChild(mHostView, VIRTUAL_CHILD_ID_2);
		info.addChild(mHostView, VIRTUAL_CHILD_ID_3);
		return info;
		}
		info = createChildrenAccessibilityNodeInfo(virtualViewId, 3);
		return info;
		}

		private AccessibilityNodeInfo createChildrenAccessibilityNodeInfo(
		int virtualViewId, int maxChildCount) {
		if (virtualViewId > maxChildCount) {
		return null;
		}
		AccessibilityNodeInfo info = AccessibilityNodeInfo.obtain(mHostView, virtualViewId);
		info.setPackageName(mPackageName);
		info.setClassName("VirtualChild" + virtualViewId + "Class");
		info.setText("Text" + virtualViewId);
		info.setContentDescription("CD" + virtualViewId);
		info.setClickable(true);
		info.setParent(mHostView);
		info.setVisibleToUser(true);
		info.setBoundsInScreen(new Rect(10 * (virtualViewId - 1), 0, 10, 10));
		return info;
		}
		@Override
		public List<AccessibilityNodeInfo> findAccessibilityNodeInfosByText(
		String text, int virtualViewId) {
		return new ArrayList<>();
		}
		@Override
		public AccessibilityNodeInfo findFocus(int focus) {
		return null;
		}
		@Override
		public void addExtraDataToAccessibilityNodeInfo(
		int virtualViewId, @NonNull AccessibilityNodeInfo info,
		@NonNull String extraDataKey, @Nullable Bundle arguments) {}
		}
		}

core/java/android/view/contentcapture/MainContentCaptureSession.java

+62 −1

Original line number	Diff line number	Diff line
		@@ -55,6 +55,8 @@ import android.util.SparseArray;
		import android.util.TimeUtils;
		import android.view.View;
		import android.view.ViewStructure;
		import android.view.accessibility.AccessibilityNodeInfo;
		import android.view.accessibility.AccessibilityNodeProvider;
		import android.view.autofill.AutofillId;
		import android.view.contentcapture.ViewNode.ViewStructureImpl;
		import android.view.contentcapture.flags.Flags;
		@@ -248,6 +250,57 @@ public final class MainContentCaptureSession extends ContentCaptureSession {
		mEventProcessQueue = new ConcurrentLinkedQueue<>();
		}

		private void notifyVirtualChildrenAppeared(@NonNull ContentCaptureSession session,
		@NonNull AutofillId hostAutofillId,
		@NonNull AccessibilityNodeProvider provider) {
		try {
		notifyVirtualChildrenAppearedHelper(hostAutofillId, hostAutofillId,
		provider, session, AccessibilityNodeProvider.HOST_VIEW_ID);
		} catch (Exception e) {
		Log.w(TAG, "Error adding virtual children", e);
		}
		}

		/**
		* Populates the {@link ViewStructure} for each virtual child,
		* and notifies the {@link ContentCaptureSession} by calling
		* {@link ContentCaptureSession#notifyViewAppeared(ViewStructure)}
		*/
		private void notifyVirtualChildrenAppearedHelper(@NonNull AutofillId hostAutofillId,
		@NonNull AutofillId parentAutofillId,
		@NonNull AccessibilityNodeProvider provider,
		@NonNull ContentCaptureSession session,
		int virtualId) {
		AccessibilityNodeInfo currentNodeInfo = provider.createAccessibilityNodeInfo(virtualId);
		if (currentNodeInfo == null) {
		return;
		}
		AutofillId currentAutofillId = session.newAutofillId(hostAutofillId, virtualId);
		ViewStructure currentViewStructure = session.newVirtualViewStructure(
		parentAutofillId, virtualId);
		currentViewStructure.setAutofillId(currentAutofillId);

		currentViewStructure.setText(currentNodeInfo.getText());
		currentViewStructure.setClassName(currentNodeInfo.getClassName() != null
		? currentNodeInfo.getClassName().toString() : "VirtualNode");
		currentViewStructure.setContentDescription(
		currentNodeInfo.getContentDescription());
		currentViewStructure.setClickable(currentNodeInfo.isClickable());
		session.notifyViewAppeared(currentViewStructure);

		final int childCount = currentNodeInfo.getChildCount();
		if (childCount == 0) {
		return;
		}

		for (int i = 0; i < childCount; i++) {
		long childNodeId = currentNodeInfo.getChildId(i);
		int childVirtualId = AccessibilityNodeInfo
		.getVirtualDescendantId(childNodeId);
		notifyVirtualChildrenAppearedHelper(hostAutofillId, currentAutofillId,
		provider, session, childVirtualId);
		}
		}
		@Override
		ContentCaptureSession getMainCaptureSession() {
		return this;
		@@ -977,7 +1030,15 @@ public final class MainContentCaptureSession extends ContentCaptureSession {
		}
		ViewStructure structure = session.newViewStructure(view);
		view.onProvideContentCaptureStructure(structure, /* flags= */ 0);

		if (Flags.enableExportAssistVirtualNodeToCcapi()
		&& view.getAccessibilityNodeProvider() != null
		&& structure.getAutofillId() != null) {
		// TODO: Move this to a background thread to improve performance.
		Trace.beginSection("notifyVirtualChildrenAppeared");
		notifyVirtualChildrenAppeared(session, structure.getAutofillId(),
		view.getAccessibilityNodeProvider());
		Trace.endSection();
		}
		structureSession.setSession(session);
		structureSession.setStructure(structure);
		}