Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 49ca40c4 authored by Zhekai Hu's avatar Zhekai Hu
Browse files

Add a new function to export virtual assist node to ccapi 

Latency:
notifyVirtualChildrenAppearedWithCustomView_mean (ns): 9258196
notifyVirtualChildrenAppearedWithCustomView_median (ns): 9274098
notifyVirtualChildrenAppearedWithCustomView_min (ns): 9096326
notifyVirtualChildrenAppearedWithCustomView_standardDeviation: 138959

Test: atest android.view.contentcapture.LoginTest#testNotifyVirtualChildrenAppearedWithCustomView
Flag: android.view.contentcapture.flags.enable_export_assist_virtual_node_to_ccapi
Bug: 430421182

Change-Id: I97ab9b05e0f696a2e8cace3c04b1e439dee6ef9b
parent f067faec

apct-tests/perftests/contentcapture/res/layout/test_export_virtual_assist_node_activity.xml

0 → 100644
+17 −0
Original line number Diff line number Diff line 
<?xml version="1.0" encoding="utf-8"?>

<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"

    android:layout_width="match_parent"

    android:layout_height="match_parent"

    android:orientation="vertical">



    <TextView

        android:layout_width="wrap_content"

        android:layout_height="wrap_content"

        android:text="a standard text view"/>



    <android.view.contentcapture.MyCustomViewWithA11yProvider

        android:id="@+id/custom_view_with_a11y_provider"

        android:layout_width="100dp"

        android:layout_height="100dp"/>



</LinearLayout>
 
 No newline at end of file

apct-tests/perftests/contentcapture/src/android/view/contentcapture/LoginTest.java

+37 −1
Original line number Diff line number Diff line
@@ -21,12 +21,12 @@ import android.content.Intent; 
import android.os.RemoteCallback;

import android.perftests.utils.BenchmarkState;

import android.view.View;



import androidx.test.filters.LargeTest;



import com.android.compatibility.common.util.ActivitiesWatcher.ActivityWatcher;

import com.android.perftests.contentcapture.R;



import org.junit.Assert;

import org.junit.Test;



@LargeTest
@@ -231,4 +231,40 @@ public class LoginTest extends AbstractContentCapturePerfTestCase { 
            state.resumeTiming();

        }

    }



    @Test

    public void testNotifyVirtualChildrenAppearedWithCustomView() throws Throwable {

        // Arrange

        MyContentCaptureService service = enableService();

        CustomTestActivity activity = launchActivity(

                R.layout.test_export_virtual_assist_node_activity, 0);

        // TODO: Add multiple views with virtual children and check the performance.

        View hostView = activity.findViewById(R.id.custom_view_with_a11y_provider);

        hostView.setImportantForContentCapture(View.IMPORTANT_FOR_CONTENT_CAPTURE_YES);

        // Expected: 1 for a11y host node (-1), 3 for virtual children (1, 2, 3)

        // 1 for host view

        int expectedViewAppeared = 5;

        long eventTimeoutMs = 20000;

        BenchmarkState state = mPerfStatusReporter.getBenchmarkState();



        // Act

        while (state.keepRunning()) {

            state.pauseTiming();

            service.clearEvents();

            // Trigger content capture structure provision.

            sInstrumentation.runOnMainSync(() -> hostView.setVisibility(View.GONE));

            sInstrumentation.waitForIdleSync();

            state.resumeTiming();

            sInstrumentation.runOnMainSync(() -> hostView.setVisibility(View.VISIBLE));

            sInstrumentation.waitForIdleSync();

            state.pauseTiming();

            service.waitForAppearedEvents(expectedViewAppeared, eventTimeoutMs);



            // Assert

            Assert.assertEquals("Expected " + expectedViewAppeared

                            + " TYPE_VIEW_APPEARED events", expectedViewAppeared,

                    service.getAppearedCount());

            state.resumeTiming();

        }

    }

}

apct-tests/perftests/contentcapture/src/android/view/contentcapture/MyContentCaptureService.java

+65 −1
Original line number Diff line number Diff line
@@ -25,9 +25,13 @@ import android.util.Pair; 
import androidx.annotation.NonNull;

import androidx.annotation.Nullable;



import java.util.ArrayList;

import java.util.List;

import java.util.Set;

import java.util.concurrent.CountDownLatch;

import java.util.concurrent.TimeUnit;

import java.util.concurrent.locks.Condition;

import java.util.concurrent.locks.ReentrantLock;



public class MyContentCaptureService extends ContentCaptureService {
@@ -37,7 +41,10 @@ public class MyContentCaptureService extends ContentCaptureService { 
            + MyContentCaptureService.class.getName();



    private static ServiceWatcher sServiceWatcher;



    private final ReentrantLock lock = new ReentrantLock();

    private final Condition eventsChanged = lock.newCondition();

    private final List<ContentCaptureEvent> mCapturedEvents = new ArrayList<>();

    private int appearedCount = 0;

    @NonNull

    public static ServiceWatcher setServiceWatcher() {

        if (sServiceWatcher != null) {
@@ -114,12 +121,69 @@ public class MyContentCaptureService extends ContentCaptureService { 
    public void onContentCaptureEvent(ContentCaptureSessionId sessionId,

            ContentCaptureEvent event) {

        Log.i(TAG, "onContentCaptureEventsRequest(session=" + sessionId + "): " + event);

        lock.lock();

        try {

            mCapturedEvents.add(event);

            if (event.getType() == ContentCaptureEvent.TYPE_VIEW_APPEARED) {

                appearedCount++;

            }

            eventsChanged.signalAll();

        } finally {

            lock.unlock();

        }

        if (sServiceWatcher != null

                && event.getType() == ContentCaptureEvent.TYPE_SESSION_PAUSED) {

            sServiceWatcher.mSessionPaused.countDown();

        }

    }



    public void clearEvents() {

        lock.lock();

        try {

            mCapturedEvents.clear();

            appearedCount = 0;

        } finally {

            lock.unlock();

        }

        Log.i(TAG, "Cleared captured events");

    }



    public List<ContentCaptureEvent> getCapturedEvents() {

        lock.lock();

        try {

            return new ArrayList<>(mCapturedEvents);

        } finally {

            lock.unlock();

        }

    }



    public int getAppearedCount() {

        lock.lock();

        try {

            return appearedCount;

        } finally {

            lock.unlock();

        }

    }



    public boolean waitForAppearedEvents(

            int expectedCount, long timeoutMillis) throws InterruptedException {

        long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis);

        lock.lock();

        try {

            while (appearedCount < expectedCount) {

                long remainingNanos = deadline - System.nanoTime();

                if (remainingNanos <= 0) {

                    return false;

                }

                eventsChanged.await(remainingNanos, TimeUnit.NANOSECONDS);

            }

            return true;

        } finally {

            lock.unlock();

        }

    }



    @Override

    public void onActivityEvent(ActivityEvent event) {

        Log.i(TAG, "onActivityEvent(): " + event);

apct-tests/perftests/contentcapture/src/android/view/contentcapture/MyCustomViewWithA11yProvider.java

0 → 100644
+116 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2025 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package android.view.contentcapture;



import android.content.Context;

import android.graphics.Rect;

import android.os.Bundle;

import android.util.AttributeSet;

import android.view.View;

import android.view.accessibility.AccessibilityNodeInfo;

import android.view.accessibility.AccessibilityNodeProvider;



import androidx.annotation.NonNull;

import androidx.annotation.Nullable;



import java.util.ArrayList;

import java.util.List;



public class MyCustomViewWithA11yProvider extends View {

    private MyAccessibilityNodeProvider mProvider;



    public MyCustomViewWithA11yProvider(Context context) {

        super(context);

        init();

    }



    public MyCustomViewWithA11yProvider(Context context, @Nullable AttributeSet attrs) {

        super(context, attrs);

        init();

    }



    public MyCustomViewWithA11yProvider(Context context,

            @Nullable AttributeSet attrs, int defStyleAttr) {

        super(context, attrs, defStyleAttr);

        init();

    }



    @Override

    public AccessibilityNodeProvider getAccessibilityNodeProvider() {

        return mProvider;

    }

    private void init() {

        mProvider = new MyAccessibilityNodeProvider(this);

    }



    private static class MyAccessibilityNodeProvider extends AccessibilityNodeProvider {

        private static final int VIRTUAL_CHILD_ID_1 = 1;

        private static final int VIRTUAL_CHILD_ID_2 = 2;

        private static final int VIRTUAL_CHILD_ID_3 = 3;

        private final View mHostView;

        private final String mPackageName;

        MyAccessibilityNodeProvider(View hostView) {

            mHostView = hostView;

            mPackageName = hostView.getContext().getPackageName();

        }



        @Override

        public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) {

            final AccessibilityNodeInfo info;

            if (virtualViewId == AccessibilityNodeProvider.HOST_VIEW_ID) {

                info = AccessibilityNodeInfo.obtain(mHostView);

                mHostView.onInitializeAccessibilityNodeInfo(info);

                info.setPackageName(mPackageName);

                info.addChild(mHostView, VIRTUAL_CHILD_ID_1);

                info.addChild(mHostView, VIRTUAL_CHILD_ID_2);

                info.addChild(mHostView, VIRTUAL_CHILD_ID_3);

                return info;

            }

            info = createChildrenAccessibilityNodeInfo(virtualViewId, 3);

            return info;

        }



        private AccessibilityNodeInfo createChildrenAccessibilityNodeInfo(

                int virtualViewId, int maxChildCount) {

            if (virtualViewId > maxChildCount) {

                return null;

            }

            AccessibilityNodeInfo info = AccessibilityNodeInfo.obtain(mHostView, virtualViewId);

            info.setPackageName(mPackageName);

            info.setClassName("VirtualChild" + virtualViewId + "Class");

            info.setText("Text" + virtualViewId);

            info.setContentDescription("CD" + virtualViewId);

            info.setClickable(true);

            info.setParent(mHostView);

            info.setVisibleToUser(true);

            info.setBoundsInScreen(new Rect(10 * (virtualViewId - 1), 0, 10, 10));

            return info;

        }

        @Override

        public List<AccessibilityNodeInfo> findAccessibilityNodeInfosByText(

                String text, int virtualViewId) {

            return new ArrayList<>();

        }

        @Override

        public AccessibilityNodeInfo findFocus(int focus) {

            return null;

        }

        @Override

        public void addExtraDataToAccessibilityNodeInfo(

                int virtualViewId, @NonNull AccessibilityNodeInfo info,

                @NonNull String extraDataKey, @Nullable Bundle arguments) {}

    }

}

core/java/android/view/contentcapture/MainContentCaptureSession.java

+62 −1
Original line number Diff line number Diff line
@@ -55,6 +55,8 @@ import android.util.SparseArray; 
import android.util.TimeUtils;

import android.view.View;

import android.view.ViewStructure;

import android.view.accessibility.AccessibilityNodeInfo;

import android.view.accessibility.AccessibilityNodeProvider;

import android.view.autofill.AutofillId;

import android.view.contentcapture.ViewNode.ViewStructureImpl;

import android.view.contentcapture.flags.Flags;
@@ -248,6 +250,57 @@ public final class MainContentCaptureSession extends ContentCaptureSession { 
        mEventProcessQueue = new ConcurrentLinkedQueue<>();

    }



    private void notifyVirtualChildrenAppeared(@NonNull ContentCaptureSession session,

            @NonNull AutofillId hostAutofillId,

            @NonNull AccessibilityNodeProvider provider) {

        try {

            notifyVirtualChildrenAppearedHelper(hostAutofillId, hostAutofillId,

                    provider, session, AccessibilityNodeProvider.HOST_VIEW_ID);

        } catch (Exception e) {

            Log.w(TAG, "Error adding virtual children", e);

        }

    }



    /**

     * Populates the {@link ViewStructure} for each virtual child,

     * and notifies the {@link ContentCaptureSession} by calling

     * {@link ContentCaptureSession#notifyViewAppeared(ViewStructure)}

     */

    private void notifyVirtualChildrenAppearedHelper(@NonNull AutofillId hostAutofillId,

            @NonNull AutofillId parentAutofillId,

            @NonNull AccessibilityNodeProvider provider,

            @NonNull ContentCaptureSession session,

            int virtualId) {

        AccessibilityNodeInfo currentNodeInfo = provider.createAccessibilityNodeInfo(virtualId);

        if (currentNodeInfo == null) {

            return;

        }

        AutofillId currentAutofillId = session.newAutofillId(hostAutofillId, virtualId);

        ViewStructure currentViewStructure = session.newVirtualViewStructure(

                parentAutofillId, virtualId);

        currentViewStructure.setAutofillId(currentAutofillId);



        currentViewStructure.setText(currentNodeInfo.getText());

        currentViewStructure.setClassName(currentNodeInfo.getClassName() != null

                ? currentNodeInfo.getClassName().toString() : "VirtualNode");

        currentViewStructure.setContentDescription(

                currentNodeInfo.getContentDescription());

        currentViewStructure.setClickable(currentNodeInfo.isClickable());

        session.notifyViewAppeared(currentViewStructure);



        final int childCount = currentNodeInfo.getChildCount();

        if (childCount == 0) {

            return;

        }



        for (int i = 0; i < childCount; i++) {

            long childNodeId = currentNodeInfo.getChildId(i);

            int childVirtualId = AccessibilityNodeInfo

                    .getVirtualDescendantId(childNodeId);

            notifyVirtualChildrenAppearedHelper(hostAutofillId, currentAutofillId,

                    provider, session, childVirtualId);

        }

    }

    @Override

    ContentCaptureSession getMainCaptureSession() {

        return this;
@@ -977,7 +1030,15 @@ public final class MainContentCaptureSession extends ContentCaptureSession { 
                    }

                    ViewStructure structure = session.newViewStructure(view);

                    view.onProvideContentCaptureStructure(structure, /* flags= */ 0);



                    if (Flags.enableExportAssistVirtualNodeToCcapi()

                            && view.getAccessibilityNodeProvider() != null

                            && structure.getAutofillId() != null) {

                        // TODO: Move this to a background thread to improve performance.

                        Trace.beginSection("notifyVirtualChildrenAppeared");

                        notifyVirtualChildrenAppeared(session, structure.getAutofillId(),

                                view.getAccessibilityNodeProvider());

                        Trace.endSection();

                    }

                    structureSession.setSession(session);

                    structureSession.setStructure(structure);

                }