Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit db4c221a authored by Dorin Drimus's avatar Dorin Drimus
Browse files

Add CaptureRequest metadata for VirtualCamera 

If per frame camera metadata support is enabled, call
the onProcesRequestWithMetadata callback with an additional
CaptureRequest parameter.

Bug: 371167033
Test: atest VirtualCameraTest
Flag: android.companion.virtualdevice.flags.virtual_camera_metadata
Change-Id: If7b217a1da0929ec39c4572c1d6b8da864841330
parent d987f495

core/api/system-current.txt

+2 −0
Original line number Diff line number Diff line
@@ -3725,6 +3725,8 @@ package android.companion.virtual.camera { 
  public interface VirtualCameraCallback {

    method @FlaggedApi("android.companion.virtualdevice.flags.virtual_camera_on_open") public default void onOpenCamera();

    method public default void onProcessCaptureRequest(int, long);

    method @FlaggedApi("android.companion.virtualdevice.flags.virtual_camera_metadata") public default void onProcessCaptureRequest(int, long, @Nullable android.hardware.camera2.CaptureRequest);

    method @FlaggedApi("android.companion.virtualdevice.flags.virtual_camera_metadata") public default void onSessionConfigured(@NonNull android.hardware.camera2.params.SessionConfiguration, @Nullable java.util.function.ObjLongConsumer<android.hardware.camera2.CaptureResult>);

    method public void onStreamClosed(int);

    method public void onStreamConfigured(int, @NonNull android.view.Surface, @IntRange(from=1) int, @IntRange(from=1) int, int);

  }

core/java/android/companion/virtual/camera/IVirtualCameraCallback.aidl

+11 −7
Original line number Diff line number Diff line
@@ -15,7 +15,7 @@ 
 */



package android.companion.virtual.camera;



import android.hardware.camera2.CaptureRequest;

import android.view.Surface;



/**
@@ -24,7 +24,7 @@ import android.view.Surface; 
 *

 * @hide

 */

interface IVirtualCameraCallback {

oneway interface IVirtualCameraCallback {



    /**

     * Called when the client application calls
@@ -38,7 +38,7 @@ interface IVirtualCameraCallback { 
     * android.hardware.camera2.CameraCaptureSession.StateCallback#onConfigured(CameraCaptureSession)}

     * is called.

     */

    oneway void onOpenCamera();

    void onOpenCamera();



    /**

     * Called when one of the requested stream has been configured by the virtual camera service and
@@ -50,7 +50,7 @@ interface IVirtualCameraCallback { 
     * @param height The height of the surface

     * @param format The pixel format of the surface

     */

    oneway void onStreamConfigured(int streamId, in Surface surface, int width, int height,

    void onStreamConfigured(int streamId, in Surface surface, int width, int height,

        int format);



    /**
@@ -64,8 +64,12 @@ interface IVirtualCameraCallback { 
     *     streamId that was given in {@link #onStreamConfigured(int, Surface, int, int, int)}

     * @param frameId The frameId that is being requested. Each request will have a different

     *     frameId, that will be increasing for each call with a particular streamId.

     * @param captureRequest The capture request metadata provided by the app in association with

     *     the requested {@code frameId}. This is {@code null} id per frame camera metadata is not

     *     enabled or if unchanged from the previous frame.

     */

    oneway void onProcessCaptureRequest(int streamId, long frameId);

    void onProcessCaptureRequest(int streamId, long frameId,

        in @nullable CaptureRequest captureRequest);



    /**

     * The stream previously configured when
@@ -75,5 +79,5 @@ interface IVirtualCameraCallback { 
     *

     * @param streamId The id of the stream that was closed.

     */

    oneway void onStreamClosed(int streamId);

    void onStreamClosed(int streamId);

}
 
 No newline at end of file

core/java/android/companion/virtual/camera/VirtualCameraCallback.java

+58 −2
Original line number Diff line number Diff line
@@ -19,14 +19,18 @@ package android.companion.virtual.camera; 
import android.annotation.FlaggedApi;

import android.annotation.IntRange;

import android.annotation.NonNull;

import android.annotation.Nullable;

import android.annotation.SystemApi;

import android.companion.virtualdevice.flags.Flags;

import android.graphics.ImageFormat;

import android.hardware.camera2.CameraCaptureSession;

import android.hardware.camera2.CaptureRequest;

import android.hardware.camera2.CaptureResult;

import android.hardware.camera2.params.SessionConfiguration;

import android.view.Surface;



import java.util.concurrent.Executor;

import java.util.function.ObjLongConsumer;



/**

 * Interface to be provided when creating a new {@link VirtualCamera} in order to receive callbacks
@@ -50,8 +54,27 @@ public interface VirtualCameraCallback { 
     * is called.

     */

    @FlaggedApi(Flags.FLAG_VIRTUAL_CAMERA_ON_OPEN)

    default void onOpenCamera() {

    }

    default void onOpenCamera() {}



    /**

     * Called when the aoo using the {@link VirtualCamera} creates a new

     * {@link CameraCaptureSession}. This callback is sent after clients open and configure the

     * capture session for the virtual camera.

     *

     * @param sessionConfiguration The session configuration provided by the app in association

     *                             with the requested capture session.

     * @param captureResultConsumer The consumer interface through which the virtual camera owner

     *                             can pass {@link android.hardware.camera2.CaptureResult}s for

     *                              each timestamp of the streams associated with this session.

     *                              This is {@code null} if the per frame metadata is not enabled

     *                              in the {@link VirtualCameraConfig} of the virtual camera.

     *

     * @see VirtualCameraConfig.Builder#setPerFrameCameraMetadataEnabled(boolean)

     * @see CameraCaptureSession

     */

    @FlaggedApi(Flags.FLAG_VIRTUAL_CAMERA_METADATA)

    default void onSessionConfigured(@NonNull SessionConfiguration sessionConfiguration,

            @Nullable ObjLongConsumer<CaptureResult> captureResultConsumer) {}



    /**

     * Called when one of the requested stream has been configured by the virtual camera service and
@@ -77,13 +100,46 @@ public interface VirtualCameraCallback { 
     * this stream that was provided during the

     * {@link #onStreamConfigured(int, Surface, int, int, int)} call.

     *

     * <p>This callback is called <b>only</b> when the support for per frame camera metadata is

     * <b>disabled</b> (default value).

     *

     * @param streamId The streamId for which the frame is requested. This corresponds to the

     *     streamId that was given in {@link #onStreamConfigured(int, Surface, int, int, int)}

     * @param frameId The frameId that is being requested. Each request will have a different

     *     frameId, that will be increasing for each call with a particular streamId.

     *

     * @see VirtualCameraConfig.Builder#setPerFrameCameraMetadataEnabled(boolean)

     * @see #onProcessCaptureRequest(int, long, CaptureRequest)

     */

    default void onProcessCaptureRequest(int streamId, long frameId) {}



    /**

     * The client application is requesting a camera frame for the given streamId and frameId with

     * the provided {@link CaptureRequest} metadata.

     *

     * <p>The virtual camera needs to write the frame data in the {@link Surface} corresponding to

     * this stream that was provided during the

     * {@link #onStreamConfigured(int, Surface, int, int, int)} call.

     *

     * <p>This callback is called <b>instead</b> of the {@link #onProcessCaptureRequest(int, long)}

     * when support for per frame camera metadata is <b>enabled</b> with

     * {@link VirtualCameraConfig.Builder#setPerFrameCameraMetadataEnabled(boolean)}.

     *

     * @param streamId The streamId for which the frame is requested. This corresponds to the

     *      streamId that was given in {@link #onStreamConfigured(int, Surface, int, int, int)}.

     * @param frameId The frameId that is being requested. Each request will have a different

     *      frameId, that will be increasing for each call with a particular streamId.

     * @param captureRequest The {@link CaptureRequest} metadata provided by the app in association

     *      with the requested frameId. It is {@code null} if there is no change from the previous

     *      {@link CaptureRequest}.

     *

     * @see VirtualCameraConfig.Builder#setPerFrameCameraMetadataEnabled(boolean)

     * @see #onProcessCaptureRequest(int, long)

     */

    @FlaggedApi(Flags.FLAG_VIRTUAL_CAMERA_METADATA)

    default void onProcessCaptureRequest(int streamId, long frameId,

            @Nullable CaptureRequest captureRequest) {}



    /**

     * The stream previously configured when

     * {@link #onStreamConfigured(int, Surface, int, int, int)} was called is now being closed and

core/java/android/companion/virtual/camera/VirtualCameraConfig.java

+29 −18
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ import android.hardware.camera2.CameraMetadata; 
import android.hardware.camera2.CaptureRequest;

import android.hardware.camera2.CaptureResult;

import android.hardware.camera2.impl.CameraMetadataNative;

import android.hardware.camera2.params.SessionConfiguration;

import android.os.Parcel;

import android.os.Parcelable;

import android.util.ArraySet;
@@ -43,6 +44,7 @@ import java.lang.annotation.Retention; 
import java.lang.annotation.RetentionPolicy;

import java.util.Set;

import java.util.concurrent.Executor;

import java.util.function.ObjLongConsumer;



/**

 * Configuration to create a new {@link VirtualCamera}.
@@ -128,7 +130,8 @@ public final class VirtualCameraConfig implements Parcelable { 
        mCallback =

                new VirtualCameraCallbackInternal(

                        requireNonNull(callback, "Missing callback"),

                        requireNonNull(executor, "Missing callback executor"));

                        requireNonNull(executor, "Missing callback executor"),

                        perFrameCameraMetadataEnabled);

        mSensorOrientation = sensorOrientation;

        mPerFrameCameraMetadataEnabled = perFrameCameraMetadataEnabled;

        mCameraCharacteristics = cameraCharacteristics;
@@ -383,24 +386,22 @@ public final class VirtualCameraConfig implements Parcelable { 
            return this;

        }



        // TODO: b/371167033 - update docs and add links to

        //  onSessionConfigured, onProcessCaptureRequest, CaptureResultConsumer

        /**

         * Declares that the virtual camera owner wants to receive and provide

         * {@link CaptureRequest} and {@link CaptureResult} for every frame.

         * Declares that the virtual camera owner wants to receive {@link CaptureRequest} and

         * can provide {@link CaptureResult} for every frame.

         *

         * <p>This changes what methods from the {@link VirtualCameraCallback} are called.

         * When enabled,

         * <p>This changes which {@link VirtualCameraCallback} methods are called. When enabled,

         * {@link VirtualCameraCallback#onProcessCaptureRequest(int, long, CaptureRequest)}

         * is called and a non null {@link CaptureResultConsumer} is received in

         * {@link VirtualCameraCallback#onSessionConfigured(SessionConfiguration, CaptureResultConsumer)}.

         * When set, the virtual camera expects the {@link CaptureResult} to be passed for each

         * frame.

         * is called and a {@code Consumer} is received in

         * {@link VirtualCameraCallback#onSessionConfigured(SessionConfiguration, ObjLongConsumer)}.

         * The {@code Consumer} takes the {@link CaptureResult} and the timestamp (as {@code long})

         * for its parameters.

         *

         * @param perFrameCameraMetadataEnabled if camera metadata is handled for each frame

         * @see onSessionConfigured

         * @see onProcessCaptureRequest

         * @see CaptureResultConsumer

         * @param perFrameCameraMetadataEnabled if set camera metadata is handled for each frame

         *

         * @see VirtualCameraCallback#onSessionConfigured

         * @see VirtualCameraCallback#onProcessCaptureRequest(int, long)

         * @see VirtualCameraCallback#onProcessCaptureRequest(int, long, CaptureRequest)

         */

        @FlaggedApi(Flags.FLAG_VIRTUAL_CAMERA_METADATA)

        @NonNull
@@ -472,12 +473,16 @@ public final class VirtualCameraConfig implements Parcelable { 


        private final VirtualCameraCallback mCallback;

        private final Executor mExecutor;

        private final boolean mPerFrameCameraMetadataEnabled;



        private VirtualCameraCallbackInternal(VirtualCameraCallback callback, Executor executor) {

        private VirtualCameraCallbackInternal(VirtualCameraCallback callback, Executor executor,

                boolean perFrameCameraMetadataEnabled) {

            mCallback = callback;

            mExecutor = executor;

            mPerFrameCameraMetadataEnabled = perFrameCameraMetadataEnabled;

        }



        @Override

        public void onOpenCamera() {

            if (Flags.virtualCameraOnOpen()) {

                mExecutor.execute(mCallback::onOpenCamera);
@@ -492,9 +497,15 @@ public final class VirtualCameraConfig implements Parcelable { 
        }



        @Override

        public void onProcessCaptureRequest(int streamId, long frameId) {

        public void onProcessCaptureRequest(int streamId, long frameId,

                CaptureRequest captureRequest) {

            if (Flags.virtualCameraMetadata() && mPerFrameCameraMetadataEnabled) {

                mExecutor.execute(

                        () -> mCallback.onProcessCaptureRequest(streamId, frameId, captureRequest));

            } else {

                mExecutor.execute(() -> mCallback.onProcessCaptureRequest(streamId, frameId));

            }

        }



        @Override

        public void onStreamClosed(int streamId) {

services/companion/java/com/android/server/companion/virtual/camera/VirtualCameraConversionUtil.java

+39 −3
Original line number Diff line number Diff line
@@ -29,11 +29,15 @@ import android.companion.virtualdevice.flags.Flags; 
import android.graphics.ImageFormat;

import android.graphics.PixelFormat;

import android.hardware.camera2.CameraCharacteristics;

import android.hardware.camera2.CaptureRequest;

import android.hardware.camera2.impl.CameraMetadataNative;

import android.os.Parcel;

import android.os.RemoteException;

import android.util.Slog;

import android.view.Surface;



import java.util.Collections;



/** Utilities to convert the client side classes to the virtual camera service ones. */

public final class VirtualCameraConversionUtil {
@@ -49,7 +53,7 @@ public final class VirtualCameraConversionUtil { 
     * @throws RemoteException If there was an issue fetching the configuration from the client.

     */

    @NonNull

    public static android.companion.virtualcamera.VirtualCameraConfiguration

    public static VirtualCameraConfiguration

            getServiceCameraConfiguration(@NonNull VirtualCameraConfig cameraConfig) {

        VirtualCameraConfiguration serviceConfiguration = new VirtualCameraConfiguration();

        serviceConfiguration.supportedStreamConfigs =
@@ -87,8 +91,23 @@ public final class VirtualCameraConversionUtil { 
            }



            @Override

            public void onProcessCaptureRequest(int streamId, int frameId) throws RemoteException {

                camera.onProcessCaptureRequest(streamId, frameId);

            public void onProcessCaptureRequest(int streamId, int frameId,

                    VirtualCameraMetadata captureRequestSettings) throws RemoteException {

                CaptureRequest captureRequest = null;



                if (Flags.virtualCameraMetadata() && captureRequestSettings != null) {

                    CameraMetadataNative metadataNative = convertToCameraMetadataNative(

                            captureRequestSettings);

                    if (metadataNative != null) {

                        // Only the settings of the CaptureRequest are useful to the VD owner app

                        captureRequest = new CaptureRequest.Builder(metadataNative,

                                false /* reprocess */,

                                0 /* reprocessableSessionId */, "" /* logicalCameraId */,

                                Collections.emptySet() /* physicalCameraIdSet */).build();

                    }

                }



                camera.onProcessCaptureRequest(streamId, frameId, captureRequest);

            }



            @Override
@@ -145,4 +164,21 @@ public final class VirtualCameraConversionUtil { 
        }

        return virtualCameraMetadata;

    }



    private static CameraMetadataNative convertToCameraMetadataNative(

            @NonNull VirtualCameraMetadata virtualCameraMetadata) {

        CameraMetadataNative cameraMetadataNative = null;

        Parcel parcel = Parcel.obtain();

        try {

            parcel.unmarshall(virtualCameraMetadata.metadata, 0,

                    virtualCameraMetadata.metadata.length);

            parcel.setDataPosition(0);

            cameraMetadataNative = CameraMetadataNative.CREATOR.createFromParcel(parcel);

        } catch (Exception e) {

            Slog.w(TAG, "Failed to convert VirtualCameraMetadata to CameraMetadataNative.");

        } finally {

            parcel.recycle();

        }

        return cameraMetadataNative;

    }

}