Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ceb6591e authored by Emilian Peev's avatar Emilian Peev
Browse files

Camera: Propagate extension results in case of dropped frames 

Capture results can be present and can be safely passed back to
client in case of frame drops within the source image reader.
Detect cases where results will get discarded, filter the tags
and notify clients accordingly.

Bug: 230790669
Test: Camera CTS
Change-Id: Ia3f77d9398165663073c01bdf8d3421fd86a67c4
parent f9ccda5c

core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java

+96 −33
Original line number Diff line number Diff line
@@ -72,7 +72,7 @@ import java.util.Set; 
import java.util.concurrent.Executor;



public final class CameraExtensionSessionImpl extends CameraExtensionSession {

    private static final int PREVIEW_QUEUE_SIZE = 3;

    private static final int PREVIEW_QUEUE_SIZE = 10;

    private static final String TAG = "CameraExtensionSessionImpl";



    private final Executor mExecutor;
@@ -1057,15 +1057,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { 
                                                mClientRequest));



                        if (mCaptureResultHandler != null) {

                            CameraMetadataNative captureResults = new CameraMetadataNative();

                            for (CaptureResult.Key key : mSupportedResultKeys) {

                                Object value = result.get(key);

                                if (value != null) {

                                    captureResults.set(key, value);

                                }

                            }

                            mCaptureResultHandler.onCaptureCompleted(timestamp,

                                    captureResults);

                                    initializeFilteredResults(result));

                        }

                    } finally {

                        Binder.restoreCallingIdentity(ident);
@@ -1126,6 +1119,11 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { 
        }



        private class ImageCallback implements OnImageAvailableListener {

            @Override

            public void onImageDropped(long timestamp) {

                notifyCaptureFailed();

            }



            @Override

            public void onImageAvailable(ImageReader reader, Image img) {

                if (mCaptureFailed) {
@@ -1159,6 +1157,9 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { 
    }



    private class ImageLoopbackCallback implements OnImageAvailableListener {

        @Override

        public void onImageDropped(long timestamp) { }



        @Override

        public void onImageAvailable(ImageReader reader, Image img) {

            img.close();
@@ -1221,7 +1222,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { 
    }



    private interface OnImageAvailableListener {

        public void onImageAvailable (ImageReader reader, Image img);

        void onImageDropped(long timestamp);

        void onImageAvailable (ImageReader reader, Image img);

    }



    private class CameraOutputImageCallback implements ImageReader.OnImageAvailableListener,
@@ -1263,6 +1265,29 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { 
            } else {

                mImageListenerMap.put(img.getTimestamp(), new Pair<>(img, null));

            }



            notifyDroppedImages(timestamp);

        }



        private void notifyDroppedImages(long timestamp) {

            Set<Long> timestamps = mImageListenerMap.keySet();

            ArrayList<Long> removedTs = new ArrayList<>();

            for (long ts : timestamps) {

                if (ts < timestamp) {

                    Log.e(TAG, "Dropped image with ts: " + ts);

                    Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.get(ts);

                    if (entry.second != null) {

                        entry.second.onImageDropped(ts);

                    }

                    if (entry.first != null) {

                        entry.first.close();

                    }

                    removedTs.add(ts);

                }

            }

            for (long ts : removedTs) {

                mImageListenerMap.remove(ts);

            }

        }



        public void registerListener(Long timestamp, OnImageAvailableListener listener) {
@@ -1291,6 +1316,12 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { 
                    entry.first.close();

                }

            }

            for (long timestamp : mImageListenerMap.keySet()) {

                Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.get(timestamp);

                if (entry.second != null) {

                    entry.second.onImageDropped(timestamp);

                }

            }

            mImageListenerMap.clear();

        }

    }
@@ -1447,7 +1478,6 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { 
            } else {

                notifyConfigurationFailure();

            }



        }



        @Override
@@ -1536,7 +1566,16 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { 
                    } else if (mPreviewProcessorType ==

                            IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) {

                        int idx = mPendingResultMap.indexOfKey(timestamp);

                        if (idx >= 0) {



                        if ((idx >= 0) && (mPendingResultMap.get(timestamp).first == null)) {

                            // Image was dropped before we can receive the capture results

                            if ((mCaptureResultHandler != null)) {

                                mCaptureResultHandler.onCaptureCompleted(timestamp,

                                        initializeFilteredResults(result));

                            }

                            discardPendingRepeatingResults(idx, mPendingResultMap, false);

                        } else  if (idx >= 0) {

                            // Image came before the capture results

                            ParcelImage parcelImage = initializeParcelImage(

                                    mPendingResultMap.get(timestamp).first);

                            try {
@@ -1563,6 +1602,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { 
                            }

                            discardPendingRepeatingResults(idx, mPendingResultMap, false);

                        } else {

                            // Image not yet available

                            notifyClient = false;

                            mPendingResultMap.put(timestamp,

                                    new Pair<>(null,
@@ -1581,16 +1621,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { 
                                                mClientRequest));

                                if ((mCaptureResultHandler != null) && (mPreviewProcessorType !=

                                        IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR)) {

                                    CameraMetadataNative captureResults =

                                            new CameraMetadataNative();

                                    for (CaptureResult.Key key : mSupportedResultKeys) {

                                        Object value = result.get(key);

                                        if (value != null) {

                                            captureResults.set(key, value);

                                        }

                                    }

                                    mCaptureResultHandler.onCaptureCompleted(timestamp,

                                            captureResults);

                                            initializeFilteredResults(result));

                                }

                            } else {

                                mExecutor.execute(
@@ -1657,8 +1689,13 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { 
            for (int i = idx; i >= 0; i--) {

                if (previewMap.valueAt(i).first != null) {

                    previewMap.valueAt(i).first.close();

                } else {

                    if (mClientNotificationsEnabled && ((i != idx) || notifyCurrentIndex)) {

                } else if (mClientNotificationsEnabled && (previewMap.valueAt(i).second != null) &&

                        ((i != idx) || notifyCurrentIndex)) {

                    TotalCaptureResult result = previewMap.valueAt(i).second;

                    Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP);

                    mCaptureResultHandler.onCaptureCompleted(timestamp,

                            initializeFilteredResults(result));



                    Log.w(TAG, "Preview frame drop with timestamp: " + previewMap.keyAt(i));

                    final long ident = Binder.clearCallingIdentity();

                    try {
@@ -1669,7 +1706,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { 
                    } finally {

                        Binder.restoreCallingIdentity(ident);

                    }

                    }



                }

                previewMap.removeAt(i);

            }
@@ -1682,6 +1719,12 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { 
                mOutputWriter = imageWriter;

            }



            @Override

            public void onImageDropped(long timestamp) {

                discardPendingRepeatingResults(mPendingResultMap.indexOfKey(timestamp),

                        mPendingResultMap, true);

            }



            @Override

            public void onImageAvailable(ImageReader reader, Image img) {

                if (img == null) {
@@ -1702,6 +1745,15 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { 


        private class ImageProcessCallback implements OnImageAvailableListener {



            @Override

            public void onImageDropped(long timestamp) {

                discardPendingRepeatingResults(mPendingResultMap.indexOfKey(timestamp),

                        mPendingResultMap, true);

                // Add an empty frame&results entry to flag that we dropped a frame

                // and valid capture results can immediately return to client.

                mPendingResultMap.put(timestamp, new Pair<>(null, null));

            }



            @Override

            public void onImageAvailable(ImageReader reader, Image img) {

                if (mPendingResultMap.size() + 1 >= PREVIEW_QUEUE_SIZE) {
@@ -1768,6 +1820,17 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { 
        }

    }



    private CameraMetadataNative initializeFilteredResults(TotalCaptureResult result) {

        CameraMetadataNative captureResults = new CameraMetadataNative();

        for (CaptureResult.Key key : mSupportedResultKeys) {

            Object value = result.get(key);

            if (value != null) {

                captureResults.set(key, value);

            }

        }

        return captureResults;

    }



    private static Size findSmallestAspectMatchedSize(@NonNull List<Size> sizes,

            @NonNull Size arSize) {

        final float TOLL = .01f;