Merge "Camera: Propagate extension results in case of dropped frames" into tm-dev (4d2ae265) · Commits · e / os / android_frameworks_base

core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java

+96 −33

Original line number	Diff line number	Diff line
		@@ -72,7 +72,7 @@ import java.util.Set;
		import java.util.concurrent.Executor;

		public final class CameraExtensionSessionImpl extends CameraExtensionSession {
		private static final int PREVIEW_QUEUE_SIZE = 3;
		private static final int PREVIEW_QUEUE_SIZE = 10;
		private static final String TAG = "CameraExtensionSessionImpl";

		private final Executor mExecutor;
		@@ -1057,15 +1057,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
		mClientRequest));

		if (mCaptureResultHandler != null) {
		CameraMetadataNative captureResults = new CameraMetadataNative();
		for (CaptureResult.Key key : mSupportedResultKeys) {
		Object value = result.get(key);
		if (value != null) {
		captureResults.set(key, value);
		}
		}
		mCaptureResultHandler.onCaptureCompleted(timestamp,
		captureResults);
		initializeFilteredResults(result));
		}
		} finally {
		Binder.restoreCallingIdentity(ident);
		@@ -1126,6 +1119,11 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
		}

		private class ImageCallback implements OnImageAvailableListener {
		@Override
		public void onImageDropped(long timestamp) {
		notifyCaptureFailed();
		}

		@Override
		public void onImageAvailable(ImageReader reader, Image img) {
		if (mCaptureFailed) {
		@@ -1159,6 +1157,9 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
		}

		private class ImageLoopbackCallback implements OnImageAvailableListener {
		@Override
		public void onImageDropped(long timestamp) { }

		@Override
		public void onImageAvailable(ImageReader reader, Image img) {
		img.close();
		@@ -1221,7 +1222,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
		}

		private interface OnImageAvailableListener {
		public void onImageAvailable (ImageReader reader, Image img);
		void onImageDropped(long timestamp);
		void onImageAvailable (ImageReader reader, Image img);
		}

		private class CameraOutputImageCallback implements ImageReader.OnImageAvailableListener,
		@@ -1263,6 +1265,29 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
		} else {
		mImageListenerMap.put(img.getTimestamp(), new Pair<>(img, null));
		}

		notifyDroppedImages(timestamp);
		}

		private void notifyDroppedImages(long timestamp) {
		Set<Long> timestamps = mImageListenerMap.keySet();
		ArrayList<Long> removedTs = new ArrayList<>();
		for (long ts : timestamps) {
		if (ts < timestamp) {
		Log.e(TAG, "Dropped image with ts: " + ts);
		Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.get(ts);
		if (entry.second != null) {
		entry.second.onImageDropped(ts);
		}
		if (entry.first != null) {
		entry.first.close();
		}
		removedTs.add(ts);
		}
		}
		for (long ts : removedTs) {
		mImageListenerMap.remove(ts);
		}
		}

		public void registerListener(Long timestamp, OnImageAvailableListener listener) {
		@@ -1291,6 +1316,12 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
		entry.first.close();
		}
		}
		for (long timestamp : mImageListenerMap.keySet()) {
		Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.get(timestamp);
		if (entry.second != null) {
		entry.second.onImageDropped(timestamp);
		}
		}
		mImageListenerMap.clear();
		}
		}
		@@ -1447,7 +1478,6 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
		} else {
		notifyConfigurationFailure();
		}

		}

		@Override
		@@ -1536,7 +1566,16 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
		} else if (mPreviewProcessorType ==
		IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) {
		int idx = mPendingResultMap.indexOfKey(timestamp);
		if (idx >= 0) {

		if ((idx >= 0) && (mPendingResultMap.get(timestamp).first == null)) {
		// Image was dropped before we can receive the capture results
		if ((mCaptureResultHandler != null)) {
		mCaptureResultHandler.onCaptureCompleted(timestamp,
		initializeFilteredResults(result));
		}
		discardPendingRepeatingResults(idx, mPendingResultMap, false);
		} else if (idx >= 0) {
		// Image came before the capture results
		ParcelImage parcelImage = initializeParcelImage(
		mPendingResultMap.get(timestamp).first);
		try {
		@@ -1563,6 +1602,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
		}
		discardPendingRepeatingResults(idx, mPendingResultMap, false);
		} else {
		// Image not yet available
		notifyClient = false;
		mPendingResultMap.put(timestamp,
		new Pair<>(null,
		@@ -1581,16 +1621,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
		mClientRequest));
		if ((mCaptureResultHandler != null) && (mPreviewProcessorType !=
		IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR)) {
		CameraMetadataNative captureResults =
		new CameraMetadataNative();
		for (CaptureResult.Key key : mSupportedResultKeys) {
		Object value = result.get(key);
		if (value != null) {
		captureResults.set(key, value);
		}
		}
		mCaptureResultHandler.onCaptureCompleted(timestamp,
		captureResults);
		initializeFilteredResults(result));
		}
		} else {
		mExecutor.execute(
		@@ -1657,8 +1689,13 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
		for (int i = idx; i >= 0; i--) {
		if (previewMap.valueAt(i).first != null) {
		previewMap.valueAt(i).first.close();
		} else {
		if (mClientNotificationsEnabled && ((i != idx) \|\| notifyCurrentIndex)) {
		} else if (mClientNotificationsEnabled && (previewMap.valueAt(i).second != null) &&
		((i != idx) \|\| notifyCurrentIndex)) {
		TotalCaptureResult result = previewMap.valueAt(i).second;
		Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP);
		mCaptureResultHandler.onCaptureCompleted(timestamp,
		initializeFilteredResults(result));

		Log.w(TAG, "Preview frame drop with timestamp: " + previewMap.keyAt(i));
		final long ident = Binder.clearCallingIdentity();
		try {
		@@ -1669,7 +1706,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
		} finally {
		Binder.restoreCallingIdentity(ident);
		}
		}

		}
		previewMap.removeAt(i);
		}
		@@ -1682,6 +1719,12 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
		mOutputWriter = imageWriter;
		}

		@Override
		public void onImageDropped(long timestamp) {
		discardPendingRepeatingResults(mPendingResultMap.indexOfKey(timestamp),
		mPendingResultMap, true);
		}

		@Override
		public void onImageAvailable(ImageReader reader, Image img) {
		if (img == null) {
		@@ -1702,6 +1745,15 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {

		private class ImageProcessCallback implements OnImageAvailableListener {

		@Override
		public void onImageDropped(long timestamp) {
		discardPendingRepeatingResults(mPendingResultMap.indexOfKey(timestamp),
		mPendingResultMap, true);
		// Add an empty frame&results entry to flag that we dropped a frame
		// and valid capture results can immediately return to client.
		mPendingResultMap.put(timestamp, new Pair<>(null, null));
		}

		@Override
		public void onImageAvailable(ImageReader reader, Image img) {
		if (mPendingResultMap.size() + 1 >= PREVIEW_QUEUE_SIZE) {
		@@ -1768,6 +1820,17 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
		}
		}

		private CameraMetadataNative initializeFilteredResults(TotalCaptureResult result) {
		CameraMetadataNative captureResults = new CameraMetadataNative();
		for (CaptureResult.Key key : mSupportedResultKeys) {
		Object value = result.get(key);
		if (value != null) {
		captureResults.set(key, value);
		}
		}
		return captureResults;
		}

		private static Size findSmallestAspectMatchedSize(@NonNull List<Size> sizes,
		@NonNull Size arSize) {
		final float TOLL = .01f;