Merge "Camera: Propagate extension results in case of dropped frames" into tm-dev am: 4d2ae2655e (7f4b2d66) · Commits · e / os / android_frameworks_base

core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java

+96 −33

Original line number	Original line	Diff line number	Diff line
	@@ -72,7 +72,7 @@ import java.util.Set;
	import java.util.concurrent.Executor;		import java.util.concurrent.Executor;

	public final class CameraExtensionSessionImpl extends CameraExtensionSession {		public final class CameraExtensionSessionImpl extends CameraExtensionSession {
	private static final int PREVIEW_QUEUE_SIZE = 3;		private static final int PREVIEW_QUEUE_SIZE = 10;
	private static final String TAG = "CameraExtensionSessionImpl";		private static final String TAG = "CameraExtensionSessionImpl";

	private final Executor mExecutor;		private final Executor mExecutor;
	@@ -1057,15 +1057,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
	mClientRequest));		mClientRequest));

	if (mCaptureResultHandler != null) {		if (mCaptureResultHandler != null) {
	CameraMetadataNative captureResults = new CameraMetadataNative();
	for (CaptureResult.Key key : mSupportedResultKeys) {
	Object value = result.get(key);
	if (value != null) {
	captureResults.set(key, value);
	}
	}
	mCaptureResultHandler.onCaptureCompleted(timestamp,		mCaptureResultHandler.onCaptureCompleted(timestamp,
	captureResults);		initializeFilteredResults(result));
	}		}
	} finally {		} finally {
	Binder.restoreCallingIdentity(ident);		Binder.restoreCallingIdentity(ident);
	@@ -1126,6 +1119,11 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
	}		}

	private class ImageCallback implements OnImageAvailableListener {		private class ImageCallback implements OnImageAvailableListener {
			@Override
			public void onImageDropped(long timestamp) {
			notifyCaptureFailed();
			}

	@Override		@Override
	public void onImageAvailable(ImageReader reader, Image img) {		public void onImageAvailable(ImageReader reader, Image img) {
	if (mCaptureFailed) {		if (mCaptureFailed) {
	@@ -1159,6 +1157,9 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
	}		}

	private class ImageLoopbackCallback implements OnImageAvailableListener {		private class ImageLoopbackCallback implements OnImageAvailableListener {
			@Override
			public void onImageDropped(long timestamp) { }

	@Override		@Override
	public void onImageAvailable(ImageReader reader, Image img) {		public void onImageAvailable(ImageReader reader, Image img) {
	img.close();		img.close();
	@@ -1221,7 +1222,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
	}		}

	private interface OnImageAvailableListener {		private interface OnImageAvailableListener {
	public void onImageAvailable (ImageReader reader, Image img);		void onImageDropped(long timestamp);
			void onImageAvailable (ImageReader reader, Image img);
	}		}

	private class CameraOutputImageCallback implements ImageReader.OnImageAvailableListener,		private class CameraOutputImageCallback implements ImageReader.OnImageAvailableListener,
	@@ -1263,6 +1265,29 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
	} else {		} else {
	mImageListenerMap.put(img.getTimestamp(), new Pair<>(img, null));		mImageListenerMap.put(img.getTimestamp(), new Pair<>(img, null));
	}		}

			notifyDroppedImages(timestamp);
			}

			private void notifyDroppedImages(long timestamp) {
			Set<Long> timestamps = mImageListenerMap.keySet();
			ArrayList<Long> removedTs = new ArrayList<>();
			for (long ts : timestamps) {
			if (ts < timestamp) {
			Log.e(TAG, "Dropped image with ts: " + ts);
			Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.get(ts);
			if (entry.second != null) {
			entry.second.onImageDropped(ts);
			}
			if (entry.first != null) {
			entry.first.close();
			}
			removedTs.add(ts);
			}
			}
			for (long ts : removedTs) {
			mImageListenerMap.remove(ts);
			}
	}		}

	public void registerListener(Long timestamp, OnImageAvailableListener listener) {		public void registerListener(Long timestamp, OnImageAvailableListener listener) {
	@@ -1291,6 +1316,12 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
	entry.first.close();		entry.first.close();
	}		}
	}		}
			for (long timestamp : mImageListenerMap.keySet()) {
			Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.get(timestamp);
			if (entry.second != null) {
			entry.second.onImageDropped(timestamp);
			}
			}
	mImageListenerMap.clear();		mImageListenerMap.clear();
	}		}
	}		}
	@@ -1447,7 +1478,6 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
	} else {		} else {
	notifyConfigurationFailure();		notifyConfigurationFailure();
	}		}

	}		}

	@Override		@Override
	@@ -1536,7 +1566,16 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
	} else if (mPreviewProcessorType ==		} else if (mPreviewProcessorType ==
	IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) {		IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) {
	int idx = mPendingResultMap.indexOfKey(timestamp);		int idx = mPendingResultMap.indexOfKey(timestamp);
	if (idx >= 0) {
			if ((idx >= 0) && (mPendingResultMap.get(timestamp).first == null)) {
			// Image was dropped before we can receive the capture results
			if ((mCaptureResultHandler != null)) {
			mCaptureResultHandler.onCaptureCompleted(timestamp,
			initializeFilteredResults(result));
			}
			discardPendingRepeatingResults(idx, mPendingResultMap, false);
			} else if (idx >= 0) {
			// Image came before the capture results
	ParcelImage parcelImage = initializeParcelImage(		ParcelImage parcelImage = initializeParcelImage(
	mPendingResultMap.get(timestamp).first);		mPendingResultMap.get(timestamp).first);
	try {		try {
	@@ -1563,6 +1602,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
	}		}
	discardPendingRepeatingResults(idx, mPendingResultMap, false);		discardPendingRepeatingResults(idx, mPendingResultMap, false);
	} else {		} else {
			// Image not yet available
	notifyClient = false;		notifyClient = false;
	mPendingResultMap.put(timestamp,		mPendingResultMap.put(timestamp,
	new Pair<>(null,		new Pair<>(null,
	@@ -1581,16 +1621,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
	mClientRequest));		mClientRequest));
	if ((mCaptureResultHandler != null) && (mPreviewProcessorType !=		if ((mCaptureResultHandler != null) && (mPreviewProcessorType !=
	IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR)) {		IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR)) {
	CameraMetadataNative captureResults =
	new CameraMetadataNative();
	for (CaptureResult.Key key : mSupportedResultKeys) {
	Object value = result.get(key);
	if (value != null) {
	captureResults.set(key, value);
	}
	}
	mCaptureResultHandler.onCaptureCompleted(timestamp,		mCaptureResultHandler.onCaptureCompleted(timestamp,
	captureResults);		initializeFilteredResults(result));
	}		}
	} else {		} else {
	mExecutor.execute(		mExecutor.execute(
	@@ -1657,8 +1689,13 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
	for (int i = idx; i >= 0; i--) {		for (int i = idx; i >= 0; i--) {
	if (previewMap.valueAt(i).first != null) {		if (previewMap.valueAt(i).first != null) {
	previewMap.valueAt(i).first.close();		previewMap.valueAt(i).first.close();
	} else {		} else if (mClientNotificationsEnabled && (previewMap.valueAt(i).second != null) &&
	if (mClientNotificationsEnabled && ((i != idx) \|\| notifyCurrentIndex)) {		((i != idx) \|\| notifyCurrentIndex)) {
			TotalCaptureResult result = previewMap.valueAt(i).second;
			Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP);
			mCaptureResultHandler.onCaptureCompleted(timestamp,
			initializeFilteredResults(result));

	Log.w(TAG, "Preview frame drop with timestamp: " + previewMap.keyAt(i));		Log.w(TAG, "Preview frame drop with timestamp: " + previewMap.keyAt(i));
	final long ident = Binder.clearCallingIdentity();		final long ident = Binder.clearCallingIdentity();
	try {		try {
	@@ -1669,7 +1706,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
	} finally {		} finally {
	Binder.restoreCallingIdentity(ident);		Binder.restoreCallingIdentity(ident);
	}		}
	}
	}		}
	previewMap.removeAt(i);		previewMap.removeAt(i);
	}		}
	@@ -1682,6 +1719,12 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
	mOutputWriter = imageWriter;		mOutputWriter = imageWriter;
	}		}

			@Override
			public void onImageDropped(long timestamp) {
			discardPendingRepeatingResults(mPendingResultMap.indexOfKey(timestamp),
			mPendingResultMap, true);
			}

	@Override		@Override
	public void onImageAvailable(ImageReader reader, Image img) {		public void onImageAvailable(ImageReader reader, Image img) {
	if (img == null) {		if (img == null) {
	@@ -1702,6 +1745,15 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {

	private class ImageProcessCallback implements OnImageAvailableListener {		private class ImageProcessCallback implements OnImageAvailableListener {

			@Override
			public void onImageDropped(long timestamp) {
			discardPendingRepeatingResults(mPendingResultMap.indexOfKey(timestamp),
			mPendingResultMap, true);
			// Add an empty frame&results entry to flag that we dropped a frame
			// and valid capture results can immediately return to client.
			mPendingResultMap.put(timestamp, new Pair<>(null, null));
			}

	@Override		@Override
	public void onImageAvailable(ImageReader reader, Image img) {		public void onImageAvailable(ImageReader reader, Image img) {
	if (mPendingResultMap.size() + 1 >= PREVIEW_QUEUE_SIZE) {		if (mPendingResultMap.size() + 1 >= PREVIEW_QUEUE_SIZE) {
	@@ -1768,6 +1820,17 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
	}		}
	}		}

			private CameraMetadataNative initializeFilteredResults(TotalCaptureResult result) {
			CameraMetadataNative captureResults = new CameraMetadataNative();
			for (CaptureResult.Key key : mSupportedResultKeys) {
			Object value = result.get(key);
			if (value != null) {
			captureResults.set(key, value);
			}
			}
			return captureResults;
			}

	private static Size findSmallestAspectMatchedSize(@NonNull List<Size> sizes,		private static Size findSmallestAspectMatchedSize(@NonNull List<Size> sizes,
	@NonNull Size arSize) {		@NonNull Size arSize) {
	final float TOLL = .01f;		final float TOLL = .01f;