camera2: Fix LEGACY mode timestamps. (91838ded) · Commits · e / os / android_frameworks_base

core/java/android/hardware/camera2/legacy/CameraDeviceState.java

+9 −4

Original line number	Diff line number	Diff line
		@@ -65,7 +65,7 @@ public class CameraDeviceState {
		void onError(int errorCode, RequestHolder holder);
		void onConfiguring();
		void onIdle();
		void onCaptureStarted(RequestHolder holder);
		void onCaptureStarted(RequestHolder holder, long timestamp);
		void onCaptureResult(CameraMetadataNative result, RequestHolder holder);
		}

		@@ -125,11 +125,12 @@ public class CameraDeviceState {
		* </p>
		*
		* @param request A {@link RequestHolder} containing the request for the current capture.
		* @param timestamp The timestamp of the capture start in nanoseconds.
		* @return {@link CameraBinderDecorator#NO_ERROR}, or an error if one has occurred.
		*/
		public synchronized int setCaptureStart(final RequestHolder request) {
		public synchronized int setCaptureStart(final RequestHolder request, long timestamp) {
		mCurrentRequest = request;
		doStateTransition(STATE_CAPTURING);
		doStateTransition(STATE_CAPTURING, timestamp);
		return mCurrentError;
		}

		@@ -180,6 +181,10 @@ public class CameraDeviceState {
		}

		private void doStateTransition(int newState) {
		doStateTransition(newState, /timestamp/0);
		}

		private void doStateTransition(int newState, final long timestamp) {
		if (DEBUG) {
		if (newState != mCurrentState) {
		Log.d(TAG, "Transitioning to state " + newState);
		@@ -250,7 +255,7 @@ public class CameraDeviceState {
		mCurrentHandler.post(new Runnable() {
		@Override
		public void run() {
		mCurrentListener.onCaptureStarted(mCurrentRequest);
		mCurrentListener.onCaptureStarted(mCurrentRequest, timestamp);
		}
		});
		}

core/java/android/hardware/camera2/legacy/CaptureCollector.java

0 → 100644

+453 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2014 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/
		package android.hardware.camera2.legacy;

		import android.hardware.camera2.impl.CameraMetadataNative;
		import android.util.Log;
		import android.util.Pair;

		import java.util.ArrayDeque;
		import java.util.concurrent.TimeUnit;
		import java.util.concurrent.locks.Condition;
		import java.util.concurrent.locks.ReentrantLock;

		/**
		* Collect timestamps and state for each {@link CaptureRequest} as it passes through
		* the Legacy camera pipeline.
		*/
		public class CaptureCollector {
		private static final String TAG = "CaptureCollector";

		private static final boolean DEBUG = Log.isLoggable(LegacyCameraDevice.DEBUG_PROP, Log.DEBUG);

		private static final int FLAG_RECEIVED_JPEG = 1;
		private static final int FLAG_RECEIVED_JPEG_TS = 2;
		private static final int FLAG_RECEIVED_PREVIEW = 4;
		private static final int FLAG_RECEIVED_PREVIEW_TS = 8;
		private static final int FLAG_RECEIVED_ALL_JPEG = FLAG_RECEIVED_JPEG \| FLAG_RECEIVED_JPEG_TS;
		private static final int FLAG_RECEIVED_ALL_PREVIEW = FLAG_RECEIVED_PREVIEW \|
		FLAG_RECEIVED_PREVIEW_TS;

		private static final int MAX_JPEGS_IN_FLIGHT = 1;

		private class CaptureHolder {
		private final RequestHolder mRequest;
		private final LegacyRequest mLegacy;
		public final boolean needsJpeg;
		public final boolean needsPreview;

		private long mTimestamp = 0;
		private int mReceivedFlags = 0;
		private boolean mHasStarted = false;

		public CaptureHolder(RequestHolder request, LegacyRequest legacyHolder) {
		mRequest = request;
		mLegacy = legacyHolder;
		needsJpeg = request.hasJpegTargets();
		needsPreview = request.hasPreviewTargets();
		}

		public boolean isPreviewCompleted() {
		return (mReceivedFlags & FLAG_RECEIVED_ALL_PREVIEW) == FLAG_RECEIVED_ALL_PREVIEW;
		}

		public boolean isJpegCompleted() {
		return (mReceivedFlags & FLAG_RECEIVED_ALL_JPEG) == FLAG_RECEIVED_ALL_JPEG;
		}

		public boolean isCompleted() {
		return (needsJpeg == isJpegCompleted()) && (needsPreview == isPreviewCompleted());
		}

		public void tryComplete() {
		if (needsPreview && isPreviewCompleted()) {
		CaptureCollector.this.onPreviewCompleted();
		}
		if (isCompleted()) {
		CaptureCollector.this.onRequestCompleted(mRequest, mLegacy, mTimestamp);
		}
		}

		public void setJpegTimestamp(long timestamp) {
		if (DEBUG) {
		Log.d(TAG, "setJpegTimestamp - called for request " + mRequest.getRequestId());
		}
		if (!needsJpeg) {
		throw new IllegalStateException(
		"setJpegTimestamp called for capture with no jpeg targets.");
		}
		if (isCompleted()) {
		throw new IllegalStateException(
		"setJpegTimestamp called on already completed request.");
		}

		mReceivedFlags \|= FLAG_RECEIVED_JPEG_TS;

		if (mTimestamp == 0) {
		mTimestamp = timestamp;
		}

		if (!mHasStarted) {
		mHasStarted = true;
		CaptureCollector.this.mDeviceState.setCaptureStart(mRequest, mTimestamp);
		}

		tryComplete();
		}

		public void setJpegProduced() {
		if (DEBUG) {
		Log.d(TAG, "setJpegProduced - called for request " + mRequest.getRequestId());
		}
		if (!needsJpeg) {
		throw new IllegalStateException(
		"setJpegProduced called for capture with no jpeg targets.");
		}
		if (isCompleted()) {
		throw new IllegalStateException(
		"setJpegProduced called on already completed request.");
		}

		mReceivedFlags \|= FLAG_RECEIVED_JPEG;
		tryComplete();
		}

		public void setPreviewTimestamp(long timestamp) {
		if (DEBUG) {
		Log.d(TAG, "setPreviewTimestamp - called for request " + mRequest.getRequestId());
		}
		if (!needsPreview) {
		throw new IllegalStateException(
		"setPreviewTimestamp called for capture with no preview targets.");
		}
		if (isCompleted()) {
		throw new IllegalStateException(
		"setPreviewTimestamp called on already completed request.");
		}

		mReceivedFlags \|= FLAG_RECEIVED_PREVIEW_TS;

		if (mTimestamp == 0) {
		mTimestamp = timestamp;
		}

		if (!needsJpeg) {
		if (!mHasStarted) {
		mHasStarted = true;
		CaptureCollector.this.mDeviceState.setCaptureStart(mRequest, mTimestamp);
		}
		}

		tryComplete();
		}

		public void setPreviewProduced() {
		if (DEBUG) {
		Log.d(TAG, "setPreviewProduced - called for request " + mRequest.getRequestId());
		}
		if (!needsPreview) {
		throw new IllegalStateException(
		"setPreviewProduced called for capture with no preview targets.");
		}
		if (isCompleted()) {
		throw new IllegalStateException(
		"setPreviewProduced called on already completed request.");
		}

		mReceivedFlags \|= FLAG_RECEIVED_PREVIEW;
		tryComplete();
		}
		}

		private final ArrayDeque<CaptureHolder> mJpegCaptureQueue;
		private final ArrayDeque<CaptureHolder> mJpegProduceQueue;
		private final ArrayDeque<CaptureHolder> mPreviewCaptureQueue;
		private final ArrayDeque<CaptureHolder> mPreviewProduceQueue;

		private final ReentrantLock mLock = new ReentrantLock();
		private final Condition mIsEmpty;
		private final Condition mPreviewsEmpty;
		private final Condition mNotFull;
		private final CameraDeviceState mDeviceState;
		private final LegacyResultMapper mMapper = new LegacyResultMapper();
		private int mInFlight = 0;
		private int mInFlightPreviews = 0;
		private final int mMaxInFlight;

		/**
		* Create a new {@link CaptureCollector} that can modify the given {@link CameraDeviceState}.
		*
		* @param maxInFlight max allowed in-flight requests.
		* @param deviceState the {@link CameraDeviceState} to update as requests are processed.
		*/
		public CaptureCollector(int maxInFlight, CameraDeviceState deviceState) {
		mMaxInFlight = maxInFlight;
		mJpegCaptureQueue = new ArrayDeque<>(MAX_JPEGS_IN_FLIGHT);
		mJpegProduceQueue = new ArrayDeque<>(MAX_JPEGS_IN_FLIGHT);
		mPreviewCaptureQueue = new ArrayDeque<>(mMaxInFlight);
		mPreviewProduceQueue = new ArrayDeque<>(mMaxInFlight);
		mIsEmpty = mLock.newCondition();
		mNotFull = mLock.newCondition();
		mPreviewsEmpty = mLock.newCondition();
		mDeviceState = deviceState;
		}

		/**
		* Queue a new request.
		*
		* <p>
		* For requests that use the Camera1 API preview output stream, this will block if there are
		* already {@code maxInFlight} requests in progress (until at least one prior request has
		* completed). For requests that use the Camera1 API jpeg callbacks, this will block until
		* all prior requests have been completed to avoid stopping preview for
		* {@link android.hardware.Camera#takePicture} before prior preview requests have been
		* completed.
		* </p>
		* @param holder the {@link RequestHolder} for this request.
		* @param legacy the {@link LegacyRequest} for this request; this will not be mutated.
		* @param timeout a timeout to use for this call.
		* @param unit the units to use for the timeout.
		* @return {@code false} if this method timed out.
		* @throws InterruptedException if this thread is interrupted.
		*/
		public boolean queueRequest(RequestHolder holder, LegacyRequest legacy, long timeout,
		TimeUnit unit)
		throws InterruptedException {
		CaptureHolder h = new CaptureHolder(holder, legacy);
		long nanos = unit.toNanos(timeout);
		final ReentrantLock lock = this.mLock;
		lock.lock();
		try {
		if (DEBUG) {
		Log.d(TAG, "queueRequest for request " + holder.getRequestId() +
		" - " + mInFlight + " requests remain in flight.");
		}
		if (h.needsJpeg) {
		// Wait for all current requests to finish before queueing jpeg.
		while (mInFlight > 0) {
		if (nanos <= 0) {
		return false;
		}
		nanos = mIsEmpty.awaitNanos(nanos);
		}
		mJpegCaptureQueue.add(h);
		mJpegProduceQueue.add(h);
		}
		if (h.needsPreview) {
		while (mInFlight >= mMaxInFlight) {
		if (nanos <= 0) {
		return false;
		}
		nanos = mNotFull.awaitNanos(nanos);
		}
		mPreviewCaptureQueue.add(h);
		mPreviewProduceQueue.add(h);
		mInFlightPreviews++;
		}

		if (!(h.needsJpeg \|\| h.needsPreview)) {
		throw new IllegalStateException("Request must target at least one output surface!");
		}

		mInFlight++;
		return true;
		} finally {
		lock.unlock();
		}
		}

		/**
		* Wait all queued requests to complete.
		*
		* @param timeout a timeout to use for this call.
		* @param unit the units to use for the timeout.
		* @return {@code false} if this method timed out.
		* @throws InterruptedException if this thread is interrupted.
		*/
		public boolean waitForEmpty(long timeout, TimeUnit unit) throws InterruptedException {
		long nanos = unit.toNanos(timeout);
		final ReentrantLock lock = this.mLock;
		lock.lock();
		try {
		while (mInFlight > 0) {
		if (nanos <= 0) {
		return false;
		}
		nanos = mIsEmpty.awaitNanos(nanos);
		}
		return true;
		} finally {
		lock.unlock();
		}
		}

		/**
		* Wait all queued requests that use the Camera1 API preview output to complete.
		*
		* @param timeout a timeout to use for this call.
		* @param unit the units to use for the timeout.
		* @return {@code false} if this method timed out.
		* @throws InterruptedException if this thread is interrupted.
		*/
		public boolean waitForPreviewsEmpty(long timeout, TimeUnit unit) throws InterruptedException {
		long nanos = unit.toNanos(timeout);
		final ReentrantLock lock = this.mLock;
		lock.lock();
		try {
		while (mInFlightPreviews > 0) {
		if (nanos <= 0) {
		return false;
		}
		nanos = mPreviewsEmpty.awaitNanos(nanos);
		}
		return true;
		} finally {
		lock.unlock();
		}
		}

		/**
		* Called to alert the {@link CaptureCollector} that the jpeg capture has begun.
		*
		* @param timestamp the time of the jpeg capture.
		* @return the {@link RequestHolder} for the request associated with this capture.
		*/
		public RequestHolder jpegCaptured(long timestamp) {
		final ReentrantLock lock = this.mLock;
		lock.lock();
		try {
		CaptureHolder h = mJpegCaptureQueue.poll();
		if (h == null) {
		Log.w(TAG, "jpegCaptured called with no jpeg request on queue!");
		return null;
		}
		h.setJpegTimestamp(timestamp);
		return h.mRequest;
		} finally {
		lock.unlock();
		}
		}

		/**
		* Called to alert the {@link CaptureCollector} that the jpeg capture has completed.
		*
		* @return a pair containing the {@link RequestHolder} and the timestamp of the capture.
		*/
		public Pair<RequestHolder, Long> jpegProduced() {
		final ReentrantLock lock = this.mLock;
		lock.lock();
		try {
		CaptureHolder h = mJpegProduceQueue.poll();
		if (h == null) {
		Log.w(TAG, "jpegProduced called with no jpeg request on queue!");
		return null;
		}
		h.setJpegProduced();
		return new Pair<>(h.mRequest, h.mTimestamp);
		} finally {
		lock.unlock();
		}
		}

		/**
		* Check if there are any pending capture requests that use the Camera1 API preview output.
		*
		* @return {@code true} if there are pending preview requests.
		*/
		public boolean hasPendingPreviewCaptures() {
		final ReentrantLock lock = this.mLock;
		lock.lock();
		try {
		return !mPreviewCaptureQueue.isEmpty();
		} finally {
		lock.unlock();
		}
		}

		/**
		* Called to alert the {@link CaptureCollector} that the preview capture has begun.
		*
		* @param timestamp the time of the preview capture.
		* @return a pair containing the {@link RequestHolder} and the timestamp of the capture.
		*/
		public Pair<RequestHolder, Long> previewCaptured(long timestamp) {
		final ReentrantLock lock = this.mLock;
		lock.lock();
		try {
		CaptureHolder h = mPreviewCaptureQueue.poll();
		if (h == null) {
		Log.w(TAG, "previewCaptured called with no preview request on queue!");
		return null;
		}
		h.setPreviewTimestamp(timestamp);
		return new Pair<>(h.mRequest, h.mTimestamp);
		} finally {
		lock.unlock();
		}
		}

		/**
		* Called to alert the {@link CaptureCollector} that the preview capture has completed.
		*
		* @return the {@link RequestHolder} for the request associated with this capture.
		*/
		public RequestHolder previewProduced() {
		final ReentrantLock lock = this.mLock;
		lock.lock();
		try {
		CaptureHolder h = mPreviewProduceQueue.poll();
		if (h == null) {
		Log.w(TAG, "previewProduced called with no preview request on queue!");
		return null;
		}
		h.setPreviewProduced();
		return h.mRequest;
		} finally {
		lock.unlock();
		}
		}

		private void onPreviewCompleted() {
		mInFlightPreviews--;
		if (mInFlightPreviews < 0) {
		throw new IllegalStateException(
		"More preview captures completed than requests queued.");
		}
		if (mInFlightPreviews == 0) {
		mPreviewsEmpty.signalAll();
		}
		}

		private void onRequestCompleted(RequestHolder request, LegacyRequest legacyHolder,
		long timestamp) {
		mInFlight--;
		if (DEBUG) {
		Log.d(TAG, "Completed request " + request.getRequestId() +
		", " + mInFlight + " requests remain in flight.");
		}
		if (mInFlight < 0) {
		throw new IllegalStateException(
		"More captures completed than requests queued.");
		}
		mNotFull.signalAll();
		if (mInFlight == 0) {
		mIsEmpty.signalAll();
		}
		CameraMetadataNative result = mMapper.cachedConvertResultMetadata(
		legacyHolder, timestamp);
		mDeviceState.setCaptureResult(request, result);
		}
		}

core/java/android/hardware/camera2/legacy/GLThreadManager.java

+19 −14

Original line number	Diff line number	Diff line
		@@ -25,6 +25,8 @@ import android.view.Surface;

		import java.util.Collection;

		import static com.android.internal.util.Preconditions.*;

		/**
		* GLThreadManager handles the thread used for rendering into the configured output surfaces.
		*/
		@@ -38,6 +40,8 @@ public class GLThreadManager {
		private static final int MSG_DROP_FRAMES = 4;
		private static final int MSG_ALLOW_FRAMES = 5;

		private CaptureCollector mCaptureCollector;

		private final SurfaceTextureRenderer mTextureRenderer;

		private final RequestHandlerThread mGLHandlerThread;
		@@ -51,10 +55,13 @@ public class GLThreadManager {
		private static class ConfigureHolder {
		public final ConditionVariable condition;
		public final Collection<Surface> surfaces;
		public final CaptureCollector collector;

		public ConfigureHolder(ConditionVariable condition, Collection<Surface> surfaces) {
		public ConfigureHolder(ConditionVariable condition, Collection<Surface> surfaces,
		CaptureCollector collector) {
		this.condition = condition;
		this.surfaces = surfaces;
		this.collector = collector;
		}
		}

		@@ -74,6 +81,7 @@ public class GLThreadManager {
		ConfigureHolder configure = (ConfigureHolder) msg.obj;
		mTextureRenderer.cleanupEGLContext();
		mTextureRenderer.configureSurfaces(configure.surfaces);
		mCaptureCollector = checkNotNull(configure.collector);
		configure.condition.open();
		mConfigured = true;
		break;
		@@ -88,7 +96,7 @@ public class GLThreadManager {
		if (!mConfigured) {
		Log.e(TAG, "Dropping frame, EGL context not configured!");
		}
		mTextureRenderer.drawIntoSurfaces((Collection<Surface>) msg.obj);
		mTextureRenderer.drawIntoSurfaces(mCaptureCollector);
		break;
		case MSG_CLEANUP:
		mTextureRenderer.cleanupEGLContext();
		@@ -158,16 +166,11 @@ public class GLThreadManager {
		}

		/**
		* Queue a new call to draw into a given set of surfaces.
		*
		* <p>
		* The set of surfaces passed here must be a subset of the set of surfaces passed in
		* the last call to {@link #setConfigurationAndWait}.
		* </p>
		*
		* @param targets a collection of {@link android.view.Surface}s to draw into.
		* Queue a new call to draw into the surfaces specified in the next available preview
		* request from the {@link CaptureCollector} passed to
		* {@link #setConfigurationAndWait(java.util.Collection, CaptureCollector)};
		*/
		public void queueNewFrame(Collection<Surface> targets) {
		public void queueNewFrame() {
		Handler handler = mGLHandlerThread.getHandler();

		/**
		@@ -175,7 +178,7 @@ public class GLThreadManager {
		* are produced, drop frames rather than allowing the queue to back up.
		*/
		if (!handler.hasMessages(MSG_NEW_FRAME)) {
		handler.sendMessage(handler.obtainMessage(MSG_NEW_FRAME, targets));
		handler.sendMessage(handler.obtainMessage(MSG_NEW_FRAME));
		} else {
		Log.e(TAG, "GLThread dropping frame. Not consuming frames quickly enough!");
		}
		@@ -186,12 +189,14 @@ public class GLThreadManager {
		* this configuration has been applied.
		*
		* @param surfaces a collection of {@link android.view.Surface}s to configure.
		* @param collector a {@link CaptureCollector} to retrieve requests from.
		*/
		public void setConfigurationAndWait(Collection<Surface> surfaces) {
		public void setConfigurationAndWait(Collection<Surface> surfaces, CaptureCollector collector) {
		checkNotNull(collector, "collector must not be null");
		Handler handler = mGLHandlerThread.getHandler();

		final ConditionVariable condition = new ConditionVariable(/closed/false);
		ConfigureHolder configure = new ConfigureHolder(condition, surfaces);
		ConfigureHolder configure = new ConfigureHolder(condition, surfaces, collector);

		Message m = handler.obtainMessage(MSG_NEW_CONFIGURATION, /arg1/0, /arg2/0, configure);
		handler.sendMessage(m);

core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java

+8 −4

Original line number	Diff line number	Diff line
		@@ -135,10 +135,9 @@ public class LegacyCameraDevice implements AutoCloseable {
		}

		@Override
		public void onCaptureStarted(RequestHolder holder) {
		public void onCaptureStarted(RequestHolder holder, final long timestamp) {
		final CaptureResultExtras extras = getExtrasFromRequest(holder);

		final long timestamp = System.nanoTime();
		mResultHandler.post(new Runnable() {
		@Override
		public void run() {
		@@ -146,7 +145,6 @@ public class LegacyCameraDevice implements AutoCloseable {
		Log.d(TAG, "doing onCaptureStarted callback.");
		}
		try {
		// TODO: Don't fake timestamp
		mDeviceCallbacks.onCaptureStarted(extras, timestamp);
		} catch (RemoteException e) {
		throw new IllegalStateException(
		@@ -167,7 +165,6 @@ public class LegacyCameraDevice implements AutoCloseable {
		Log.d(TAG, "doing onCaptureResult callback.");
		}
		try {
		// TODO: Don't fake metadata
		mDeviceCallbacks.onResultReceived(result, extras);
		} catch (RemoteException e) {
		throw new IllegalStateException(
		@@ -483,6 +480,12 @@ public class LegacyCameraDevice implements AutoCloseable {
		return new Size(dimens[0], dimens[1]);
		}

		static void setNextTimestamp(Surface surface, long timestamp)
		throws BufferQueueAbandonedException {
		checkNotNull(surface);
		LegacyExceptionUtils.throwOnError(nativeSetNextTimestamp(surface, timestamp));
		}

		private static native int nativeDetectSurfaceType(Surface surface);

		private static native int nativeDetectSurfaceDimens(Surface surface,
		@@ -506,4 +509,5 @@ public class LegacyCameraDevice implements AutoCloseable {
		private static native int nativeDetectTextureDimens(SurfaceTexture surfaceTexture,
		/out/int[/2/] dimens);

		private static native int nativeSetNextTimestamp(Surface surface, long timestamp);
		}

core/java/android/hardware/camera2/legacy/RequestThreadManager.java

+64 −56

File changed.

Preview size limit exceeded, changes collapsed.