Merge "Cap pts gap between adjacent frames to specified value" (0520e05f) · Commits · e / os / android_frameworks_av

include/media/IOMX.h

+1 −0

Original line number	Diff line number	Diff line
		@@ -142,6 +142,7 @@ public:
		enum InternalOptionType {
		INTERNAL_OPTION_SUSPEND, // data is a bool
		INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY, // data is an int64_t
		INTERNAL_OPTION_MAX_TIMESTAMP_GAP, // data is int64_t
		};
		virtual status_t setInternalOption(
		node_id node,

include/media/stagefright/ACodec.h

+1 −0

Original line number	Diff line number	Diff line
		@@ -202,6 +202,7 @@ private:
		int32_t mMetaDataBuffersToSubmit;

		int64_t mRepeatFrameDelayUs;
		int64_t mMaxPtsGapUs;

		status_t setCyclicIntraMacroblockRefresh(const sp<AMessage> &msg, int32_t mode);
		status_t allocateBuffersOnPort(OMX_U32 portIndex);

media/libstagefright/ACodec.cpp

+21 −1

Original line number	Diff line number	Diff line
		@@ -372,7 +372,8 @@ ACodec::ACodec()
		mDequeueCounter(0),
		mStoreMetaDataInOutputBuffers(false),
		mMetaDataBuffersToSubmit(0),
		mRepeatFrameDelayUs(-1ll) {
		mRepeatFrameDelayUs(-1ll),
		mMaxPtsGapUs(-1l) {
		mUninitializedState = new UninitializedState(this);
		mLoadedState = new LoadedState(this);
		mLoadedToIdleState = new LoadedToIdleState(this);
		@@ -1114,6 +1115,10 @@ status_t ACodec::configureCodec(
		&mRepeatFrameDelayUs)) {
		mRepeatFrameDelayUs = -1ll;
		}

		if (!msg->findInt64("max-pts-gap-to-encoder", &mMaxPtsGapUs)) {
		mMaxPtsGapUs = -1l;
		}
		}

		// Always try to enable dynamic output buffers on native surface
		@@ -3921,6 +3926,21 @@ void ACodec::LoadedState::onCreateInputSurface(
		}
		}

		if (err == OK && mCodec->mMaxPtsGapUs > 0l) {
		err = mCodec->mOMX->setInternalOption(
		mCodec->mNode,
		kPortIndexInput,
		IOMX::INTERNAL_OPTION_MAX_TIMESTAMP_GAP,
		&mCodec->mMaxPtsGapUs,
		sizeof(mCodec->mMaxPtsGapUs));

		if (err != OK) {
		ALOGE("[%s] Unable to configure max timestamp gap (err %d)",
		mCodec->mComponentName.c_str(),
		err);
		}
		}

		if (err == OK) {
		notify->setObject("input-surface",
		new BufferProducerWrapper(bufferProducer));

media/libstagefright/omx/GraphicBufferSource.cpp

+99 −1

Original line number	Diff line number	Diff line
		@@ -42,7 +42,11 @@ GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance,
		mEndOfStream(false),
		mEndOfStreamSent(false),
		mRepeatAfterUs(-1ll),
		mMaxTimestampGapUs(-1ll),
		mPrevOriginalTimeUs(-1ll),
		mPrevModifiedTimeUs(-1ll),
		mRepeatLastFrameGeneration(0),
		mRepeatLastFrameTimestamp(-1ll),
		mLatestSubmittedBufferId(-1),
		mLatestSubmittedBufferFrameNum(0),
		mLatestSubmittedBufferUseCount(0),
		@@ -299,6 +303,32 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) {
		return;
		}

		void GraphicBufferSource::codecBufferFilled(OMX_BUFFERHEADERTYPE* header) {
		Mutex::Autolock autoLock(mMutex);

		if (mMaxTimestampGapUs > 0ll
		&& !(header->nFlags & OMX_BUFFERFLAG_CODECCONFIG)) {
		ssize_t index = mOriginalTimeUs.indexOfKey(header->nTimeStamp);
		if (index >= 0) {
		ALOGV("OUT timestamp: %lld -> %lld",
		header->nTimeStamp, mOriginalTimeUs[index]);
		header->nTimeStamp = mOriginalTimeUs[index];
		mOriginalTimeUs.removeItemsAt(index);
		} else {
		// giving up the effort as encoder doesn't appear to preserve pts
		ALOGW("giving up limiting timestamp gap (pts = %lld)",
		header->nTimeStamp);
		mMaxTimestampGapUs = -1ll;
		}
		if (mOriginalTimeUs.size() > BufferQueue::NUM_BUFFER_SLOTS) {
		// something terribly wrong must have happened, giving up...
		ALOGE("mOriginalTimeUs has too many entries (%d)",
		mOriginalTimeUs.size());
		mMaxTimestampGapUs = -1ll;
		}
		}
		}

		void GraphicBufferSource::suspend(bool suspend) {
		Mutex::Autolock autoLock(mMutex);

		@@ -431,6 +461,7 @@ bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() {
		BufferQueue::BufferItem item;
		item.mBuf = mLatestSubmittedBufferId;
		item.mFrameNumber = mLatestSubmittedBufferFrameNum;
		item.mTimestamp = mRepeatLastFrameTimestamp;

		status_t err = submitBuffer_l(item, cbi);

		@@ -440,6 +471,20 @@ bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() {

		++mLatestSubmittedBufferUseCount;

		/* repeat last frame up to kRepeatLastFrameCount times.
		* in case of static scene, a single repeat might not get rid of encoder
		* ghosting completely, refresh a couple more times to get better quality
		*/
		if (--mRepeatLastFrameCount > 0) {
		mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;

		if (mReflector != NULL) {
		sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id());
		msg->setInt32("generation", ++mRepeatLastFrameGeneration);
		msg->post(mRepeatAfterUs);
		}
		}

		return true;
		}

		@@ -460,8 +505,11 @@ void GraphicBufferSource::setLatestSubmittedBuffer_l(

		mLatestSubmittedBufferId = item.mBuf;
		mLatestSubmittedBufferFrameNum = item.mFrameNumber;
		mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;

		mLatestSubmittedBufferUseCount = 1;
		mRepeatBufferDeferred = false;
		mRepeatLastFrameCount = kRepeatLastFrameCount;

		if (mReflector != NULL) {
		sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id());
		@@ -497,6 +545,39 @@ status_t GraphicBufferSource::signalEndOfInputStream() {
		return OK;
		}

		int64_t GraphicBufferSource::getTimestamp(const BufferQueue::BufferItem &item) {
		int64_t timeUs = item.mTimestamp / 1000;

		if (mMaxTimestampGapUs > 0ll) {
		/* Cap timestamp gap between adjacent frames to specified max
		*
		* In the scenario of cast mirroring, encoding could be suspended for
		* prolonged periods. Limiting the pts gap to workaround the problem
		* where encoder's rate control logic produces huge frames after a
		* long period of suspension.
		*/

		int64_t originalTimeUs = timeUs;
		if (mPrevOriginalTimeUs >= 0ll) {
		if (originalTimeUs < mPrevOriginalTimeUs) {
		// Drop the frame if it's going backward in time. Bad timestamp
		// could disrupt encoder's rate control completely.
		ALOGV("Dropping frame that's going backward in time");
		return -1;
		}
		int64_t timestampGapUs = originalTimeUs - mPrevOriginalTimeUs;
		timeUs = (timestampGapUs < mMaxTimestampGapUs ?
		timestampGapUs : mMaxTimestampGapUs) + mPrevModifiedTimeUs;
		}
		mPrevOriginalTimeUs = originalTimeUs;
		mPrevModifiedTimeUs = timeUs;
		mOriginalTimeUs.add(timeUs, originalTimeUs);
		ALOGV("IN timestamp: %lld -> %lld", originalTimeUs, timeUs);
		}

		return timeUs;
		}

		status_t GraphicBufferSource::submitBuffer_l(
		const BufferQueue::BufferItem &item, int cbi) {
		ALOGV("submitBuffer_l cbi=%d", cbi);
		@@ -513,9 +594,15 @@ status_t GraphicBufferSource::submitBuffer_l(
		memcpy(data, &type, 4);
		memcpy(data + 4, &handle, sizeof(buffer_handle_t));

		int64_t timeUs = getTimestamp(item);
		if (timeUs < 0ll) {
		ALOGE("Dropping frame with bad timestamp");
		return UNKNOWN_ERROR;
		}

		status_t err = mNodeInstance->emptyDirectBuffer(header, 0,
		4 + sizeof(buffer_handle_t), OMX_BUFFERFLAG_ENDOFFRAME,
		item.mTimestamp / 1000);
		timeUs);
		if (err != OK) {
		ALOGW("WARNING: emptyDirectBuffer failed: 0x%x", err);
		codecBuffer.mGraphicBuffer = NULL;
		@@ -664,6 +751,17 @@ status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs(
		return OK;
		}

		status_t GraphicBufferSource::setMaxTimestampGapUs(int64_t maxGapUs) {
		Mutex::Autolock autoLock(mMutex);

		if (mExecuting \|\| maxGapUs <= 0ll) {
		return INVALID_OPERATION;
		}

		mMaxTimestampGapUs = maxGapUs;

		return OK;
		}
		void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) {
		switch (msg->what()) {
		case kWhatRepeatLastFrame:

media/libstagefright/omx/GraphicBufferSource.h

+24 −1

Original line number	Diff line number	Diff line
		@@ -87,6 +87,10 @@ public:
		// fill it with a new frame of data; otherwise, just mark it as available.
		void codecBufferEmptied(OMX_BUFFERHEADERTYPE* header);

		// Called when omx_message::FILL_BUFFER_DONE is received. (Currently the
		// buffer source will fix timestamp in the header if needed.)
		void codecBufferFilled(OMX_BUFFERHEADERTYPE* header);

		// This is called after the last input frame has been submitted. We
		// need to submit an empty buffer with the EOS flag set. If we don't
		// have a codec buffer ready, we just set the mEndOfStream flag.
		@@ -105,6 +109,15 @@ public:
		// state and once this behaviour is specified it cannot be reset.
		status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs);

		// When set, the timestamp fed to the encoder will be modified such that
		// the gap between two adjacent frames is capped at maxGapUs. Timestamp
		// will be restored to the original when the encoded frame is returned to
		// the client.
		// This is to solve a problem in certain real-time streaming case, where
		// encoder's rate control logic produces huge frames after a long period
		// of suspension on input.
		status_t setMaxTimestampGapUs(int64_t maxGapUs);

		protected:
		// BufferQueue::ConsumerListener interface, called when a new frame of
		// data is available. If we're executing and a codec buffer is
		@@ -165,6 +178,7 @@ private:

		void setLatestSubmittedBuffer_l(const BufferQueue::BufferItem &item);
		bool repeatLatestSubmittedBuffer_l();
		int64_t getTimestamp(const BufferQueue::BufferItem &item);

		// Lock, covers all member variables.
		mutable Mutex mMutex;
		@@ -206,13 +220,22 @@ private:
		enum {
		kWhatRepeatLastFrame,
		};

		enum {
		kRepeatLastFrameCount = 10,
		};
		int64_t mRepeatAfterUs;
		int64_t mMaxTimestampGapUs;

		KeyedVector<int64_t, int64_t> mOriginalTimeUs;
		int64_t mPrevOriginalTimeUs;
		int64_t mPrevModifiedTimeUs;

		sp<ALooper> mLooper;
		sp<AHandlerReflector<GraphicBufferSource> > mReflector;

		int32_t mRepeatLastFrameGeneration;
		int64_t mRepeatLastFrameTimestamp;
		int32_t mRepeatLastFrameCount;

		int mLatestSubmittedBufferId;
		uint64_t mLatestSubmittedBufferFrameNum;