Merge "Heif decoder API for decoding image sequences" (01ad43e6) · Commits · e / os / android_frameworks_av

media/libheif/HeifDecoderImpl.cpp

+180 −75

Original line number	Diff line number	Diff line
		@@ -31,6 +31,7 @@
		#include <private/media/VideoFrame.h>
		#include <utils/Log.h>
		#include <utils/RefBase.h>
		#include <vector>

		HeifDecoder* createHeifDecoder() {
		return new android::HeifDecoderImpl();
		@@ -38,6 +39,23 @@ HeifDecoder* createHeifDecoder() {

		namespace android {

		void initFrameInfo(HeifFrameInfo info, const VideoFrame videoFrame) {
		info->mWidth = videoFrame->mWidth;
		info->mHeight = videoFrame->mHeight;
		info->mRotationAngle = videoFrame->mRotationAngle;
		info->mBytesPerPixel = videoFrame->mBytesPerPixel;
		// TODO: retrieve per-frame duration from extractor/metadataretriever.
		info->mDurationUs = 33333;
		if (videoFrame->mIccSize > 0) {
		info->mIccData.assign(
		videoFrame->getFlattenedIccData(),
		videoFrame->getFlattenedIccData() + videoFrame->mIccSize);
		} else {
		// clear old Icc data if there is no Icc data.
		info->mIccData.clear();
		}
		}

		/*
		* HeifDataSource
		*
		@@ -293,11 +311,11 @@ HeifDecoderImpl::HeifDecoderImpl() :
		// it's not, default to HAL_PIXEL_FORMAT_RGB_565.
		mOutputColor(HAL_PIXEL_FORMAT_RGB_565),
		mCurScanline(0),
		mWidth(0),
		mHeight(0),
		mTotalScanline(0),
		mFrameDecoded(false),
		mHasImage(false),
		mHasVideo(false),
		mSequenceLength(0),
		mAvailableLines(0),
		mNumSlices(1),
		mSliceHeight(0),
		@@ -336,25 +354,21 @@ bool HeifDecoderImpl::init(HeifStream* stream, HeifFrameInfo* frameInfo) {

		mHasImage = hasImage && !strcasecmp(hasImage, "yes");
		mHasVideo = hasVideo && !strcasecmp(hasVideo, "yes");
		sp<IMemory> sharedMem;

		HeifFrameInfo* defaultInfo = nullptr;
		if (mHasImage) {
		// image index < 0 to retrieve primary image
		sharedMem = mRetriever->getImageAtIndex(
		sp<IMemory> sharedMem = mRetriever->getImageAtIndex(
		-1, mOutputColor, true /metaOnly/);
		} else if (mHasVideo) {
		sharedMem = mRetriever->getFrameAtTime(0,
		MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC,
		mOutputColor, true /metaOnly/);
		}

		if (sharedMem == nullptr \|\| sharedMem->pointer() == nullptr) {
		ALOGE("getFrameAtTime: videoFrame is a nullptr");
		ALOGE("init: videoFrame is a nullptr");
		return false;
		}

		VideoFrame* videoFrame = static_cast<VideoFrame*>(sharedMem->pointer());

		ALOGV("Meta dimension %dx%d, display %dx%d, angle %d, iccSize %d",
		ALOGV("Image dimension %dx%d, display %dx%d, angle %d, iccSize %d",
		videoFrame->mWidth,
		videoFrame->mHeight,
		videoFrame->mDisplayWidth,
		@@ -362,22 +376,72 @@ bool HeifDecoderImpl::init(HeifStream* stream, HeifFrameInfo* frameInfo) {
		videoFrame->mRotationAngle,
		videoFrame->mIccSize);

		if (frameInfo != nullptr) {
		frameInfo->set(
		initFrameInfo(&mImageInfo, videoFrame);

		if (videoFrame->mTileHeight >= 512) {
		// Try decoding in slices only if the image has tiles and is big enough.
		mSliceHeight = videoFrame->mTileHeight;
		ALOGV("mSliceHeight %u", mSliceHeight);
		}

		defaultInfo = &mImageInfo;
		}

		if (mHasVideo) {
		sp<IMemory> sharedMem = mRetriever->getFrameAtTime(0,
		MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC,
		mOutputColor, true /metaOnly/);

		if (sharedMem == nullptr \|\| sharedMem->pointer() == nullptr) {
		ALOGE("init: videoFrame is a nullptr");
		return false;
		}

		VideoFrame* videoFrame = static_cast<VideoFrame*>(sharedMem->pointer());

		ALOGV("Sequence dimension %dx%d, display %dx%d, angle %d, iccSize %d",
		videoFrame->mWidth,
		videoFrame->mHeight,
		videoFrame->mDisplayWidth,
		videoFrame->mDisplayHeight,
		videoFrame->mRotationAngle,
		videoFrame->mBytesPerPixel,
		videoFrame->mIccSize,
		videoFrame->getFlattenedIccData());
		videoFrame->mIccSize);

		initFrameInfo(&mSequenceInfo, videoFrame);

		mSequenceLength = atoi(mRetriever->extractMetadata(METADATA_KEY_VIDEO_FRAME_COUNT));

		if (defaultInfo == nullptr) {
		defaultInfo = &mSequenceInfo;
		}
		mWidth = videoFrame->mWidth;
		mHeight = videoFrame->mHeight;
		if (mHasImage && videoFrame->mTileHeight >= 512 && mWidth >= 3000 && mHeight >= 2000 ) {
		// Try decoding in slices only if the image has tiles and is big enough.
		mSliceHeight = videoFrame->mTileHeight;
		mNumSlices = (videoFrame->mHeight + mSliceHeight - 1) / mSliceHeight;
		ALOGV("mSliceHeight %u, mNumSlices %zu", mSliceHeight, mNumSlices);
		}

		if (defaultInfo == nullptr) {
		ALOGD("No valid image or sequence available");
		return false;
		}

		if (frameInfo != nullptr) {
		frameInfo = defaultInfo;
		}

		// default total scanline, this might change if decodeSequence() is used
		mTotalScanline = defaultInfo->mHeight;

		return true;
		}

		bool HeifDecoderImpl::getSequenceInfo(
		HeifFrameInfo* frameInfo, size_t *frameCount) {
		ALOGV("%s", __FUNCTION__);
		if (!mHasVideo) {
		return false;
		}
		if (frameInfo != nullptr) {
		*frameInfo = mSequenceInfo;
		}
		if (frameCount != nullptr) {
		*frameCount = mSequenceLength;
		}
		return true;
		}
		@@ -416,11 +480,11 @@ bool HeifDecoderImpl::decodeAsync() {
		ALOGV("decodeAsync(): decoding slice %zu", i);
		size_t top = i * mSliceHeight;
		size_t bottom = (i + 1) * mSliceHeight;
		if (bottom > mHeight) {
		bottom = mHeight;
		if (bottom > mImageInfo.mHeight) {
		bottom = mImageInfo.mHeight;
		}
		sp<IMemory> frameMemory = mRetriever->getImageRectAtIndex(
		-1, mOutputColor, 0, top, mWidth, bottom);
		-1, mOutputColor, 0, top, mImageInfo.mWidth, bottom);
		{
		Mutex::Autolock autolock(mLock);

		@@ -452,10 +516,19 @@ bool HeifDecoderImpl::decode(HeifFrameInfo* frameInfo) {
		// See if we want to decode in slices to allow client to start
		// scanline processing in parallel with decode. If this fails
		// we fallback to decoding the full frame.
		if (mHasImage && mNumSlices > 1) {
		if (mHasImage) {
		if (mSliceHeight >= 512 &&
		mImageInfo.mWidth >= 3000 &&
		mImageInfo.mHeight >= 2000 ) {
		// Try decoding in slices only if the image has tiles and is big enough.
		mNumSlices = (mImageInfo.mHeight + mSliceHeight - 1) / mSliceHeight;
		ALOGV("mSliceHeight %u, mNumSlices %zu", mSliceHeight, mNumSlices);
		}

		if (mNumSlices > 1) {
		// get first slice and metadata
		sp<IMemory> frameMemory = mRetriever->getImageRectAtIndex(
		-1, mOutputColor, 0, 0, mWidth, mSliceHeight);
		-1, mOutputColor, 0, 0, mImageInfo.mWidth, mSliceHeight);

		if (frameMemory == nullptr \|\| frameMemory->pointer() == nullptr) {
		ALOGE("decode: metadata is a nullptr");
		@@ -465,15 +538,8 @@ bool HeifDecoderImpl::decode(HeifFrameInfo* frameInfo) {
		VideoFrame* videoFrame = static_cast<VideoFrame*>(frameMemory->pointer());

		if (frameInfo != nullptr) {
		frameInfo->set(
		videoFrame->mWidth,
		videoFrame->mHeight,
		videoFrame->mRotationAngle,
		videoFrame->mBytesPerPixel,
		videoFrame->mIccSize,
		videoFrame->getFlattenedIccData());
		initFrameInfo(frameInfo, videoFrame);
		}

		mFrameMemory = frameMemory;
		mAvailableLines = mSliceHeight;
		mThread = new DecodeThread(this);
		@@ -481,7 +547,6 @@ bool HeifDecoderImpl::decode(HeifFrameInfo* frameInfo) {
		mFrameDecoded = true;
		return true;
		}

		// Fallback to decode without slicing
		mThread.clear();
		mNumSlices = 1;
		@@ -489,6 +554,7 @@ bool HeifDecoderImpl::decode(HeifFrameInfo* frameInfo) {
		mAvailableLines = 0;
		mFrameMemory.clear();
		}
		}

		if (mHasImage) {
		// image index < 0 to retrieve primary image
		@@ -520,13 +586,8 @@ bool HeifDecoderImpl::decode(HeifFrameInfo* frameInfo) {
		videoFrame->mSize);

		if (frameInfo != nullptr) {
		frameInfo->set(
		videoFrame->mWidth,
		videoFrame->mHeight,
		videoFrame->mRotationAngle,
		videoFrame->mBytesPerPixel,
		videoFrame->mIccSize,
		videoFrame->getFlattenedIccData());
		initFrameInfo(frameInfo, videoFrame);

		}
		mFrameDecoded = true;

		@@ -536,6 +597,50 @@ bool HeifDecoderImpl::decode(HeifFrameInfo* frameInfo) {
		return true;
		}

		bool HeifDecoderImpl::decodeSequence(int frameIndex, HeifFrameInfo* frameInfo) {
		ALOGV("%s: frame index %d", __FUNCTION__, frameIndex);
		if (!mHasVideo) {
		return false;
		}

		if (frameIndex < 0 \|\| frameIndex >= mSequenceLength) {
		ALOGE("invalid frame index: %d, total frames %zu", frameIndex, mSequenceLength);
		return false;
		}

		mCurScanline = 0;

		// set total scanline to sequence height now
		mTotalScanline = mSequenceInfo.mHeight;

		mFrameMemory = mRetriever->getFrameAtIndex(frameIndex, mOutputColor);
		if (mFrameMemory == nullptr \|\| mFrameMemory->pointer() == nullptr) {
		ALOGE("decode: videoFrame is a nullptr");
		return false;
		}

		VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->pointer());
		if (videoFrame->mSize == 0 \|\|
		mFrameMemory->size() < videoFrame->getFlattenedSize()) {
		ALOGE("decode: videoFrame size is invalid");
		return false;
		}

		ALOGV("Decoded dimension %dx%d, display %dx%d, angle %d, rowbytes %d, size %d",
		videoFrame->mWidth,
		videoFrame->mHeight,
		videoFrame->mDisplayWidth,
		videoFrame->mDisplayHeight,
		videoFrame->mRotationAngle,
		videoFrame->mRowBytes,
		videoFrame->mSize);

		if (frameInfo != nullptr) {
		initFrameInfo(frameInfo, videoFrame);
		}
		return true;
		}

		bool HeifDecoderImpl::getScanlineInner(uint8_t* dst) {
		if (mFrameMemory == nullptr \|\| mFrameMemory->pointer() == nullptr) {
		return false;
		@@ -547,7 +652,7 @@ bool HeifDecoderImpl::getScanlineInner(uint8_t* dst) {
		}

		bool HeifDecoderImpl::getScanline(uint8_t* dst) {
		if (mCurScanline >= mHeight) {
		if (mCurScanline >= mTotalScanline) {
		ALOGE("no more scanline available");
		return false;
		}
		@@ -567,8 +672,8 @@ bool HeifDecoderImpl::getScanline(uint8_t* dst) {
		size_t HeifDecoderImpl::skipScanlines(size_t count) {
		uint32_t oldScanline = mCurScanline;
		mCurScanline += count;
		if (mCurScanline > mHeight) {
		mCurScanline = mHeight;
		if (mCurScanline > mTotalScanline) {
		mCurScanline = mTotalScanline;
		}
		return (mCurScanline > oldScanline) ? (mCurScanline - oldScanline) : 0;
		}

media/libheif/HeifDecoderImpl.h

+8 −2

Original line number	Diff line number	Diff line
		@@ -40,12 +40,16 @@ public:

		bool init(HeifStream* stream, HeifFrameInfo* frameInfo) override;

		bool getSequenceInfo(HeifFrameInfo* frameInfo, size_t *frameCount) override;

		bool getEncodedColor(HeifEncodedColor* outColor) const override;

		bool setOutputColor(HeifColorFormat heifColor) override;

		bool decode(HeifFrameInfo* frameInfo) override;

		bool decodeSequence(int frameIndex, HeifFrameInfo* frameInfo) override;

		bool getScanline(uint8_t* dst) override;

		size_t skipScanlines(size_t count) override;
		@@ -56,13 +60,15 @@ private:
		sp<IDataSource> mDataSource;
		sp<MediaMetadataRetriever> mRetriever;
		sp<IMemory> mFrameMemory;
		HeifFrameInfo mImageInfo;
		HeifFrameInfo mSequenceInfo;
		android_pixel_format_t mOutputColor;
		size_t mCurScanline;
		uint32_t mWidth;
		uint32_t mHeight;
		size_t mTotalScanline;
		bool mFrameDecoded;
		bool mHasImage;
		bool mHasVideo;
		size_t mSequenceLength;

		// Slice decoding only
		Mutex mLock;

media/libheif/include/HeifDecoderAPI.h

+28 −34

Original line number	Diff line number	Diff line
		@@ -17,7 +17,7 @@
		#ifndef _HEIF_DECODER_API_
		#define _HEIF_DECODER_API_

		#include <memory>
		#include <vector>

		/*
		* The output color pixel format of heif decoder.
		@@ -40,41 +40,13 @@ typedef enum {
		/*
		* Represents a color converted (RGB-based) video frame
		*/
		struct HeifFrameInfo
		{
		HeifFrameInfo() :
		mWidth(0), mHeight(0), mRotationAngle(0), mBytesPerPixel(0),
		mIccSize(0), mIccData(nullptr) {}

		// update the frame info, will make a copy of \|iccData\| internally
		void set(uint32_t width, uint32_t height, int32_t rotation, uint32_t bpp,
		uint32_t iccSize, uint8_t* iccData) {
		mWidth = width;
		mHeight = height;
		mRotationAngle = rotation;
		mBytesPerPixel = bpp;

		if (mIccData != nullptr) {
		mIccData.reset(nullptr);
		}
		mIccSize = iccSize;
		if (iccSize > 0) {
		mIccData.reset(new uint8_t[iccSize]);
		if (mIccData.get() != nullptr) {
		memcpy(mIccData.get(), iccData, iccSize);
		} else {
		mIccSize = 0;
		}
		}
		}

		// Intentional public access modifiers:
		struct HeifFrameInfo {
		uint32_t mWidth;
		uint32_t mHeight;
		int32_t mRotationAngle; // Rotation angle, clockwise, should be multiple of 90
		uint32_t mBytesPerPixel; // Number of bytes for one pixel
		uint32_t mIccSize; // Number of bytes in mIccData
		std::unique_ptr<uint8_t[]> mIccData; // Actual ICC data, memory is owned by this structure
		int64_t mDurationUs; // Duration of the frame in us
		std::vector<uint8_t> mIccData; // ICC data array
		};

		/*
		@@ -113,8 +85,8 @@ struct HeifStream {
		virtual size_t getLength() const = 0;

		private:
		HeifStream(const HeifFrameInfo&) = delete;
		HeifStream& operator=(const HeifFrameInfo&) = delete;
		HeifStream(const HeifStream&) = delete;
		HeifStream& operator=(const HeifStream&) = delete;
		};

		/*
		@@ -145,6 +117,14 @@ struct HeifDecoder {
		*/
		virtual bool init(HeifStream* stream, HeifFrameInfo* frameInfo) = 0;

		/*
		* Returns true if the stream contains an image sequence and false otherwise.
		* \|frameInfo\| will be filled with information of pictures in the sequence
		* and \|frameCount\| the length of the sequence upon success and unmodified
		* upon failure.
		*/
		virtual bool getSequenceInfo(HeifFrameInfo* frameInfo, size_t *frameCount) = 0;

		/*
		* Decode the picture internally, returning whether it succeeded. \|frameInfo\|
		* will be filled with information of the primary picture upon success and
		@@ -155,6 +135,20 @@ struct HeifDecoder {
		*/
		virtual bool decode(HeifFrameInfo* frameInfo) = 0;

		/*
		* Decode the picture from the image sequence at index \|frameIndex\|.
		* \|frameInfo\| will be filled with information of the decoded picture upon
		* success and unmodified upon failure.
		*
		* \|frameIndex\| is the 0-based index of the video frame to retrieve. The frame
		* index must be that of a valid frame. The total number of frames available for
		* retrieval was reported via getSequenceInfo().
		*
		* After this succeeded, getScanline can be called to read the scanlines
		* that were decoded.
		*/
		virtual bool decodeSequence(int frameIndex, HeifFrameInfo* frameInfo) = 0;

		/*
		* Read the next scanline (in top-down order), returns true upon success
		* and false otherwise.