Merge "media, mainline: Add basic AVIF decoding support" (d49109c3) · Commits · e / os / android_frameworks_av

media/extractors/mp4/ItemTable.cpp

+61 −9

Original line number	Diff line number	Diff line
		@@ -76,6 +76,7 @@ struct ImageItem {
		size_t size;
		sp<ABuffer> hvcc;
		sp<ABuffer> icc;
		sp<ABuffer> av1c;

		Vector<uint32_t> thumbnails;
		Vector<uint32_t> dimgRefs;
		@@ -764,6 +765,39 @@ status_t HvccBox::parse(off64_t offset, size_t size) {
		return OK;
		}

		struct Av1cBox : public Box, public ItemProperty {
		Av1cBox(DataSourceHelper *source) :
		Box(source, FOURCC("av1C")) {}

		status_t parse(off64_t offset, size_t size) override;

		void attachTo(ImageItem &image) const override {
		image.av1c = mAv1c;
		}

		private:
		sp<ABuffer> mAv1c;
		};

		status_t Av1cBox::parse(off64_t offset, size_t size) {
		ALOGV("%s: offset %lld, size %zu", __FUNCTION__, (long long)offset, size);

		mAv1c = new ABuffer(size);

		if (mAv1c->data() == NULL) {
		ALOGE("b/28471206");
		return NO_MEMORY;
		}

		if (source()->readAt(offset, mAv1c->data(), size) < (ssize_t)size) {
		return ERROR_IO;
		}

		ALOGV("property av1C");

		return OK;
		}

		struct IrotBox : public Box, public ItemProperty {
		IrotBox(DataSourceHelper *source) :
		Box(source, FOURCC("irot")), mAngle(0) {}
		@@ -957,6 +991,11 @@ status_t IpcoBox::onChunkData(uint32_t type, off64_t offset, size_t size) {
		itemProperty = new ColrBox(source());
		break;
		}
		case FOURCC("av1C"):
		{
		itemProperty = new Av1cBox(source());
		break;
		}
		default:
		{
		// push dummy to maintain correct item property index
		@@ -1203,8 +1242,9 @@ status_t IinfBox::onChunkData(uint32_t type, off64_t offset, size_t size) {

		//////////////////////////////////////////////////////////////////

		ItemTable::ItemTable(DataSourceHelper *source)
		ItemTable::ItemTable(DataSourceHelper *source, bool isHeif)
		: mDataSource(source),
		mIsHeif(isHeif),
		mPrimaryItemId(0),
		mIdatOffset(0),
		mIdatSize(0),
		@@ -1363,7 +1403,8 @@ status_t ItemTable::buildImageItemsIfPossible(uint32_t type) {
		// 'Exif': EXIF metadata
		if (info.itemType != FOURCC("grid") &&
		info.itemType != FOURCC("hvc1") &&
		info.itemType != FOURCC("Exif")) {
		info.itemType != FOURCC("Exif") &&
		info.itemType != FOURCC("av01")) {
		continue;
		}

		@@ -1509,7 +1550,9 @@ AMediaFormat *ItemTable::getImageMeta(const uint32_t imageIndex) {
		}

		AMediaFormat *meta = AMediaFormat_new();
		AMediaFormat_setString(meta, AMEDIAFORMAT_KEY_MIME, MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC);
		AMediaFormat_setString(
		meta, AMEDIAFORMAT_KEY_MIME,
		mIsHeif ? MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC : MEDIA_MIMETYPE_IMAGE_AVIF);

		if (image->itemId == mPrimaryItemId) {
		AMediaFormat_setInt32(meta, AMEDIAFORMAT_KEY_IS_DEFAULT, 1);
		@@ -1539,7 +1582,7 @@ AMediaFormat *ItemTable::getImageMeta(const uint32_t imageIndex) {
		ssize_t thumbItemIndex = mItemIdToItemMap.indexOfKey(image->thumbnails[0]);
		if (thumbItemIndex >= 0) {
		const ImageItem &thumbnail = mItemIdToItemMap[thumbItemIndex];

		// TODO(vigneshv): Handle thumbnail for AVIF.
		if (thumbnail.hvcc != NULL) {
		AMediaFormat_setInt32(meta,
		AMEDIAFORMAT_KEY_THUMBNAIL_WIDTH, thumbnail.width);
		@@ -1574,12 +1617,21 @@ AMediaFormat *ItemTable::getImageMeta(const uint32_t imageIndex) {
		AMEDIAFORMAT_KEY_MAX_INPUT_SIZE, image->width * image->height * 3 / 2);
		}

		if (mIsHeif) {
		if (image->hvcc == NULL) {
		ALOGE("%s: hvcc is missing for image[%u]!", __FUNCTION__, imageIndex);
		return NULL;
		}
		AMediaFormat_setBuffer(meta,
		AMEDIAFORMAT_KEY_CSD_HEVC, image->hvcc->data(), image->hvcc->size());
		} else {
		if (image->av1c == NULL) {
		ALOGE("%s: av1c is missing for image[%u]!", __FUNCTION__, imageIndex);
		return NULL;
		}
		AMediaFormat_setBuffer(meta,
		AMEDIAFORMAT_KEY_CSD_0, image->av1c->data(), image->av1c->size());
		}

		if (image->icc != NULL) {
		AMediaFormat_setBuffer(meta,

media/extractors/mp4/ItemTable.h

+4 −2

Original line number	Diff line number	Diff line
		@@ -42,12 +42,12 @@ struct ItemReference;

		/*
		* ItemTable keeps track of all image items (including coded images, grids and
		* tiles) inside a HEIF still image (ISO/IEC FDIS 23008-12.2:2017(E)).
		* tiles) inside a HEIF/AVIF still image (ISO/IEC FDIS 23008-12.2:2017(E)).
		*/

		class ItemTable : public RefBase {
		public:
		explicit ItemTable(DataSourceHelper *source);
		ItemTable(DataSourceHelper *source, bool isHeif);

		status_t parse(uint32_t type, off64_t offset, size_t size);

		@@ -65,6 +65,8 @@ protected:

		private:
		DataSourceHelper *mDataSource;
		// If this is true, then this item table is for a HEIF image. Otherwise it is for an AVIF image.
		bool mIsHeif;

		KeyedVector<uint32_t, ItemLoc> mItemLocs;
		Vector<ItemInfo> mItemInfos;

media/extractors/mp4/MPEG4Extractor.cpp

+31 −11

Original line number	Diff line number	Diff line
		@@ -149,6 +149,7 @@ private:
		uint8_t *mSrcBuffer;

		bool mIsHeif;
		bool mIsAvif;
		bool mIsAudio;
		bool mIsUsac = false;
		sp<ItemTable> mItemTable;
		@@ -414,6 +415,7 @@ MPEG4Extractor::MPEG4Extractor(DataSourceHelper source, const char mime)
		mIsHeif(false),
		mHasMoovBox(false),
		mPreferHeif(mime != NULL && !strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_HEIF)),
		mIsAvif(false),
		mFirstTrack(NULL),
		mLastTrack(NULL) {
		ALOGV("mime=%s, mPreferHeif=%d", mime, mPreferHeif);
		@@ -670,7 +672,7 @@ status_t MPEG4Extractor::readMetaData() {
		}
		}

		if (mIsHeif && (mItemTable != NULL) && (mItemTable->countImages() > 0)) {
		if ((mIsAvif \|\| mIsHeif) && (mItemTable != NULL) && (mItemTable->countImages() > 0)) {
		off64_t exifOffset;
		size_t exifSize;
		if (mItemTable->getExifOffsetAndSize(&exifOffset, &exifSize) == OK) {
		@@ -696,7 +698,7 @@ status_t MPEG4Extractor::readMetaData() {
		}
		mInitCheck = OK;

		ALOGV("adding HEIF image track %u", imageIndex);
		ALOGV("adding %s image track %u", mIsHeif ? "HEIF" : "AVIF", imageIndex);
		Track *track = new Track;
		if (mLastTrack != NULL) {
		mLastTrack->next = track;
		@@ -722,6 +724,10 @@ status_t MPEG4Extractor::readMetaData() {
		MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC) != NULL) {
		AMediaFormat_setString(mFileMetaData,
		AMEDIAFORMAT_KEY_MIME, MEDIA_MIMETYPE_CONTAINER_HEIF);
		} else if (findTrackByMimePrefix(
		MEDIA_MIMETYPE_IMAGE_AVIF) != NULL) {
		AMediaFormat_setString(mFileMetaData,
		AMEDIAFORMAT_KEY_MIME, MEDIA_MIMETYPE_IMAGE_AVIF);
		} else {
		AMediaFormat_setString(mFileMetaData,
		AMEDIAFORMAT_KEY_MIME, "application/octet-stream");
		@@ -2576,9 +2582,9 @@ status_t MPEG4Extractor::parseChunk(off64_t *offset, int depth) {
		case FOURCC("iref"):
		case FOURCC("ipro"):
		{
		if (mIsHeif) {
		if (mIsHeif \|\| mIsAvif) {
		if (mItemTable == NULL) {
		mItemTable = new ItemTable(mDataSource);
		mItemTable = new ItemTable(mDataSource, mIsHeif);
		}
		status_t err = mItemTable->parse(
		chunk_type, data_offset, chunk_data_size);
		@@ -3019,14 +3025,20 @@ status_t MPEG4Extractor::parseChunk(off64_t *offset, int depth) {
		mIsHeif = true;
		brandSet.erase(FOURCC("mif1"));
		brandSet.erase(FOURCC("heic"));
		} else if (brandSet.count(FOURCC("avif")) > 0 \|\|
		brandSet.count(FOURCC("avis")) > 0) {
		ALOGV("identified AVIF image");
		mIsAvif = true;
		brandSet.erase(FOURCC("avif"));
		brandSet.erase(FOURCC("avis"));
		}

		if (!brandSet.empty()) {
		// This means that the file should have moov box.
		// It could be any iso files (mp4, heifs, etc.)
		mHasMoovBox = true;
		if (mIsHeif) {
		ALOGV("identified HEIF image with other tracks");
		if (mIsHeif \|\| mIsAvif) {
		ALOGV("identified %s image with other tracks", mIsHeif ? "HEIF" : "AVIF");
		}
		}
		}
		@@ -4364,7 +4376,8 @@ MediaTrackHelper *MPEG4Extractor::getTrack(size_t index) {
		if (size != 24 \|\| ((ptr[0] != 1 \|\| ptr[1] != 0) && (ptr[0] != 2 \|\| ptr[1] != 1))) {
		return NULL;
		}
		} else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AV1)) {
		} else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AV1)
		\|\| !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_AVIF)) {
		void *data;
		size_t size;
		if (!AMediaFormat_getBuffer(track->meta, AMEDIAFORMAT_KEY_CSD_0, &data, &size)) {
		@@ -4376,6 +4389,9 @@ MediaTrackHelper *MPEG4Extractor::getTrack(size_t index) {
		if (size < 5 \|\| ptr[0] != 0x81) { // configurationVersion == 1
		return NULL;
		}
		if (!strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_AVIF)) {
		itemTable = mItemTable;
		}
		} else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_VP9)) {
		void *data;
		size_t size;
		@@ -4937,7 +4953,6 @@ MPEG4Source::MPEG4Source(
		mStarted(false),
		mBuffer(NULL),
		mSrcBuffer(NULL),
		mIsHeif(itemTable != NULL),
		mItemTable(itemTable),
		mElstShiftStartTicks(elstShiftStartTicks),
		mElstInitialEmptyEditTicks(elstInitialEmptyEditTicks) {
		@@ -4972,6 +4987,8 @@ MPEG4Source::MPEG4Source(
		!strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC);
		mIsAC4 = !strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AC4);
		mIsDolbyVision = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_DOLBY_VISION);
		mIsHeif = !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC) && mItemTable != NULL;
		mIsAvif = !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_AVIF) && mItemTable != NULL;

		if (mIsAVC) {
		void *data;
		@@ -5966,7 +5983,7 @@ media_status_t MPEG4Source::read(

		if (options && options->getSeekTo(&seekTimeUs, &mode)) {
		ALOGV("seekTimeUs:%" PRId64, seekTimeUs);
		if (mIsHeif) {
		if (mIsHeif \|\| mIsAvif) {
		CHECK(mSampleTable == NULL);
		CHECK(mItemTable != NULL);
		int32_t imageIndex;
		@@ -6111,7 +6128,7 @@ media_status_t MPEG4Source::read(
		newBuffer = true;

		status_t err;
		if (!mIsHeif) {
		if (!mIsHeif && !mIsAvif) {
		err = mSampleTable->getMetaDataForSample(mCurrentSampleIndex, &offset, &size,
		(uint64_t*)&cts, &isSyncSample, &stts);
		if(err == OK) {
		@@ -6750,7 +6767,8 @@ static bool LegacySniffMPEG4(DataSourceHelper source, float confidence) {
		\|\| !memcmp(header, "ftypM4A ", 8) \|\| !memcmp(header, "ftypf4v ", 8)
		\|\| !memcmp(header, "ftypkddi", 8) \|\| !memcmp(header, "ftypM4VP", 8)
		\|\| !memcmp(header, "ftypmif1", 8) \|\| !memcmp(header, "ftypheic", 8)
		\|\| !memcmp(header, "ftypmsf1", 8) \|\| !memcmp(header, "ftyphevc", 8)) {
		\|\| !memcmp(header, "ftypmsf1", 8) \|\| !memcmp(header, "ftyphevc", 8)
		\|\| !memcmp(header, "ftypavif", 8) \|\| !memcmp(header, "ftypavis", 8)) {
		*confidence = 0.4;

		return true;
		@@ -6786,6 +6804,8 @@ static bool isCompatibleBrand(uint32_t fourcc) {
		FOURCC("heic"), // HEIF image
		FOURCC("msf1"), // HEIF image sequence
		FOURCC("hevc"), // HEIF image sequence
		FOURCC("avif"), // AVIF image
		FOURCC("avis"), // AVIF image sequence
		};

		for (size_t i = 0;

media/extractors/mp4/MPEG4Extractor.h

+1 −0

Original line number	Diff line number	Diff line
		@@ -144,6 +144,7 @@ private:
		bool mIsHeif;
		bool mHasMoovBox;
		bool mPreferHeif;
		bool mIsAvif;

		Track mFirstTrack, mLastTrack;