Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f89bf3c8 authored by Vignesh Venkatasubramanian's avatar Vignesh Venkatasubramanian
Browse files

media, mainline: Add basic AVIF decoding support 

This CL contains files that are part of mainline.

Parse the AVIF boxes and pass the data along to a compatible AV1
decoder (similar to how HEIC frames are passed to the HEVC
decoder).

Test: BitmapFactory.decodeByteArray() is now able to decode valid AVIF images.
Test: HEIF related CTS tests still pass.

CTS tests to come in a different CL.

Bug: 141654151

Change-Id: I84a01db859281a1b14746fbb97c247f427d440f8
parent b79b20f8

media/extractors/mp4/ItemTable.cpp

+61 −9
Original line number Diff line number Diff line
@@ -76,6 +76,7 @@ struct ImageItem { 
    size_t size;

    sp<ABuffer> hvcc;

    sp<ABuffer> icc;

    sp<ABuffer> av1c;



    Vector<uint32_t> thumbnails;

    Vector<uint32_t> dimgRefs;
@@ -764,6 +765,39 @@ status_t HvccBox::parse(off64_t offset, size_t size) { 
    return OK;

}



struct Av1cBox : public Box, public ItemProperty {

    Av1cBox(DataSourceHelper *source) :

        Box(source, FOURCC("av1C")) {}



    status_t parse(off64_t offset, size_t size) override;



    void attachTo(ImageItem &image) const override {

        image.av1c = mAv1c;

    }



private:

    sp<ABuffer> mAv1c;

};



status_t Av1cBox::parse(off64_t offset, size_t size) {

    ALOGV("%s: offset %lld, size %zu", __FUNCTION__, (long long)offset, size);



    mAv1c = new ABuffer(size);



    if (mAv1c->data() == NULL) {

        ALOGE("b/28471206");

        return NO_MEMORY;

    }



    if (source()->readAt(offset, mAv1c->data(), size) < (ssize_t)size) {

        return ERROR_IO;

    }



    ALOGV("property av1C");



    return OK;

}



struct IrotBox : public Box, public ItemProperty {

    IrotBox(DataSourceHelper *source) :

        Box(source, FOURCC("irot")), mAngle(0) {}
@@ -957,6 +991,11 @@ status_t IpcoBox::onChunkData(uint32_t type, off64_t offset, size_t size) { 
            itemProperty = new ColrBox(source());

            break;

        }

        case FOURCC("av1C"):

        {

            itemProperty = new Av1cBox(source());

            break;

        }

        default:

        {

            // push dummy to maintain correct item property index
@@ -1203,8 +1242,9 @@ status_t IinfBox::onChunkData(uint32_t type, off64_t offset, size_t size) { 


//////////////////////////////////////////////////////////////////



ItemTable::ItemTable(DataSourceHelper *source)

ItemTable::ItemTable(DataSourceHelper *source, bool isHeif)

    : mDataSource(source),

      mIsHeif(isHeif),

      mPrimaryItemId(0),

      mIdatOffset(0),

      mIdatSize(0),
@@ -1363,7 +1403,8 @@ status_t ItemTable::buildImageItemsIfPossible(uint32_t type) { 
        //   'Exif': EXIF metadata

        if (info.itemType != FOURCC("grid") &&

            info.itemType != FOURCC("hvc1") &&

            info.itemType != FOURCC("Exif")) {

            info.itemType != FOURCC("Exif") &&

            info.itemType != FOURCC("av01")) {

            continue;

        }
@@ -1509,7 +1550,9 @@ AMediaFormat *ItemTable::getImageMeta(const uint32_t imageIndex) { 
    }



    AMediaFormat *meta = AMediaFormat_new();

    AMediaFormat_setString(meta, AMEDIAFORMAT_KEY_MIME, MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC);

    AMediaFormat_setString(

        meta, AMEDIAFORMAT_KEY_MIME,

        mIsHeif ? MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC : MEDIA_MIMETYPE_IMAGE_AVIF);



    if (image->itemId == mPrimaryItemId) {

        AMediaFormat_setInt32(meta, AMEDIAFORMAT_KEY_IS_DEFAULT, 1);
@@ -1539,7 +1582,7 @@ AMediaFormat *ItemTable::getImageMeta(const uint32_t imageIndex) { 
        ssize_t thumbItemIndex = mItemIdToItemMap.indexOfKey(image->thumbnails[0]);

        if (thumbItemIndex >= 0) {

            const ImageItem &thumbnail = mItemIdToItemMap[thumbItemIndex];



            // TODO(vigneshv): Handle thumbnail for AVIF.

            if (thumbnail.hvcc != NULL) {

                AMediaFormat_setInt32(meta,

                        AMEDIAFORMAT_KEY_THUMBNAIL_WIDTH, thumbnail.width);
@@ -1574,12 +1617,21 @@ AMediaFormat *ItemTable::getImageMeta(const uint32_t imageIndex) { 
                AMEDIAFORMAT_KEY_MAX_INPUT_SIZE, image->width * image->height * 3 / 2);

    }



    if (mIsHeif) {

        if (image->hvcc == NULL) {

            ALOGE("%s: hvcc is missing for image[%u]!", __FUNCTION__, imageIndex);

            return NULL;

        }

        AMediaFormat_setBuffer(meta,

                AMEDIAFORMAT_KEY_CSD_HEVC, image->hvcc->data(), image->hvcc->size());

    } else {

        if (image->av1c == NULL) {

            ALOGE("%s: av1c is missing for image[%u]!", __FUNCTION__, imageIndex);

            return NULL;

        }

        AMediaFormat_setBuffer(meta,

                AMEDIAFORMAT_KEY_CSD_0, image->av1c->data(), image->av1c->size());

    }



    if (image->icc != NULL) {

        AMediaFormat_setBuffer(meta,

media/extractors/mp4/ItemTable.h

+4 −2
Original line number Diff line number Diff line
@@ -42,12 +42,12 @@ struct ItemReference; 


/*

 * ItemTable keeps track of all image items (including coded images, grids and

 * tiles) inside a HEIF still image (ISO/IEC FDIS 23008-12.2:2017(E)).

 * tiles) inside a HEIF/AVIF still image (ISO/IEC FDIS 23008-12.2:2017(E)).

 */



class ItemTable : public RefBase {

public:

    explicit ItemTable(DataSourceHelper *source);

    ItemTable(DataSourceHelper *source, bool isHeif);



    status_t parse(uint32_t type, off64_t offset, size_t size);
@@ -65,6 +65,8 @@ protected: 


private:

    DataSourceHelper *mDataSource;

    // If this is true, then this item table is for a HEIF image. Otherwise it is for an AVIF image.

    bool mIsHeif;



    KeyedVector<uint32_t, ItemLoc> mItemLocs;

    Vector<ItemInfo> mItemInfos;

media/extractors/mp4/MPEG4Extractor.cpp

+31 −11
Original line number Diff line number Diff line
@@ -149,6 +149,7 @@ private: 
    uint8_t *mSrcBuffer;



    bool mIsHeif;

    bool mIsAvif;

    bool mIsAudio;

    bool mIsUsac = false;

    sp<ItemTable> mItemTable;
@@ -414,6 +415,7 @@ MPEG4Extractor::MPEG4Extractor(DataSourceHelper *source, const char *mime) 
      mIsHeif(false),

      mHasMoovBox(false),

      mPreferHeif(mime != NULL && !strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_HEIF)),

      mIsAvif(false),

      mFirstTrack(NULL),

      mLastTrack(NULL) {

    ALOGV("mime=%s, mPreferHeif=%d", mime, mPreferHeif);
@@ -670,7 +672,7 @@ status_t MPEG4Extractor::readMetaData() { 
        }

    }



    if (mIsHeif && (mItemTable != NULL) && (mItemTable->countImages() > 0)) {

    if ((mIsAvif || mIsHeif) && (mItemTable != NULL) && (mItemTable->countImages() > 0)) {

        off64_t exifOffset;

        size_t exifSize;

        if (mItemTable->getExifOffsetAndSize(&exifOffset, &exifSize) == OK) {
@@ -696,7 +698,7 @@ status_t MPEG4Extractor::readMetaData() { 
            }

            mInitCheck = OK;



            ALOGV("adding HEIF image track %u", imageIndex);

            ALOGV("adding %s image track %u", mIsHeif ? "HEIF" : "AVIF", imageIndex);

            Track *track = new Track;

            if (mLastTrack != NULL) {

                mLastTrack->next = track;
@@ -722,6 +724,10 @@ status_t MPEG4Extractor::readMetaData() { 
                MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC) != NULL) {

            AMediaFormat_setString(mFileMetaData,

                    AMEDIAFORMAT_KEY_MIME, MEDIA_MIMETYPE_CONTAINER_HEIF);

        } else if (findTrackByMimePrefix(

                MEDIA_MIMETYPE_IMAGE_AVIF) != NULL) {

            AMediaFormat_setString(mFileMetaData,

                    AMEDIAFORMAT_KEY_MIME, MEDIA_MIMETYPE_IMAGE_AVIF);

        } else {

            AMediaFormat_setString(mFileMetaData,

                    AMEDIAFORMAT_KEY_MIME, "application/octet-stream");
@@ -2576,9 +2582,9 @@ status_t MPEG4Extractor::parseChunk(off64_t *offset, int depth) { 
        case FOURCC("iref"):

        case FOURCC("ipro"):

        {

            if (mIsHeif) {

            if (mIsHeif || mIsAvif) {

                if (mItemTable == NULL) {

                    mItemTable = new ItemTable(mDataSource);

                    mItemTable = new ItemTable(mDataSource, mIsHeif);

                }

                status_t err = mItemTable->parse(

                        chunk_type, data_offset, chunk_data_size);
@@ -3019,14 +3025,20 @@ status_t MPEG4Extractor::parseChunk(off64_t *offset, int depth) { 
                    mIsHeif = true;

                    brandSet.erase(FOURCC("mif1"));

                    brandSet.erase(FOURCC("heic"));

                } else if (brandSet.count(FOURCC("avif")) > 0 ||

                       brandSet.count(FOURCC("avis")) > 0) {

                    ALOGV("identified AVIF image");

                    mIsAvif = true;

                    brandSet.erase(FOURCC("avif"));

                    brandSet.erase(FOURCC("avis"));

                }



                if (!brandSet.empty()) {

                    // This means that the file should have moov box.

                    // It could be any iso files (mp4, heifs, etc.)

                    mHasMoovBox = true;

                    if (mIsHeif) {

                        ALOGV("identified HEIF image with other tracks");

                    if (mIsHeif || mIsAvif) {

                        ALOGV("identified %s image with other tracks", mIsHeif ? "HEIF" : "AVIF");

                    }

                }

            }
@@ -4364,7 +4376,8 @@ MediaTrackHelper *MPEG4Extractor::getTrack(size_t index) { 
        if (size != 24 || ((ptr[0] != 1 || ptr[1] != 0) && (ptr[0] != 2 || ptr[1] != 1))) {

            return NULL;

        }

   } else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AV1)) {

   } else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AV1)

           || !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_AVIF)) {

        void *data;

        size_t size;

        if (!AMediaFormat_getBuffer(track->meta, AMEDIAFORMAT_KEY_CSD_0, &data, &size)) {
@@ -4376,6 +4389,9 @@ MediaTrackHelper *MPEG4Extractor::getTrack(size_t index) { 
        if (size < 5 || ptr[0] != 0x81) {  // configurationVersion == 1

            return NULL;

        }

        if (!strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_AVIF)) {

            itemTable = mItemTable;

        }

    } else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_VP9)) {

        void *data;

        size_t size;
@@ -4937,7 +4953,6 @@ MPEG4Source::MPEG4Source( 
      mStarted(false),

      mBuffer(NULL),

      mSrcBuffer(NULL),

      mIsHeif(itemTable != NULL),

      mItemTable(itemTable),

      mElstShiftStartTicks(elstShiftStartTicks),

      mElstInitialEmptyEditTicks(elstInitialEmptyEditTicks) {
@@ -4972,6 +4987,8 @@ MPEG4Source::MPEG4Source( 
              !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC);

    mIsAC4 = !strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AC4);

    mIsDolbyVision = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_DOLBY_VISION);

    mIsHeif = !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC) && mItemTable != NULL;

    mIsAvif = !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_AVIF) && mItemTable != NULL;



    if (mIsAVC) {

        void *data;
@@ -5966,7 +5983,7 @@ media_status_t MPEG4Source::read( 


    if (options && options->getSeekTo(&seekTimeUs, &mode)) {

        ALOGV("seekTimeUs:%" PRId64, seekTimeUs);

        if (mIsHeif) {

        if (mIsHeif || mIsAvif) {

            CHECK(mSampleTable == NULL);

            CHECK(mItemTable != NULL);

            int32_t imageIndex;
@@ -6111,7 +6128,7 @@ media_status_t MPEG4Source::read( 
        newBuffer = true;



        status_t err;

        if (!mIsHeif) {

        if (!mIsHeif && !mIsAvif) {

            err = mSampleTable->getMetaDataForSample(mCurrentSampleIndex, &offset, &size,

                                                    (uint64_t*)&cts, &isSyncSample, &stts);

            if(err == OK) {
@@ -6750,7 +6767,8 @@ static bool LegacySniffMPEG4(DataSourceHelper *source, float *confidence) { 
        || !memcmp(header, "ftypM4A ", 8) || !memcmp(header, "ftypf4v ", 8)

        || !memcmp(header, "ftypkddi", 8) || !memcmp(header, "ftypM4VP", 8)

        || !memcmp(header, "ftypmif1", 8) || !memcmp(header, "ftypheic", 8)

        || !memcmp(header, "ftypmsf1", 8) || !memcmp(header, "ftyphevc", 8)) {

        || !memcmp(header, "ftypmsf1", 8) || !memcmp(header, "ftyphevc", 8)

        || !memcmp(header, "ftypavif", 8) || !memcmp(header, "ftypavis", 8)) {

        *confidence = 0.4;



        return true;
@@ -6786,6 +6804,8 @@ static bool isCompatibleBrand(uint32_t fourcc) { 
        FOURCC("heic"),  // HEIF image

        FOURCC("msf1"),  // HEIF image sequence

        FOURCC("hevc"),  // HEIF image sequence

        FOURCC("avif"),  // AVIF image

        FOURCC("avis"),  // AVIF image sequence

    };



    for (size_t i = 0;

media/extractors/mp4/MPEG4Extractor.h

+1 −0
Original line number Diff line number Diff line
@@ -144,6 +144,7 @@ private: 
    bool mIsHeif;

    bool mHasMoovBox;

    bool mPreferHeif;

    bool mIsAvif;



    Track *mFirstTrack, *mLastTrack;