Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0c1407f7 authored by Chong Zhang's avatar Chong Zhang
Browse files

Decode one row of tiles at a time for image that has tiles 

Add an api to IMediaMetadataRetriever to decode image rect.
It will reuse the same full frame IMemory, and decode only
the requested rect. For now, StagefrightMetadataRetriever
will only allow decoding of rect that's a full row of tiles,
and the requested must be issued sequentially (i.e. no
arbitrary rects). When the extract side is fixed to allow
seeking by tiles, it can be extended to allow arbitrary
rects.

This allows HeifDecoderImpl (on client side) to start
processing the getScanlines in parallel with the decoding.

Test: CTS MediaMetadataRetrieverTest;
Manual testing of HEIF decoding of files with or without tiles;
Manual testing of HEIF thumbnails generation in Downloads app.

bug: 78475896
Change-Id: I820b21cdf33f80593ee6092d8e1ba68b3beb65dd
parent 0c1f9148

include/private/media/VideoFrame.h

+4 −0
Original line number Diff line number Diff line
@@ -37,9 +37,11 @@ public: 
    // will calculate frame buffer size if |hasData| is set to true.

    VideoFrame(uint32_t width, uint32_t height,

            uint32_t displayWidth, uint32_t displayHeight,

            uint32_t tileWidth, uint32_t tileHeight,

            uint32_t angle, uint32_t bpp, bool hasData, size_t iccSize):

        mWidth(width), mHeight(height),

        mDisplayWidth(displayWidth), mDisplayHeight(displayHeight),

        mTileWidth(tileWidth), mTileHeight(tileHeight),

        mRotationAngle(angle), mBytesPerPixel(bpp), mRowBytes(bpp * width),

        mSize(hasData ? (bpp * width * height) : 0),

        mIccSize(iccSize), mReserved(0) {
@@ -74,6 +76,8 @@ public: 
    uint32_t mHeight;          // Decoded image height before rotation

    uint32_t mDisplayWidth;    // Display width before rotation

    uint32_t mDisplayHeight;   // Display height before rotation

    uint32_t mTileWidth;       // Tile width (0 if image doesn't have grid)

    uint32_t mTileHeight;      // Tile height (0 if image doesn't have grid)

    int32_t  mRotationAngle;   // Rotation angle, clockwise, should be multiple of 90

    uint32_t mBytesPerPixel;   // Number of bytes per pixel

    uint32_t mRowBytes;        // Number of bytes per row before rotation

media/libheif/HeifDecoderImpl.cpp

+134 −19
Original line number Diff line number Diff line
@@ -271,17 +271,43 @@ status_t HeifDataSource::getSize(off64_t* size) { 


/////////////////////////////////////////////////////////////////////////



struct HeifDecoderImpl::DecodeThread : public Thread {

    explicit DecodeThread(HeifDecoderImpl *decoder) : mDecoder(decoder) {}



private:

    HeifDecoderImpl* mDecoder;



    bool threadLoop();



    DISALLOW_EVIL_CONSTRUCTORS(DecodeThread);

};



bool HeifDecoderImpl::DecodeThread::threadLoop() {

    return mDecoder->decodeAsync();

}



/////////////////////////////////////////////////////////////////////////



HeifDecoderImpl::HeifDecoderImpl() :

    // output color format should always be set via setOutputColor(), in case

    // it's not, default to HAL_PIXEL_FORMAT_RGB_565.

    mOutputColor(HAL_PIXEL_FORMAT_RGB_565),

    mCurScanline(0),

    mWidth(0),

    mHeight(0),

    mFrameDecoded(false),

    mHasImage(false),

    mHasVideo(false) {

    mHasVideo(false),

    mAvailableLines(0),

    mNumSlices(1),

    mSliceHeight(0),

    mAsyncDecodeDone(false) {

}



HeifDecoderImpl::~HeifDecoderImpl() {

    if (mThread != nullptr) {

        mThread->join();

    }

}



bool HeifDecoderImpl::init(HeifStream* stream, HeifFrameInfo* frameInfo) {
@@ -310,22 +336,23 @@ bool HeifDecoderImpl::init(HeifStream* stream, HeifFrameInfo* frameInfo) { 


    mHasImage = hasImage && !strcasecmp(hasImage, "yes");

    mHasVideo = hasVideo && !strcasecmp(hasVideo, "yes");

    sp<IMemory> sharedMem;

    if (mHasImage) {

        // image index < 0 to retrieve primary image

        mFrameMemory = mRetriever->getImageAtIndex(

        sharedMem = mRetriever->getImageAtIndex(

                -1, mOutputColor, true /*metaOnly*/);

    } else if (mHasVideo) {

        mFrameMemory = mRetriever->getFrameAtTime(0,

        sharedMem = mRetriever->getFrameAtTime(0,

                MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC,

                mOutputColor, true /*metaOnly*/);

    }



    if (mFrameMemory == nullptr || mFrameMemory->pointer() == nullptr) {

    if (sharedMem == nullptr || sharedMem->pointer() == nullptr) {

        ALOGE("getFrameAtTime: videoFrame is a nullptr");

        return false;

    }



    VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->pointer());

    VideoFrame* videoFrame = static_cast<VideoFrame*>(sharedMem->pointer());



    ALOGV("Meta dimension %dx%d, display %dx%d, angle %d, iccSize %d",

            videoFrame->mWidth,
@@ -344,6 +371,14 @@ bool HeifDecoderImpl::init(HeifStream* stream, HeifFrameInfo* frameInfo) { 
                videoFrame->mIccSize,

                videoFrame->getFlattenedIccData());

    }

    mWidth = videoFrame->mWidth;

    mHeight = videoFrame->mHeight;

    if (mHasImage && videoFrame->mTileHeight >= 512 && mWidth >= 3000 && mHeight >= 2000 ) {

        // Try decoding in slices only if the image has tiles and is big enough.

        mSliceHeight = videoFrame->mTileHeight;

        mNumSlices = (videoFrame->mHeight + mSliceHeight - 1) / mSliceHeight;

        ALOGV("mSliceHeight %u, mNumSlices %zu", mSliceHeight, mNumSlices);

    }

    return true;

}
@@ -376,6 +411,36 @@ bool HeifDecoderImpl::setOutputColor(HeifColorFormat heifColor) { 
    return false;

}



bool HeifDecoderImpl::decodeAsync() {

    for (size_t i = 1; i < mNumSlices; i++) {

        ALOGV("decodeAsync(): decoding slice %zu", i);

        size_t top = i * mSliceHeight;

        size_t bottom = (i + 1) * mSliceHeight;

        if (bottom > mHeight) {

            bottom = mHeight;

        }

        sp<IMemory> frameMemory = mRetriever->getImageRectAtIndex(

                -1, mOutputColor, 0, top, mWidth, bottom);

        {

            Mutex::Autolock autolock(mLock);



            if (frameMemory == nullptr || frameMemory->pointer() == nullptr) {

                mAsyncDecodeDone = true;

                mScanlineReady.signal();

                break;

            }

            mFrameMemory = frameMemory;

            mAvailableLines = bottom;

            ALOGV("decodeAsync(): available lines %zu", mAvailableLines);

            mScanlineReady.signal();

        }

    }

    // Aggressive clear to avoid holding on to resources

    mRetriever.clear();

    mDataSource.clear();

    return false;

}



bool HeifDecoderImpl::decode(HeifFrameInfo* frameInfo) {

    // reset scanline pointer

    mCurScanline = 0;
@@ -384,6 +449,47 @@ bool HeifDecoderImpl::decode(HeifFrameInfo* frameInfo) { 
        return true;

    }



    // See if we want to decode in slices to allow client to start

    // scanline processing in parallel with decode. If this fails

    // we fallback to decoding the full frame.

    if (mHasImage && mNumSlices > 1) {

        // get first slice and metadata

        sp<IMemory> frameMemory = mRetriever->getImageRectAtIndex(

                -1, mOutputColor, 0, 0, mWidth, mSliceHeight);



        if (frameMemory == nullptr || frameMemory->pointer() == nullptr) {

            ALOGE("decode: metadata is a nullptr");

            return false;

        }



        VideoFrame* videoFrame = static_cast<VideoFrame*>(frameMemory->pointer());



        if (frameInfo != nullptr) {

            frameInfo->set(

                    videoFrame->mWidth,

                    videoFrame->mHeight,

                    videoFrame->mRotationAngle,

                    videoFrame->mBytesPerPixel,

                    videoFrame->mIccSize,

                    videoFrame->getFlattenedIccData());

        }



        mFrameMemory = frameMemory;

        mAvailableLines = mSliceHeight;

        mThread = new DecodeThread(this);

        if (mThread->run("HeifDecode", ANDROID_PRIORITY_FOREGROUND) == OK) {

            mFrameDecoded = true;

            return true;

        }



        // Fallback to decode without slicing

        mThread.clear();

        mNumSlices = 1;

        mSliceHeight = 0;

        mAvailableLines = 0;

        mFrameMemory.clear();

    }



    if (mHasImage) {

        // image index < 0 to retrieve primary image

        mFrameMemory = mRetriever->getImageAtIndex(-1, mOutputColor);
@@ -393,14 +499,14 @@ bool HeifDecoderImpl::decode(HeifFrameInfo* frameInfo) { 
    }



    if (mFrameMemory == nullptr || mFrameMemory->pointer() == nullptr) {

        ALOGE("getFrameAtTime: videoFrame is a nullptr");

        ALOGE("decode: videoFrame is a nullptr");

        return false;

    }



    VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->pointer());

    if (videoFrame->mSize == 0 ||

            mFrameMemory->size() < videoFrame->getFlattenedSize()) {

        ALOGE("getFrameAtTime: videoFrame size is invalid");

        ALOGE("decode: videoFrame size is invalid");

        return false;

    }
@@ -424,36 +530,45 @@ bool HeifDecoderImpl::decode(HeifFrameInfo* frameInfo) { 
    }

    mFrameDecoded = true;



    // Aggressive clear to avoid holding on to resources

    // Aggressively clear to avoid holding on to resources

    mRetriever.clear();

    mDataSource.clear();

    return true;

}



bool HeifDecoderImpl::getScanline(uint8_t* dst) {

bool HeifDecoderImpl::getScanlineInner(uint8_t* dst) {

    if (mFrameMemory == nullptr || mFrameMemory->pointer() == nullptr) {

        return false;

    }

    VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->pointer());

    if (mCurScanline >= videoFrame->mHeight) {

        ALOGE("no more scanline available");

        return false;

    }

    uint8_t* src = videoFrame->getFlattenedData() + videoFrame->mRowBytes * mCurScanline++;

    memcpy(dst, src, videoFrame->mBytesPerPixel * videoFrame->mWidth);

    return true;

}



size_t HeifDecoderImpl::skipScanlines(size_t count) {

    if (mFrameMemory == nullptr || mFrameMemory->pointer() == nullptr) {

        return 0;

bool HeifDecoderImpl::getScanline(uint8_t* dst) {

    if (mCurScanline >= mHeight) {

        ALOGE("no more scanline available");

        return false;

    }



    if (mNumSlices > 1) {

        Mutex::Autolock autolock(mLock);



        while (!mAsyncDecodeDone && mCurScanline >= mAvailableLines) {

            mScanlineReady.wait(mLock);

        }

        return (mCurScanline < mAvailableLines) ? getScanlineInner(dst) : false;

    }



    return getScanlineInner(dst);

}

    VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->pointer());



size_t HeifDecoderImpl::skipScanlines(size_t count) {

    uint32_t oldScanline = mCurScanline;

    mCurScanline += count;

    if (mCurScanline > videoFrame->mHeight) {

        mCurScanline = videoFrame->mHeight;

    if (mCurScanline > mHeight) {

        mCurScanline = mHeight;

    }

    return (mCurScanline > oldScanline) ? (mCurScanline - oldScanline) : 0;

}

media/libheif/HeifDecoderImpl.h

+18 −0
Original line number Diff line number Diff line
@@ -19,6 +19,8 @@ 


#include "include/HeifDecoderAPI.h"

#include <system/graphics.h>

#include <utils/Condition.h>

#include <utils/Mutex.h>

#include <utils/RefBase.h>



namespace android {
@@ -49,14 +51,30 @@ public: 
    size_t skipScanlines(size_t count) override;



private:

    struct DecodeThread;



    sp<IDataSource> mDataSource;

    sp<MediaMetadataRetriever> mRetriever;

    sp<IMemory> mFrameMemory;

    android_pixel_format_t mOutputColor;

    size_t mCurScanline;

    uint32_t mWidth;

    uint32_t mHeight;

    bool mFrameDecoded;

    bool mHasImage;

    bool mHasVideo;



    // Slice decoding only

    Mutex mLock;

    Condition mScanlineReady;

    sp<DecodeThread> mThread;

    size_t mAvailableLines;

    size_t mNumSlices;

    uint32_t mSliceHeight;

    bool mAsyncDecodeDone;



    bool decodeAsync();

    bool getScanlineInner(uint8_t* dst);

};



} // namespace android

media/libmedia/IMediaMetadataRetriever.cpp

+53 −0
Original line number Diff line number Diff line
@@ -69,6 +69,7 @@ enum { 
    SET_DATA_SOURCE_CALLBACK,

    GET_FRAME_AT_TIME,

    GET_IMAGE_AT_INDEX,

    GET_IMAGE_RECT_AT_INDEX,

    GET_FRAME_AT_INDEX,

    EXTRACT_ALBUM_ART,

    EXTRACT_METADATA,
@@ -187,6 +188,30 @@ public: 
        return interface_cast<IMemory>(reply.readStrongBinder());

    }



    sp<IMemory> getImageRectAtIndex(

            int index, int colorFormat, int left, int top, int right, int bottom)

    {

        ALOGV("getImageRectAtIndex: index %d, colorFormat(%d) rect {%d, %d, %d, %d}",

                index, colorFormat, left, top, right, bottom);

        Parcel data, reply;

        data.writeInterfaceToken(IMediaMetadataRetriever::getInterfaceDescriptor());

        data.writeInt32(index);

        data.writeInt32(colorFormat);

        data.writeInt32(left);

        data.writeInt32(top);

        data.writeInt32(right);

        data.writeInt32(bottom);

#ifndef DISABLE_GROUP_SCHEDULE_HACK

        sendSchedPolicy(data);

#endif

        remote()->transact(GET_IMAGE_RECT_AT_INDEX, data, &reply);

        status_t ret = reply.readInt32();

        if (ret != NO_ERROR) {

            return NULL;

        }

        return interface_cast<IMemory>(reply.readStrongBinder());

    }



    status_t getFrameAtIndex(std::vector<sp<IMemory> > *frames,

            int frameIndex, int numFrames, int colorFormat, bool metaOnly)

    {
@@ -375,6 +400,34 @@ status_t BnMediaMetadataRetriever::onTransact( 
#endif

            return NO_ERROR;

        } break;



        case GET_IMAGE_RECT_AT_INDEX: {

            CHECK_INTERFACE(IMediaMetadataRetriever, data, reply);

            int index = data.readInt32();

            int colorFormat = data.readInt32();

            int left = data.readInt32();

            int top = data.readInt32();

            int right = data.readInt32();

            int bottom = data.readInt32();

            ALOGV("getImageRectAtIndex: index(%d), colorFormat(%d), rect {%d, %d, %d, %d}",

                    index, colorFormat, left, top, right, bottom);

#ifndef DISABLE_GROUP_SCHEDULE_HACK

            setSchedPolicy(data);

#endif

            sp<IMemory> bitmap = getImageRectAtIndex(

                    index, colorFormat, left, top, right, bottom);

            if (bitmap != 0) {  // Don't send NULL across the binder interface

                reply->writeInt32(NO_ERROR);

                reply->writeStrongBinder(IInterface::asBinder(bitmap));

            } else {

                reply->writeInt32(UNKNOWN_ERROR);

            }

#ifndef DISABLE_GROUP_SCHEDULE_HACK

            restoreSchedPolicy();

#endif

            return NO_ERROR;

        } break;



        case GET_FRAME_AT_INDEX: {

            CHECK_INTERFACE(IMediaMetadataRetriever, data, reply);

            int frameIndex = data.readInt32();

media/libmedia/include/media/IMediaMetadataRetriever.h

+2 −0
Original line number Diff line number Diff line
@@ -46,6 +46,8 @@ public: 
            int64_t timeUs, int option, int colorFormat, bool metaOnly) = 0;

    virtual sp<IMemory>     getImageAtIndex(

            int index, int colorFormat, bool metaOnly, bool thumbnail) = 0;

    virtual sp<IMemory>     getImageRectAtIndex(

            int index, int colorFormat, int left, int top, int right, int bottom) = 0;

    virtual status_t        getFrameAtIndex(

            std::vector<sp<IMemory> > *frames,

            int frameIndex, int numFrames, int colorFormat, bool metaOnly) = 0;