Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 65842db0 authored by Wei Jia's avatar Wei Jia
Browse files

Merge commit 'b46eb8d9' into HEAD 

SoftAVCEnc: check requested memory size before allocation.

Bug: 20674674
(cherry picked from commit f6fe4340)

Change-Id: Ib52a26de912fd5a9a08dd9948885fb4a9b32e9e6
parents e4ac35fa b46eb8d9

media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp

+8 −0
Original line number Diff line number Diff line
@@ -234,6 +234,10 @@ OMX_ERRORTYPE SoftAVCEncoder::initEncParams() { 
            || mStoreMetaDataInBuffers) {

        // Color conversion is needed.

        free(mInputFrameData);

        if (((uint64_t)mVideoWidth * mVideoHeight) > ((uint64_t)INT32_MAX / 3)) {

            ALOGE("Buffer size is too big.");

            return OMX_ErrorUndefined;

        }

        mInputFrameData =

            (uint8_t *) malloc((mVideoWidth * mVideoHeight * 3 ) >> 1);

        CHECK(mInputFrameData != NULL);
@@ -255,6 +259,10 @@ OMX_ERRORTYPE SoftAVCEncoder::initEncParams() { 
    int32_t nMacroBlocks = ((((mVideoWidth + 15) >> 4) << 4) *

            (((mVideoHeight + 15) >> 4) << 4)) >> 8;

    CHECK(mSliceGroup == NULL);

    if (nMacroBlocks > SIZE_MAX / sizeof(uint32_t)) {

        ALOGE("requested memory size is too big.");

        return OMX_ErrorUndefined;

    }

    mSliceGroup = (uint32_t *) malloc(sizeof(uint32_t) * nMacroBlocks);

    CHECK(mSliceGroup != NULL);

    for (int ii = 0, idx = 0; ii < nMacroBlocks; ++ii) {