Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 3fd96683 authored by Wei Jia's avatar Wei Jia
Browse files

SoftAVCEnc: check requested memory size before allocation. 

Bug: 20674674

Change-Id: If80186a7b9078e575d389220f3bebe9f7630a956
(cherry picked from commit f6fe4340)
parent fc8c226b

media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp

+16 −0
Original line number Original line Diff line number Diff line
@@ -34,6 +34,10 @@ 




#include "SoftAVCEncoder.h"

#include "SoftAVCEncoder.h"





#ifndef INT32_MAX

#define INT32_MAX   2147483647

#endif



namespace android {

namespace android {





template<class T>

template<class T>
@@ -257,6 +261,10 @@ OMX_ERRORTYPE SoftAVCEncoder::initEncParams() { 
    if (mVideoColorFormat == OMX_COLOR_FormatYUV420SemiPlanar) {

    if (mVideoColorFormat == OMX_COLOR_FormatYUV420SemiPlanar) {

        // Color conversion is needed.

        // Color conversion is needed.

        CHECK(mInputFrameData == NULL);

        CHECK(mInputFrameData == NULL);

        if (((uint64_t)mVideoWidth * mVideoHeight) > ((uint64_t)INT32_MAX / 3)) {

            ALOGE("Buffer size is too big.");

            return OMX_ErrorUndefined;

        }

        mInputFrameData =

        mInputFrameData =

            (uint8_t *) malloc((mVideoWidth * mVideoHeight * 3 ) >> 1);

            (uint8_t *) malloc((mVideoWidth * mVideoHeight * 3 ) >> 1);

        CHECK(mInputFrameData != NULL);

        CHECK(mInputFrameData != NULL);
@@ -278,6 +286,10 @@ OMX_ERRORTYPE SoftAVCEncoder::initEncParams() { 
    int32_t nMacroBlocks = ((((mVideoWidth + 15) >> 4) << 4) *

    int32_t nMacroBlocks = ((((mVideoWidth + 15) >> 4) << 4) *

            (((mVideoHeight + 15) >> 4) << 4)) >> 8;

            (((mVideoHeight + 15) >> 4) << 4)) >> 8;

    CHECK(mSliceGroup == NULL);

    CHECK(mSliceGroup == NULL);

    if ((size_t)nMacroBlocks > SIZE_MAX / sizeof(uint32_t)) {

        ALOGE("requested memory size is too big.");

        return OMX_ErrorUndefined;

    }

    mSliceGroup = (uint32_t *) malloc(sizeof(uint32_t) * nMacroBlocks);

    mSliceGroup = (uint32_t *) malloc(sizeof(uint32_t) * nMacroBlocks);

    CHECK(mSliceGroup != NULL);

    CHECK(mSliceGroup != NULL);

    for (int ii = 0, idx = 0; ii < nMacroBlocks; ++ii) {

    for (int ii = 0, idx = 0; ii < nMacroBlocks; ++ii) {
@@ -698,6 +710,10 @@ OMX_ERRORTYPE SoftAVCEncoder::internalSetParameter( 
            if (mStoreMetaDataInBuffers) {

            if (mStoreMetaDataInBuffers) {

                mVideoColorFormat == OMX_COLOR_FormatYUV420SemiPlanar;

                mVideoColorFormat == OMX_COLOR_FormatYUV420SemiPlanar;

                if (mInputFrameData == NULL) {

                if (mInputFrameData == NULL) {

                    if (((uint64_t)mVideoWidth * mVideoHeight) > ((uint64_t)INT32_MAX / 3)) {

                        ALOGE("Buffer size is too big.");

                        return OMX_ErrorUndefined;

                    }

                    mInputFrameData =

                    mInputFrameData =

                            (uint8_t *) malloc((mVideoWidth * mVideoHeight * 3 ) >> 1);

                            (uint8_t *) malloc((mVideoWidth * mVideoHeight * 3 ) >> 1);

                }

                }