Merge "libstagefright: check memory size for overflow before allocation." into mnc-dev (ca097142) · Commits · e / os / android_frameworks_av

media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp

+39 −0

Original line number	Diff line number	Diff line
		@@ -95,6 +95,11 @@ OSCL_EXPORT_REF Bool PVInitVideoDecoder(VideoDecControls decCtrl, uint8 volbuf
		#ifdef DEC_INTERNAL_MEMORY_OPT
		video->vol = (Vol **) IMEM_VOL;
		#else
		if ((size_t)nLayers > SIZE_MAX / sizeof(Vol *)) {
		status = PV_FALSE;
		goto fail;
		}

		video->vol = (Vol *) oscl_malloc(nLayers sizeof(Vol *));
		#endif
		if (video->vol == NULL) status = PV_FALSE;
		@@ -128,6 +133,11 @@ OSCL_EXPORT_REF Bool PVInitVideoDecoder(VideoDecControls decCtrl, uint8 volbuf
		else oscl_memset(video->prevVop, 0, sizeof(Vop));
		video->memoryUsage += (sizeof(Vop) * 2);

		if ((size_t)nLayers > SIZE_MAX / sizeof(Vop *)) {
		status = PV_FALSE;
		goto fail;
		}

		video->vopHeader = (Vop *) oscl_malloc(sizeof(Vop ) * nLayers);
		#endif
		if (video->vopHeader == NULL) status = PV_FALSE;
		@@ -277,6 +287,7 @@ OSCL_EXPORT_REF Bool PVInitVideoDecoder(VideoDecControls decCtrl, uint8 volbuf
		status = PV_FALSE;
		}

		fail:
		if (status == PV_FALSE) PVCleanUpVideoDecoder(decCtrl);

		return status;
		@@ -305,6 +316,10 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay
		video->nMBPerRow * video->nMBPerCol;
		}

		if (((uint64_t)video->width * video->height) > (uint64_t)INT32_MAX / sizeof(PIXEL)) {
		return PV_FALSE;
		}

		size = (int32)sizeof(PIXEL) * video->width * video->height;
		#ifdef PV_MEMORY_POOL
		decCtrl->size = size;
		@@ -320,6 +335,9 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay
		video->prevVop->uChan = video->prevVop->yChan + size;
		video->prevVop->vChan = video->prevVop->uChan + (size >> 2);
		#else
		if (size > INT32_MAX / 3 * 2) {
		return PV_FALSE;
		}
		video->currVop->yChan = (PIXEL ) oscl_malloc(size 3 / 2); /* Allocate memory for all VOP OKA 3/2/1*/
		if (video->currVop->yChan == NULL) status = PV_FALSE;

		@@ -347,6 +365,10 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay
		{
		oscl_memset(video->prevEnhcVop, 0, sizeof(Vop));
		#ifndef PV_MEMORY_POOL
		if (size > INT32_MAX / 3 * 2) {
		return PV_FALSE;
		}

		video->prevEnhcVop->yChan = (PIXEL ) oscl_malloc(size 3 / 2); /* Allocate memory for all VOP OKA 3/2/1*/
		if (video->prevEnhcVop->yChan == NULL) status = PV_FALSE;
		video->prevEnhcVop->uChan = video->prevEnhcVop->yChan + size;
		@@ -403,10 +425,17 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay
		if (video->acPredFlag == NULL) status = PV_FALSE;
		video->memoryUsage += (nTotalMB);

		if ((size_t)nTotalMB > SIZE_MAX / sizeof(typeDCStore)) {
		return PV_FALSE;
		}
		video->predDC = (typeDCStore ) oscl_malloc(nTotalMB sizeof(typeDCStore));
		if (video->predDC == NULL) status = PV_FALSE;
		video->memoryUsage += (nTotalMB * sizeof(typeDCStore));

		if (nMBPerRow > INT32_MAX - 1
		\|\| (size_t)(nMBPerRow + 1) > SIZE_MAX / sizeof(typeDCACStore)) {
		return PV_FALSE;
		}
		video->predDCAC_col = (typeDCACStore ) oscl_malloc((nMBPerRow + 1) sizeof(typeDCACStore));
		if (video->predDCAC_col == NULL) status = PV_FALSE;
		video->memoryUsage += ((nMBPerRow + 1) * sizeof(typeDCACStore));
		@@ -422,6 +451,10 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay
		video->headerInfo.CBP = (uint8 *) oscl_malloc(nTotalMB);
		if (video->headerInfo.CBP == NULL) status = PV_FALSE;
		video->memoryUsage += nTotalMB;

		if ((size_t)nTotalMB > SIZE_MAX / sizeof(int16)) {
		return PV_FALSE;
		}
		video->QPMB = (int16 ) oscl_malloc(nTotalMB sizeof(int16));
		if (video->QPMB == NULL) status = PV_FALSE;
		video->memoryUsage += (nTotalMB * sizeof(int));
		@@ -439,6 +472,9 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay
		video->memoryUsage += sizeof(MacroBlock);
		}
		/* Allocating motion vector space */
		if ((size_t)nTotalMB > SIZE_MAX / (sizeof(MOT) * 4)) {
		return PV_FALSE;
		}
		video->motX = (MOT ) oscl_malloc(sizeof(MOT) 4 * nTotalMB);
		if (video->motX == NULL) status = PV_FALSE;
		video->motY = (MOT ) oscl_malloc(sizeof(MOT) 4 * nTotalMB);
		@@ -472,6 +508,9 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay
		}

		#else
		if (nTotalMB > INT32_MAX / 6) {
		return PV_FALSE;
		}
		video->pstprcTypCur = (uint8 ) oscl_malloc(nTotalMB 6);
		video->memoryUsage += (nTotalMB * 6);
		if (video->pstprcTypCur == NULL)

media/libstagefright/codecs/m4v_h263/enc/src/mp4enc_api.cpp

+26 −0

Original line number	Diff line number	Diff line
		@@ -610,6 +610,10 @@ OSCL_EXPORT_REF Bool PVInitVideoEncoder(VideoEncControls *encoderControl, Vid
		max = temp_w * temp_h;
		max_width = ((temp_w + 15) >> 4) << 4;
		max_height = ((temp_h + 15) >> 4) << 4;
		if (((uint64_t)max_width * max_height) > (uint64_t)INT32_MAX
		\|\| temp_w > INT32_MAX - 15 \|\| temp_h > INT32_MAX - 15) {
		goto CLEAN_UP;
		}
		nTotalMB = ((max_width * max_height) >> 8);
		}

		@@ -654,6 +658,9 @@ OSCL_EXPORT_REF Bool PVInitVideoEncoder(VideoEncControls *encoderControl, Vid

		/* Allocating motion vector space and interpolation memory*/

		if ((size_t)nTotalMB > SIZE_MAX / sizeof(MOT *)) {
		goto CLEAN_UP;
		}
		video->mot = (MOT *)M4VENC_MALLOC(sizeof(MOT ) * nTotalMB);
		if (video->mot == NULL) goto CLEAN_UP;

		@@ -676,11 +683,17 @@ OSCL_EXPORT_REF Bool PVInitVideoEncoder(VideoEncControls *encoderControl, Vid
		/* so that compilers can generate faster code to indexing the */
		/* data inside (by using << instead of ). 04/14/2000. /
		/* 5/29/01, use decoder lib ACDC prediction memory scheme. */
		if ((size_t)nTotalMB > SIZE_MAX / sizeof(typeDCStore)) {
		goto CLEAN_UP;
		}
		video->predDC = (typeDCStore ) M4VENC_MALLOC(nTotalMB sizeof(typeDCStore));
		if (video->predDC == NULL) goto CLEAN_UP;

		if (!video->encParams->H263_Enabled)
		{
		if ((size_t)((max_width >> 4) + 1) > SIZE_MAX / sizeof(typeDCACStore)) {
		goto CLEAN_UP;
		}
		video->predDCAC_col = (typeDCACStore ) M4VENC_MALLOC(((max_width >> 4) + 1) sizeof(typeDCACStore));
		if (video->predDCAC_col == NULL) goto CLEAN_UP;

		@@ -688,6 +701,9 @@ OSCL_EXPORT_REF Bool PVInitVideoEncoder(VideoEncControls *encoderControl, Vid
		/* the rest will be used for storing horizontal (row) AC coefficients */
		video->predDCAC_row = video->predDCAC_col + 1; /* ACDC */

		if ((size_t)nTotalMB > SIZE_MAX / sizeof(Int)) {
		goto CLEAN_UP;
		}
		video->acPredFlag = (Int ) M4VENC_MALLOC(nTotalMB sizeof(Int)); /* Memory for acPredFlag */
		if (video->acPredFlag == NULL) goto CLEAN_UP;
		}
		@@ -741,8 +757,15 @@ OSCL_EXPORT_REF Bool PVInitVideoEncoder(VideoEncControls *encoderControl, Vid
		offset = (pitch << 4) + 16;
		max_height += 32;
		}
		if (((uint64_t)pitch * max_height) > (uint64_t)INT32_MAX) {
		goto CLEAN_UP;
		}
		size = pitch * max_height;

		if (size > INT32_MAX - (size >> 1)
		\|\| (size_t)(size + (size >> 1)) > SIZE_MAX / sizeof(PIXEL)) {
		goto CLEAN_UP;
		}
		video->currVop->yChan = (PIXEL )M4VENC_MALLOC(sizeof(PIXEL) (size + (size >> 1))); /* Memory for currVop Y */
		if (video->currVop->yChan == NULL) goto CLEAN_UP;
		video->currVop->uChan = video->currVop->yChan + size;/* Memory for currVop U */
		@@ -841,6 +864,9 @@ OSCL_EXPORT_REF Bool PVInitVideoEncoder(VideoEncControls *encoderControl, Vid
		/* /// End /////////////////////////////////////// */


		if ((size_t)nLayers > SIZE_MAX / sizeof(Vol *)) {
		goto CLEAN_UP;
		}
		video->vol = (Vol *)M4VENC_MALLOC(nLayers sizeof(Vol )); / Memory for VOL pointers */

		/* Memory allocation and Initialization of Vols and writing of headers */