h264dec: check for overflows when calculating allocation size. am: a3dd713 (9f3762d8) · Commits · e / os / android_frameworks_av

media/libstagefright/codecs/on2/h264dec/inc/H264SwDecApi.h

+1 −1

Original line number	Diff line number	Diff line
		@@ -161,7 +161,7 @@ extern "C"
		void H264SwDecTrace(char *);

		/* function prototype for memory allocation */
		void* H264SwDecMalloc(u32 size);
		void* H264SwDecMalloc(u32 size, u32 num);

		/* function prototype for memory free */
		void H264SwDecFree(void *ptr);

+6 −3

Original line number	Diff line number	Diff line
		@@ -700,18 +700,21 @@ void H264SwDecTrace(char *string)
		library function malloc for allocation of memory.

		------------------------------------------------------------------------------*/
		void* H264SwDecMalloc(u32 size)
		void* H264SwDecMalloc(u32 size, u32 num)
		{
		if (size > UINT32_MAX / num) {
		return NULL;
		}

		#if defined(CHECK_MEMORY_USAGE)
		/* Note that if the decoder has to free and reallocate some of the buffers
		* the total value will be invalid */
		static u32 numBytes = 0;
		numBytes += size;
		numBytes += size * num;
		DEBUG(("Allocated %d bytes, total %d\n", size, numBytes));
		#endif

		return malloc(size);
		return malloc(size * num);
		}

		/*------------------------------------------------------------------------------

+6 −3

Original line number	Diff line number	Diff line
		@@ -85,7 +85,7 @@ int main(int argc, char **argv)
		rewind(finput);

		/* allocate memory for stream buffer, exit if unsuccessful */
		byteStrm = byteStrmStart = (u8 )H264SwDecMalloc(sizeof(u8)strmLen);
		byteStrm = byteStrmStart = (u8 *)H264SwDecMalloc(sizeof(u8), strmLen);
		if (byteStrm == NULL)
		{
		printf("UNABLE TO ALLOCATE MEMORY\n");
		@@ -298,9 +298,12 @@ void H264SwDecTrace(char *string)
		library function malloc for allocation of memory.

		------------------------------------------------------------------------------*/
		void* H264SwDecMalloc(u32 size)
		void* H264SwDecMalloc(u32 size, u32 num)
		{
		return malloc(size);
		if (size > UINT32_MAX / num) {
		return NULL;
		}
		return malloc(size * num);
		}

		/*------------------------------------------------------------------------------

+10 −3

Original line number	Diff line number	Diff line
		@@ -35,6 +35,8 @@
		/*------------------------------------------------------------------------------
		1. Include headers
		------------------------------------------------------------------------------*/
		#include <log/log.h>

		#include <stdlib.h>
		#include "basetype.h"
		#include "h264bsd_container.h"
		@@ -75,8 +77,13 @@ H264DEC_EVALUATION Compile evaluation version, restricts number of frames
		void H264SwDecTrace(char *string) {
		}

		void* H264SwDecMalloc(u32 size) {
		return malloc(size);
		void* H264SwDecMalloc(u32 size, u32 num) {
		if (size > UINT32_MAX / num) {
		ALOGE("can't allocate %u * %u bytes", size, num);
		android_errorWriteLog(0x534e4554, "27855419");
		return NULL;
		}
		return malloc(size * num);
		}

		void H264SwDecFree(void *ptr) {
		@@ -140,7 +147,7 @@ H264SwDecRet H264SwDecInit(H264SwDecInst *decInst, u32 noOutputReordering)
		return(H264SWDEC_PARAM_ERR);
		}

		pDecCont = (decContainer_t *)H264SwDecMalloc(sizeof(decContainer_t));
		pDecCont = (decContainer_t *)H264SwDecMalloc(sizeof(decContainer_t), 1);

		if (pDecCont == NULL)
		{

+5 −2

Original line number	Diff line number	Diff line
		@@ -413,9 +413,12 @@ void H264SwDecTrace(char *string)
		Function name: H264SwDecmalloc

		------------------------------------------------------------------------------*/
		void* H264SwDecMalloc(u32 size)
		void* H264SwDecMalloc(u32 size, u32 num)
		{
		return malloc(size);
		if (size > UINT32_MAX / num) {
		return NULL;
		}
		return malloc(size * num);
		}

		/*------------------------------------------------------------------------------