h264dec: check for overflows when calculating allocation size. am: a3dd713 (9f3762d8) · Commits · e / os / android_frameworks_av

media/libstagefright/codecs/on2/h264dec/inc/H264SwDecApi.h

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -161,7 +161,7 @@ extern "C"
	void H264SwDecTrace(char *);		void H264SwDecTrace(char *);

	/* function prototype for memory allocation */		/* function prototype for memory allocation */
	void* H264SwDecMalloc(u32 size);		void* H264SwDecMalloc(u32 size, u32 num);

	/* function prototype for memory free */		/* function prototype for memory free */
	void H264SwDecFree(void *ptr);		void H264SwDecFree(void *ptr);

+6 −3

Original line number	Original line	Diff line number	Diff line
	@@ -700,18 +700,21 @@ void H264SwDecTrace(char *string)
	library function malloc for allocation of memory.		library function malloc for allocation of memory.

	------------------------------------------------------------------------------*/		------------------------------------------------------------------------------*/
	void* H264SwDecMalloc(u32 size)		void* H264SwDecMalloc(u32 size, u32 num)
	{		{
			if (size > UINT32_MAX / num) {
			return NULL;
			}

	#if defined(CHECK_MEMORY_USAGE)		#if defined(CHECK_MEMORY_USAGE)
	/* Note that if the decoder has to free and reallocate some of the buffers		/* Note that if the decoder has to free and reallocate some of the buffers
	* the total value will be invalid */		* the total value will be invalid */
	static u32 numBytes = 0;		static u32 numBytes = 0;
	numBytes += size;		numBytes += size * num;
	DEBUG(("Allocated %d bytes, total %d\n", size, numBytes));		DEBUG(("Allocated %d bytes, total %d\n", size, numBytes));
	#endif		#endif

	return malloc(size);		return malloc(size * num);
	}		}

	/*------------------------------------------------------------------------------		/*------------------------------------------------------------------------------

+6 −3

Original line number	Original line	Diff line number	Diff line
	@@ -85,7 +85,7 @@ int main(int argc, char **argv)
	rewind(finput);		rewind(finput);

	/* allocate memory for stream buffer, exit if unsuccessful */		/* allocate memory for stream buffer, exit if unsuccessful */
	byteStrm = byteStrmStart = (u8 )H264SwDecMalloc(sizeof(u8)strmLen);		byteStrm = byteStrmStart = (u8 *)H264SwDecMalloc(sizeof(u8), strmLen);
	if (byteStrm == NULL)		if (byteStrm == NULL)
	{		{
	printf("UNABLE TO ALLOCATE MEMORY\n");		printf("UNABLE TO ALLOCATE MEMORY\n");
	@@ -298,9 +298,12 @@ void H264SwDecTrace(char *string)
	library function malloc for allocation of memory.		library function malloc for allocation of memory.

	------------------------------------------------------------------------------*/		------------------------------------------------------------------------------*/
	void* H264SwDecMalloc(u32 size)		void* H264SwDecMalloc(u32 size, u32 num)
	{		{
	return malloc(size);		if (size > UINT32_MAX / num) {
			return NULL;
			}
			return malloc(size * num);
	}		}

	/*------------------------------------------------------------------------------		/*------------------------------------------------------------------------------

+10 −3

Original line number	Original line	Diff line number	Diff line
	@@ -35,6 +35,8 @@
	/*------------------------------------------------------------------------------		/*------------------------------------------------------------------------------
	1. Include headers		1. Include headers
	------------------------------------------------------------------------------*/		------------------------------------------------------------------------------*/
			#include <log/log.h>

	#include <stdlib.h>		#include <stdlib.h>
	#include "basetype.h"		#include "basetype.h"
	#include "h264bsd_container.h"		#include "h264bsd_container.h"
	@@ -75,8 +77,13 @@ H264DEC_EVALUATION Compile evaluation version, restricts number of frames
	void H264SwDecTrace(char *string) {		void H264SwDecTrace(char *string) {
	}		}

	void* H264SwDecMalloc(u32 size) {		void* H264SwDecMalloc(u32 size, u32 num) {
	return malloc(size);		if (size > UINT32_MAX / num) {
			ALOGE("can't allocate %u * %u bytes", size, num);
			android_errorWriteLog(0x534e4554, "27855419");
			return NULL;
			}
			return malloc(size * num);
	}		}

	void H264SwDecFree(void *ptr) {		void H264SwDecFree(void *ptr) {
	@@ -140,7 +147,7 @@ H264SwDecRet H264SwDecInit(H264SwDecInst *decInst, u32 noOutputReordering)
	return(H264SWDEC_PARAM_ERR);		return(H264SWDEC_PARAM_ERR);
	}		}

	pDecCont = (decContainer_t *)H264SwDecMalloc(sizeof(decContainer_t));		pDecCont = (decContainer_t *)H264SwDecMalloc(sizeof(decContainer_t), 1);

	if (pDecCont == NULL)		if (pDecCont == NULL)
	{		{

+5 −2

Original line number	Original line	Diff line number	Diff line
	@@ -413,9 +413,12 @@ void H264SwDecTrace(char *string)
	Function name: H264SwDecmalloc		Function name: H264SwDecmalloc

	------------------------------------------------------------------------------*/		------------------------------------------------------------------------------*/
	void* H264SwDecMalloc(u32 size)		void* H264SwDecMalloc(u32 size, u32 num)
	{		{
	return malloc(size);		if (size > UINT32_MAX / num) {
			return NULL;
			}
			return malloc(size * num);
	}		}

	/*------------------------------------------------------------------------------		/*------------------------------------------------------------------------------