Integer overflow in fx_basic.h and fx_memory.h

BUG=382656
R=palmer@chromium.org

Review URL: https://codereview.chromium.org/334573007

core/include/fxcrt/fx_basic.h

diff --git a/core/include/fxcrt/fx_basic.h b/core/include/fxcrt/fx_basic.h
index 378d6c3..bdb1599 100644
--- a/core/include/fxcrt/fx_basic.h
+++ b/core/include/fxcrt/fx_basic.h
@@ -6,6 +6,10 @@
 
 #ifndef _FX_BASIC_H_
 #define _FX_BASIC_H_
+#ifndef _STDINT_H_ 
+#define _STDINT_H_
+#include <stdint.h>
+#endif
 #ifndef _FX_SYSTEM_H_
 #include "fx_system.h"
 #endif
@@ -18,6 +22,7 @@
 #ifndef _FX_STREAM_H_
 #include "fx_stream.h"
 #endif
+ 
 class CFX_BinaryBuf : public CFX_Object
 {
 public:
@@ -776,6 +781,9 @@
         if (data_size > FixedSize) {
             m_pData = FX_Allocator_Alloc(m_pAllocator, DataType, data_size);
         } else {
+            if (FixedSize > SIZE_MAX/sizeof(DataType))
+                return;
+
             FXSYS_memset32(m_Data, 0, sizeof(DataType)*FixedSize);
         }
     }
@@ -788,6 +796,10 @@
         if (data_size > FixedSize) {
             m_pData = FX_Allocator_Alloc(m_pAllocator, DataType, data_size);
         } else {
+
+            if (FixedSize > SIZE_MAX/sizeof(DataType))
+                return;
+
             FXSYS_memset32(m_Data, 0, sizeof(DataType)*FixedSize);
         }
     }

core/include/fxcrt/fx_memory.h

diff --git a/core/include/fxcrt/fx_memory.h b/core/include/fxcrt/fx_memory.h
index cf795fa..1869ccc 100644
--- a/core/include/fxcrt/fx_memory.h
+++ b/core/include/fxcrt/fx_memory.h
@@ -117,13 +117,13 @@
 #endif
 typedef struct _IFX_Allocator {
 
-    void*	(*m_AllocDebug)(struct _IFX_Allocator* pAllocator, size_t size, FX_LPCSTR file, int line);
+    void*	(*m_AllocDebug)(struct _IFX_Allocator* pAllocator, size_t num, size_t size, FX_LPCSTR file, int line);
 
-    void*	(*m_Alloc)(struct _IFX_Allocator* pAllocator, size_t size);
+    void*	(*m_Alloc)(struct _IFX_Allocator* pAllocator, size_t num, size_t size);
 
-    void*	(*m_ReallocDebug)(struct _IFX_Allocator* pAllocator, void* p, size_t size, FX_LPCSTR file, int line);
+    void*	(*m_ReallocDebug)(struct _IFX_Allocator* pAllocator, void* p, size_t num, size_t size, FX_LPCSTR file, int line);
 
-    void*	(*m_Realloc)(struct _IFX_Allocator* pAllocator, void* p, size_t size);
+    void*	(*m_Realloc)(struct _IFX_Allocator* pAllocator, void* p, size_t num, size_t size);
 
     void	(*m_Free)(struct _IFX_Allocator* pAllocator, void* p);
 } IFX_Allocator;
@@ -134,17 +134,17 @@
 #ifdef _DEBUG
 
 #define FX_Allocator_Alloc(fxAllocator, type, size) \
-    ((fxAllocator) ? (type*)(fxAllocator)->m_AllocDebug((fxAllocator), (size) * sizeof(type), __FILE__, __LINE__) : (FX_Alloc(type, size)))
+    ((fxAllocator) ? (type*)(fxAllocator)->m_AllocDebug((fxAllocator), (size), sizeof(type), __FILE__, __LINE__) : (FX_Alloc(type, size)))
 
 #define FX_Allocator_Realloc(fxAllocator, type, ptr, new_size) \
-    ((fxAllocator) ? (type*)(fxAllocator)->m_ReallocDebug((fxAllocator), (ptr), (new_size) * sizeof(type), __FILE__, __LINE__) : (FX_Realloc(type, ptr, new_size)))
+    ((fxAllocator) ? (type*)(fxAllocator)->m_ReallocDebug((fxAllocator), (ptr), (new_size) , sizeof(type), __FILE__, __LINE__) : (FX_Realloc(type, ptr, new_size)))
 #else
 
 #define FX_Allocator_Alloc(fxAllocator, type, size) \
-    ((fxAllocator) ? (type*)(fxAllocator)->m_Alloc((fxAllocator), (size) * sizeof(type)) : (FX_Alloc(type, size)))
+    ((fxAllocator) ? (type*)(fxAllocator)->m_Alloc((fxAllocator), (size), sizeof(type)) : (FX_Alloc(type, size)))
 
 #define FX_Allocator_Realloc(fxAllocator, type, ptr, new_size) \
-    ((fxAllocator) ? (type*)(fxAllocator)->m_Realloc((fxAllocator), (ptr), (new_size) * sizeof(type)) : (FX_Realloc(type, ptr, new_size)))
+    ((fxAllocator) ? (type*)(fxAllocator)->m_Realloc((fxAllocator), (ptr), (new_size), sizeof(type)) : (FX_Realloc(type, ptr, new_size)))
 #endif
 #define FX_Allocator_Free(fxAllocator, ptr) \
     ((fxAllocator) ? (fxAllocator)->m_Free((fxAllocator), (ptr)) : (FX_Free(ptr)))

core/src/fxcrt/fx_basic_memmgr.cpp

diff --git a/core/src/fxcrt/fx_basic_memmgr.cpp b/core/src/fxcrt/fx_basic_memmgr.cpp
index 5c862a2..1021ab7 100644
--- a/core/src/fxcrt/fx_basic_memmgr.cpp
+++ b/core/src/fxcrt/fx_basic_memmgr.cpp
@@ -6,6 +6,7 @@
 
 #include "../../include/fxcrt/fx_basic.h"
 #include "mem_int.h"
+
 void FXMEM_DestroyFoxitMgr(FXMEM_FoxitMgr* pFoxitMgr)
 {
     if (pFoxitMgr == NULL) {
@@ -25,20 +26,36 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-static void* _DefAllocDebug(IFX_Allocator* pAllocator, size_t size, FX_LPCSTR filename, int line)
+static void* _DefAllocDebug(IFX_Allocator* pAllocator, size_t num, size_t size, FX_LPCSTR filename, int line)
 {
+    if (size == 0 || num > SIZE_MAX/size)
+        return NULL;
+
+    size = size * num; 
     return ((FX_DefAllocator*)pAllocator)->m_pFoxitMgr->AllocDebug(size, 0, filename, line);
 }
-static void* _DefAlloc(IFX_Allocator* pAllocator, size_t size)
+static void* _DefAlloc(IFX_Allocator* pAllocator, size_t num, size_t size)
 {
+    if (size == 0 || num > SIZE_MAX/size)
+        return NULL;
+
+    size = size * num;
     return ((FX_DefAllocator*)pAllocator)->m_pFoxitMgr->Alloc(size, 0);
 }
-static void* _DefReallocDebug(IFX_Allocator* pAllocator, void* p, size_t size, FX_LPCSTR filename, int line)
+static void* _DefReallocDebug(IFX_Allocator* pAllocator, void* p, size_t new_num, size_t size, FX_LPCSTR filename, int line)
 {
+    if (size == 0 || new_num > SIZE_MAX/size)
+        return NULL;
+
+    size = size * new_num;
     return ((FX_DefAllocator*)pAllocator)->m_pFoxitMgr->ReallocDebug(p, size, 0, filename, line);
 }
-static void* _DefRealloc(IFX_Allocator* pAllocator, void* p, size_t size)
+static void* _DefRealloc(IFX_Allocator* pAllocator, void* p, size_t new_num, size_t size)
 {
+    if (size == 0 || new_num > SIZE_MAX/size)
+        return NULL;
+
+    size = size * new_num;
     return ((FX_DefAllocator*)pAllocator)->m_pFoxitMgr->Realloc(p, size, 0);
 }
 static void _DefFree(IFX_Allocator* pAllocator, void* p)
@@ -193,7 +210,7 @@
 }
 void* CFX_AllocObject::operator new(size_t size, IFX_Allocator* pAllocator, FX_LPCSTR filename, int line)
 {
-    void* p = pAllocator ? pAllocator->m_AllocDebug(pAllocator, size, filename, line) :
+    void* p = pAllocator ? pAllocator->m_AllocDebug(pAllocator, size, 1, filename, line) :
               g_pDefFoxitMgr->AllocDebug(size, 0, filename, line);
     ((CFX_AllocObject*)p)->m_pAllocator = pAllocator;
     return p;
@@ -208,7 +225,7 @@
 }
 void* CFX_AllocObject::operator new(size_t size, IFX_Allocator* pAllocator)
 {
-    void* p = pAllocator ? pAllocator->m_Alloc(pAllocator, size) : g_pDefFoxitMgr->Alloc(size, 0);
+    void* p = pAllocator ? pAllocator->m_Alloc(pAllocator, size, 1) : g_pDefFoxitMgr->Alloc(size, 0);
     ((CFX_AllocObject*)p)->m_pAllocator = pAllocator;
     return p;
 }
@@ -229,21 +246,37 @@
     }
 }
 extern "C" {
-    static void* _GOPAllocDebug(IFX_Allocator* pAllocator, size_t size, FX_LPCSTR file, int line)
+    static void* _GOPAllocDebug(IFX_Allocator* pAllocator, size_t num, size_t size, FX_LPCSTR file, int line)
     {
+        if (size == 0 || num > SIZE_MAX/size)
+            return NULL;
+
+        size = size * num;
         return ((CFX_GrowOnlyPool*)pAllocator)->Alloc(size);
     }
-    static void* _GOPAlloc(IFX_Allocator* pAllocator, size_t size)
+    static void* _GOPAlloc(IFX_Allocator* pAllocator, size_t num, size_t size)
     {
+        if (size == 0 || num > SIZE_MAX/size)
+            return NULL;
+
+        size = size * num;
         return ((CFX_GrowOnlyPool*)pAllocator)->Alloc(size);
     }
-    static void* _GOPReallocDebug(IFX_Allocator* pAllocator, void* p, size_t new_size, FX_LPCSTR file, int line)
+    static void* _GOPReallocDebug(IFX_Allocator* pAllocator, void* p, size_t new_num, size_t size, FX_LPCSTR file, int line)
     {
-        return ((CFX_GrowOnlyPool*)pAllocator)->Realloc(p, new_size);
+        if (size == 0 || new_num > SIZE_MAX/size)
+            return NULL;
+
+        size = size * new_num;
+        return ((CFX_GrowOnlyPool*)pAllocator)->Realloc(p, size);
     }
-    static void* _GOPRealloc(IFX_Allocator* pAllocator, void* p, size_t new_size)
+    static void* _GOPRealloc(IFX_Allocator* pAllocator, void* p, size_t new_num, size_t size)
     {
-        return ((CFX_GrowOnlyPool*)pAllocator)->Realloc(p, new_size);
+        if (size == 0 || new_num > SIZE_MAX/size)
+            return NULL;
+
+        size = size * new_num;
+        return ((CFX_GrowOnlyPool*)pAllocator)->Realloc(p, size);
     }
     static void _GOPFree(IFX_Allocator* pAllocator, void* p)
     {
@@ -297,7 +330,11 @@
         pTrunk = pTrunk->m_pNext;
     }
     size_t alloc_size = size > m_TrunkSize ? size : m_TrunkSize;
-    pTrunk = (_FX_GrowOnlyTrunk*)m_pAllocator->m_Alloc(m_pAllocator, sizeof(_FX_GrowOnlyTrunk) + alloc_size);
+    
+    if (alloc_size > SIZE_MAX - sizeof(_FX_GrowOnlyTrunk) )
+        return NULL;
+
+    pTrunk = (_FX_GrowOnlyTrunk*)m_pAllocator->m_Alloc(m_pAllocator, sizeof(_FX_GrowOnlyTrunk) + alloc_size, 1);
     pTrunk->m_Size = alloc_size;
     pTrunk->m_Allocated = size;
     pTrunk->m_pNext = (_FX_GrowOnlyTrunk*)m_pFirstTrunk;