Limit bitmap memory usage in PDFiumFuzzerHelper

The fuzzer may try to create very large bitmaps and reach an OOM
condition. Limit the amount of memory used for bitmap allocation to
avoid this.

Bug: 458896741
Change-Id: I7820d27faa373e2ee1633c9af7e0cc776e64e678
Reviewed-on: https://pdfium-review.googlesource.com/c/pdfium/+/137891
Reviewed-by: Tom Sepez <tsepez@chromium.org>
Commit-Queue: Lei Zhang <thestig@chromium.org>
diff --git a/testing/fuzzers/pdfium_fuzzer_helper.cc b/testing/fuzzers/pdfium_fuzzer_helper.cc
index e9198ed..6c4de47 100644
--- a/testing/fuzzers/pdfium_fuzzer_helper.cc
+++ b/testing/fuzzers/pdfium_fuzzer_helper.cc
@@ -20,6 +20,8 @@
 #include "core/fxcrt/check_op.h"
 #include "core/fxcrt/compiler_specific.h"
 #include "core/fxcrt/fx_memcpy_wrappers.h"
+#include "core/fxcrt/fx_safe_types.h"
+#include "core/fxcrt/notreached.h"
 #include "core/fxcrt/numerics/checked_math.h"
 #include "core/fxcrt/span.h"
 #include "public/cpp/fpdf_scopers.h"
@@ -121,6 +123,29 @@
   };
 }
 
+int GetBytesPerPixelForBitmapFormat(int bitmap_format) {
+  switch (bitmap_format) {
+    case FPDFBitmap_Gray:
+      return 1;
+    case FPDFBitmap_BGR:
+      return 3;
+    case FPDFBitmap_BGRx:
+    case FPDFBitmap_BGRA:
+    case FPDFBitmap_BGRA_Premul:
+      return 4;
+    default:
+      NOTREACHED();
+  }
+}
+
+bool CheckImageSize(int width, int height, int bitmap_format) {
+  static constexpr uint32_t kMemLimitBytes = 1024 * 1024 * 1024;  // 1 GB.
+  FX_SAFE_UINT32 mem = width;
+  mem *= height;
+  mem *= GetBytesPerPixelForBitmapFormat(bitmap_format);
+  return mem.IsValid() && mem.ValueOrDie() <= kMemLimitBytes;
+}
+
 }  // namespace
 
 PDFiumFuzzerHelper::PDFiumFuzzerHelper() = default;
@@ -241,9 +266,12 @@
 
   FormActionHandler(form, doc, page.get());
 
-  const double scale = 1.0;
-  int width = static_cast<int>(FPDF_GetPageWidthF(page.get()) * scale);
-  int height = static_cast<int>(FPDF_GetPageHeightF(page.get()) * scale);
+  int width = static_cast<int>(FPDF_GetPageWidthF(page.get()));
+  int height = static_cast<int>(FPDF_GetPageHeightF(page.get()));
+  if (!CheckImageSize(width, height, options.bitmap_format)) {
+    return false;
+  }
+
   ScopedFPDFBitmap bitmap(FPDFBitmap_CreateEx(
       width, height, options.bitmap_format, nullptr, /*stride=*/0));
   if (bitmap) {