Cherry pick an OpenJPEG fix for DWT integer overflows.
Apply OpenJPEG's commit badbd93af92836c7a38ef069d410a829e2575ce2 to the
local copy of OpenJPEG.
Bug: chromium:1244495
Change-Id: Idb8d7b46ea38a9dd349fd88c080bb37da0b247dd
Reviewed-on: https://pdfium-review.googlesource.com/c/pdfium/+/90730
Reviewed-by: Tom Sepez <tsepez@chromium.org>
Commit-Queue: Lei Zhang <thestig@chromium.org>
diff --git a/third_party/libopenjpeg20/0040-dwt_overflows.patch b/third_party/libopenjpeg20/0040-dwt_overflows.patch
new file mode 100644
index 0000000..588cd3f
--- /dev/null
+++ b/third_party/libopenjpeg20/0040-dwt_overflows.patch
@@ -0,0 +1,124 @@
+commit badbd93af92836c7a38ef069d410a829e2575ce2
+Author: Even Rouault <even.rouault@spatialys.com>
+Date: Fri Sep 3 15:17:56 2021 +0200
+
+ Avoid integer overflows in DWT. Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=11700 and https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=30646
+
+diff --git a/src/lib/openjp2/dwt.c b/src/lib/openjp2/dwt.c
+index 4164ba09..824a9fc4 100644
+--- a/src/lib/openjp2/dwt.c
++++ b/src/lib/openjp2/dwt.c
+@@ -385,7 +385,8 @@ static void opj_idwt53_h_cas0(OPJ_INT32* tmp,
+ s0n = s1n - ((d1c + d1n + 2) >> 2);
+
+ tmp[i ] = s0c;
+- tmp[i + 1] = d1c + ((s0c + s0n) >> 1);
++ tmp[i + 1] = opj_int_add_no_overflow(d1c, opj_int_add_no_overflow(s0c,
++ s0n) >> 1);
+ }
+
+ tmp[i] = s0n;
+@@ -450,7 +451,7 @@ static void opj_idwt53_h_cas1(OPJ_INT32* tmp,
+
+ dn = in_odd[j] - ((s1 + s2 + 2) >> 2);
+ tmp[i ] = dc;
+- tmp[i + 1] = s1 + ((dn + dc) >> 1);
++ tmp[i + 1] = opj_int_add_no_overflow(s1, opj_int_add_no_overflow(dn, dc) >> 1);
+
+ dc = dn;
+ s1 = s2;
+@@ -796,7 +797,8 @@ static void opj_idwt3_v_cas0(OPJ_INT32* tmp,
+ s1n = tiledp_col[(OPJ_SIZE_T)(j + 1) * stride];
+ d1n = tiledp_col[(OPJ_SIZE_T)(sn + j + 1) * stride];
+
+- s0n = s1n - ((d1c + d1n + 2) >> 2);
++ s0n = opj_int_sub_no_overflow(s1n,
++ opj_int_add_no_overflow(opj_int_add_no_overflow(d1c, d1n), 2) >> 2);
+
+ tmp[i ] = s0c;
+ tmp[i + 1] = d1c + ((s0c + s0n) >> 1);
+@@ -2343,10 +2345,13 @@ static void opj_dwt_decode_partial_1(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn,
+ OPJ_S(0) /= 2;
+ } else {
+ for (i = win_l_x0; i < win_l_x1; i++) {
+- OPJ_D(i) -= (OPJ_SS_(i) + OPJ_SS_(i + 1) + 2) >> 2;
++ OPJ_D(i) = opj_int_sub_no_overflow(OPJ_D(i),
++ opj_int_add_no_overflow(opj_int_add_no_overflow(OPJ_SS_(i), OPJ_SS_(i + 1)),
++ 2) >> 2);
+ }
+ for (i = win_h_x0; i < win_h_x1; i++) {
+- OPJ_S(i) += (OPJ_DD_(i) + OPJ_DD_(i - 1)) >> 1;
++ OPJ_S(i) = opj_int_add_no_overflow(OPJ_S(i),
++ opj_int_add_no_overflow(OPJ_DD_(i), OPJ_DD_(i - 1)) >> 1);
+ }
+ }
+ }
+@@ -2484,12 +2489,17 @@ static void opj_dwt_decode_partial_1_parallel(OPJ_INT32 *a,
+ } else {
+ for (i = win_l_x0; i < win_l_x1; i++) {
+ for (off = 0; off < 4; off++) {
+- OPJ_D_off(i, off) -= (OPJ_SS__off(i, off) + OPJ_SS__off(i + 1, off) + 2) >> 2;
++ OPJ_D_off(i, off) = opj_int_sub_no_overflow(
++ OPJ_D_off(i, off),
++ opj_int_add_no_overflow(
++ opj_int_add_no_overflow(OPJ_SS__off(i, off), OPJ_SS__off(i + 1, off)), 2) >> 2);
+ }
+ }
+ for (i = win_h_x0; i < win_h_x1; i++) {
+ for (off = 0; off < 4; off++) {
+- OPJ_S_off(i, off) += (OPJ_DD__off(i, off) + OPJ_DD__off(i - 1, off)) >> 1;
++ OPJ_S_off(i, off) = opj_int_add_no_overflow(
++ OPJ_S_off(i, off),
++ opj_int_add_no_overflow(OPJ_DD__off(i, off), OPJ_DD__off(i - 1, off)) >> 1);
+ }
+ }
+ }
+diff --git a/src/lib/openjp2/opj_intmath.h b/src/lib/openjp2/opj_intmath.h
+index afe69d90..f8cc6139 100644
+--- a/src/lib/openjp2/opj_intmath.h
++++ b/src/lib/openjp2/opj_intmath.h
+@@ -276,6 +276,44 @@ static INLINE OPJ_INT32 opj_int_fix_mul_t1(OPJ_INT32 a, OPJ_INT32 b)
+ return (OPJ_INT32)(temp >> (13 + 11 - T1_NMSEDEC_FRACBITS)) ;
+ }
+
++/**
++Addtion two signed integers with a wrap-around behaviour.
++Assumes complement-to-two signed integers.
++@param a
++@param b
++@return Returns a + b
++*/
++static INLINE OPJ_INT32 opj_int_add_no_overflow(OPJ_INT32 a, OPJ_INT32 b)
++{
++ void* pa = &a;
++ void* pb = &b;
++ OPJ_UINT32* upa = (OPJ_UINT32*)pa;
++ OPJ_UINT32* upb = (OPJ_UINT32*)pb;
++ OPJ_UINT32 ures = *upa + *upb;
++ void* pures = &ures;
++ OPJ_INT32* ipres = (OPJ_INT32*)pures;
++ return *ipres;
++}
++
++/**
++Subtract two signed integers with a wrap-around behaviour.
++Assumes complement-to-two signed integers.
++@param a
++@param b
++@return Returns a - b
++*/
++static INLINE OPJ_INT32 opj_int_sub_no_overflow(OPJ_INT32 a, OPJ_INT32 b)
++{
++ void* pa = &a;
++ void* pb = &b;
++ OPJ_UINT32* upa = (OPJ_UINT32*)pa;
++ OPJ_UINT32* upb = (OPJ_UINT32*)pb;
++ OPJ_UINT32 ures = *upa - *upb;
++ void* pures = &ures;
++ OPJ_INT32* ipres = (OPJ_INT32*)pures;
++ return *ipres;
++}
++
+ /* ----------------------------------------------------------------------- */
+ /*@}*/
+
diff --git a/third_party/libopenjpeg20/README.pdfium b/third_party/libopenjpeg20/README.pdfium
index c62780c..ce299ea 100644
--- a/third_party/libopenjpeg20/README.pdfium
+++ b/third_party/libopenjpeg20/README.pdfium
@@ -28,3 +28,4 @@
0034-opj_malloc.patch: PDFium changes in opj_malloc.
0035-opj_image_data_free.patch: Use the right free function in opj_jp2_apply_pclr.
0039-opj_mqc_renorme.patch: Remove unused opj_mqc_renorme().
+0040-dwt_overflows.patch: Avoid integer overflows in DWT.
diff --git a/third_party/libopenjpeg20/dwt.c b/third_party/libopenjpeg20/dwt.c
index a36b7ed..67a0b9f 100644
--- a/third_party/libopenjpeg20/dwt.c
+++ b/third_party/libopenjpeg20/dwt.c
@@ -390,7 +390,8 @@
s0n = s1n - ((d1c + d1n + 2) >> 2);
tmp[i ] = s0c;
- tmp[i + 1] = d1c + ((s0c + s0n) >> 1);
+ tmp[i + 1] = opj_int_add_no_overflow(d1c, opj_int_add_no_overflow(s0c,
+ s0n) >> 1);
}
tmp[i] = s0n;
@@ -455,7 +456,7 @@
dn = in_odd[j] - ((s1 + s2 + 2) >> 2);
tmp[i ] = dc;
- tmp[i + 1] = s1 + ((dn + dc) >> 1);
+ tmp[i + 1] = opj_int_add_no_overflow(s1, opj_int_add_no_overflow(dn, dc) >> 1);
dc = dn;
s1 = s2;
@@ -801,7 +802,8 @@
s1n = tiledp_col[(OPJ_SIZE_T)(j + 1) * stride];
d1n = tiledp_col[(OPJ_SIZE_T)(sn + j + 1) * stride];
- s0n = s1n - ((d1c + d1n + 2) >> 2);
+ s0n = opj_int_sub_no_overflow(s1n,
+ opj_int_add_no_overflow(opj_int_add_no_overflow(d1c, d1n), 2) >> 2);
tmp[i ] = s0c;
tmp[i + 1] = d1c + ((s0c + s0n) >> 1);
@@ -2350,10 +2352,13 @@
OPJ_S(0) /= 2;
} else {
for (i = win_l_x0; i < win_l_x1; i++) {
- OPJ_D(i) -= (OPJ_SS_(i) + OPJ_SS_(i + 1) + 2) >> 2;
+ OPJ_D(i) = opj_int_sub_no_overflow(OPJ_D(i),
+ opj_int_add_no_overflow(opj_int_add_no_overflow(OPJ_SS_(i), OPJ_SS_(i + 1)),
+ 2) >> 2);
}
for (i = win_h_x0; i < win_h_x1; i++) {
- OPJ_S(i) += (OPJ_DD_(i) + OPJ_DD_(i - 1)) >> 1;
+ OPJ_S(i) = opj_int_add_no_overflow(OPJ_S(i),
+ opj_int_add_no_overflow(OPJ_DD_(i), OPJ_DD_(i - 1)) >> 1);
}
}
}
@@ -2491,12 +2496,17 @@
} else {
for (i = win_l_x0; i < win_l_x1; i++) {
for (off = 0; off < 4; off++) {
- OPJ_D_off(i, off) -= (OPJ_SS__off(i, off) + OPJ_SS__off(i + 1, off) + 2) >> 2;
+ OPJ_D_off(i, off) = opj_int_sub_no_overflow(
+ OPJ_D_off(i, off),
+ opj_int_add_no_overflow(
+ opj_int_add_no_overflow(OPJ_SS__off(i, off), OPJ_SS__off(i + 1, off)), 2) >> 2);
}
}
for (i = win_h_x0; i < win_h_x1; i++) {
for (off = 0; off < 4; off++) {
- OPJ_S_off(i, off) += (OPJ_DD__off(i, off) + OPJ_DD__off(i - 1, off)) >> 1;
+ OPJ_S_off(i, off) = opj_int_add_no_overflow(
+ OPJ_S_off(i, off),
+ opj_int_add_no_overflow(OPJ_DD__off(i, off), OPJ_DD__off(i - 1, off)) >> 1);
}
}
}
diff --git a/third_party/libopenjpeg20/opj_intmath.h b/third_party/libopenjpeg20/opj_intmath.h
index afe69d9..f8cc613 100644
--- a/third_party/libopenjpeg20/opj_intmath.h
+++ b/third_party/libopenjpeg20/opj_intmath.h
@@ -276,6 +276,44 @@
return (OPJ_INT32)(temp >> (13 + 11 - T1_NMSEDEC_FRACBITS)) ;
}
+/**
+Addtion two signed integers with a wrap-around behaviour.
+Assumes complement-to-two signed integers.
+@param a
+@param b
+@return Returns a + b
+*/
+static INLINE OPJ_INT32 opj_int_add_no_overflow(OPJ_INT32 a, OPJ_INT32 b)
+{
+ void* pa = &a;
+ void* pb = &b;
+ OPJ_UINT32* upa = (OPJ_UINT32*)pa;
+ OPJ_UINT32* upb = (OPJ_UINT32*)pb;
+ OPJ_UINT32 ures = *upa + *upb;
+ void* pures = &ures;
+ OPJ_INT32* ipres = (OPJ_INT32*)pures;
+ return *ipres;
+}
+
+/**
+Subtract two signed integers with a wrap-around behaviour.
+Assumes complement-to-two signed integers.
+@param a
+@param b
+@return Returns a - b
+*/
+static INLINE OPJ_INT32 opj_int_sub_no_overflow(OPJ_INT32 a, OPJ_INT32 b)
+{
+ void* pa = &a;
+ void* pb = &b;
+ OPJ_UINT32* upa = (OPJ_UINT32*)pa;
+ OPJ_UINT32* upb = (OPJ_UINT32*)pb;
+ OPJ_UINT32 ures = *upa - *upb;
+ void* pures = &ures;
+ OPJ_INT32* ipres = (OPJ_INT32*)pures;
+ return *ipres;
+}
+
/* ----------------------------------------------------------------------- */
/*@}*/
