Fix an assertion failure in OpenJPEG.
Cherrypick the fix [1] from upstream OpenJPEG.
[1] https://github.com/uclouvain/openjpeg/pull/1436
Bug: chromium:1343737
Change-Id: Ie92a9b8b81dc105df8aa8159480d2f0ba0ccc569
Reviewed-on: https://pdfium-review.googlesource.com/c/pdfium/+/96471
Reviewed-by: Tom Sepez <tsepez@chromium.org>
Commit-Queue: Lei Zhang <thestig@chromium.org>
diff --git a/third_party/libopenjpeg/0043-mel_init.patch b/third_party/libopenjpeg/0043-mel_init.patch
new file mode 100644
index 0000000..7197bac
--- /dev/null
+++ b/third_party/libopenjpeg/0043-mel_init.patch
@@ -0,0 +1,59 @@
+commit 4da04cd3e88a0280be526e16077c540a45cbbfa8
+Author: Aous Naman <aous72@yahoo.com>
+Date: Fri Aug 12 02:29:40 2022 +1000
+
+ Replace the assert in mel_init to an if statement to address an issue with fuzzing. (#1436)
+
+ Modified the mel_init code to replace the assert statement with an if statement, returning false when an incorrect sequence of bytes are encountered in the MEL segment. Similar code should be added to the main MEL decoding subrountine, but the change is more involved; in any case, an incorrect sequence produces incorrect results, but should not be harmful or cause a crash.
+
+diff --git a/src/lib/openjp2/ht_dec.c b/src/lib/openjp2/ht_dec.c
+index a803d1bb..62a6c9e1 100644
+--- a/src/lib/openjp2/ht_dec.c
++++ b/src/lib/openjp2/ht_dec.c
+@@ -294,7 +294,7 @@ void mel_decode(dec_mel_t *melp)
+ * @param [in] scup is the length of MEL+VLC segments
+ */
+ static INLINE
+-void mel_init(dec_mel_t *melp, OPJ_UINT8* bbuf, int lcup, int scup)
++OPJ_BOOL mel_init(dec_mel_t *melp, OPJ_UINT8* bbuf, int lcup, int scup)
+ {
+ int num;
+ int i;
+@@ -316,7 +316,9 @@ void mel_init(dec_mel_t *melp, OPJ_UINT8* bbuf, int lcup, int scup)
+ OPJ_UINT64 d;
+ int d_bits;
+
+- assert(melp->unstuff == OPJ_FALSE || melp->data[0] <= 0x8F);
++ if (melp->unstuff == OPJ_TRUE && melp->data[0] > 0x8F) {
++ return OPJ_FALSE;
++ }
+ d = (melp->size > 0) ? *melp->data : 0xFF; // if buffer is consumed
+ // set data to 0xFF
+ if (melp->size == 1) {
+@@ -332,6 +334,7 @@ void mel_init(dec_mel_t *melp, OPJ_UINT8* bbuf, int lcup, int scup)
+ }
+ melp->tmp <<= (64 - melp->bits); //push all the way up so the first bit
+ // is the MSB
++ return OPJ_TRUE;
+ }
+
+ //************************************************************************/
+@@ -1374,7 +1377,17 @@ OPJ_BOOL opj_t1_ht_decode_cblk(opj_t1_t *t1,
+ }
+
+ // init structures
+- mel_init(&mel, coded_data, lcup, scup);
++ if (mel_init(&mel, coded_data, lcup, scup) == OPJ_FALSE) {
++ if (p_manager_mutex) {
++ opj_mutex_lock(p_manager_mutex);
++ }
++ opj_event_msg(p_manager, EVT_ERROR, "Malformed HT codeblock. "
++ "Incorrect MEL segment sequence.\n");
++ if (p_manager_mutex) {
++ opj_mutex_unlock(p_manager_mutex);
++ }
++ return OPJ_FALSE;
++ }
+ rev_init(&vlc, coded_data, lcup, scup);
+ frwd_init(&magsgn, coded_data, lcup - scup, 0xFF);
+ if (num_passes > 1) { // needs to be tested
diff --git a/third_party/libopenjpeg/README.pdfium b/third_party/libopenjpeg/README.pdfium
index 0dc36dc..5b029ab 100644
--- a/third_party/libopenjpeg/README.pdfium
+++ b/third_party/libopenjpeg/README.pdfium
@@ -30,3 +30,4 @@
0039-opj_mqc_renorme.patch: Remove unused opj_mqc_renorme().
0041-remove_opj_clock.patch: Remove unused opj_clock.h include.
0042-popcnt-windows-arm64.patch: Backport to fix Windows arm64 build.
+0043-mel_init.patch: Backport fix for assertion failure in mel_init().
diff --git a/third_party/libopenjpeg/ht_dec.c b/third_party/libopenjpeg/ht_dec.c
index e2f3afd..1a38172 100644
--- a/third_party/libopenjpeg/ht_dec.c
+++ b/third_party/libopenjpeg/ht_dec.c
@@ -294,7 +294,7 @@
* @param [in] scup is the length of MEL+VLC segments
*/
static INLINE
-void mel_init(dec_mel_t *melp, OPJ_UINT8* bbuf, int lcup, int scup)
+OPJ_BOOL mel_init(dec_mel_t *melp, OPJ_UINT8* bbuf, int lcup, int scup)
{
int num;
int i;
@@ -316,7 +316,9 @@
OPJ_UINT64 d;
int d_bits;
- assert(melp->unstuff == OPJ_FALSE || melp->data[0] <= 0x8F);
+ if (melp->unstuff == OPJ_TRUE && melp->data[0] > 0x8F) {
+ return OPJ_FALSE;
+ }
d = (melp->size > 0) ? *melp->data : 0xFF; // if buffer is consumed
// set data to 0xFF
if (melp->size == 1) {
@@ -332,6 +334,7 @@
}
melp->tmp <<= (64 - melp->bits); //push all the way up so the first bit
// is the MSB
+ return OPJ_TRUE;
}
//************************************************************************/
@@ -1374,7 +1377,17 @@
}
// init structures
- mel_init(&mel, coded_data, lcup, scup);
+ if (mel_init(&mel, coded_data, lcup, scup) == OPJ_FALSE) {
+ if (p_manager_mutex) {
+ opj_mutex_lock(p_manager_mutex);
+ }
+ opj_event_msg(p_manager, EVT_ERROR, "Malformed HT codeblock. "
+ "Incorrect MEL segment sequence.\n");
+ if (p_manager_mutex) {
+ opj_mutex_unlock(p_manager_mutex);
+ }
+ return OPJ_FALSE;
+ }
rev_init(&vlc, coded_data, lcup, scup);
frwd_init(&magsgn, coded_data, lcup - scup, 0xFF);
if (num_passes > 1) { // needs to be tested
