Add support for per-test fuzzy matching
Adds support for a new SUPPRESSIONS_EXACT_MATCHING file that
"suppresses" exact matching for individual tests, instead falling back
to fuzzy matching.
Fixed: pdfium:1991
Change-Id: I2658bb51d97f465f42dbb13650842998546090c1
Reviewed-on: https://pdfium-review.googlesource.com/c/pdfium/+/104350
Reviewed-by: Nigi <nigi@chromium.org>
Commit-Queue: K. Moon <kmoon@chromium.org>
diff --git a/testing/SUPPRESSIONS_EXACT_MATCHING b/testing/SUPPRESSIONS_EXACT_MATCHING
new file mode 100644
index 0000000..374549d
--- /dev/null
+++ b/testing/SUPPRESSIONS_EXACT_MATCHING
@@ -0,0 +1,20 @@
+# Copyright 2023 The PDFium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# List of tests to use fuzzy instead of exact matching, one per line.
+# There are four space-separated columns per line
+# Each column (except column 0) can contain a comma-separated list of values.
+#
+# Column 0: test file name
+# Column 1: platform: *, win, mac, linux
+# Column 2: v8 support: *, nov8, v8
+# Column 3: xfa support: *, noxfa, xfa
+# Column 4: rendering support: *, agg, skia
+#
+# All columns on a line on a line must match, but filenames may be repeated
+# on subsequent lines to suppress more cases. Within each column, any one of
+# the comma-separated values must match in order for the colum to "match".
+# The filenames and keywords are case-sensitive.
+#
+# Try to keep the file alphabetized within each category of test.
diff --git a/testing/tools/pngdiffer.py b/testing/tools/pngdiffer.py
index 91468bb..2044a34 100755
--- a/testing/tools/pngdiffer.py
+++ b/testing/tools/pngdiffer.py
@@ -10,6 +10,9 @@
import subprocess
import sys
+EXACT_MATCHING = 'exact'
+FUZZY_MATCHING = 'fuzzy'
+
_PNG_OPTIMIZER = 'optipng'
_COMMON_SUFFIX_ORDER = ('_{os}', '')
@@ -68,10 +71,12 @@
except subprocess.CalledProcessError as e:
return e
- def _RunImageCompareCommand(self, image_diff):
+ def _RunImageCompareCommand(self, image_diff, image_matching_algorithm):
cmd = [self.pdfium_diff_path]
if self.reverse_byte_order:
cmd.append('--reverse-byte-order')
+ if image_matching_algorithm == FUZZY_MATCHING:
+ cmd.append('--fuzzy')
cmd.extend([image_diff.actual_path, image_diff.expected_path])
return self._RunCommand(cmd)
@@ -82,7 +87,8 @@
image_diff.expected_path, image_diff.diff_path
])
- def ComputeDifferences(self, input_filename, source_dir, working_dir):
+ def ComputeDifferences(self, input_filename, source_dir, working_dir,
+ image_matching_algorithm):
"""Computes differences between actual and expected image files.
Returns:
@@ -102,7 +108,8 @@
if os.path.exists(expected_path):
page_diff.expected_path = expected_path
- compare_error = self._RunImageCompareCommand(page_diff)
+ compare_error = self._RunImageCompareCommand(page_diff,
+ image_matching_algorithm)
if compare_error:
page_diff.reason = str(compare_error)
@@ -115,7 +122,8 @@
# Validate that no other paths match.
for unexpected_path in path_templates.GetExpectedPaths(page)[1:]:
page_diff.expected_path = unexpected_path
- if not self._RunImageCompareCommand(page_diff):
+ if not self._RunImageCompareCommand(page_diff,
+ image_matching_algorithm):
page_diff.reason = f'Also matches {unexpected_path}'
break
page_diff.expected_path = expected_path
@@ -129,7 +137,8 @@
return image_diffs
- def Regenerate(self, input_filename, source_dir, working_dir):
+ def Regenerate(self, input_filename, source_dir, working_dir,
+ image_matching_algorithm):
path_templates = _PathTemplates(input_filename, source_dir, working_dir,
self.os_name, self.suffix_order)
for page in itertools.count():
@@ -142,7 +151,8 @@
# Match against all expected page images.
for index, expected_path in enumerate(expected_paths):
page_diff.expected_path = expected_path
- if not self._RunImageCompareCommand(page_diff):
+ if not self._RunImageCompareCommand(page_diff,
+ image_matching_algorithm):
if first_match is None:
first_match = index
last_match = index
diff --git a/testing/tools/suppressor.py b/testing/tools/suppressor.py
index a7146c5..989f4dd 100755
--- a/testing/tools/suppressor.py
+++ b/testing/tools/suppressor.py
@@ -6,6 +6,7 @@
import os
import common
+import pngdiffer
class Suppressor:
@@ -17,6 +18,8 @@
self.suppression_set = self._LoadSuppressedSet('SUPPRESSIONS', finder)
self.image_suppression_set = self._LoadSuppressedSet(
'SUPPRESSIONS_IMAGE_DIFF', finder)
+ self.exact_matching_suppression_set = self._LoadSuppressedSet(
+ 'SUPPRESSIONS_EXACT_MATCHING', finder)
def _LoadSuppressedSet(self, suppressions_filename, finder):
v8_option = "v8" if self.has_v8 else "nov8"
@@ -70,3 +73,9 @@
print("%s image diff comparison is suppressed" % input_filename)
return True
return False
+
+ def GetImageMatchingAlgorithm(self, input_filename):
+ if input_filename in self.exact_matching_suppression_set:
+ print(f"{input_filename} image diff comparison is fuzzy")
+ return pngdiffer.FUZZY_MATCHING
+ return pngdiffer.EXACT_MATCHING
diff --git a/testing/tools/test_runner.py b/testing/tools/test_runner.py
index 8ff288d..ad348aa 100644
--- a/testing/tools/test_runner.py
+++ b/testing/tools/test_runner.py
@@ -539,6 +539,10 @@
return _per_process_state.test_suppressor.IsImageDiffSuppressed(
self.input_filename)
+ def GetImageMatchingAlgorithm(self):
+ return _per_process_state.test_suppressor.GetImageMatchingAlgorithm(
+ self.input_filename)
+
def RunCommand(self, command, stdout=None):
"""Runs a test command.
@@ -609,9 +613,11 @@
return
if self.IsResultSuppressed() or self.IsImageDiffSuppressed():
return
- _per_process_state.image_differ.Regenerate(self.input_filename,
- self.source_dir,
- self.working_dir)
+ _per_process_state.image_differ.Regenerate(
+ self.input_filename,
+ self.source_dir,
+ self.working_dir,
+ image_matching_algorithm=self.GetImageMatchingAlgorithm())
def Generate(self):
input_event_path = os.path.join(self.source_dir, f'{self.test_id}.evt')
@@ -727,7 +733,10 @@
if self.actual_images:
image_diffs = _per_process_state.image_differ.ComputeDifferences(
- self.input_filename, self.source_dir, self.working_dir)
+ self.input_filename,
+ self.source_dir,
+ self.working_dir,
+ image_matching_algorithm=self.GetImageMatchingAlgorithm())
if image_diffs:
test_result.status = result_types.FAIL
test_result.reason = 'Images differ'
