Add --regenerate_expected option to test_runner.py.
After a change in rendering, corpus and pixel test expected pngs
need to be regenerated manually. This option reduces the toil by
automating renaming and moving the new expected files over the old
ones.
Change-Id: I9d490369ccf946d4d4567e440b7b5252469eec46
Reviewed-on: https://pdfium-review.googlesource.com/16451
Commit-Queue: Henrique Nakashima <hnakashima@chromium.org>
Reviewed-by: Nicolás Peña Moreno <npm@chromium.org>
diff --git a/testing/tools/pngdiffer.py b/testing/tools/pngdiffer.py
index a1533b8..95acb19 100755
--- a/testing/tools/pngdiffer.py
+++ b/testing/tools/pngdiffer.py
@@ -4,32 +4,26 @@
# found in the LICENSE file.
import os
+import shutil
import sys
import common
class PNGDiffer():
- ACTUAL_TEMPLATE = '.pdf.%d.png'
- EXPECTED_TEMPLATE = '_expected' + ACTUAL_TEMPLATE
- PLATFORM_EXPECTED_TEMPLATE = '_expected_%s' + ACTUAL_TEMPLATE
-
def __init__(self, finder):
self.pdfium_diff_path = finder.ExecutablePath('pdfium_diff')
self.os_name = finder.os_name
def GetActualFiles(self, input_filename, source_dir, working_dir):
actual_paths = []
- template_paths = self._GetTemplatePaths(
- input_filename, source_dir, working_dir)
- actual_path_template = template_paths[0];
- expected_path_template = template_paths[1]
- platform_expected_path_template = template_paths[2]
+ path_templates = PathTemplates(input_filename, source_dir, working_dir)
+
i = 0
while True:
- actual_path = actual_path_template % i
- expected_path = expected_path_template % i
- platform_expected_path = (
- platform_expected_path_template % (self.os_name, i))
+ actual_path = path_templates.GetActualPath(i)
+ expected_path = path_templates.GetExpectedPath(i)
+ platform_expected_path = path_templates.GetPlatformExpectedPath(
+ self.os_name, i)
if os.path.exists(platform_expected_path):
expected_path = platform_expected_path
elif not os.path.exists(expected_path):
@@ -39,19 +33,16 @@
return actual_paths
def HasDifferences(self, input_filename, source_dir, working_dir):
- template_paths = self._GetTemplatePaths(
- input_filename, source_dir, working_dir)
- actual_path_template = template_paths[0];
- expected_path_template = template_paths[1]
- platform_expected_path_template = template_paths[2]
+ path_templates = PathTemplates(input_filename, source_dir, working_dir)
+
i = 0
while True:
- actual_path = actual_path_template % i
- expected_path = expected_path_template % i
+ actual_path = path_templates.GetActualPath(i)
+ expected_path = path_templates.GetExpectedPath(i)
# PDFium tests should be platform independent. Platform based results are
# used to capture platform dependent implementations.
- platform_expected_path = (
- platform_expected_path_template % (self.os_name, i))
+ platform_expected_path = path_templates.GetPlatformExpectedPath(
+ self.os_name, i)
if (not os.path.exists(expected_path) and
not os.path.exists(platform_expected_path)):
if i == 0:
@@ -75,11 +66,54 @@
i += 1
return False
- def _GetTemplatePaths(self, input_filename, source_dir, working_dir):
+ def Regenerate(self, input_filename, source_dir, working_dir):
+ path_templates = PathTemplates(input_filename, source_dir, working_dir)
+
+ page = 0
+ while True:
+ # Loop through the generated page images. Stop when there is a page
+ # missing a png, which means the document ended.
+ actual_path = path_templates.GetActualPath(page)
+ if not os.path.isfile(actual_path):
+ break
+
+ platform_expected_path = path_templates.GetPlatformExpectedPath(
+ self.os_name, page)
+
+ # If there is a platform expected png, we will overwrite it. Otherwise,
+ # overwrite the generic png.
+ if os.path.exists(platform_expected_path):
+ expected_path = platform_expected_path
+ else:
+ expected_path = path_templates.GetExpectedPath(page)
+
+ if os.path.exists(expected_path):
+ shutil.copyfile(actual_path, expected_path)
+
+ page += 1
+
+
+ACTUAL_TEMPLATE = '.pdf.%d.png'
+EXPECTED_TEMPLATE = '_expected' + ACTUAL_TEMPLATE
+PLATFORM_EXPECTED_TEMPLATE = '_expected_%s' + ACTUAL_TEMPLATE
+
+
+class PathTemplates(object):
+
+ def __init__(self, input_filename, source_dir, working_dir):
input_root, _ = os.path.splitext(input_filename)
- actual_path = os.path.join(working_dir, input_root + self.ACTUAL_TEMPLATE)
- expected_path = os.path.join(
- source_dir, input_root + self.EXPECTED_TEMPLATE)
- platform_expected_path = os.path.join(
- source_dir, input_root + self.PLATFORM_EXPECTED_TEMPLATE)
- return (actual_path, expected_path, platform_expected_path)
+ self.actual_path_template = os.path.join(working_dir,
+ input_root + ACTUAL_TEMPLATE)
+ self.expected_path = os.path.join(
+ source_dir, input_root + EXPECTED_TEMPLATE)
+ self.platform_expected_path = os.path.join(
+ source_dir, input_root + PLATFORM_EXPECTED_TEMPLATE)
+
+ def GetActualPath(self, page):
+ return self.actual_path_template % page
+
+ def GetExpectedPath(self, page):
+ return self.expected_path % page
+
+ def GetPlatformExpectedPath(self, platform, page):
+ return self.platform_expected_path % (platform, page)
diff --git a/testing/tools/test_runner.py b/testing/tools/test_runner.py
index 86e0563..ea8f2b8 100644
--- a/testing/tools/test_runner.py
+++ b/testing/tools/test_runner.py
@@ -81,6 +81,11 @@
if actual_images:
if self.image_differ.HasDifferences(input_filename, source_dir,
self.working_dir):
+ if (self.options.regenerate_expected
+ and not self.test_suppressor.IsResultSuppressed(input_filename)
+ and not self.test_suppressor.IsImageDiffSuppressed(input_filename)):
+ self.image_differ.Regenerate(input_filename, source_dir,
+ self.working_dir)
return False, results
else:
if (self.enforce_expected_images
@@ -171,12 +176,15 @@
parser.add_option('--gold_ignore_hashes', default='', dest="gold_ignore_hashes",
help='Path to a file with MD5 hashes we wish to ignore.')
+ parser.add_option('--regenerate_expected', action="store_true", dest="regenerate_expected",
+ help='Regenerates expected images.')
+
parser.add_option('--ignore_errors', action="store_true", dest="ignore_errors",
help='Prevents the return value from being non-zero when image comparison fails.')
- options, args = parser.parse_args()
+ self.options, self.args = parser.parse_args()
- finder = common.DirectoryFinder(options.build_dir)
+ finder = common.DirectoryFinder(self.options.build_dir)
self.fixup_path = finder.ScriptPath('fixup_pdf_template.py')
self.text_diff_path = finder.ScriptPath('text_diff.py')
@@ -206,8 +214,8 @@
self.test_cases = []
self.execution_suppressed_cases = []
input_file_re = re.compile('^.+[.](in|pdf)$')
- if args:
- for file_name in args:
+ if self.args:
+ for file_name in self.args:
file_name.replace('.pdf', '.in')
input_path = os.path.join(walk_from_dir, file_name)
if not os.path.isfile(input_path):
@@ -233,16 +241,16 @@
# Collect Gold results if an output directory was named.
self.gold_results = None
- if options.gold_output_dir:
+ if self.options.gold_output_dir:
self.gold_results = gold.GoldResults('pdfium',
- options.gold_output_dir,
- options.gold_properties,
- options.gold_key,
- options.gold_ignore_hashes)
+ self.options.gold_output_dir,
+ self.options.gold_properties,
+ self.options.gold_key,
+ self.options.gold_ignore_hashes)
- if options.num_workers > 1 and len(self.test_cases) > 1:
+ if self.options.num_workers > 1 and len(self.test_cases) > 1:
try:
- pool = multiprocessing.Pool(options.num_workers)
+ pool = multiprocessing.Pool(self.options.num_workers)
worker_func = functools.partial(TestOneFileParallel, self)
worker_results = pool.imap(worker_func, self.test_cases)
@@ -282,7 +290,7 @@
self._PrintSummary()
if self.failures:
- if not options.ignore_errors:
+ if not self.options.ignore_errors:
return 1
return 0
