Skia gold in pdfium
Backwards compatible with current pdfium recipe, so when this lands all
builds will run skia gold tests, alongside the current non-gold tests.
Gold failures will not fail the step or build.
Changed out optparse to argparse since optparse is now deprecated.
Bug: pdfium:1642
Change-Id: Ie38dcf45e526c81aae2179ca62c672ce8f38a6a1
Reviewed-on: https://pdfium-review.googlesource.com/c/pdfium/+/77930
Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Reviewed-by: Daniel Hosseinian <dhoss@chromium.org>
diff --git a/.gitignore b/.gitignore
index 2acea3e..af0aa39 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,7 @@
/tools/clang
/tools/code_coverage
/tools/memory
+/tools/skia_goldctl
/v8
/xcodebuild
.DS_Store
diff --git a/DEPS b/DEPS
index 2ab889f..dbd7158 100644
--- a/DEPS
+++ b/DEPS
@@ -203,6 +203,40 @@
Var('chromium_git') + '/chromium/src/tools/memory@' +
Var('tools_memory_revision'),
+ # TODO(crbug.com/pdfium/1650): Set up autorollers for goldctl.
+ 'tools/skia_goldctl/linux': {
+ 'packages': [
+ {
+ 'package': 'skia/tools/goldctl/linux-amd64',
+ 'version': 'git_revision:9476457a4a8acb6b45c61c11fa49dd2e9fccc10b',
+ }
+ ],
+ 'dep_type': 'cipd',
+ 'condition': 'checkout_linux',
+ },
+
+ 'tools/skia_goldctl/mac': {
+ 'packages': [
+ {
+ 'package': 'skia/tools/goldctl/mac-amd64',
+ 'version': 'git_revision:9476457a4a8acb6b45c61c11fa49dd2e9fccc10b',
+ }
+ ],
+ 'dep_type': 'cipd',
+ 'condition': 'checkout_mac',
+ },
+
+ 'tools/skia_goldctl/win': {
+ 'packages': [
+ {
+ 'package': 'skia/tools/goldctl/windows-amd64',
+ 'version': 'git_revision:9476457a4a8acb6b45c61c11fa49dd2e9fccc10b',
+ }
+ ],
+ 'dep_type': 'cipd',
+ 'condition': 'checkout_win',
+ },
+
'v8':
Var('chromium_git') + '/v8/v8.git@' + Var('v8_revision'),
}
diff --git a/testing/tools/gold.py b/testing/tools/gold.py
deleted file mode 100644
index c686bba..0000000
--- a/testing/tools/gold.py
+++ /dev/null
@@ -1,284 +0,0 @@
-# Copyright 2015 The PDFium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import json
-import os
-import shlex
-import shutil
-import ssl
-import urllib2
-
-
-def _ParseKeyValuePairs(kv_str):
- """
- Parses a string of the type 'key1 value1 key2 value2' into a dict.
- """
- kv_pairs = shlex.split(kv_str)
- if len(kv_pairs) % 2:
- raise ValueError('Uneven number of key/value pairs. Got %s' % kv_str)
- return {kv_pairs[i]: kv_pairs[i + 1] for i in xrange(0, len(kv_pairs), 2)}
-
-
-# This module downloads a json provided by Skia Gold with the expected baselines
-# for each test file.
-#
-# The expected format for the json is:
-# {
-# "commit": {
-# "author": "John Doe (jdoe@chromium.org)",
-# "commit_time": 1510598123,
-# "hash": "cee39e6e90c219cc91f2c94a912a06977f4461a0"
-# },
-# "master": {
-# "abc.pdf.1": {
-# "0ec3d86f545052acd7c9a16fde8ca9d4": 1,
-# "80455b71673becc9fbc100d6da56ca65": 1,
-# "b68e2ecb80090b4502ec89ad1be2322c": 1
-# },
-# "defgh.pdf.0": {
-# "01e020cd4cd05c6738e479a46a506044": 1,
-# "b68e2ecb80090b4502ec89ad1be2322c": 1
-# }
-# },
-# "changeLists": {
-# "18499" : {
-# "abc.pdf.1": {
-# "d5dd649124cf1779152253dc8fb239c5": 1,
-# "42a270581930579cdb0f28674972fb1a": 1,
-# }
-# }
-# }
-# }
-class GoldBaseline(object):
-
- def __init__(self, properties_str):
- """
- properties_str is a string with space separated key/value pairs that
- is used to find the cl number for which to baseline
- """
- self._properties = _ParseKeyValuePairs(properties_str)
- self._baselines = self._LoadSkiaGoldBaselines()
-
- def _LoadSkiaGoldBaselines(self):
- """
- Download the baseline json and return a list of the two baselines that
- should be used to match hashes (master and cl#).
- """
- GOLD_BASELINE_URL = 'https://pdfium-gold.skia.org/json/baseline'
-
- # If we have an issue number add it to the baseline URL
- cl_number_str = self._properties.get('issue', None)
- url = GOLD_BASELINE_URL + ('/' + cl_number_str if cl_number_str else '')
-
- json_data = ''
- MAX_TIMEOUT = 33 # 5 tries. (2, 4, 8, 16, 32)
- timeout = 2
- while True:
- try:
- response = urllib2.urlopen(url, timeout=timeout)
- c_type = response.headers.get('Content-type', '')
- EXPECTED_CONTENT_TYPE = 'application/json'
- if c_type != EXPECTED_CONTENT_TYPE:
- raise ValueError('Invalid content type. Got %s instead of %s' %
- (c_type, EXPECTED_CONTENT_TYPE))
- json_data = response.read()
- break # If this line is reached, then no exception occurred.
- except (ssl.SSLError, urllib2.HTTPError, urllib2.URLError) as e:
- timeout *= 2
- if timeout < MAX_TIMEOUT:
- continue
- print('Error: Unable to read skia gold json from %s: %s' % (url, e))
- return None
-
- try:
- data = json.loads(json_data)
- except ValueError as e:
- print 'Error: Malformed json read from %s: %s' % (url, e)
- return None
-
- return data.get('master', {})
-
- # Return values for MatchLocalResult().
- MATCH = 'match'
- MISMATCH = 'mismatch'
- NO_BASELINE = 'no_baseline'
- BASELINE_DOWNLOAD_FAILED = 'baseline_download_failed'
-
- def MatchLocalResult(self, test_name, md5_hash):
- """
- Match a locally generated hash of a test cases rendered image with the
- expected hashes downloaded in the baselines json.
-
- Each baseline is a dict mapping the test case name to a dict with the
- expected hashes as keys. Therefore, this list of baselines should be
- searched until the test case name is found, then the hash should be matched
- with the options in that dict. If the hashes don't match, it should be
- considered a failure and we should not continue searching the baseline list.
-
- Returns MATCH if the md5 provided matches the ones in the baseline json,
- MISMATCH if it does not, NO_BASELINE if the test case has no baseline, or
- BASELINE_DOWNLOAD_FAILED if the baseline could not be downloaded and parsed.
- """
- if self._baselines is None:
- return GoldBaseline.BASELINE_DOWNLOAD_FAILED
-
- found_test_case = False
- if test_name in self._baselines:
- found_test_case = True
- if md5_hash in self._baselines[test_name]:
- return GoldBaseline.MATCH
-
- return (GoldBaseline.MISMATCH
- if found_test_case else GoldBaseline.NO_BASELINE)
-
-
-# This module collects and writes output in a format expected by the
-# Gold baseline tool. Based on meta data provided explicitly and by
-# adding a series of test results it can be used to produce
-# a JSON file that is uploaded to Google Storage and ingested by Gold.
-#
-# The output will look similar this:
-#
-# {
-# "build_number" : "2",
-# "gitHash" : "a4a338179013b029d6dd55e737b5bd648a9fb68c",
-# "key" : {
-# "arch" : "arm64",
-# "compiler" : "Clang",
-# },
-# "results" : [
-# {
-# "key" : {
-# "config" : "vk",
-# "name" : "yuv_nv12_to_rgb_effect",
-# "source_type" : "gm"
-# },
-# "md5" : "7db34da246868d50ab9ddd776ce6d779",
-# "options" : {
-# "ext" : "png",
-# "gamma_correct" : "no"
-# }
-# },
-# {
-# "key" : {
-# "config" : "vk",
-# "name" : "yuv_to_rgb_effect",
-# "source_type" : "gm"
-# },
-# "md5" : "0b955f387740c66eb23bf0e253c80d64",
-# "options" : {
-# "ext" : "png",
-# "gamma_correct" : "no"
-# }
-# }
-# ],
-# }
-#
-class GoldResults(object):
-
- def __init__(self, source_type, output_dir, properties_str, key_str,
- ignore_hashes_file):
- """
- source_type is the source_type (=corpus) field used for all results.
- output_dir is the directory where the resulting images are copied and
- the dm.json file is written. If the directory exists it will
- be removed and recreated.
- properties_str is a string with space separated key/value pairs that
- is used to set the top level fields in the output JSON file.
- key_str is a string with space separated key/value pairs that
- is used to set the 'key' field in the output JSON file.
- ignore_hashes_file is a file that contains a list of image hashes
- that should be ignored.
- """
- self._source_type = source_type
- self._properties = _ParseKeyValuePairs(properties_str)
- self._properties['key'] = _ParseKeyValuePairs(key_str)
- self._results = []
- self._passfail = []
- self._output_dir = output_dir
-
- # make sure the output directory exists and is empty.
- if os.path.exists(output_dir):
- shutil.rmtree(output_dir, ignore_errors=True)
- os.makedirs(output_dir)
-
- self._ignore_hashes = set()
- if ignore_hashes_file:
- with open(ignore_hashes_file, 'r') as ig_file:
- hashes = [x.strip() for x in ig_file.readlines() if x.strip()]
- self._ignore_hashes = set(hashes)
-
- def AddTestResult(self, testName, md5Hash, outputImagePath, matchResult):
- # If the hash is in the list of hashes to ignore then we don'try
- # make a copy, but add it to the result.
- imgExt = os.path.splitext(outputImagePath)[1].lstrip('.')
- if md5Hash not in self._ignore_hashes:
- # Copy the image to <output_dir>/<md5Hash>.<image_extension>
- if not imgExt:
- raise ValueError('File %s does not have an extension' % outputImagePath)
- newFilePath = os.path.join(self._output_dir, md5Hash + '.' + imgExt)
- shutil.copy2(outputImagePath, newFilePath)
-
- # Add an entry to the list of test results
- self._results.append({
- 'key': {
- 'name': testName,
- 'source_type': self._source_type,
- },
- 'md5': md5Hash,
- 'options': {
- 'ext': imgExt,
- 'gamma_correct': 'no'
- }
- })
-
- self._passfail.append((testName, matchResult))
-
- def WriteResults(self):
- self._properties.update({'results': self._results})
-
- output_file_name = os.path.join(self._output_dir, 'dm.json')
- with open(output_file_name, 'wb') as outfile:
- json.dump(self._properties, outfile, indent=1)
- outfile.write('\n')
-
- output_file_name = os.path.join(self._output_dir, 'passfail.json')
- with open(output_file_name, 'wb') as outfile:
- json.dump(self._passfail, outfile, indent=1)
- outfile.write('\n')
-
-
-# Produce example output for manual testing.
-def _Example():
- # Create a test directory with three empty 'image' files.
- test_dir = './testdirectory'
- if not os.path.exists(test_dir):
- os.makedirs(test_dir)
- open(os.path.join(test_dir, 'image1.png'), 'wb').close()
- open(os.path.join(test_dir, 'image2.png'), 'wb').close()
- open(os.path.join(test_dir, 'image3.png'), 'wb').close()
-
- # Create an instance and add results.
- prop_str = 'build_number 2 "builder name" Builder-Name gitHash ' \
- 'a4a338179013b029d6dd55e737b5bd648a9fb68c'
-
- key_str = 'arch arm64 compiler Clang configuration Debug'
-
- hash_file = os.path.join(test_dir, 'ignore_hashes.txt')
- with open(hash_file, 'wb') as f:
- f.write('\n'.join(['hash-1', 'hash-4']) + '\n')
-
- output_dir = './output_directory'
- gr = GoldResults('pdfium', output_dir, prop_str, key_str, hash_file)
- gr.AddTestResult('test-1', 'hash-1', os.path.join(test_dir, 'image1.png'),
- GoldBaseline.MATCH)
- gr.AddTestResult('test-2', 'hash-2', os.path.join(test_dir, 'image2.png'),
- GoldBaseline.MATCH)
- gr.AddTestResult('test-3', 'hash-3', os.path.join(test_dir, 'image3.png'),
- GoldBaseline.MISMATCH)
- gr.WriteResults()
-
-
-if __name__ == '__main__':
- _Example()
diff --git a/testing/tools/skia_gold/__init__.py b/testing/tools/skia_gold/__init__.py
new file mode 100644
index 0000000..3415b87
--- /dev/null
+++ b/testing/tools/skia_gold/__init__.py
@@ -0,0 +1,4 @@
+# pylint: disable=relative-import
+import path_util
+
+path_util.AddDirToPathIfNeeded(path_util.GetPDFiumDir(), 'build')
diff --git a/testing/tools/skia_gold/path_util.py b/testing/tools/skia_gold/path_util.py
new file mode 100644
index 0000000..94c9297
--- /dev/null
+++ b/testing/tools/skia_gold/path_util.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# Copyright 2021 The PDFium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import sys
+
+
+def AddDirToPathIfNeeded(*path_parts):
+ path = os.path.abspath(os.path.join(*path_parts))
+ if os.path.isdir(path) and path not in sys.path:
+ sys.path.append(path)
+
+
+def GetPDFiumDir():
+ if not GetPDFiumDir.pdfium_dir:
+ # Expect |skia_gold_dir| to be .../pdfium/testing/tools/skia_gold.
+ skia_gold_dir = os.path.dirname(os.path.realpath(__file__))
+ tools_dir = os.path.dirname(skia_gold_dir)
+ testing_dir = os.path.dirname(tools_dir)
+ if (os.path.basename(tools_dir) != 'tools' or
+ os.path.basename(testing_dir) != 'testing'):
+ raise RuntimeError(
+ 'Confused, can not find pdfium root directory, aborting.')
+ GetPDFiumDir.pdfium_dir = os.path.dirname(testing_dir)
+ return GetPDFiumDir.pdfium_dir
+
+
+GetPDFiumDir.pdfium_dir = None
diff --git a/testing/tools/skia_gold/pdfium_skia_gold_properties.py b/testing/tools/skia_gold/pdfium_skia_gold_properties.py
new file mode 100644
index 0000000..b90d2c3
--- /dev/null
+++ b/testing/tools/skia_gold/pdfium_skia_gold_properties.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# Copyright 2021 The PDFium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""PDFium implementation of //build/skia_gold_common/skia_gold_properties.py."""
+
+# pylint: disable=relative-import
+import subprocess
+import sys
+
+import path_util
+from skia_gold_common import skia_gold_properties
+
+
+class PDFiumSkiaGoldProperties(skia_gold_properties.SkiaGoldProperties):
+
+ @staticmethod
+ def _GetGitOriginMasterHeadSha1():
+ try:
+ return subprocess.check_output(['git', 'rev-parse', 'origin/master'],
+ shell=_IsWin(),
+ cwd=path_util.GetPDFiumDir()).strip()
+ except subprocess.CalledProcessError:
+ return None
+
+
+def _IsWin():
+ return sys.platform == 'win32'
diff --git a/testing/tools/skia_gold/pdfium_skia_gold_session.py b/testing/tools/skia_gold/pdfium_skia_gold_session.py
new file mode 100644
index 0000000..9f7d198
--- /dev/null
+++ b/testing/tools/skia_gold/pdfium_skia_gold_session.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# Copyright 2021 The PDFium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""PDFium implementation of //build/skia_gold_common/skia_gold_session.py."""
+
+# pylint: disable=relative-import
+from skia_gold_common import output_managerless_skia_gold_session as omsgs
+
+
+# ComparisonResults nested inside the SkiaGoldSession causes issues with
+# multiprocessing and pickling, so it was moved out here.
+class PDFiumComparisonResults(object):
+ """Struct-like object for storing results of an image comparison."""
+
+ def __init__(self):
+ self.public_triage_link = None
+ self.internal_triage_link = None
+ self.triage_link_omission_reason = None
+ self.local_diff_given_image = None
+ self.local_diff_closest_image = None
+ self.local_diff_diff_image = None
+
+
+class PDFiumSkiaGoldSession(omsgs.OutputManagerlessSkiaGoldSession):
+
+ def _GetDiffGoldInstance(self):
+ return str(self._instance)
+
+ def ComparisonResults(self):
+ return PDFiumComparisonResults()
diff --git a/testing/tools/skia_gold/pdfium_skia_gold_session_manager.py b/testing/tools/skia_gold/pdfium_skia_gold_session_manager.py
new file mode 100644
index 0000000..4eed413
--- /dev/null
+++ b/testing/tools/skia_gold/pdfium_skia_gold_session_manager.py
@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+# Copyright 2021 The PDFium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""PDFium implementation of
+//build/skia_gold_common/skia_gold_session_manager.py."""
+
+# pylint: disable=relative-import
+import pdfium_skia_gold_session
+from skia_gold_common import skia_gold_session_manager as sgsm
+
+SKIA_PDF_INSTANCE = 'pdfium'
+
+
+class PDFiumSkiaGoldSessionManager(sgsm.SkiaGoldSessionManager):
+
+ @staticmethod
+ def GetSessionClass():
+ return pdfium_skia_gold_session.PDFiumSkiaGoldSession
+
+ @staticmethod
+ def _GetDefaultInstance():
+ return SKIA_PDF_INSTANCE
diff --git a/testing/tools/skia_gold/skia_gold.py b/testing/tools/skia_gold/skia_gold.py
new file mode 100644
index 0000000..d5067b8
--- /dev/null
+++ b/testing/tools/skia_gold/skia_gold.py
@@ -0,0 +1,223 @@
+# Copyright 2021 The PDFium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+from __future__ import print_function
+
+import os
+import logging
+import shlex
+import shutil
+
+# pylint: disable=relative-import
+import pdfium_skia_gold_properties
+import pdfium_skia_gold_session_manager
+
+GS_BUCKET = 'skia-pdfium-gm'
+
+
+def _ParseKeyValuePairs(kv_str):
+ """
+ Parses a string of the type 'key1 value1 key2 value2' into a dict.
+ """
+ kv_pairs = shlex.split(kv_str)
+ if len(kv_pairs) % 2:
+ raise ValueError('Uneven number of key/value pairs. Got %s' % kv_str)
+ return {kv_pairs[i]: kv_pairs[i + 1] for i in xrange(0, len(kv_pairs), 2)}
+
+
+def add_skia_gold_args(parser):
+ group = parser.add_argument_group('Skia Gold Arguments')
+ group.add_argument(
+ '--git-revision', help='Revision being tested.', default=None)
+ group.add_argument(
+ '--gerrit-issue',
+ help='For Skia Gold integration. Gerrit issue ID.',
+ default='')
+ group.add_argument(
+ '--gerrit-patchset',
+ help='For Skia Gold integration. Gerrit patch set number.',
+ default='')
+ group.add_argument(
+ '--buildbucket-id',
+ help='For Skia Gold integration. Buildbucket build ID.',
+ default='')
+ group.add_argument(
+ '--bypass-skia-gold-functionality',
+ action='store_true',
+ default=False,
+ help='Bypass all interaction with Skia Gold, effectively disabling the '
+ 'image comparison portion of any tests that use Gold. Only meant to '
+ 'be used in case a Gold outage occurs and cannot be fixed quickly.')
+ local_group = group.add_mutually_exclusive_group()
+ local_group.add_argument(
+ '--local-pixel-tests',
+ action='store_true',
+ default=None,
+ help='Specifies to run the test harness in local run mode or not. When '
+ 'run in local mode, uploading to Gold is disabled and links to '
+ 'help with local debugging are output. Running in local mode also '
+ 'implies --no-luci-auth. If both this and --no-local-pixel-tests are '
+ 'left unset, the test harness will attempt to detect whether it is '
+ 'running on a workstation or not and set this option accordingly.')
+ local_group.add_argument(
+ '--no-local-pixel-tests',
+ action='store_false',
+ dest='local_pixel_tests',
+ help='Specifies to run the test harness in non-local (bot) mode. When '
+ 'run in this mode, data is actually uploaded to Gold and triage links '
+ 'arge generated. If both this and --local-pixel-tests are left unset, '
+ 'the test harness will attempt to detect whether it is running on a '
+ 'workstation or not and set this option accordingly.')
+ group.add_argument(
+ '--no-luci-auth',
+ action='store_true',
+ default=False,
+ help='Don\'t use the service account provided by LUCI for '
+ 'authentication for Skia Gold, instead relying on gsutil to be '
+ 'pre-authenticated. Meant for testing locally instead of on the bots.')
+
+ group.add_argument(
+ '--gold_key',
+ default='',
+ dest="gold_key",
+ help='Key value pairs of config data such like the hardware/software '
+ 'configuration the image was produced on.')
+ group.add_argument(
+ '--gold_output_dir',
+ default='',
+ dest="gold_output_dir",
+ help='Path to the dir where diff output image files are saved, '
+ 'if running locally. If this is a tryjob run, will contain link to skia '
+ 'gold CL triage link.')
+
+
+def clear_gold_output_dir(output_dir):
+ # make sure the output directory exists and is empty.
+ if os.path.exists(output_dir):
+ shutil.rmtree(output_dir, ignore_errors=True)
+ os.makedirs(output_dir)
+
+
+class SkiaGoldTester(object):
+
+ def __init__(self, source_type, skia_gold_args, process_name=None):
+ """
+ source_type: source_type (=corpus) field used for all results.
+ skia_gold_args: Parsed arguments from argparse.ArgumentParser.
+ process_name: Unique name of current process, if multiprocessing is on.
+ """
+ self._source_type = source_type
+ self._output_dir = skia_gold_args.gold_output_dir
+ # goldctl is not thread safe, so each process needs its own directory
+ if process_name:
+ self._output_dir = os.path.join(skia_gold_args.gold_output_dir,
+ process_name)
+ clear_gold_output_dir(self._output_dir)
+ self._keys = _ParseKeyValuePairs(skia_gold_args.gold_key)
+ self._old_gold_props = _ParseKeyValuePairs(skia_gold_args.gold_properties)
+ self._skia_gold_args = skia_gold_args
+ self._skia_gold_session_manager = None
+ self._skia_gold_properties = None
+
+ def WriteCLTriageLink(self, link):
+ # pdfium recipe will read from this file and display the link in the step
+ # presentation
+ assert isinstance(link, str)
+ output_file_name = os.path.join(self._output_dir, 'cl_triage_link.txt')
+ if os.path.exists(output_file_name):
+ os.remove(output_file_name)
+ with open(output_file_name, 'wb') as outfile:
+ outfile.write(link)
+
+ def GetSkiaGoldProperties(self):
+ if not self._skia_gold_properties:
+ if self._old_gold_props:
+ self._skia_gold_args.git_revision = self._old_gold_props['gitHash']
+ self._skia_gold_args.gerrit_issue = self._old_gold_props['issue']
+ self._skia_gold_args.gerrit_patchset = self._old_gold_props['patchset']
+ self._skia_gold_args.buildbucket_id = \
+ self._old_gold_props['buildbucket_build_id']
+
+ if self._skia_gold_args.local_pixel_tests is None:
+ self._skia_gold_args.local_pixel_tests = 'SWARMING_SERVER' \
+ not in os.environ
+
+ self._skia_gold_properties = pdfium_skia_gold_properties\
+ .PDFiumSkiaGoldProperties(self._skia_gold_args)
+ return self._skia_gold_properties
+
+ def GetSkiaGoldSessionManager(self):
+ if not self._skia_gold_session_manager:
+ self._skia_gold_session_manager = pdfium_skia_gold_session_manager\
+ .PDFiumSkiaGoldSessionManager(self._output_dir,
+ self.GetSkiaGoldProperties())
+ return self._skia_gold_session_manager
+
+ def IsTryjobRun(self):
+ return self.GetSkiaGoldProperties().IsTryjobRun()
+
+ def GetCLTriageLink(self):
+ return 'https://pdfium-gold.skia.org/search?issue={issue}&crs=gerrit&'\
+ 'corpus={source_type}'.format(
+ issue=self.GetSkiaGoldProperties().issue, source_type=self._source_type)
+
+ def UploadTestResultToSkiaGold(self, image_name, image_path):
+ gold_properties = self.GetSkiaGoldProperties()
+ use_luci = not (gold_properties.local_pixel_tests or
+ gold_properties.no_luci_auth)
+ gold_session = self.GetSkiaGoldSessionManager()\
+ .GetSkiaGoldSession(self._keys, corpus=self._source_type,
+ bucket=GS_BUCKET)
+
+ status, error = gold_session.RunComparison(
+ name=image_name, png_file=image_path, use_luci=use_luci)
+
+ status_codes =\
+ self.GetSkiaGoldSessionManager().GetSessionClass().StatusCodes
+ if status == status_codes.SUCCESS:
+ return True
+ elif status == status_codes.AUTH_FAILURE:
+ logging.error('Gold authentication failed with output %s', error)
+ elif status == status_codes.INIT_FAILURE:
+ logging.error('Gold initialization failed with output %s', error)
+ elif status == status_codes.COMPARISON_FAILURE_REMOTE:
+ logging.error('Remote comparison failed. See outputted triage links.')
+ elif status == status_codes.COMPARISON_FAILURE_LOCAL:
+ logging.error('Local comparison failed. Local diff files:')
+ _OutputLocalDiffFiles(gold_session, image_name)
+ print()
+ elif status == status_codes.LOCAL_DIFF_FAILURE:
+ logging.error(
+ 'Local comparison failed and an error occurred during diff '
+ 'generation: %s', error)
+ # There might be some files, so try outputting them.
+ logging.error('Local diff files:')
+ _OutputLocalDiffFiles(gold_session, image_name)
+ print()
+ else:
+ logging.error(
+ 'Given unhandled SkiaGoldSession StatusCode %s with error %s', status,
+ error)
+
+ return False
+
+
+def _OutputLocalDiffFiles(gold_session, image_name):
+ """Logs the local diff image files from the given SkiaGoldSession.
+
+ Args:
+ gold_session: A skia_gold_session.SkiaGoldSession instance to pull files
+ from.
+ image_name: A string containing the name of the image/test that was
+ compared.
+ """
+ given_file = gold_session.GetGivenImageLink(image_name)
+ closest_file = gold_session.GetClosestImageLink(image_name)
+ diff_file = gold_session.GetDiffImageLink(image_name)
+ failure_message = 'Unable to retrieve link'
+ logging.error('Generated image for %s: %s', image_name, given_file or
+ failure_message)
+ logging.error('Closest image for %s: %s', image_name, closest_file or
+ failure_message)
+ logging.error('Diff image for %s: %s', image_name, diff_file or
+ failure_message)
diff --git a/testing/tools/test_runner.py b/testing/tools/test_runner.py
index f4b45ac..77ac19f 100644
--- a/testing/tools/test_runner.py
+++ b/testing/tools/test_runner.py
@@ -3,9 +3,11 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+from __future__ import print_function
+
+import argparse
import functools
import multiprocessing
-import optparse
import os
import re
import shutil
@@ -14,9 +16,10 @@
# pylint: disable=relative-import
import common
-import gold
import pngdiffer
import suppressor
+from skia_gold import skia_gold
+
# Arbitrary timestamp, expressed in seconds since the epoch, used to make sure
# that tests that depend on the current time are stable. Happens to be the
@@ -48,6 +51,17 @@
raise KeyboardInterruptError()
+def RunSkiaWrapper(this, img_path_input_filename):
+ """Wrapper to call RunSkia() and redirect output to stdout"""
+ img_path, input_filename = img_path_input_filename
+ multiprocessing_name = multiprocessing.current_process().name
+ try:
+ test_name, skia_success = this.RunSkia(img_path, multiprocessing_name)
+ return test_name, skia_success, input_filename
+ except KeyboardInterrupt:
+ raise KeyboardInterruptError()
+
+
def DeleteFiles(files):
"""Utility function to delete a list of files"""
for f in files:
@@ -67,6 +81,25 @@
self.delete_output_on_success = False
self.enforce_expected_images = False
self.oneshot_renderer = False
+ self.skia_tester = None
+
+ def GetSkiaGoldTester(self, process_name=None):
+ if not self.skia_tester:
+ self.skia_tester = skia_gold.SkiaGoldTester(
+ source_type=self.test_type,
+ skia_gold_args=self.options,
+ process_name=process_name)
+ return self.skia_tester
+
+ def RunSkia(self, img_path, process_name=None):
+ skia_tester = self.GetSkiaGoldTester(process_name=process_name)
+ # The output filename without image extension becomes the test name.
+ # For example, "/path/to/.../testing/corpus/example_005.pdf.0.png"
+ # becomes "example_005.pdf.0".
+ test_name = os.path.splitext(os.path.split(img_path)[1])[0]
+ skia_success = skia_tester.UploadTestResultToSkiaGold(test_name, img_path)
+ sys.stdout.flush()
+ return test_name, skia_success
# GenerateAndTest returns a tuple <success, outputfiles> where
# success is a boolean indicating whether the tests passed comparison
@@ -87,7 +120,7 @@
pdf_path)
if raised_exception is not None:
- print 'FAILURE: %s; %s' % (input_filename, raised_exception)
+ print('FAILURE: {}; {}'.format(input_filename, raised_exception))
return False, []
results = []
@@ -100,7 +133,7 @@
raised_exception, results = self.TestPixel(pdf_path, use_ahem)
if raised_exception is not None:
- print 'FAILURE: %s; %s' % (input_filename, raised_exception)
+ print('FAILURE: {}; {}'.format(input_filename, raised_exception))
return False, results
if actual_images:
@@ -112,7 +145,7 @@
if (self.enforce_expected_images and
not self.test_suppressor.IsImageDiffSuppressed(input_filename)):
self.RegenerateIfNeeded_(input_filename, source_dir)
- print 'FAILURE: %s; Missing expected images' % input_filename
+ print('FAILURE: {}; Missing expected images'.format(input_filename))
return False, results
if self.delete_output_on_success:
@@ -221,26 +254,7 @@
return common.RunCommandExtractHashedFiles(cmd_to_run)
def HandleResult(self, input_filename, input_path, result):
- success, image_paths = result
-
- if image_paths:
- for img_path, md5_hash in image_paths:
- # The output filename without image extension becomes the test name.
- # For example, "/path/to/.../testing/corpus/example_005.pdf.0.png"
- # becomes "example_005.pdf.0".
- test_name = os.path.splitext(os.path.split(img_path)[1])[0]
-
- matched = "suppressed"
- if not self.test_suppressor.IsResultSuppressed(input_filename):
- matched = self.gold_baseline.MatchLocalResult(test_name, md5_hash)
- if matched == gold.GoldBaseline.MISMATCH:
- print 'Skia Gold hash mismatch for test case: %s' % test_name
- elif matched == gold.GoldBaseline.NO_BASELINE:
- print 'No Skia Gold baseline found for test case: %s' % test_name
-
- if self.gold_results:
- self.gold_results.AddTestResult(test_name, md5_hash, img_path,
- matched)
+ success, _ = result
if self.test_suppressor.IsResultSuppressed(input_filename):
self.result_suppressed_cases.append(input_filename)
@@ -254,60 +268,54 @@
# Running a test defines a number of attributes on the fly.
# pylint: disable=attribute-defined-outside-init
- parser = optparse.OptionParser()
+ parser = argparse.ArgumentParser()
- parser.add_option(
+ parser.add_argument(
'--build-dir',
default=os.path.join('out', 'Debug'),
help='relative path from the base source directory')
- parser.add_option(
+ parser.add_argument(
'-j',
default=multiprocessing.cpu_count(),
dest='num_workers',
- type='int',
+ type=int,
help='run NUM_WORKERS jobs in parallel')
- parser.add_option(
+ parser.add_argument(
'--disable-javascript',
action="store_true",
dest="disable_javascript",
help='Prevents JavaScript from executing in PDF files.')
- parser.add_option(
+ parser.add_argument(
'--disable-xfa',
action="store_true",
dest="disable_xfa",
help='Prevents processing XFA forms.')
- parser.add_option(
+ parser.add_argument(
+ '--run-skia-gold',
+ action='store_true',
+ default=False,
+ help='When flag is on, skia gold tests will be run.')
+
+ # TODO: Remove when pdfium recipe stops passing this argument
+ parser.add_argument(
'--gold_properties',
default='',
dest="gold_properties",
help='Key value pairs that are written to the top level '
'of the JSON file that is ingested by Gold.')
- parser.add_option(
- '--gold_key',
- default='',
- dest="gold_key",
- help='Key value pairs that are added to the "key" field '
- 'of the JSON file that is ingested by Gold.')
-
- parser.add_option(
- '--gold_output_dir',
- default='',
- dest="gold_output_dir",
- help='Path of where to write the JSON output to be '
- 'uploaded to Gold.')
-
- parser.add_option(
+ # TODO: Remove when pdfium recipe stops passing this argument
+ parser.add_argument(
'--gold_ignore_hashes',
default='',
dest="gold_ignore_hashes",
help='Path to a file with MD5 hashes we wish to ignore.')
- parser.add_option(
+ parser.add_argument(
'--regenerate_expected',
default='',
dest="regenerate_expected",
@@ -316,24 +324,26 @@
'"platform" to regenerate only platform-specific '
'expected pngs.')
- parser.add_option(
+ parser.add_argument(
'--reverse-byte-order',
action='store_true',
dest="reverse_byte_order",
help='Run image-based tests using --reverse-byte-order.')
- parser.add_option(
+ parser.add_argument(
'--ignore_errors',
action="store_true",
dest="ignore_errors",
help='Prevents the return value from being non-zero '
'when image comparison fails.')
- self.options, self.args = parser.parse_args()
+ skia_gold.add_skia_gold_args(parser)
+
+ self.options, self.inputted_file_paths = parser.parse_known_args()
if (self.options.regenerate_expected and
self.options.regenerate_expected not in ['all', 'platform']):
- print 'FAILURE: --regenerate_expected must be "all" or "platform"'
+ print('FAILURE: --regenerate_expected must be "all" or "platform"')
return 1
finder = common.DirectoryFinder(self.options.build_dir)
@@ -349,8 +359,9 @@
self.pdfium_test_path = finder.ExecutablePath('pdfium_test')
if not os.path.exists(self.pdfium_test_path):
- print "FAILURE: Can't find test executable '%s'" % self.pdfium_test_path
- print 'Use --build-dir to specify its location.'
+ print("FAILURE: Can't find test executable '{}'".format(
+ self.pdfium_test_path))
+ print('Use --build-dir to specify its location.')
return 1
self.working_dir = finder.WorkingDir(os.path.join('testing', self.test_dir))
@@ -367,22 +378,21 @@
error_message = self.image_differ.CheckMissingTools(
self.options.regenerate_expected)
if error_message:
- print "FAILURE: %s" % error_message
+ print('FAILURE:', error_message)
return 1
- self.gold_baseline = gold.GoldBaseline(self.options.gold_properties)
walk_from_dir = finder.TestingDir(test_dir)
self.test_cases = []
self.execution_suppressed_cases = []
input_file_re = re.compile('^.+[.](in|pdf)$')
- if self.args:
- for file_name in self.args:
+ if self.inputted_file_paths:
+ for file_name in self.inputted_file_paths:
file_name.replace('.pdf', '.in')
input_path = os.path.join(walk_from_dir, file_name)
if not os.path.isfile(input_path):
- print "Can't find test file '%s'" % file_name
+ print("Can't find test file '{}'".format(file_name))
return 1
self.test_cases.append((os.path.basename(input_path),
@@ -401,28 +411,39 @@
self.test_cases.sort()
self.failures = []
self.surprises = []
+ self.skia_gold_successes = []
+ self.skia_gold_unexpected_successes = []
+ self.skia_gold_failures = []
self.result_suppressed_cases = []
- # Collect Gold results if an output directory was named.
- self.gold_results = None
- if self.options.gold_output_dir:
- self.gold_results = gold.GoldResults(
- self.test_type, self.options.gold_output_dir,
- self.options.gold_properties, self.options.gold_key,
- self.options.gold_ignore_hashes)
-
+ gold_results = []
if self.options.num_workers > 1 and len(self.test_cases) > 1:
try:
pool = multiprocessing.Pool(self.options.num_workers)
worker_func = functools.partial(TestOneFileParallel, self)
worker_results = pool.imap(worker_func, self.test_cases)
+ skia_gold_inputs = []
for worker_result in worker_results:
result, input_filename, source_dir = worker_result
input_path = os.path.join(source_dir, input_filename)
self.HandleResult(input_filename, input_path, result)
+ if self.test_type not in TEXT_TESTS and self.options.run_skia_gold:
+ _, image_paths = result
+ if image_paths:
+ path_filename_tuples = [
+ (path, input_filename) for path, _ in image_paths
+ ]
+ skia_gold_inputs.extend(path_filename_tuples)
+
+ if skia_gold_inputs and self.test_type not in TEXT_TESTS:
+ gold_worker_func = functools.partial(RunSkiaWrapper, self)
+ # Clear out top level gold output directory before starting
+ skia_gold.clear_gold_output_dir(self.options.gold_output_dir)
+ gold_results = pool.imap(gold_worker_func, skia_gold_inputs)
+
except KeyboardInterrupt:
pool.terminate()
finally:
@@ -435,20 +456,47 @@
self.HandleResult(input_filename,
os.path.join(input_file_dir, input_filename), result)
- if self.gold_results:
- self.gold_results.WriteResults()
+ _, image_paths = result
+ if image_paths and self.test_type not in TEXT_TESTS:
+ # Clear out top level gold output directory before starting
+ skia_gold.clear_gold_output_dir(self.options.gold_output_dir)
+ for img_path, _ in image_paths:
+ test_name, skia_success = self.RunSkia(img_path)
+ gold_results.append((test_name, skia_success, input_filename))
+
+ for r in gold_results:
+ test_name, skia_success, input_filename = r
+ if skia_success:
+ if self.test_suppressor.IsResultSuppressed(input_filename):
+ self.skia_gold_unexpected_successes.append(test_name)
+ else:
+ self.skia_gold_successes.append(test_name)
+ else:
+ self.skia_gold_failures.append(test_name)
if self.surprises:
self.surprises.sort()
- print '\n\nUnexpected Successes:'
+ print('\nUnexpected Successes:')
for surprise in self.surprises:
- print surprise
+ print(surprise)
if self.failures:
self.failures.sort()
- print '\n\nSummary of Failures:'
+ print('\nSummary of Failures:')
for failure in self.failures:
- print failure
+ print(failure)
+
+ if self.skia_gold_unexpected_successes:
+ self.skia_gold_failures.sort()
+ print('\nUnexpected Skia Gold Successes:')
+ for surprise in self.skia_gold_unexpected_successes:
+ print(surprise)
+
+ if self.skia_gold_failures:
+ self.skia_gold_failures.sort()
+ print('\nSummary of Skia Gold Failures:')
+ for failure in self.skia_gold_failures:
+ print(failure)
self._PrintSummary()
@@ -464,14 +512,28 @@
number_suppressed = len(self.result_suppressed_cases)
number_successes = number_test_cases - number_failures - number_suppressed
number_surprises = len(self.surprises)
- print
- print 'Test cases executed: %d' % number_test_cases
- print ' Successes: %d' % number_successes
- print ' Suppressed: %d' % number_suppressed
- print ' Surprises: %d' % number_surprises
- print ' Failures: %d' % number_failures
- print
- print 'Test cases not executed: %d' % len(self.execution_suppressed_cases)
+ print('\nTest cases executed:', number_test_cases)
+ print(' Successes:', number_successes)
+ print(' Suppressed:', number_suppressed)
+ print(' Surprises:', number_surprises)
+ print(' Failures:', number_failures)
+ if self.test_type not in TEXT_TESTS:
+ number_gold_failures = len(self.skia_gold_failures)
+ number_gold_successes = len(self.skia_gold_successes)
+ number_gold_surprises = len(self.skia_gold_unexpected_successes)
+ number_total_gold_tests = sum(
+ [number_gold_failures, number_gold_successes, number_gold_surprises])
+ print('\nSkia Gold Test cases executed:', number_total_gold_tests)
+ print(' Skia Gold Successes:', number_gold_successes)
+ print(' Skia Gold Surprises:', number_gold_surprises)
+ print(' Skia Gold Failures:', number_gold_failures)
+ skia_tester = self.GetSkiaGoldTester()
+ if self.skia_gold_failures and skia_tester.IsTryjobRun():
+ cl_triage_link = skia_tester.GetCLTriageLink()
+ print(' Triage link for CL:', cl_triage_link)
+ skia_tester.WriteCLTriageLink(cl_triage_link)
+ print()
+ print('Test cases not executed:', len(self.execution_suppressed_cases))
def SetDeleteOutputOnSuccess(self, new_value):
"""Set whether to delete generated output if the test passes."""