Don't pickle TestRunner
Avoids pickling TestRunner by separating state used to run individual
tests from state used to orchestrate the test suite as a whole.
This change has a relatively large delta due to moving code from the
TestRunner class to the _PerProcessState class, but most of the changes
focus on moving properties and initialization code from TestRunner to
_PerProcessConfig or _PerProcessState, along with necessary adjustments
to any call sites in TestRunner.
Bug: pdfium:1641
Change-Id: Ic648f1e72c2bf0f591c2e8c8eff4047b87d0df46
Reviewed-on: https://pdfium-review.googlesource.com/c/pdfium/+/99610
Commit-Queue: K. Moon <kmoon@chromium.org>
Reviewed-by: Lei Zhang <thestig@chromium.org>
diff --git a/testing/tools/test_runner.py b/testing/tools/test_runner.py
index 08e0783..7251174 100644
--- a/testing/tools/test_runner.py
+++ b/testing/tools/test_runner.py
@@ -35,26 +35,7 @@
class KeyboardInterruptError(Exception):
- pass
-
-
-# This is a class, rather than a closure, to make it picklable.
-class _WrapKeyboardInterrupt:
- """Wraps `KeyboardInterrupt` thrown from a callable.
-
- This wrapper prevents `KeyboardInterrupt` from escaping the wrapped callable,
- by wrapping the exception in a custom `KeyboardInterruptError` exception
- instead.
- """
-
- def __init__(self, wrapped):
- self.wrapped = wrapped
-
- def __call__(self, *args):
- try:
- return self.wrapped(*args)
- except KeyboardInterrupt as exc:
- raise KeyboardInterruptError() from exc
+ """Custom exception used to wrap `KeyboardInterrupt` exceptions."""
def DeleteFiles(files):
@@ -64,7 +45,6 @@
os.remove(f)
-# TODO(crbug.com/pdfium/1641): Don't pickle TestRunner.
class TestRunner:
def __init__(self, dirname):
@@ -72,10 +52,427 @@
# which all correspond directly to the type for the test being run. In the
# future if there are tests that don't have this clean correspondence, then
# an argument for the type will need to be added.
- self.test_dir = dirname
- self.test_type = dirname
- self.delete_output_on_success = False
- self.enforce_expected_images = False
+ self.per_process_config = _PerProcessConfig(
+ test_dir=dirname, test_type=dirname)
+
+ @property
+ def options(self):
+ return self.per_process_config.options
+
+ def IsSkiaGoldEnabled(self):
+ return (self.options.run_skia_gold and
+ not self.per_process_config.test_type in TEXT_TESTS)
+
+ def IsExecutionSuppressed(self, input_path):
+ return self.per_process_state.test_suppressor.IsExecutionSuppressed(
+ input_path)
+
+ def IsResultSuppressed(self, input_filename):
+ return self.per_process_state.test_suppressor.IsResultSuppressed(
+ input_filename)
+
+ def HandleResult(self, test_case, test_result):
+ input_filename = os.path.basename(test_case.input_path)
+
+ if self.IsResultSuppressed(input_filename):
+ self.result_suppressed_cases.append(input_filename)
+ if test_result.IsPass():
+ self.surprises.append(test_case.input_path)
+
+ # There isn't an actual status for succeeded-but-ignored, so use the
+ # "abort" status to differentiate this from failed-but-ignored.
+ #
+ # Note that this appears as a preliminary failure in Gerrit.
+ result_status = result_types.UNKNOWN
+ else:
+ # There isn't an actual status for failed-but-ignored, so use the
+ # "skip" status to differentiate this from succeeded-but-ignored.
+ result_status = result_types.SKIP
+ else:
+ if test_result.IsPass():
+ result_status = result_types.PASS
+ else:
+ self.failures.append(test_case.input_path)
+ result_status = result_types.FAIL
+
+ if self.resultdb:
+ # TODO(crbug.com/pdfium/1916): Populate more ResultDB fields.
+ self.resultdb.Post(
+ test_id=test_result.test_id,
+ status=result_status,
+ duration=None,
+ test_log=None,
+ test_file=None)
+
+ def Run(self):
+ # Running a test defines a number of attributes on the fly.
+ # pylint: disable=attribute-defined-outside-init
+
+ relative_test_dir = self.per_process_config.test_dir
+ if relative_test_dir != 'corpus':
+ relative_test_dir = os.path.join('resources', relative_test_dir)
+
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument(
+ '--build-dir',
+ default=os.path.join('out', 'Debug'),
+ help='relative path from the base source directory')
+
+ parser.add_argument(
+ '-j',
+ default=multiprocessing.cpu_count(),
+ dest='num_workers',
+ type=int,
+ help='run NUM_WORKERS jobs in parallel')
+
+ parser.add_argument(
+ '--disable-javascript',
+ action="store_true",
+ dest="disable_javascript",
+ help='Prevents JavaScript from executing in PDF files.')
+
+ parser.add_argument(
+ '--disable-xfa',
+ action="store_true",
+ dest="disable_xfa",
+ help='Prevents processing XFA forms.')
+
+ parser.add_argument(
+ '--render-oneshot',
+ action="store_true",
+ dest="render_oneshot",
+ help='Sets whether to use the oneshot renderer.')
+
+ parser.add_argument(
+ '--run-skia-gold',
+ action='store_true',
+ default=False,
+ help='When flag is on, skia gold tests will be run.')
+
+ # TODO: Remove when pdfium recipe stops passing this argument
+ parser.add_argument(
+ '--gold_properties',
+ default='',
+ dest="gold_properties",
+ help='Key value pairs that are written to the top level '
+ 'of the JSON file that is ingested by Gold.')
+
+ # TODO: Remove when pdfium recipe stops passing this argument
+ parser.add_argument(
+ '--gold_ignore_hashes',
+ default='',
+ dest="gold_ignore_hashes",
+ help='Path to a file with MD5 hashes we wish to ignore.')
+
+ parser.add_argument(
+ '--regenerate_expected',
+ default='',
+ dest="regenerate_expected",
+ help='Regenerates expected images. Valid values are '
+ '"all" to regenerate all expected pngs, and '
+ '"platform" to regenerate only platform-specific '
+ 'expected pngs.')
+
+ parser.add_argument(
+ '--reverse-byte-order',
+ action='store_true',
+ dest="reverse_byte_order",
+ help='Run image-based tests using --reverse-byte-order.')
+
+ parser.add_argument(
+ '--ignore_errors',
+ action="store_true",
+ dest="ignore_errors",
+ help='Prevents the return value from being non-zero '
+ 'when image comparison fails.')
+
+ parser.add_argument(
+ 'inputted_file_paths',
+ nargs='*',
+ help='Path to test files to run, relative to '
+ f'testing/{relative_test_dir}. If omitted, runs all test files under '
+ f'testing/{relative_test_dir}.',
+ metavar='relative/test/path')
+
+ skia_gold.add_skia_gold_args(parser)
+
+ self.per_process_config.options = parser.parse_args()
+
+ if (self.options.regenerate_expected and
+ self.options.regenerate_expected not in ['all', 'platform']):
+ print('FAILURE: --regenerate_expected must be "all" or "platform"')
+ return 1
+
+ finder = self.per_process_config.NewFinder()
+ pdfium_test_path = self.per_process_config.GetPdfiumTestPath(finder)
+ if not os.path.exists(pdfium_test_path):
+ print(f"FAILURE: Can't find test executable '{pdfium_test_path}'")
+ print('Use --build-dir to specify its location.')
+ return 1
+ self.per_process_config.InitializeFeatures(pdfium_test_path)
+
+ self.per_process_state = _PerProcessState(self.per_process_config)
+ shutil.rmtree(self.per_process_state.working_dir, ignore_errors=True)
+ os.makedirs(self.per_process_state.working_dir)
+
+ error_message = self.per_process_state.image_differ.CheckMissingTools(
+ self.options.regenerate_expected)
+ if error_message:
+ print('FAILURE:', error_message)
+ return 1
+
+ self.resultdb = result_sink.TryInitClient()
+ if self.resultdb:
+ print('Detected ResultSink environment')
+
+ # Collect test cases.
+ walk_from_dir = finder.TestingDir(relative_test_dir)
+
+ self.test_cases = TestCaseManager()
+ self.execution_suppressed_cases = []
+ input_file_re = re.compile('^.+[.](in|pdf)$')
+ if self.options.inputted_file_paths:
+ for file_name in self.options.inputted_file_paths:
+ input_path = os.path.join(walk_from_dir, file_name)
+ if not os.path.isfile(input_path):
+ print(f"Can't find test file '{file_name}'")
+ return 1
+
+ self.test_cases.NewTestCase(input_path)
+ else:
+ for file_dir, _, filename_list in os.walk(walk_from_dir):
+ for input_filename in filename_list:
+ if input_file_re.match(input_filename):
+ input_path = os.path.join(file_dir, input_filename)
+ if self.IsExecutionSuppressed(input_path):
+ self.execution_suppressed_cases.append(input_path)
+ continue
+ if not os.path.isfile(input_path):
+ continue
+
+ self.test_cases.NewTestCase(input_path)
+
+ # Execute test cases.
+ self.failures = []
+ self.surprises = []
+ self.skia_gold_successes = []
+ self.skia_gold_unexpected_successes = []
+ self.skia_gold_failures = []
+ self.result_suppressed_cases = []
+
+ if self.IsSkiaGoldEnabled():
+ assert self.options.gold_output_dir
+ # Clear out and create top level gold output directory before starting
+ skia_gold.clear_gold_output_dir(self.options.gold_output_dir)
+
+ with multiprocessing.Pool(
+ processes=self.options.num_workers,
+ initializer=_InitializePerProcessState,
+ initargs=[self.per_process_config]) as pool:
+ skia_gold_test_cases = TestCaseManager()
+ for result in pool.imap(_RunPdfiumTest, self.test_cases):
+ self.HandleResult(self.test_cases.GetTestCase(result.test_id), result)
+
+ if self.IsSkiaGoldEnabled():
+ for artifact in result.image_artifacts:
+ # The output filename without image extension becomes the test ID.
+ # For example, "/path/to/.../testing/corpus/example_005.pdf.0.png"
+ # becomes "example_005.pdf.0".
+ skia_gold_test_cases.NewTestCase(artifact.image_path)
+
+ for result in pool.imap(_RunSkiaTest, skia_gold_test_cases):
+ if result.IsPass():
+ test_case = skia_gold_test_cases.GetTestCase(result.test_id)
+ if self.IsResultSuppressed(test_case.input_path):
+ self.skia_gold_unexpected_successes.append(result.test_id)
+ else:
+ self.skia_gold_successes.append(result.test_id)
+ else:
+ self.skia_gold_failures.append(result.test_id)
+
+ # Report test results.
+ #
+ # For some reason, summary will be cut off from stdout on windows if
+ # _PrintSummary() is called at the end
+ # TODO(crbug.com/pdfium/1657): Once resolved, move _PrintSummary() back
+ # down to the end
+ self._PrintSummary()
+
+ if self.surprises:
+ self.surprises.sort()
+ print('\nUnexpected Successes:')
+ for surprise in self.surprises:
+ print(surprise)
+
+ if self.failures:
+ self.failures.sort()
+ print('\nSummary of Failures:')
+ for failure in self.failures:
+ print(failure)
+
+ if self.skia_gold_unexpected_successes:
+ self.skia_gold_unexpected_successes.sort()
+ print('\nUnexpected Skia Gold Successes:')
+ for surprise in self.skia_gold_unexpected_successes:
+ print(surprise)
+
+ if self.skia_gold_failures:
+ self.skia_gold_failures.sort()
+ print('\nSummary of Skia Gold Failures:')
+ for failure in self.skia_gold_failures:
+ print(failure)
+
+ if self.failures:
+ if not self.options.ignore_errors:
+ return 1
+
+ return 0
+
+ def _PrintSummary(self):
+ number_test_cases = len(self.test_cases)
+ number_failures = len(self.failures)
+ number_suppressed = len(self.result_suppressed_cases)
+ number_successes = number_test_cases - number_failures - number_suppressed
+ number_surprises = len(self.surprises)
+ print('\nTest cases executed:', number_test_cases)
+ print(' Successes:', number_successes)
+ print(' Suppressed:', number_suppressed)
+ print(' Surprises:', number_surprises)
+ print(' Failures:', number_failures)
+ if self.IsSkiaGoldEnabled():
+ number_gold_failures = len(self.skia_gold_failures)
+ number_gold_successes = len(self.skia_gold_successes)
+ number_gold_surprises = len(self.skia_gold_unexpected_successes)
+ number_total_gold_tests = sum(
+ [number_gold_failures, number_gold_successes, number_gold_surprises])
+ print('\nSkia Gold Test cases executed:', number_total_gold_tests)
+ print(' Skia Gold Successes:', number_gold_successes)
+ print(' Skia Gold Surprises:', number_gold_surprises)
+ print(' Skia Gold Failures:', number_gold_failures)
+ skia_tester = self.per_process_state.GetSkiaGoldTester()
+ if self.skia_gold_failures and skia_tester.IsTryjobRun():
+ cl_triage_link = skia_tester.GetCLTriageLink()
+ print(' Triage link for CL:', cl_triage_link)
+ skia_tester.WriteCLTriageLink(cl_triage_link)
+ print()
+ print('Test cases not executed:', len(self.execution_suppressed_cases))
+
+ def SetDeleteOutputOnSuccess(self, new_value):
+ """Set whether to delete generated output if the test passes."""
+ self.per_process_config.delete_output_on_success = new_value
+
+ def SetEnforceExpectedImages(self, new_value):
+ """Set whether to enforce that each test case provide an expected image."""
+ self.per_process_config.enforce_expected_images = new_value
+
+
+def _RunPdfiumTest(test_case):
+ """Runs a PDFium test case."""
+ try:
+ return _per_process_state.GenerateAndTest(test_case)
+ except KeyboardInterrupt as exc:
+ raise KeyboardInterruptError() from exc
+
+
+def _RunSkiaTest(test_case):
+ """Runs a Skia Gold test case."""
+ try:
+ skia_tester = _per_process_state.GetSkiaGoldTester()
+ skia_success = skia_tester.UploadTestResultToSkiaGold(
+ test_case.test_id, test_case.input_path)
+ sys.stdout.flush()
+
+ return test_case.NewResult(
+ result_types.PASS if skia_success else result_types.FAIL)
+ except KeyboardInterrupt as exc:
+ raise KeyboardInterruptError() from exc
+
+
+# `_PerProcessState` singleton. This is initialized when creating the
+# `multiprocessing.Pool()`. `TestRunner.Run()` creates its own separate
+# instance of `_PerProcessState` as well.
+_per_process_state = None
+
+
+def _InitializePerProcessState(config):
+ """Initializes the `_per_process_state` singleton."""
+ global _per_process_state
+ assert not _per_process_state
+ _per_process_state = _PerProcessState(config)
+
+
+@dataclass
+class _PerProcessConfig:
+ """Configuration for initializing `_PerProcessState`.
+
+ Attributes:
+ test_dir: The name of the test directory.
+ test_type: The test type.
+ delete_output_on_success: Whether to delete output on success.
+ enforce_expected_images: Whether to enforce expected images.
+ options: The dictionary of command line options.
+ features: The list of features supported by `pdfium_test`.
+ """
+ test_dir: str
+ test_type: str
+ delete_output_on_success: bool = False
+ enforce_expected_images: bool = False
+ options: dict = None
+ features: list = None
+
+ def NewFinder(self):
+ return common.DirectoryFinder(self.options.build_dir)
+
+ def GetPdfiumTestPath(self, finder):
+ return finder.ExecutablePath('pdfium_test')
+
+ def InitializeFeatures(self, pdfium_test_path):
+ output = subprocess.check_output([pdfium_test_path, '--show-config'])
+ self.features = output.decode('utf-8').strip().split(',')
+
+
+class _PerProcessState:
+ """State defined per process."""
+
+ def __init__(self, config):
+ self.test_dir = config.test_dir
+ self.test_type = config.test_type
+ self.delete_output_on_success = config.delete_output_on_success
+ self.enforce_expected_images = config.enforce_expected_images
+ self.options = config.options
+ self.features = config.features
+
+ finder = config.NewFinder()
+ self.pdfium_test_path = config.GetPdfiumTestPath(finder)
+ self.fixup_path = finder.ScriptPath('fixup_pdf_template.py')
+ self.text_diff_path = finder.ScriptPath('text_diff.py')
+ self.font_dir = os.path.join(finder.TestingDir(), 'resources', 'fonts')
+ self.third_party_font_dir = finder.ThirdPartyFontsDir()
+
+ self.source_dir = finder.TestingDir()
+ self.working_dir = finder.WorkingDir(os.path.join('testing', self.test_dir))
+
+ self.test_suppressor = suppressor.Suppressor(
+ finder, self.features, self.options.disable_javascript,
+ self.options.disable_xfa)
+ self.image_differ = pngdiffer.PNGDiffer(finder, self.features,
+ self.options.reverse_byte_order)
+
+ self.process_name = multiprocessing.current_process().name
+ self.skia_tester = None
+
+ def __getstate__(self):
+ raise RuntimeError('Cannot pickle per-process state')
+
+ def GetSkiaGoldTester(self):
+ """Gets the `SkiaGoldTester` singleton for this worker."""
+ if not self.skia_tester:
+ self.skia_tester = skia_gold.SkiaGoldTester(
+ source_type=self.test_type,
+ skia_gold_args=self.options,
+ process_name=self.process_name)
+ return self.skia_tester
def GenerateAndTest(self, test_case):
"""Generate test input and run pdfium_test."""
@@ -245,368 +642,6 @@
]
return raised_exception, results
- def HandleResult(self, test_case, test_result):
- input_filename = os.path.basename(test_case.input_path)
-
- if self.test_suppressor.IsResultSuppressed(input_filename):
- self.result_suppressed_cases.append(input_filename)
- if test_result.IsPass():
- self.surprises.append(test_case.input_path)
-
- # There isn't an actual status for succeeded-but-ignored, so use the
- # "abort" status to differentiate this from failed-but-ignored.
- #
- # Note that this appears as a preliminary failure in Gerrit.
- result_status = result_types.UNKNOWN
- else:
- # There isn't an actual status for failed-but-ignored, so use the
- # "skip" status to differentiate this from succeeded-but-ignored.
- result_status = result_types.SKIP
- else:
- if test_result.IsPass():
- result_status = result_types.PASS
- else:
- self.failures.append(test_case.input_path)
- result_status = result_types.FAIL
-
- if self.resultdb:
- # TODO(crbug.com/pdfium/1916): Populate more ResultDB fields.
- self.resultdb.Post(
- test_id=test_result.test_id,
- status=result_status,
- duration=None,
- test_log=None,
- test_file=None)
-
- def Run(self):
- # Running a test defines a number of attributes on the fly.
- # pylint: disable=attribute-defined-outside-init
-
- if self.test_dir == 'corpus':
- relative_test_dir = self.test_dir
- else:
- relative_test_dir = os.path.join('resources', self.test_dir)
-
- parser = argparse.ArgumentParser()
-
- parser.add_argument(
- '--build-dir',
- default=os.path.join('out', 'Debug'),
- help='relative path from the base source directory')
-
- parser.add_argument(
- '-j',
- default=multiprocessing.cpu_count(),
- dest='num_workers',
- type=int,
- help='run NUM_WORKERS jobs in parallel')
-
- parser.add_argument(
- '--disable-javascript',
- action="store_true",
- dest="disable_javascript",
- help='Prevents JavaScript from executing in PDF files.')
-
- parser.add_argument(
- '--disable-xfa',
- action="store_true",
- dest="disable_xfa",
- help='Prevents processing XFA forms.')
-
- parser.add_argument(
- '--render-oneshot',
- action="store_true",
- dest="render_oneshot",
- help='Sets whether to use the oneshot renderer.')
-
- parser.add_argument(
- '--run-skia-gold',
- action='store_true',
- default=False,
- help='When flag is on, skia gold tests will be run.')
-
- # TODO: Remove when pdfium recipe stops passing this argument
- parser.add_argument(
- '--gold_properties',
- default='',
- dest="gold_properties",
- help='Key value pairs that are written to the top level '
- 'of the JSON file that is ingested by Gold.')
-
- # TODO: Remove when pdfium recipe stops passing this argument
- parser.add_argument(
- '--gold_ignore_hashes',
- default='',
- dest="gold_ignore_hashes",
- help='Path to a file with MD5 hashes we wish to ignore.')
-
- parser.add_argument(
- '--regenerate_expected',
- default='',
- dest="regenerate_expected",
- help='Regenerates expected images. Valid values are '
- '"all" to regenerate all expected pngs, and '
- '"platform" to regenerate only platform-specific '
- 'expected pngs.')
-
- parser.add_argument(
- '--reverse-byte-order',
- action='store_true',
- dest="reverse_byte_order",
- help='Run image-based tests using --reverse-byte-order.')
-
- parser.add_argument(
- '--ignore_errors',
- action="store_true",
- dest="ignore_errors",
- help='Prevents the return value from being non-zero '
- 'when image comparison fails.')
-
- parser.add_argument(
- 'inputted_file_paths',
- nargs='*',
- help='Path to test files to run, relative to '
- f'testing/{relative_test_dir}. If omitted, runs all test files under '
- f'testing/{relative_test_dir}.',
- metavar='relative/test/path')
-
- skia_gold.add_skia_gold_args(parser)
-
- self.options = parser.parse_args()
-
- if (self.options.regenerate_expected and
- self.options.regenerate_expected not in ['all', 'platform']):
- print('FAILURE: --regenerate_expected must be "all" or "platform"')
- return 1
-
- finder = common.DirectoryFinder(self.options.build_dir)
- self.fixup_path = finder.ScriptPath('fixup_pdf_template.py')
- self.text_diff_path = finder.ScriptPath('text_diff.py')
- self.font_dir = os.path.join(finder.TestingDir(), 'resources', 'fonts')
- self.third_party_font_dir = finder.ThirdPartyFontsDir()
-
- self.source_dir = finder.TestingDir()
-
- self.pdfium_test_path = finder.ExecutablePath('pdfium_test')
- if not os.path.exists(self.pdfium_test_path):
- print("FAILURE: Can't find test executable '{}'".format(
- self.pdfium_test_path))
- print('Use --build-dir to specify its location.')
- return 1
-
- self.working_dir = finder.WorkingDir(os.path.join('testing', self.test_dir))
- shutil.rmtree(self.working_dir, ignore_errors=True)
- os.makedirs(self.working_dir)
-
- self.features = subprocess.check_output(
- [self.pdfium_test_path,
- '--show-config']).decode('utf-8').strip().split(',')
- self.test_suppressor = suppressor.Suppressor(
- finder, self.features, self.options.disable_javascript,
- self.options.disable_xfa)
- self.image_differ = pngdiffer.PNGDiffer(finder, self.features,
- self.options.reverse_byte_order)
- error_message = self.image_differ.CheckMissingTools(
- self.options.regenerate_expected)
- if error_message:
- print('FAILURE:', error_message)
- return 1
-
- self.resultdb = result_sink.TryInitClient()
- if self.resultdb:
- print('Detected ResultSink environment')
-
- # Collect test cases.
- walk_from_dir = finder.TestingDir(relative_test_dir)
-
- self.test_cases = TestCaseManager()
- self.execution_suppressed_cases = []
- input_file_re = re.compile('^.+[.](in|pdf)$')
- if self.options.inputted_file_paths:
- for file_name in self.options.inputted_file_paths:
- input_path = os.path.join(walk_from_dir, file_name)
- if not os.path.isfile(input_path):
- print(f"Can't find test file '{file_name}'")
- return 1
-
- self.test_cases.NewTestCase(input_path)
- else:
- for file_dir, _, filename_list in os.walk(walk_from_dir):
- for input_filename in filename_list:
- if input_file_re.match(input_filename):
- input_path = os.path.join(file_dir, input_filename)
- if self.test_suppressor.IsExecutionSuppressed(input_path):
- self.execution_suppressed_cases.append(input_path)
- continue
- if not os.path.isfile(input_path):
- continue
-
- self.test_cases.NewTestCase(input_path)
-
- # Execute test cases.
- self.failures = []
- self.surprises = []
- self.skia_gold_successes = []
- self.skia_gold_unexpected_successes = []
- self.skia_gold_failures = []
- self.result_suppressed_cases = []
-
- if self.test_type not in TEXT_TESTS and self.options.run_skia_gold:
- assert self.options.gold_output_dir
- # Clear out and create top level gold output directory before starting
- skia_gold.clear_gold_output_dir(self.options.gold_output_dir)
-
- per_process_state_args = [self.test_type, self.options]
- per_process_state = _PerProcessState(*per_process_state_args)
- with multiprocessing.Pool(
- processes=self.options.num_workers,
- initializer=_InitializePerProcessState,
- initargs=per_process_state_args) as pool:
- skia_gold_test_cases = TestCaseManager()
- for result in pool.imap(
- _WrapKeyboardInterrupt(self.GenerateAndTest), self.test_cases):
- self.HandleResult(self.test_cases.GetTestCase(result.test_id), result)
-
- if self.test_type not in TEXT_TESTS and self.options.run_skia_gold:
- for artifact in result.image_artifacts:
- # The output filename without image extension becomes the test ID.
- # For example, "/path/to/.../testing/corpus/example_005.pdf.0.png"
- # becomes "example_005.pdf.0".
- skia_gold_test_cases.NewTestCase(artifact.image_path)
-
- for result in pool.imap(
- _WrapKeyboardInterrupt(_RunSkiaTest), skia_gold_test_cases):
- if result.IsPass():
- test_case = skia_gold_test_cases.GetTestCase(result.test_id)
- if self.test_suppressor.IsResultSuppressed(test_case.input_path):
- self.skia_gold_unexpected_successes.append(result.test_id)
- else:
- self.skia_gold_successes.append(result.test_id)
- else:
- self.skia_gold_failures.append(result.test_id)
-
- # Report test results.
- #
- # For some reason, summary will be cut off from stdout on windows if
- # _PrintSummary() is called at the end
- # TODO(crbug.com/pdfium/1657): Once resolved, move _PrintSummary() back
- # down to the end
- self._PrintSummary(per_process_state)
-
- if self.surprises:
- self.surprises.sort()
- print('\nUnexpected Successes:')
- for surprise in self.surprises:
- print(surprise)
-
- if self.failures:
- self.failures.sort()
- print('\nSummary of Failures:')
- for failure in self.failures:
- print(failure)
-
- if self.skia_gold_unexpected_successes:
- self.skia_gold_unexpected_successes.sort()
- print('\nUnexpected Skia Gold Successes:')
- for surprise in self.skia_gold_unexpected_successes:
- print(surprise)
-
- if self.skia_gold_failures:
- self.skia_gold_failures.sort()
- print('\nSummary of Skia Gold Failures:')
- for failure in self.skia_gold_failures:
- print(failure)
-
- if self.failures:
- if not self.options.ignore_errors:
- return 1
-
- return 0
-
- def _PrintSummary(self, per_process_state):
- number_test_cases = len(self.test_cases)
- number_failures = len(self.failures)
- number_suppressed = len(self.result_suppressed_cases)
- number_successes = number_test_cases - number_failures - number_suppressed
- number_surprises = len(self.surprises)
- print('\nTest cases executed:', number_test_cases)
- print(' Successes:', number_successes)
- print(' Suppressed:', number_suppressed)
- print(' Surprises:', number_surprises)
- print(' Failures:', number_failures)
- if self.test_type not in TEXT_TESTS and self.options.run_skia_gold:
- number_gold_failures = len(self.skia_gold_failures)
- number_gold_successes = len(self.skia_gold_successes)
- number_gold_surprises = len(self.skia_gold_unexpected_successes)
- number_total_gold_tests = sum(
- [number_gold_failures, number_gold_successes, number_gold_surprises])
- print('\nSkia Gold Test cases executed:', number_total_gold_tests)
- print(' Skia Gold Successes:', number_gold_successes)
- print(' Skia Gold Surprises:', number_gold_surprises)
- print(' Skia Gold Failures:', number_gold_failures)
- skia_tester = per_process_state.GetSkiaGoldTester()
- if self.skia_gold_failures and skia_tester.IsTryjobRun():
- cl_triage_link = skia_tester.GetCLTriageLink()
- print(' Triage link for CL:', cl_triage_link)
- skia_tester.WriteCLTriageLink(cl_triage_link)
- print()
- print('Test cases not executed:', len(self.execution_suppressed_cases))
-
- def SetDeleteOutputOnSuccess(self, new_value):
- """Set whether to delete generated output if the test passes."""
- self.delete_output_on_success = new_value
-
- def SetEnforceExpectedImages(self, new_value):
- """Set whether to enforce that each test case provide an expected image."""
- self.enforce_expected_images = new_value
-
-
-def _RunSkiaTest(test_case):
- """Runs a Skia Gold test case."""
- skia_tester = _per_process_state.GetSkiaGoldTester()
- skia_success = skia_tester.UploadTestResultToSkiaGold(test_case.test_id,
- test_case.input_path)
- sys.stdout.flush()
-
- return test_case.NewResult(
- result_types.PASS if skia_success else result_types.FAIL)
-
-
-# `_PerProcessState` singleton. This is initialized when creating the
-# `multiprocessing.Pool()`. `TestRunner.Run()` creates its own separate
-# instance of `_PerProcessState` as well.
-_per_process_state = None
-
-
-def _InitializePerProcessState(test_type, options):
- """Initializes the `_per_process_state` singleton."""
- global _per_process_state
- assert not _per_process_state
- _per_process_state = _PerProcessState(test_type, options)
-
-
-class _PerProcessState:
- """State defined per process."""
-
- def __init__(self, test_type, options):
- self.test_type = test_type
- self.options = options
- self.process_name = multiprocessing.current_process().name
-
- self.skia_tester = None
-
- def __getstate__(self):
- raise RuntimeError('Cannot pickle per-process state')
-
- def GetSkiaGoldTester(self):
- """Gets the `SkiaGoldTester` singleton for this worker."""
- if not self.skia_tester:
- self.skia_tester = skia_gold.SkiaGoldTester(
- source_type=self.test_type,
- skia_gold_args=self.options,
- process_name=self.process_name)
- return self.skia_tester
-
@dataclass
class TestCase:
