| #!/usr/bin/env python | |
| # Copyright 2017 The PDFium Authors. All rights reserved. | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| """Looks for performance regressions on all pushes since the last run. | |
| Run this nightly to have a periodical check for performance regressions. | |
| Stores the results for each run and last checkpoint in a results directory. | |
| """ | |
| import argparse | |
| import datetime | |
| import json | |
| import os | |
| import sys | |
| from common import PrintWithTime | |
| from common import RunCommandPropagateErr | |
| from githelper import GitHelper | |
| from safetynet_conclusions import PrintConclusionsDictHumanReadable | |
| class JobContext(object): | |
| """Context for a single run, including name and directory paths.""" | |
| def __init__(self, args): | |
| self.datetime = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') | |
| self.results_dir = args.results_dir | |
| self.last_revision_covered_file = os.path.join(self.results_dir, | |
| 'last_revision_covered') | |
| self.run_output_dir = os.path.join(self.results_dir, | |
| 'profiles_%s' % self.datetime) | |
| self.run_output_log_file = os.path.join(self.results_dir, | |
| '%s.log' % self.datetime) | |
| class JobRun(object): | |
| """A single run looking for regressions since the last one.""" | |
| def __init__(self, args, context): | |
| """Constructor. | |
| Args: | |
| args: Namespace with arguments passed to the script. | |
| context: JobContext for this run. | |
| """ | |
| self.git = GitHelper() | |
| self.args = args | |
| self.context = context | |
| def Run(self): | |
| """Searches for regressions. | |
| Will only write a checkpoint when first run, and on all subsequent runs | |
| a comparison is done against the last checkpoint. | |
| Returns: | |
| Exit code for the script: 0 if no significant changes are found; 1 if | |
| there was an error in the comparison; 3 if there was a regression; 4 if | |
| there was an improvement and no regression. | |
| """ | |
| pdfium_src_dir = os.path.join( | |
| os.path.dirname(__file__), | |
| os.path.pardir, | |
| os.path.pardir) | |
| os.chdir(pdfium_src_dir) | |
| if not self.git.IsCurrentBranchClean() and not self.args.no_checkout: | |
| PrintWithTime('Current branch is not clean, aborting') | |
| return 1 | |
| branch_to_restore = self.git.GetCurrentBranchName() | |
| if not self.args.no_checkout: | |
| self.git.FetchOriginMaster() | |
| self.git.Checkout('origin/master') | |
| # Make sure results dir exists | |
| if not os.path.exists(self.context.results_dir): | |
| os.makedirs(self.context.results_dir) | |
| if not os.path.exists(self.context.last_revision_covered_file): | |
| result = self._InitialRun() | |
| else: | |
| with open(self.context.last_revision_covered_file) as f: | |
| last_revision_covered = f.read().strip() | |
| result = self._IncrementalRun(last_revision_covered) | |
| self.git.Checkout(branch_to_restore) | |
| return result | |
| def _InitialRun(self): | |
| """Initial run, just write a checkpoint. | |
| Returns: | |
| Exit code for the script. | |
| """ | |
| current = self.git.GetCurrentBranchHash() | |
| PrintWithTime('Initial run, current is %s' % current) | |
| self._WriteCheckpoint(current) | |
| PrintWithTime('All set up, next runs will be incremental and perform ' | |
| 'comparisons') | |
| return 0 | |
| def _IncrementalRun(self, last_revision_covered): | |
| """Incremental run, compare against last checkpoint and update it. | |
| Args: | |
| last_revision_covered: String with hash for last checkpoint. | |
| Returns: | |
| Exit code for the script. | |
| """ | |
| current = self.git.GetCurrentBranchHash() | |
| PrintWithTime('Incremental run, current is %s, last is %s' | |
| % (current, last_revision_covered)) | |
| if not os.path.exists(self.context.run_output_dir): | |
| os.makedirs(self.context.run_output_dir) | |
| if current == last_revision_covered: | |
| PrintWithTime('No changes seen, finishing job') | |
| output_info = { | |
| 'metadata': self._BuildRunMetadata(last_revision_covered, | |
| current, | |
| False)} | |
| self._WriteRawJson(output_info) | |
| return 0 | |
| # Run compare | |
| cmd = ['testing/tools/safetynet_compare.py', | |
| '--this-repo', | |
| '--machine-readable', | |
| '--branch-before=%s' % last_revision_covered, | |
| '--output-dir=%s' % self.context.run_output_dir] | |
| cmd.extend(self.args.input_paths) | |
| json_output = RunCommandPropagateErr(cmd) | |
| if json_output is None: | |
| return 1 | |
| output_info = json.loads(json_output) | |
| run_metadata = self._BuildRunMetadata(last_revision_covered, | |
| current, | |
| True) | |
| output_info.setdefault('metadata', {}).update(run_metadata) | |
| self._WriteRawJson(output_info) | |
| PrintConclusionsDictHumanReadable(output_info, | |
| colored=(not self.args.output_to_log | |
| and not self.args.no_color), | |
| key='after') | |
| status = 0 | |
| if output_info['summary']['improvement']: | |
| PrintWithTime('Improvement detected.') | |
| status = 4 | |
| if output_info['summary']['regression']: | |
| PrintWithTime('Regression detected.') | |
| status = 3 | |
| if status == 0: | |
| PrintWithTime('Nothing detected.') | |
| self._WriteCheckpoint(current) | |
| return status | |
| def _WriteRawJson(self, output_info): | |
| json_output_file = os.path.join(self.context.run_output_dir, 'raw.json') | |
| with open(json_output_file, 'w') as f: | |
| json.dump(output_info, f) | |
| def _BuildRunMetadata(self, revision_before, revision_after, | |
| comparison_performed): | |
| return { | |
| 'datetime': self.context.datetime, | |
| 'revision_before': revision_before, | |
| 'revision_after': revision_after, | |
| 'comparison_performed': comparison_performed, | |
| } | |
| def _WriteCheckpoint(self, checkpoint): | |
| if not self.args.no_checkpoint: | |
| with open(self.context.last_revision_covered_file, 'w') as f: | |
| f.write(checkpoint + '\n') | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('results_dir', | |
| help='where to write the job results') | |
| parser.add_argument('input_paths', nargs='+', | |
| help='pdf files or directories to search for pdf files ' | |
| 'to run as test cases') | |
| parser.add_argument('--no-checkout', action='store_true', | |
| help='whether to skip checking out origin/master. Use ' | |
| 'for script debugging.') | |
| parser.add_argument('--no-checkpoint', action='store_true', | |
| help='whether to skip writing the new checkpoint. Use ' | |
| 'for script debugging.') | |
| parser.add_argument('--no-color', action='store_true', | |
| help='whether to write output without color escape ' | |
| 'codes.') | |
| parser.add_argument('--output-to-log', action='store_true', | |
| help='whether to write output to a log file') | |
| args = parser.parse_args() | |
| job_context = JobContext(args) | |
| if args.output_to_log: | |
| log_file = open(job_context.run_output_log_file, 'w') | |
| sys.stdout = log_file | |
| sys.stderr = log_file | |
| run = JobRun(args, job_context) | |
| result = run.Run() | |
| if args.output_to_log: | |
| log_file.close() | |
| return result | |
| if __name__ == '__main__': | |
| sys.exit(main()) | |