blob: c76ce44e7ab47d3609f284a07d112f0934c13ae1 [file] [log] [blame]
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -04001#!/usr/bin/env python
2# Copyright 2017 The PDFium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -04005"""Compares the performance of two versions of the pdfium code."""
6
7import argparse
8import functools
Henrique Nakashima52489ee2018-05-09 21:37:42 +00009import glob
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -040010import json
11import multiprocessing
12import os
13import re
14import shutil
15import subprocess
16import sys
17import tempfile
18
K Moon8de7d57d2019-12-05 19:32:33 +000019# pylint: disable=relative-import
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -040020from common import GetBooleanGnArg
Henrique Nakashima0da39e62017-08-15 14:37:58 -040021from common import PrintErr
22from common import RunCommandPropagateErr
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -040023from githelper import GitHelper
24from safetynet_conclusions import ComparisonConclusions
25from safetynet_conclusions import PrintConclusionsDictHumanReadable
26from safetynet_conclusions import RATING_IMPROVEMENT
27from safetynet_conclusions import RATING_REGRESSION
Henrique Nakashima52489ee2018-05-09 21:37:42 +000028from safetynet_image import ImageComparison
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -040029
30
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -040031def RunSingleTestCaseParallel(this, run_label, build_dir, test_case):
32 result = this.RunSingleTestCase(run_label, build_dir, test_case)
33 return (test_case, result)
34
35
36class CompareRun(object):
37 """A comparison between two branches of pdfium."""
38
39 def __init__(self, args):
40 self.git = GitHelper()
41 self.args = args
42 self._InitPaths()
43
44 def _InitPaths(self):
45 if self.args.this_repo:
Henrique Nakashima0da39e62017-08-15 14:37:58 -040046 self.safe_script_dir = self.args.build_dir
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -040047 else:
Henrique Nakashima0da39e62017-08-15 14:37:58 -040048 self.safe_script_dir = os.path.join('testing', 'tools')
49
50 self.safe_measure_script_path = os.path.abspath(
Lei Zhang30543372019-11-19 19:02:30 +000051 os.path.join(self.safe_script_dir, 'safetynet_measure.py'))
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -040052
53 input_file_re = re.compile('^.+[.]pdf$')
54 self.test_cases = []
55 for input_path in self.args.input_paths:
56 if os.path.isfile(input_path):
57 self.test_cases.append(input_path)
58 elif os.path.isdir(input_path):
59 for file_dir, _, filename_list in os.walk(input_path):
60 for input_filename in filename_list:
61 if input_file_re.match(input_filename):
62 file_path = os.path.join(file_dir, input_filename)
63 if os.path.isfile(file_path):
64 self.test_cases.append(file_path)
65
66 self.after_build_dir = self.args.build_dir
67 if self.args.build_dir_before:
68 self.before_build_dir = self.args.build_dir_before
69 else:
70 self.before_build_dir = self.after_build_dir
71
72 def Run(self):
73 """Runs comparison by checking out branches, building and measuring them.
74
75 Returns:
76 Exit code for the script.
77 """
78 if self.args.this_repo:
79 self._FreezeMeasureScript()
80
81 if self.args.branch_after:
82 if self.args.this_repo:
83 before, after = self._ProfileTwoOtherBranchesInThisRepo(
Lei Zhang30543372019-11-19 19:02:30 +000084 self.args.branch_before, self.args.branch_after)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -040085 else:
Lei Zhang30543372019-11-19 19:02:30 +000086 before, after = self._ProfileTwoOtherBranches(self.args.branch_before,
87 self.args.branch_after)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -040088 elif self.args.branch_before:
89 if self.args.this_repo:
90 before, after = self._ProfileCurrentAndOtherBranchInThisRepo(
91 self.args.branch_before)
92 else:
93 before, after = self._ProfileCurrentAndOtherBranch(
94 self.args.branch_before)
95 else:
96 if self.args.this_repo:
97 before, after = self._ProfileLocalChangesAndCurrentBranchInThisRepo()
98 else:
99 before, after = self._ProfileLocalChangesAndCurrentBranch()
100
101 conclusions = self._DrawConclusions(before, after)
102 conclusions_dict = conclusions.GetOutputDict()
Henrique Nakashimaf76741e2017-09-07 12:16:05 -0400103 conclusions_dict.setdefault('metadata', {})['profiler'] = self.args.profiler
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400104
105 self._PrintConclusions(conclusions_dict)
106
107 self._CleanUp(conclusions)
108
Henrique Nakashima52489ee2018-05-09 21:37:42 +0000109 if self.args.png_dir:
110 image_comparison = ImageComparison(
Lei Zhang30543372019-11-19 19:02:30 +0000111 self.after_build_dir, self.args.png_dir, ('before', 'after'),
112 self.args.num_workers, self.args.png_threshold)
Henrique Nakashima82cb8092018-05-12 04:58:47 +0000113 image_comparison.Run(open_in_browser=not self.args.machine_readable)
Henrique Nakashima52489ee2018-05-09 21:37:42 +0000114
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400115 return 0
116
117 def _FreezeMeasureScript(self):
118 """Freezes a version of the measuring script.
119
120 This is needed to make sure we are comparing the pdfium library changes and
121 not script changes that may happen between the two branches.
122 """
Henrique Nakashima0da39e62017-08-15 14:37:58 -0400123 self.__FreezeFile(os.path.join('testing', 'tools', 'safetynet_measure.py'))
124 self.__FreezeFile(os.path.join('testing', 'tools', 'common.py'))
125
K Moon8de7d57d2019-12-05 19:32:33 +0000126 def __FreezeFile(self, filename):
127 RunCommandPropagateErr(['cp', filename, self.safe_script_dir],
Henrique Nakashima0da39e62017-08-15 14:37:58 -0400128 exit_status_on_error=1)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400129
130 def _ProfileTwoOtherBranchesInThisRepo(self, before_branch, after_branch):
131 """Profiles two branches that are not the current branch.
132
133 This is done in the local repository and changes may not be restored if the
134 script fails or is interrupted.
135
136 after_branch does not need to descend from before_branch, they will be
137 measured the same way
138
139 Args:
140 before_branch: One branch to profile.
141 after_branch: Other branch to profile.
142
143 Returns:
144 A tuple (before, after), where each of before and after is a dict
Lei Zhang72d34be2018-01-12 05:27:49 +0000145 mapping a test case name to the profiling values for that test case
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400146 in the given branch.
147 """
148 branch_to_restore = self.git.GetCurrentBranchName()
149
150 self._StashLocalChanges()
151
152 self._CheckoutBranch(after_branch)
Henrique Nakashima26fbb032018-09-07 16:48:32 +0000153 self._BuildCurrentBranch(self.after_build_dir)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400154 after = self._MeasureCurrentBranch('after', self.after_build_dir)
155
156 self._CheckoutBranch(before_branch)
Henrique Nakashima26fbb032018-09-07 16:48:32 +0000157 self._BuildCurrentBranch(self.before_build_dir)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400158 before = self._MeasureCurrentBranch('before', self.before_build_dir)
159
160 self._CheckoutBranch(branch_to_restore)
161 self._RestoreLocalChanges()
162
163 return before, after
164
165 def _ProfileTwoOtherBranches(self, before_branch, after_branch):
166 """Profiles two branches that are not the current branch.
167
168 This is done in new, cloned repositories, therefore it is safer but slower
169 and requires downloads.
170
171 after_branch does not need to descend from before_branch, they will be
172 measured the same way
173
174 Args:
175 before_branch: One branch to profile.
176 after_branch: Other branch to profile.
177
178 Returns:
179 A tuple (before, after), where each of before and after is a dict
Lei Zhang72d34be2018-01-12 05:27:49 +0000180 mapping a test case name to the profiling values for that test case
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400181 in the given branch.
182 """
Lei Zhang30543372019-11-19 19:02:30 +0000183 after = self._ProfileSeparateRepo('after', self.after_build_dir,
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400184 after_branch)
Lei Zhang30543372019-11-19 19:02:30 +0000185 before = self._ProfileSeparateRepo('before', self.before_build_dir,
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400186 before_branch)
187 return before, after
188
189 def _ProfileCurrentAndOtherBranchInThisRepo(self, other_branch):
190 """Profiles the current branch (with uncommitted changes) and another one.
191
192 This is done in the local repository and changes may not be restored if the
193 script fails or is interrupted.
194
195 The current branch does not need to descend from other_branch.
196
197 Args:
198 other_branch: Other branch to profile that is not the current.
199
200 Returns:
201 A tuple (before, after), where each of before and after is a dict
Lei Zhang72d34be2018-01-12 05:27:49 +0000202 mapping a test case name to the profiling values for that test case
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400203 in the given branch. The current branch is considered to be "after" and
204 the other branch is considered to be "before".
205 """
206 branch_to_restore = self.git.GetCurrentBranchName()
207
Henrique Nakashima26fbb032018-09-07 16:48:32 +0000208 self._BuildCurrentBranch(self.after_build_dir)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400209 after = self._MeasureCurrentBranch('after', self.after_build_dir)
210
211 self._StashLocalChanges()
212
213 self._CheckoutBranch(other_branch)
Henrique Nakashima26fbb032018-09-07 16:48:32 +0000214 self._BuildCurrentBranch(self.before_build_dir)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400215 before = self._MeasureCurrentBranch('before', self.before_build_dir)
216
217 self._CheckoutBranch(branch_to_restore)
218 self._RestoreLocalChanges()
219
220 return before, after
221
222 def _ProfileCurrentAndOtherBranch(self, other_branch):
223 """Profiles the current branch (with uncommitted changes) and another one.
224
225 This is done in new, cloned repositories, therefore it is safer but slower
226 and requires downloads.
227
228 The current branch does not need to descend from other_branch.
229
230 Args:
231 other_branch: Other branch to profile that is not the current. None will
232 compare to the same branch.
233
234 Returns:
235 A tuple (before, after), where each of before and after is a dict
Lei Zhang72d34be2018-01-12 05:27:49 +0000236 mapping a test case name to the profiling values for that test case
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400237 in the given branch. The current branch is considered to be "after" and
238 the other branch is considered to be "before".
239 """
Henrique Nakashima26fbb032018-09-07 16:48:32 +0000240 self._BuildCurrentBranch(self.after_build_dir)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400241 after = self._MeasureCurrentBranch('after', self.after_build_dir)
242
Lei Zhang30543372019-11-19 19:02:30 +0000243 before = self._ProfileSeparateRepo('before', self.before_build_dir,
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400244 other_branch)
245
246 return before, after
247
248 def _ProfileLocalChangesAndCurrentBranchInThisRepo(self):
249 """Profiles the current branch with and without uncommitted changes.
250
251 This is done in the local repository and changes may not be restored if the
252 script fails or is interrupted.
253
254 Returns:
255 A tuple (before, after), where each of before and after is a dict
Lei Zhang72d34be2018-01-12 05:27:49 +0000256 mapping a test case name to the profiling values for that test case
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400257 using the given version. The current branch without uncommitted changes is
258 considered to be "before" and with uncommitted changes is considered to be
259 "after".
260 """
Henrique Nakashima26fbb032018-09-07 16:48:32 +0000261 self._BuildCurrentBranch(self.after_build_dir)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400262 after = self._MeasureCurrentBranch('after', self.after_build_dir)
263
264 pushed = self._StashLocalChanges()
265 if not pushed and not self.args.build_dir_before:
266 PrintErr('Warning: No local changes to compare')
267
268 before_build_dir = self.before_build_dir
269
Henrique Nakashima26fbb032018-09-07 16:48:32 +0000270 self._BuildCurrentBranch(before_build_dir)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400271 before = self._MeasureCurrentBranch('before', before_build_dir)
272
273 self._RestoreLocalChanges()
274
275 return before, after
276
277 def _ProfileLocalChangesAndCurrentBranch(self):
278 """Profiles the current branch with and without uncommitted changes.
279
280 This is done in new, cloned repositories, therefore it is safer but slower
281 and requires downloads.
282
283 Returns:
284 A tuple (before, after), where each of before and after is a dict
Lei Zhang72d34be2018-01-12 05:27:49 +0000285 mapping a test case name to the profiling values for that test case
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400286 using the given version. The current branch without uncommitted changes is
287 considered to be "before" and with uncommitted changes is considered to be
288 "after".
289 """
290 return self._ProfileCurrentAndOtherBranch(other_branch=None)
291
292 def _ProfileSeparateRepo(self, run_label, relative_build_dir, branch):
293 """Profiles a branch in a a temporary git repository.
294
295 Args:
296 run_label: String to differentiate this version of the code in output
297 files from other versions.
298 relative_build_dir: Path to the build dir in the current working dir to
299 clone build args from.
300 branch: Branch to checkout in the new repository. None will
301 profile the same branch checked out in the original repo.
302 Returns:
Lei Zhang72d34be2018-01-12 05:27:49 +0000303 A dict mapping each test case name to the profiling values for that
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400304 test case.
305 """
Lei Zhang30543372019-11-19 19:02:30 +0000306 build_dir = self._CreateTempRepo('repo_%s' % run_label, relative_build_dir,
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400307 branch)
308
Henrique Nakashima26fbb032018-09-07 16:48:32 +0000309 self._BuildCurrentBranch(build_dir)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400310 return self._MeasureCurrentBranch(run_label, build_dir)
311
312 def _CreateTempRepo(self, dir_name, relative_build_dir, branch):
313 """Clones a temporary git repository out of the current working dir.
314
315 Args:
316 dir_name: Name for the temporary repository directory
317 relative_build_dir: Path to the build dir in the current working dir to
318 clone build args from.
319 branch: Branch to checkout in the new repository. None will keep checked
320 out the same branch as the local repo.
321 Returns:
322 Path to the build directory of the new repository.
323 """
324 cwd = os.getcwd()
325
326 repo_dir = tempfile.mkdtemp(suffix='-%s' % dir_name)
327 src_dir = os.path.join(repo_dir, 'pdfium')
328
329 self.git.CloneLocal(os.getcwd(), src_dir)
330
331 if branch is not None:
332 os.chdir(src_dir)
333 self.git.Checkout(branch)
334
335 os.chdir(repo_dir)
336 PrintErr('Syncing...')
337
Lei Zhang30543372019-11-19 19:02:30 +0000338 cmd = [
339 'gclient', 'config', '--unmanaged',
340 'https://pdfium.googlesource.com/pdfium.git'
341 ]
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400342 if self.args.cache_dir:
343 cmd.append('--cache-dir=%s' % self.args.cache_dir)
Henrique Nakashima0da39e62017-08-15 14:37:58 -0400344 RunCommandPropagateErr(cmd, exit_status_on_error=1)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400345
Henrique Nakashima5fea8ca2018-09-07 17:13:51 +0000346 RunCommandPropagateErr(['gclient', 'sync', '--force'],
347 exit_status_on_error=1)
Henrique Nakashima0da39e62017-08-15 14:37:58 -0400348
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400349 PrintErr('Done.')
350
351 build_dir = os.path.join(src_dir, relative_build_dir)
352 os.makedirs(build_dir)
353 os.chdir(src_dir)
354
355 source_gn_args = os.path.join(cwd, relative_build_dir, 'args.gn')
356 dest_gn_args = os.path.join(build_dir, 'args.gn')
357 shutil.copy(source_gn_args, dest_gn_args)
358
Henrique Nakashima0da39e62017-08-15 14:37:58 -0400359 RunCommandPropagateErr(['gn', 'gen', relative_build_dir],
360 exit_status_on_error=1)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400361
362 os.chdir(cwd)
363
364 return build_dir
365
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400366 def _CheckoutBranch(self, branch):
367 PrintErr("Checking out branch '%s'" % branch)
368 self.git.Checkout(branch)
369
370 def _StashLocalChanges(self):
371 PrintErr('Stashing local changes')
372 return self.git.StashPush()
373
374 def _RestoreLocalChanges(self):
375 PrintErr('Restoring local changes')
376 self.git.StashPopAll()
377
Henrique Nakashima26fbb032018-09-07 16:48:32 +0000378 def _BuildCurrentBranch(self, build_dir):
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400379 """Synchronizes and builds the current version of pdfium.
380
381 Args:
382 build_dir: String with path to build directory
383 """
384 PrintErr('Syncing...')
Henrique Nakashima5fea8ca2018-09-07 17:13:51 +0000385 RunCommandPropagateErr(['gclient', 'sync', '--force'],
386 exit_status_on_error=1)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400387 PrintErr('Done.')
388
Henrique Nakashima0da39e62017-08-15 14:37:58 -0400389 PrintErr('Building...')
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400390 cmd = ['ninja', '-C', build_dir, 'pdfium_test']
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400391 if GetBooleanGnArg('use_goma', build_dir):
392 cmd.extend(['-j', '250'])
Henrique Nakashima0da39e62017-08-15 14:37:58 -0400393 RunCommandPropagateErr(cmd, stdout_has_errors=True, exit_status_on_error=1)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400394 PrintErr('Done.')
395
396 def _MeasureCurrentBranch(self, run_label, build_dir):
397 PrintErr('Measuring...')
398 if self.args.num_workers > 1 and len(self.test_cases) > 1:
399 results = self._RunAsync(run_label, build_dir)
400 else:
401 results = self._RunSync(run_label, build_dir)
402 PrintErr('Done.')
403
404 return results
405
406 def _RunSync(self, run_label, build_dir):
407 """Profiles the test cases synchronously.
408
409 Args:
410 run_label: String to differentiate this version of the code in output
411 files from other versions.
412 build_dir: String with path to build directory
413
414 Returns:
Lei Zhang72d34be2018-01-12 05:27:49 +0000415 A dict mapping each test case name to the profiling values for that
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400416 test case.
417 """
418 results = {}
419
420 for test_case in self.test_cases:
421 result = self.RunSingleTestCase(run_label, build_dir, test_case)
422 if result is not None:
423 results[test_case] = result
424
425 return results
426
427 def _RunAsync(self, run_label, build_dir):
428 """Profiles the test cases asynchronously.
429
430 Uses as many workers as configured by --num-workers.
431
432 Args:
433 run_label: String to differentiate this version of the code in output
434 files from other versions.
435 build_dir: String with path to build directory
436
437 Returns:
Lei Zhang72d34be2018-01-12 05:27:49 +0000438 A dict mapping each test case name to the profiling values for that
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400439 test case.
440 """
441 results = {}
442 pool = multiprocessing.Pool(self.args.num_workers)
Lei Zhang30543372019-11-19 19:02:30 +0000443 worker_func = functools.partial(RunSingleTestCaseParallel, self, run_label,
444 build_dir)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400445
446 try:
447 # The timeout is a workaround for http://bugs.python.org/issue8296
448 # which prevents KeyboardInterrupt from working.
449 one_year_in_seconds = 3600 * 24 * 365
Lei Zhang30543372019-11-19 19:02:30 +0000450 worker_results = (
451 pool.map_async(worker_func, self.test_cases).get(one_year_in_seconds))
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400452 for worker_result in worker_results:
453 test_case, result = worker_result
454 if result is not None:
455 results[test_case] = result
456 except KeyboardInterrupt:
457 pool.terminate()
458 sys.exit(1)
459 else:
460 pool.close()
461
462 pool.join()
463
464 return results
465
466 def RunSingleTestCase(self, run_label, build_dir, test_case):
467 """Profiles a single test case.
468
469 Args:
470 run_label: String to differentiate this version of the code in output
471 files from other versions.
472 build_dir: String with path to build directory
473 test_case: Path to the test case.
474
475 Returns:
476 The measured profiling value for that test case.
477 """
Lei Zhang30543372019-11-19 19:02:30 +0000478 command = [
479 self.safe_measure_script_path, test_case,
480 '--build-dir=%s' % build_dir
481 ]
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400482
483 if self.args.interesting_section:
484 command.append('--interesting-section')
485
486 if self.args.profiler:
487 command.append('--profiler=%s' % self.args.profiler)
488
489 profile_file_path = self._GetProfileFilePath(run_label, test_case)
490 if profile_file_path:
491 command.append('--output-path=%s' % profile_file_path)
492
Henrique Nakashima52489ee2018-05-09 21:37:42 +0000493 if self.args.png_dir:
494 command.append('--png')
495
496 if self.args.pages:
497 command.extend(['--pages', self.args.pages])
498
Henrique Nakashima0da39e62017-08-15 14:37:58 -0400499 output = RunCommandPropagateErr(command)
500
501 if output is None:
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400502 return None
503
Henrique Nakashima52489ee2018-05-09 21:37:42 +0000504 if self.args.png_dir:
505 self._MoveImages(test_case, run_label)
506
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400507 # Get the time number as output, making sure it's just a number
508 output = output.strip()
509 if re.match('^[0-9]+$', output):
510 return int(output)
511
512 return None
513
Henrique Nakashima52489ee2018-05-09 21:37:42 +0000514 def _MoveImages(self, test_case, run_label):
515 png_dir = os.path.join(self.args.png_dir, run_label)
516 if not os.path.exists(png_dir):
517 os.makedirs(png_dir)
518
519 test_case_dir, test_case_filename = os.path.split(test_case)
520 test_case_png_matcher = '%s.*.png' % test_case_filename
Lei Zhang30543372019-11-19 19:02:30 +0000521 for output_png in glob.glob(
522 os.path.join(test_case_dir, test_case_png_matcher)):
Henrique Nakashima52489ee2018-05-09 21:37:42 +0000523 shutil.move(output_png, png_dir)
524
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400525 def _GetProfileFilePath(self, run_label, test_case):
526 if self.args.output_dir:
Lei Zhang30543372019-11-19 19:02:30 +0000527 output_filename = (
528 'callgrind.out.%s.%s' % (test_case.replace('/', '_'), run_label))
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400529 return os.path.join(self.args.output_dir, output_filename)
530 else:
531 return None
532
533 def _DrawConclusions(self, times_before_branch, times_after_branch):
534 """Draws conclusions comparing results of test runs in two branches.
535
536 Args:
Lei Zhang72d34be2018-01-12 05:27:49 +0000537 times_before_branch: A dict mapping each test case name to the
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400538 profiling values for that test case in the branch to be considered
539 as the baseline.
Lei Zhang72d34be2018-01-12 05:27:49 +0000540 times_after_branch: A dict mapping each test case name to the
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400541 profiling values for that test case in the branch to be considered
542 as the new version.
543
544 Returns:
545 ComparisonConclusions with all test cases processed.
546 """
547 conclusions = ComparisonConclusions(self.args.threshold_significant)
548
549 for test_case in sorted(self.test_cases):
550 before = times_before_branch.get(test_case)
551 after = times_after_branch.get(test_case)
552 conclusions.ProcessCase(test_case, before, after)
553
554 return conclusions
555
556 def _PrintConclusions(self, conclusions_dict):
557 """Prints the conclusions as the script output.
558
559 Depending on the script args, this can output a human or a machine-readable
560 version of the conclusions.
561
562 Args:
563 conclusions_dict: Dict to print returned from
564 ComparisonConclusions.GetOutputDict().
565 """
566 if self.args.machine_readable:
Henrique Nakashima0da39e62017-08-15 14:37:58 -0400567 print json.dumps(conclusions_dict)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400568 else:
569 PrintConclusionsDictHumanReadable(
570 conclusions_dict, colored=True, key=self.args.case_order)
571
572 def _CleanUp(self, conclusions):
573 """Removes profile output files for uninteresting cases.
574
575 Cases without significant regressions or improvements and considered
576 uninteresting.
577
578 Args:
579 conclusions: A ComparisonConclusions.
580 """
581 if not self.args.output_dir:
582 return
583
584 if self.args.profiler != 'callgrind':
585 return
586
587 for case_result in conclusions.GetCaseResults().values():
588 if case_result.rating not in [RATING_REGRESSION, RATING_IMPROVEMENT]:
589 self._CleanUpOutputFile('before', case_result.case_name)
590 self._CleanUpOutputFile('after', case_result.case_name)
591
592 def _CleanUpOutputFile(self, run_label, case_name):
593 """Removes one profile output file.
594
595 If the output file does not exist, fails silently.
596
597 Args:
598 run_label: String to differentiate a version of the code in output
599 files from other versions.
600 case_name: String identifying test case for which to remove the output
601 file.
602 """
603 try:
604 os.remove(self._GetProfileFilePath(run_label, case_name))
605 except OSError:
606 pass
607
608
609def main():
610 parser = argparse.ArgumentParser()
Lei Zhang30543372019-11-19 19:02:30 +0000611 parser.add_argument(
612 'input_paths',
613 nargs='+',
614 help='pdf files or directories to search for pdf files '
615 'to run as test cases')
616 parser.add_argument(
617 '--branch-before',
618 help='git branch to use as "before" for comparison. '
619 'Omitting this will use the current branch '
620 'without uncommitted changes as the baseline.')
621 parser.add_argument(
622 '--branch-after',
623 help='git branch to use as "after" for comparison. '
624 'Omitting this will use the current branch '
625 'with uncommitted changes.')
626 parser.add_argument(
627 '--build-dir',
628 default=os.path.join('out', 'Release'),
629 help='relative path from the base source directory '
630 'to the build directory')
631 parser.add_argument(
632 '--build-dir-before',
633 help='relative path from the base source directory '
634 'to the build directory for the "before" branch, if '
635 'different from the build directory for the '
636 '"after" branch')
637 parser.add_argument(
638 '--cache-dir',
639 default=None,
640 help='directory with a new or preexisting cache for '
641 'downloads. Default is to not use a cache.')
642 parser.add_argument(
643 '--this-repo',
644 action='store_true',
645 help='use the repository where the script is instead of '
646 'checking out a temporary one. This is faster and '
647 'does not require downloads, but although it '
648 'restores the state of the local repo, if the '
649 'script is killed or crashes the changes can remain '
650 'stashed and you may be on another branch.')
651 parser.add_argument(
652 '--profiler',
653 default='callgrind',
654 help='which profiler to use. Supports callgrind, '
655 'perfstat, and none. Default is callgrind.')
656 parser.add_argument(
657 '--interesting-section',
658 action='store_true',
659 help='whether to measure just the interesting section or '
660 'the whole test harness. Limiting to only the '
661 'interesting section does not work on Release since '
662 'the delimiters are optimized out')
663 parser.add_argument(
664 '--pages',
665 help='selects some pages to be rendered. Page numbers '
666 'are 0-based. "--pages A" will render only page A. '
667 '"--pages A-B" will render pages A to B '
668 '(inclusive).')
669 parser.add_argument(
670 '--num-workers',
671 default=multiprocessing.cpu_count(),
672 type=int,
673 help='run NUM_WORKERS jobs in parallel')
674 parser.add_argument(
675 '--output-dir', help='directory to write the profile data output files')
676 parser.add_argument(
677 '--png-dir',
678 default=None,
679 help='outputs pngs to the specified directory that can '
680 'be compared with a static html generated. Will '
681 'affect performance measurements.')
682 parser.add_argument(
683 '--png-threshold',
684 default=0.0,
685 type=float,
686 help='Requires --png-dir. Threshold above which a png '
687 'is considered to have changed.')
688 parser.add_argument(
689 '--threshold-significant',
690 default=0.02,
691 type=float,
692 help='variations in performance above this factor are '
693 'considered significant')
694 parser.add_argument(
695 '--machine-readable',
696 action='store_true',
697 help='whether to get output for machines. If enabled the '
698 'output will be a json with the format specified in '
699 'ComparisonConclusions.GetOutputDict(). Default is '
700 'human-readable.')
701 parser.add_argument(
702 '--case-order',
703 default=None,
704 help='what key to use when sorting test cases in the '
705 'output. Accepted values are "after", "before", '
706 '"ratio" and "rating". Default is sorting by test '
707 'case path.')
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400708
709 args = parser.parse_args()
710
711 # Always start at the pdfium src dir, which is assumed to be two level above
712 # this script.
713 pdfium_src_dir = os.path.join(
Lei Zhang30543372019-11-19 19:02:30 +0000714 os.path.dirname(__file__), os.path.pardir, os.path.pardir)
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400715 os.chdir(pdfium_src_dir)
716
717 git = GitHelper()
718
719 if args.branch_after and not args.branch_before:
720 PrintErr('--branch-after requires --branch-before to be specified.')
721 return 1
722
723 if args.branch_after and not git.BranchExists(args.branch_after):
724 PrintErr('Branch "%s" does not exist' % args.branch_after)
725 return 1
726
727 if args.branch_before and not git.BranchExists(args.branch_before):
728 PrintErr('Branch "%s" does not exist' % args.branch_before)
729 return 1
730
731 if args.output_dir:
732 args.output_dir = os.path.expanduser(args.output_dir)
733 if not os.path.isdir(args.output_dir):
734 PrintErr('"%s" is not a directory' % args.output_dir)
735 return 1
736
Henrique Nakashima52489ee2018-05-09 21:37:42 +0000737 if args.png_dir:
738 args.png_dir = os.path.expanduser(args.png_dir)
739 if not os.path.isdir(args.png_dir):
740 PrintErr('"%s" is not a directory' % args.png_dir)
741 return 1
742
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400743 if args.threshold_significant <= 0.0:
744 PrintErr('--threshold-significant should receive a positive float')
745 return 1
746
Henrique Nakashima52489ee2018-05-09 21:37:42 +0000747 if args.png_threshold:
748 if not args.png_dir:
749 PrintErr('--png-threshold requires --png-dir to be specified.')
750 return 1
751
752 if args.png_threshold <= 0.0:
753 PrintErr('--png-threshold should receive a positive float')
754 return 1
755
756 if args.pages:
757 if not re.match(r'^\d+(-\d+)?$', args.pages):
758 PrintErr('Supported formats for --pages are "--pages 7" and '
759 '"--pages 3-6"')
760 return 1
761
Henrique Nakashimaf24fc1e2017-08-03 13:29:22 -0400762 run = CompareRun(args)
763 return run.Run()
764
765
766if __name__ == '__main__':
767 sys.exit(main())