| #!/usr/bin/env python3 | |
| # Copyright 2026 The PDFium Authors | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| """Finds performance step functions (regressions or improvements) in CI.""" | |
| import argparse | |
| import collections | |
| import concurrent.futures | |
| import datetime | |
| import json | |
| import os | |
| import statistics | |
| import subprocess | |
| import sys | |
| import tempfile | |
| def ParseISO8601(date_str): | |
| """Parses an ISO-8601 date or date-time string into a UTC datetime.""" | |
| norm_str = date_str[:-1] + '+00:00' if date_str.endswith('Z') else date_str | |
| dt = datetime.datetime.fromisoformat(norm_str) | |
| if dt.tzinfo is None: | |
| dt = dt.replace(tzinfo=datetime.timezone.utc) | |
| else: | |
| dt = dt.astimezone(datetime.timezone.utc) | |
| return dt | |
| def GetCIBuilders(): | |
| """Returns a list of all CI builders for the pdfium project.""" | |
| cmd = ['bb', 'builders', 'pdfium/ci'] | |
| proc = subprocess.run(cmd, capture_output=True, text=True, check=False) | |
| if proc.returncode != 0: | |
| err = proc.stderr.strip() | |
| if err: | |
| print(f'Warning: "bb builders" failed: {err}', file=sys.stderr) | |
| return [] | |
| builders = [] | |
| for line in proc.stdout.splitlines(): | |
| line = line.strip() | |
| if line.startswith('pdfium/ci/'): | |
| builders.append(line.replace('pdfium/ci/', '')) | |
| return sorted(builders) | |
| def GetBuildsForBuilder(builder, since_dt, until_dt): | |
| """Fetches successful builds for `builder` within the date range.""" | |
| predicate = json.dumps({ | |
| 'builder': { | |
| 'project': 'pdfium', | |
| 'bucket': 'ci', | |
| 'builder': builder, | |
| }, | |
| 'status': 'SUCCESS', | |
| 'createTime': { | |
| 'startTime': since_dt.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), | |
| 'endTime': until_dt.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), | |
| }, | |
| }) | |
| cmd = ['bb', 'ls', '-predicate', predicate, '-json'] | |
| builds = [] | |
| proc = subprocess.Popen( | |
| cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) | |
| with proc: | |
| for line in proc.stdout: | |
| line = line.strip() | |
| if not line: | |
| continue | |
| try: | |
| d = json.loads(line) | |
| create_time = d.get('createTime') or d.get('startTime') | |
| if not create_time: | |
| continue | |
| ct = ParseISO8601(create_time) | |
| if ct > until_dt or ct < since_dt: | |
| continue | |
| build_id = d.get('id') | |
| build_num = d.get('number') | |
| commit = (d.get('input') or {}).get('gitilesCommit', {}).get('id', '') | |
| if not commit: | |
| for t in d.get('tags') or []: | |
| val = t.get('value') or '' | |
| if t.get('key') == 'buildset' and 'commit/gitiles' in val: | |
| commit = val.split('/')[-1] | |
| break | |
| if build_id: | |
| builds.append({ | |
| 'id': build_id, | |
| 'number': build_num, | |
| 'commit': commit, | |
| 'create_time': create_time, | |
| }) | |
| except (json.JSONDecodeError, ValueError): | |
| pass | |
| proc.wait() | |
| if proc.returncode != 0: | |
| err = proc.stderr.read().strip() | |
| if err: | |
| print(f'Warning: "bb ls" failed for {builder}: {err}', file=sys.stderr) | |
| # Buildbucket returns newest first; reverse so order is chronological. | |
| builds.reverse() | |
| return builds | |
| def QueryResultDBForBuilds(build_ids, batch_size=25): | |
| """Queries ResultDB in batches and yields (build_id, test_result) tuples.""" | |
| for i in range(0, len(build_ids), batch_size): | |
| batch = build_ids[i:i + batch_size] | |
| inv_args = [f'build-{bid}' for bid in batch] | |
| cmd = ['rdb', 'query', '-json', '-tr-fields', 'testId,duration,variant' | |
| ] + inv_args | |
| proc = subprocess.Popen( | |
| cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) | |
| with proc: | |
| for line in proc.stdout: | |
| line = line.strip() | |
| if not line: | |
| continue | |
| try: | |
| data = json.loads(line) | |
| tr = data.get('testResult') | |
| inv_id = (data.get('invocationId') or '').replace('build-', '') | |
| if tr and inv_id: | |
| yield inv_id, tr | |
| except json.JSONDecodeError: | |
| pass | |
| proc.wait() | |
| if proc.returncode != 0: | |
| err = proc.stderr.read().strip() | |
| if err: | |
| print(f'Warning: "rdb query" failed: {err}', file=sys.stderr) | |
| SUITE_ALIASES = { | |
| 'unittest': 'pdfium_unittests', | |
| 'unittests': 'pdfium_unittests', | |
| 'embedder': 'pdfium_embeddertests', | |
| 'embeddertest': 'pdfium_embeddertests', | |
| 'embeddertests': 'pdfium_embeddertests', | |
| } | |
| def MatchesSuiteFilter(suite, suite_filters, exact=False): | |
| """Returns True if `suite` matches any filter in `suite_filters`.""" | |
| if not suite_filters: | |
| return True | |
| suite_lower = suite.lower() | |
| if exact: | |
| return suite_lower in ( | |
| SUITE_ALIASES.get(sf.lower(), sf.lower()) for sf in suite_filters) | |
| # Strip '_javascript_disabled' when matching general filters like 'javascript' | |
| # so that 'corpus_javascript_disabled' and 'pixel_javascript_disabled' are not | |
| # mistakenly matched by '--javascript'. | |
| base_suite = suite_lower.replace('_javascript_disabled', '') | |
| for sf in suite_filters: | |
| if 'javascript_disabled' in sf: | |
| if sf in suite_lower: | |
| return True | |
| elif sf in base_suite: | |
| return True | |
| return False | |
| def ExtractTestData(build_ids, | |
| suite_filters=None, | |
| exact_suite=False, | |
| use_cache=True, | |
| verbose=False): | |
| """Collects suite sums, test counts, and test durations across `build_ids`.""" | |
| test_timings = collections.defaultdict(lambda: collections.defaultdict(dict)) | |
| suite_totals = collections.defaultdict(lambda: collections.defaultdict(float)) | |
| suite_counts = collections.defaultdict(lambda: collections.defaultdict(int)) | |
| cache_dir = os.path.join(tempfile.gettempdir(), 'pdfium_rdb_cache') | |
| if use_cache: | |
| os.makedirs(cache_dir, exist_ok=True) | |
| needed_build_ids = [] | |
| for bid in build_ids: | |
| cache_path = os.path.join(cache_dir, f'{bid}.json') if use_cache else None | |
| if cache_path and os.path.isfile(cache_path): | |
| try: | |
| with open(cache_path, 'r', encoding='utf-8') as f: | |
| build_data = json.load(f) | |
| for suite, tests in build_data.items(): | |
| if suite_filters and not MatchesSuiteFilter( | |
| suite, suite_filters, exact=exact_suite): | |
| continue | |
| for test_id, dur in tests.items(): | |
| test_timings[bid][suite][test_id] = dur | |
| suite_totals[bid][suite] += dur | |
| suite_counts[bid][suite] += 1 | |
| continue | |
| except (json.JSONDecodeError, OSError): | |
| pass | |
| needed_build_ids.append(bid) | |
| if needed_build_ids: | |
| if verbose: | |
| print( | |
| f'Querying ResultDB for {len(needed_build_ids)} builds...', | |
| file=sys.stderr, | |
| flush=True) | |
| new_raw = collections.defaultdict( | |
| lambda: collections.defaultdict(lambda: collections.defaultdict(list))) | |
| for bid, tr in QueryResultDBForBuilds(needed_build_ids): | |
| variant = (tr.get('variant') or {}).get('def') or {} | |
| suite = variant.get('test_suite') or 'unknown' | |
| test_id = tr.get('testId') or 'unknown' | |
| dur_str = tr.get('duration') or '0s' | |
| try: | |
| dur = float(dur_str.rstrip('s')) if dur_str.endswith('s') else 0.0 | |
| except ValueError: | |
| dur = 0.0 | |
| new_raw[bid][suite][test_id].append(dur) | |
| for bid in needed_build_ids: | |
| suites = new_raw[bid] | |
| build_cache_data = {} | |
| for suite, tests in suites.items(): | |
| build_cache_data[suite] = {} | |
| include_suite = not suite_filters or MatchesSuiteFilter( | |
| suite, suite_filters, exact=exact_suite) | |
| for test_id, durs in tests.items(): | |
| mean_dur = durs[0] if len(durs) == 1 else (sum(durs) / len(durs)) | |
| build_cache_data[suite][test_id] = mean_dur | |
| if include_suite: | |
| test_timings[bid][suite][test_id] = mean_dur | |
| suite_totals[bid][suite] += mean_dur | |
| suite_counts[bid][suite] += 1 | |
| if use_cache and build_cache_data: | |
| cache_path = os.path.join(cache_dir, f'{bid}.json') | |
| try: | |
| with open(cache_path, 'w', encoding='utf-8') as f: | |
| json.dump(build_cache_data, f) | |
| except OSError: | |
| pass | |
| return suite_totals, suite_counts, test_timings | |
| def DetectSuiteSteps(runs, | |
| suite_totals, | |
| suite_counts=None, | |
| threshold_pct=15.0, | |
| min_delta=2.0, | |
| persistence_runs=2, | |
| suite_filters=None, | |
| exact_suite=False): | |
| """Detects persistent step functions in suite durations. | |
| A step occurs at index `i` if the median of `runs[i : i + 1 + | |
| persistence_runs]` differs from the pre-change baseline (median of | |
| `runs[i - 3 : i]`) by at least `threshold_pct` and `min_delta` seconds, | |
| and every run in the post window maintains that shift. | |
| """ | |
| all_suites = sorted( | |
| set().union(*(suite_totals[r['id']].keys() for r in runs))) | |
| if suite_filters: | |
| all_suites = [ | |
| s for s in all_suites | |
| if MatchesSuiteFilter(s, suite_filters, exact=exact_suite) | |
| ] | |
| post_window_size = persistence_runs + 1 | |
| step_events = [] | |
| for suite in all_suites: | |
| # Filter to runs where this suite had test results. | |
| suite_runs = [r for r in runs if suite in suite_totals[r['id']]] | |
| if len(suite_runs) < 3 + post_window_size: | |
| continue | |
| durations = [suite_totals[r['id']][suite] for r in suite_runs] | |
| counts = [ | |
| suite_counts[r['id']][suite] if suite_counts else 0 for r in suite_runs | |
| ] | |
| idx = 3 | |
| limit = len(suite_runs) - post_window_size | |
| while idx <= limit: | |
| pre_window = durations[idx - 3:idx] | |
| post_window = durations[idx:idx + post_window_size] | |
| m_pre = statistics.median(pre_window) | |
| m_post = statistics.median(post_window) | |
| delta = m_post - m_pre | |
| pct = (delta / m_pre * 100.0) if m_pre > 0.0 else 0.0 | |
| if abs(pct) >= threshold_pct and abs(delta) >= min_delta: | |
| c_pre = statistics.median(counts[idx - 3:idx]) | |
| c_post = statistics.median(counts[idx:idx + post_window_size]) | |
| if delta > 0: | |
| # Regression: all post-runs must remain elevated. | |
| min_allowed = m_pre * (1.0 + (threshold_pct / 200.0)) | |
| if all(d >= min_allowed for d in post_window): | |
| step_events.append({ | |
| 'type': 'REGRESSION', | |
| 'suite': suite, | |
| 'run_index': idx, | |
| 'pre_runs': suite_runs[idx - 3:idx], | |
| 'trigger_run': suite_runs[idx], | |
| 'prev_run': suite_runs[idx - 1], | |
| 'future_runs': suite_runs[idx + 1:idx + post_window_size], | |
| 'm_pre': m_pre, | |
| 'm_post': m_post, | |
| 'delta': delta, | |
| 'pct': pct, | |
| 'count_pre': int(c_pre), | |
| 'count_post': int(c_post), | |
| }) | |
| idx += post_window_size | |
| continue | |
| else: | |
| # Improvement: all post-runs must remain lower. | |
| max_allowed = m_pre * (1.0 - (threshold_pct / 200.0)) | |
| if all(d <= max_allowed for d in post_window): | |
| step_events.append({ | |
| 'type': 'IMPROVEMENT', | |
| 'suite': suite, | |
| 'run_index': idx, | |
| 'pre_runs': suite_runs[idx - 3:idx], | |
| 'trigger_run': suite_runs[idx], | |
| 'prev_run': suite_runs[idx - 1], | |
| 'future_runs': suite_runs[idx + 1:idx + post_window_size], | |
| 'm_pre': m_pre, | |
| 'm_post': m_post, | |
| 'delta': delta, | |
| 'pct': pct, | |
| 'count_pre': int(c_pre), | |
| 'count_post': int(c_post), | |
| }) | |
| idx += post_window_size | |
| continue | |
| idx += 1 | |
| return step_events | |
| def FindCulpritTests(step_event, | |
| test_timings, | |
| top_n=5, | |
| test_threshold_pct=30.0, | |
| test_min_delta=0.2): | |
| """Finds individual tests that persistently shifted across the step.""" | |
| suite = step_event['suite'] | |
| pre_runs = step_event.get('pre_runs', [step_event['prev_run']]) | |
| pre_ids = [r['id'] for r in pre_runs] | |
| trigger_id = step_event['trigger_run']['id'] | |
| future_ids = [r['id'] for r in step_event['future_runs']] | |
| post_ids = [trigger_id] + future_ids | |
| all_test_ids = set() | |
| for bid in pre_ids + post_ids: | |
| all_test_ids.update(test_timings[bid].get(suite, {}).keys()) | |
| candidates = [] | |
| is_regression = step_event['type'] == 'REGRESSION' | |
| for test_id in all_test_ids: | |
| pre_durs = [ | |
| test_timings[bid][suite][test_id] | |
| for bid in pre_ids | |
| if test_id in test_timings[bid].get(suite, {}) | |
| ] | |
| post_durs = [ | |
| test_timings[bid][suite][test_id] | |
| for bid in post_ids | |
| if test_id in test_timings[bid].get(suite, {}) | |
| ] | |
| # Test must run at least once in each epoch to compare durations. | |
| if not pre_durs or not post_durs: | |
| continue | |
| t_pre = statistics.median(pre_durs) | |
| m_test_post = statistics.median(post_durs) | |
| t_delta = m_test_post - t_pre | |
| t_pct = (t_delta / t_pre * 100.0) if t_pre > 0.0 else 0.0 | |
| if abs(t_pct) >= test_threshold_pct and abs(t_delta) >= test_min_delta: | |
| if is_regression and t_delta > 0 and all(d >= t_pre for d in post_durs): | |
| candidates.append({ | |
| 'test_id': test_id, | |
| 'pre_dur': t_pre, | |
| 'post_durs': post_durs, | |
| 'm_post': m_test_post, | |
| 'delta': t_delta, | |
| 'pct': t_pct, | |
| }) | |
| elif (not is_regression and t_delta < 0 and | |
| all(d <= t_pre for d in post_durs)): | |
| candidates.append({ | |
| 'test_id': test_id, | |
| 'pre_dur': t_pre, | |
| 'post_durs': post_durs, | |
| 'm_post': m_test_post, | |
| 'delta': t_delta, | |
| 'pct': t_pct, | |
| }) | |
| # Rank by absolute delta descending. | |
| candidates.sort(key=lambda x: abs(x['delta']), reverse=True) | |
| total_candidate_delta = sum(c['delta'] for c in candidates) | |
| return { | |
| 'candidates': candidates[:top_n], | |
| 'total_count': len(candidates), | |
| 'total_delta': total_candidate_delta, | |
| } | |
| def AnalyzeBuilder(builder, | |
| since_dt, | |
| until_dt, | |
| threshold_pct, | |
| min_delta, | |
| persistence_runs, | |
| top_n_culprits, | |
| test_threshold_pct=30.0, | |
| test_min_delta=0.2, | |
| suite_filters=None, | |
| exact_suite=False, | |
| use_cache=True, | |
| verbose=False): | |
| """Fetches and evaluates performance step functions for `builder`.""" | |
| if verbose: | |
| print( | |
| f'Fetching builds for {builder} between {since_dt.isoformat()} ' | |
| f'and {until_dt.isoformat()}...', | |
| file=sys.stderr, | |
| flush=True) | |
| builds = GetBuildsForBuilder(builder, since_dt, until_dt) | |
| if len(builds) < 3 + 1 + persistence_runs: | |
| if verbose: | |
| print( | |
| f'Insufficient builds found for {builder} ({len(builds)} builds).', | |
| file=sys.stderr) | |
| return [] | |
| build_ids = [b['id'] for b in builds] | |
| suite_totals, suite_counts, test_timings = ExtractTestData( | |
| build_ids, | |
| suite_filters=suite_filters, | |
| exact_suite=exact_suite, | |
| use_cache=use_cache, | |
| verbose=verbose) | |
| step_events = DetectSuiteSteps( | |
| runs=builds, | |
| suite_totals=suite_totals, | |
| suite_counts=suite_counts, | |
| threshold_pct=threshold_pct, | |
| min_delta=min_delta, | |
| persistence_runs=persistence_runs, | |
| suite_filters=suite_filters, | |
| exact_suite=exact_suite) | |
| results = [] | |
| for step in step_events: | |
| culprit_info = FindCulpritTests( | |
| step_event=step, | |
| test_timings=test_timings, | |
| top_n=top_n_culprits, | |
| test_threshold_pct=test_threshold_pct, | |
| test_min_delta=test_min_delta) | |
| results.append({ | |
| 'builder': builder, | |
| 'step': step, | |
| 'culprits': culprit_info['candidates'], | |
| 'culprit_total_count': culprit_info['total_count'], | |
| 'culprit_total_delta': culprit_info['total_delta'], | |
| }) | |
| return results | |
| def GetCommitRangeUrl(prev_commit, trigger_commit): | |
| """Returns a Gitiles log URL between `prev_commit` and `trigger_commit`.""" | |
| if prev_commit and trigger_commit and prev_commit != trigger_commit: | |
| return (f'https://pdfium.googlesource.com/pdfium/+log/' | |
| f'{prev_commit[:10]}..{trigger_commit[:10]}') | |
| return '' | |
| def FormatEventText(event): | |
| """Formats a single step function event for text display.""" | |
| builder = event['builder'] | |
| step = event['step'] | |
| culprits = event['culprits'] | |
| culprit_total_count = event.get('culprit_total_count', len(culprits)) | |
| culprit_total_delta = event.get('culprit_total_delta', | |
| sum(c['delta'] for c in culprits)) | |
| trigger = step['trigger_run'] | |
| prev = step['prev_run'] | |
| future = step['future_runs'] | |
| stype = step['type'] | |
| sign = '+' if step['delta'] > 0 else '' | |
| trigger_commit = trigger.get('commit') or '' | |
| prev_commit = prev.get('commit') or '' | |
| range_url = GetCommitRangeUrl(prev_commit, trigger_commit) | |
| range_line = f' Commit Range: {range_url}\n' if range_url else '' | |
| test_count_line = '' | |
| c_pre = step.get('count_pre', 0) | |
| c_post = step.get('count_post', 0) | |
| if c_pre or c_post: | |
| c_delta = c_post - c_pre | |
| csign = '+' if c_delta > 0 else '' | |
| test_count_line = f' Tests Run: {c_pre} -> {c_post} ({csign}{c_delta} tests)\n' | |
| header = (f'=== [{stype}] builder: {builder} | suite: {step["suite"]} ===\n' | |
| f' Trigger Build: #{trigger.get("number") or "unknown"} ' | |
| f'(commit: {(trigger_commit or "unknown")[:10]})\n' | |
| f' Pre Build: #{prev.get("number") or "unknown"} ' | |
| f'(commit: {(prev_commit or "unknown")[:10]})\n' | |
| f'{range_line}' | |
| f' Date: {trigger.get("create_time") or "unknown"}\n' | |
| f'{test_count_line}' | |
| f' Suite Shift: {step["m_pre"]:.2f}s -> {step["m_post"]:.2f}s ' | |
| f'({sign}{step["pct"]:.1f}%, {sign}{step["delta"]:.2f}s)\n' | |
| f' Persisted in: ' + | |
| ', '.join(f'#{r.get("number") or "?"}' for r in future)) | |
| lines = [header] | |
| if culprits: | |
| csign = '+' if culprit_total_delta > 0 else '' | |
| lines.append( | |
| f' Culprit Tests (showing top {len(culprits)} of {culprit_total_count}, ' | |
| f'accounting for {csign}{culprit_total_delta:.2f}s of {sign}{step["delta"]:.2f}s shift):' | |
| ) | |
| for c in culprits: | |
| c_item_sign = '+' if c['delta'] > 0 else '' | |
| post_str = ', '.join(f'{d:.2f}s' for d in c['post_durs']) | |
| lines.append( | |
| f' - {c["test_id"]}\n' | |
| f' duration: {c["pre_dur"]:.2f}s -> ({post_str}) ' | |
| f'[{c_item_sign}{c["pct"]:.1f}%, {c_item_sign}{c["delta"]:.2f}s]') | |
| else: | |
| lines.append( | |
| ' Culprit Tests: None met threshold (diffuse across entire suite)') | |
| return '\n'.join(lines) | |
| def main(): | |
| parser = argparse.ArgumentParser( | |
| description='Find performance step functions across CI builders.') | |
| parser.add_argument( | |
| '--since', | |
| help='Start date/time in ISO-8601 format (e.g. "2026-09-14" or ' | |
| '"2026-09-14T00:00:00Z"). Defaults to 7 days ago.') | |
| parser.add_argument( | |
| '--until', | |
| help='End date/time in ISO-8601 format (e.g. "2026-09-21" or ' | |
| '"2026-09-21T23:59:59Z"). Defaults to current time.') | |
| parser.add_argument( | |
| '--builder', | |
| '-b', | |
| help='Specific CI builder to inspect (e.g. "linux", or comma-separated ' | |
| 'list). Defaults to "linux" if omitted and --all-builders is not set.') | |
| parser.add_argument( | |
| '--all-builders', | |
| action='store_true', | |
| help='Inspect all active CI builders in pdfium/ci.') | |
| parser.add_argument( | |
| '--threshold', | |
| type=float, | |
| default=15.0, | |
| help='Relative threshold percentage to flag a step function ' | |
| '(default: 15.0)') | |
| parser.add_argument( | |
| '--min-delta', | |
| type=float, | |
| default=2.0, | |
| help='Minimum absolute change in seconds to flag a step function ' | |
| '(default: 2.0)') | |
| parser.add_argument( | |
| '--persistence', | |
| type=int, | |
| default=2, | |
| help='Number of future runs required to confirm persistence (default: 2)') | |
| parser.add_argument( | |
| '--top-culprits', | |
| type=int, | |
| default=5, | |
| help='Number of individual culprit tests to report per step (default: 5)') | |
| parser.add_argument( | |
| '--test-threshold', | |
| type=float, | |
| default=30.0, | |
| help='Relative percentage threshold for individual culprit tests ' | |
| '(default: 30.0)') | |
| parser.add_argument( | |
| '--test-min-delta', | |
| type=float, | |
| default=0.2, | |
| help='Minimum absolute change in seconds for individual culprit tests ' | |
| '(default: 0.2)') | |
| parser.add_argument( | |
| '--suite', | |
| '-s', | |
| help='Filter to specific test suite(s), comma-separated or substring ' | |
| '(e.g. "unittests", "embeddertests", "javascript", "corpus", "pixel")') | |
| parser.add_argument( | |
| '--exact-suite', | |
| action='store_true', | |
| help='Require exact suite name match instead of substring matching') | |
| parser.add_argument( | |
| '--unittest', | |
| '--unittests', | |
| action='store_true', | |
| help='Shortcut to filter for unit tests') | |
| parser.add_argument( | |
| '--corpus', | |
| action='store_true', | |
| help='Shortcut to filter for corpus tests') | |
| parser.add_argument( | |
| '--pixel', action='store_true', help='Shortcut to filter for pixel tests') | |
| parser.add_argument( | |
| '--embeddertest', | |
| '--embeddertests', | |
| action='store_true', | |
| help='Shortcut to filter for embedder tests') | |
| parser.add_argument( | |
| '--javascript', | |
| action='store_true', | |
| help='Shortcut to filter for javascript tests') | |
| parser.add_argument( | |
| '--concurrency', | |
| '-j', | |
| type=int, | |
| default=4, | |
| help='Number of concurrent builders to process in parallel (default: 4)') | |
| parser.add_argument( | |
| '--no-cache', | |
| action='store_true', | |
| help='Disable local disk caching of ResultDB build queries') | |
| parser.add_argument( | |
| '--json', action='store_true', help='Output results as JSON') | |
| parser.add_argument( | |
| '--verbose', | |
| '-v', | |
| action='store_true', | |
| help='Print verbose progress messages') | |
| args = parser.parse_args() | |
| if args.persistence < 1: | |
| print('Error: --persistence must be at least 1.', file=sys.stderr) | |
| return 1 | |
| if args.threshold <= 0.0: | |
| print('Error: --threshold must be positive.', file=sys.stderr) | |
| return 1 | |
| if args.min_delta < 0.0: | |
| print('Error: --min-delta must be non-negative.', file=sys.stderr) | |
| return 1 | |
| if args.test_threshold <= 0.0: | |
| print('Error: --test-threshold must be positive.', file=sys.stderr) | |
| return 1 | |
| if args.test_min_delta < 0.0: | |
| print('Error: --test-min-delta must be non-negative.', file=sys.stderr) | |
| return 1 | |
| now = datetime.datetime.now(datetime.timezone.utc) | |
| if args.until: | |
| try: | |
| until_dt = ParseISO8601(args.until) | |
| if len(args.until) <= 10: | |
| until_dt = until_dt.replace( | |
| hour=23, minute=59, second=59, microsecond=999999) | |
| except ValueError as e: | |
| print(f'Error: Invalid ISO-8601 format for --until: {e}', file=sys.stderr) | |
| return 1 | |
| else: | |
| until_dt = now | |
| if args.since: | |
| try: | |
| since_dt = ParseISO8601(args.since) | |
| except ValueError as e: | |
| print(f'Error: Invalid ISO-8601 format for --since: {e}', file=sys.stderr) | |
| return 1 | |
| else: | |
| since_dt = now - datetime.timedelta(days=7) | |
| if since_dt >= until_dt: | |
| print( | |
| 'Error: --since date must be earlier than --until date.', | |
| file=sys.stderr) | |
| return 1 | |
| suite_filters = [] | |
| if args.suite: | |
| suite_filters.extend( | |
| [s.strip().lower() for s in args.suite.split(',') if s.strip()]) | |
| if args.unittest: | |
| suite_filters.append('unittests') | |
| if args.corpus: | |
| suite_filters.append('corpus') | |
| if args.pixel: | |
| suite_filters.append('pixel') | |
| if args.embeddertest: | |
| suite_filters.append('embeddertests') | |
| if args.javascript: | |
| suite_filters.append('javascript') | |
| if args.all_builders: | |
| builders = GetCIBuilders() | |
| elif args.builder: | |
| builders = [b.strip() for b in args.builder.split(',') if b.strip()] | |
| else: | |
| builders = ['linux'] | |
| if not builders: | |
| print('Error: No builders specified or found.', file=sys.stderr) | |
| return 1 | |
| if not args.json: | |
| print(f'Analyzing builds from {since_dt.isoformat()} to ' | |
| f'{until_dt.isoformat()} for builders: {", ".join(builders)}') | |
| if suite_filters: | |
| print(f'Suite filters: {", ".join(suite_filters)}') | |
| print(f'Threshold: {args.threshold}%, Min Delta: {args.min_delta}s, ' | |
| f'Persistence: {args.persistence} future runs\n') | |
| all_events = [] | |
| if len(builders) > 1 and args.concurrency > 1: | |
| with concurrent.futures.ThreadPoolExecutor( | |
| max_workers=min(args.concurrency, len(builders))) as executor: | |
| futures = { | |
| executor.submit( | |
| AnalyzeBuilder, | |
| builder=b, | |
| since_dt=since_dt, | |
| until_dt=until_dt, | |
| threshold_pct=args.threshold, | |
| min_delta=args.min_delta, | |
| persistence_runs=args.persistence, | |
| top_n_culprits=args.top_culprits, | |
| test_threshold_pct=args.test_threshold, | |
| test_min_delta=args.test_min_delta, | |
| suite_filters=suite_filters, | |
| exact_suite=args.exact_suite, | |
| use_cache=not args.no_cache, | |
| verbose=args.verbose): | |
| b for b in builders | |
| } | |
| for future in concurrent.futures.as_completed(futures): | |
| b = futures[future] | |
| try: | |
| events = future.result() | |
| all_events.extend(events) | |
| except Exception as e: | |
| print(f'Error analyzing builder {b}: {e}', file=sys.stderr) | |
| else: | |
| for b in builders: | |
| try: | |
| events = AnalyzeBuilder( | |
| builder=b, | |
| since_dt=since_dt, | |
| until_dt=until_dt, | |
| threshold_pct=args.threshold, | |
| min_delta=args.min_delta, | |
| persistence_runs=args.persistence, | |
| top_n_culprits=args.top_culprits, | |
| test_threshold_pct=args.test_threshold, | |
| test_min_delta=args.test_min_delta, | |
| suite_filters=suite_filters, | |
| exact_suite=args.exact_suite, | |
| use_cache=not args.no_cache, | |
| verbose=args.verbose) | |
| all_events.extend(events) | |
| except Exception as e: | |
| print(f'Error analyzing builder {b}: {e}', file=sys.stderr) | |
| # Sort events deterministically by builder and trigger time. | |
| all_events.sort(key=lambda e: (e['builder'], e['step']['trigger_run'].get( | |
| 'create_time') or '')) | |
| if args.json: | |
| json_output = [] | |
| for event in all_events: | |
| step = event['step'] | |
| c_pre = step.get('count_pre') | |
| c_post = step.get('count_post') | |
| c_delta = (c_post - | |
| c_pre) if (c_pre is not None and c_post is not None) else None | |
| json_output.append({ | |
| 'builder': | |
| event['builder'], | |
| 'type': | |
| step['type'], | |
| 'suite': | |
| step['suite'], | |
| 'trigger_build_number': | |
| step['trigger_run'].get('number'), | |
| 'trigger_build_id': | |
| step['trigger_run'].get('id'), | |
| 'trigger_commit': | |
| step['trigger_run'].get('commit'), | |
| 'trigger_time': | |
| step['trigger_run'].get('create_time'), | |
| 'pre_build_number': | |
| step['prev_run'].get('number'), | |
| 'pre_commit': | |
| step['prev_run'].get('commit'), | |
| 'commit_range_url': | |
| GetCommitRangeUrl(step['prev_run'].get('commit') or '', | |
| step['trigger_run'].get('commit') or ''), | |
| 'test_count_pre': | |
| c_pre, | |
| 'test_count_post': | |
| c_post, | |
| 'test_count_delta': | |
| c_delta, | |
| 'median_pre_seconds': | |
| step['m_pre'], | |
| 'median_post_seconds': | |
| step['m_post'], | |
| 'delta_seconds': | |
| step['delta'], | |
| 'percent_change': | |
| step['pct'], | |
| 'future_build_numbers': [ | |
| r.get('number') for r in step['future_runs'] | |
| ], | |
| 'culprits': | |
| event['culprits'], | |
| 'culprit_total_count': | |
| event.get('culprit_total_count'), | |
| 'culprit_total_delta': | |
| event.get('culprit_total_delta'), | |
| }) | |
| print(json.dumps(json_output, indent=2)) | |
| return 0 | |
| if not all_events: | |
| print('No persistent performance step functions detected.') | |
| return 0 | |
| for event in all_events: | |
| print(FormatEventText(event)) | |
| print() | |
| print(f'Total step events found: {len(all_events)}') | |
| return 0 | |
| if __name__ == '__main__': | |
| sys.exit(main()) |