blob: bb2d4e1396dce6b2949379ca85174e95bebd5212 [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2017 The PDFium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Generates a coverage report for given binaries using llvm-gcov & lcov.
Requires llvm-cov 3.5 or later.
Requires lcov 1.11 or later.
Requires that 'use_coverage = true' is set in args.gn.
"""
import argparse
from collections import namedtuple
import pprint
import os
import re
import subprocess
import sys
# 'binary' is the file that is to be run for the test.
# 'use_test_runner' indicates if 'binary' depends on test_runner.py and thus
# requires special handling.
TestSpec = namedtuple('TestSpec', 'binary, use_test_runner')
# All of the coverage tests that the script knows how to run.
COVERAGE_TESTS = {
'pdfium_unittests': TestSpec('pdfium_unittests', False),
'pdfium_embeddertests': TestSpec('pdfium_embeddertests', False),
'corpus_tests': TestSpec('run_corpus_tests.py', True),
'javascript_tests': TestSpec('run_javascript_tests.py', True),
'pixel_tests': TestSpec('run_pixel_tests.py', True),
}
# Coverage tests that are known to take a long time to run, so are not in the
# default set. The user must either explicitly invoke these tests or pass in
# --slow.
SLOW_TESTS = ['corpus_tests', 'javascript_tests', 'pixel_tests']
class CoverageExecutor(object):
def __init__(self, parser, args):
"""Initialize executor based on the current script environment
Args:
parser: argparse.ArgumentParser for handling improper inputs.
args: Dictionary of arguments passed into the calling script.
"""
self.dry_run = args['dry_run']
self.verbose = args['verbose']
llvm_cov = self.determine_proper_llvm_cov()
if not llvm_cov:
print 'Unable to find appropriate llvm-cov to use'
sys.exit(1)
self.lcov_env = os.environ
self.lcov_env['LLVM_COV_BIN'] = llvm_cov
self.lcov = self.determine_proper_lcov()
if not self.lcov:
print 'Unable to find appropriate lcov to use'
sys.exit(1)
self.coverage_files = set()
self.source_directory = args['source_directory']
if not os.path.isdir(self.source_directory):
parser.error("'%s' needs to be a directory" % self.source_directory)
self.build_directory = args['build_directory']
if not os.path.isdir(self.build_directory):
parser.error("'%s' needs to be a directory" % self.build_directory)
self.coverage_tests = self.calculate_coverage_tests(args)
if not self.coverage_tests:
parser.error(
'No valid tests in set to be run. This is likely due to bad command '
'line arguments')
if not self.boolean_gn_arg('use_coverage'):
parser.error(
'use_coverage does not appear to be set to true for build, but is '
'needed')
self.use_goma = self.boolean_gn_arg('use_goma')
self.output_directory = args['output_directory']
if not os.path.exists(self.output_directory):
if not self.dry_run:
os.makedirs(self.output_directory)
elif not os.path.isdir(self.output_directory):
parser.error('%s exists, but is not a directory' % self.output_directory)
self.coverage_totals_path = os.path.join(self.output_directory,
'pdfium_totals.info')
def boolean_gn_arg(self, arg):
"""Extract the value of a boolean flag in args.gn"""
cwd = os.getcwd()
os.chdir(self.build_directory)
gn_args_output = self.check_output(
['gn', 'args', '.', '--list=%s' % arg, '--short'])
os.chdir(cwd)
arg_match_output = re.match('%s = (.*)' % arg, gn_args_output).group(1)
if self.verbose:
print "Found '%s' for value of %s" % (arg_match_output, arg)
return arg_match_output == 'true'
def check_output(self, args, dry_run=False, env=None):
"""Dry run aware wrapper of subprocess.check_output()"""
if dry_run:
print "Would have run '%s'" % ' '.join(args)
return ''
output = subprocess.check_output(args, env=env)
if self.verbose:
print "check_output(%s) returned '%s'" % (args, output)
return output
def call(self, args, dry_run=False, env=None):
"""Dry run aware wrapper of subprocess.call()"""
if dry_run:
print "Would have run '%s'" % ' '.join(args)
return 0
output = subprocess.call(args, env=env)
if self.verbose:
print 'call(%s) returned %s' % (args, output)
return output
def call_lcov(self, args, dry_run=False, needs_directory=True):
"""Wrapper to call lcov that adds appropriate arguments as needed."""
lcov_args = [
self.lcov, '--config-file',
os.path.join(self.source_directory, 'tools', 'coverage', 'lcovrc'),
'--gcov-tool',
os.path.join(self.source_directory, 'tools', 'coverage', 'llvm-gcov')
]
if needs_directory:
lcov_args.extend(['--directory', self.source_directory])
if not self.verbose:
lcov_args.append('--quiet')
lcov_args.extend(args)
return self.call(lcov_args, dry_run=dry_run, env=self.lcov_env)
def calculate_coverage_tests(self, args):
"""Determine which tests should be run."""
testing_tools_directory = os.path.join(self.source_directory, 'testing',
'tools')
coverage_tests = {}
for name in COVERAGE_TESTS.keys():
test_spec = COVERAGE_TESTS[name]
if test_spec.use_test_runner:
binary_path = os.path.join(testing_tools_directory, test_spec.binary)
else:
binary_path = os.path.join(self.build_directory, test_spec.binary)
coverage_tests[name] = TestSpec(binary_path, test_spec.use_test_runner)
if args['tests']:
return {name: spec
for name, spec in coverage_tests.iteritems() if name in args['tests']}
elif not args['slow']:
return {name: spec
for name, spec in coverage_tests.iteritems() if name not in SLOW_TESTS}
else:
return coverage_tests
def find_acceptable_binary(self, binary_name, version_regex,
min_major_version, min_minor_version):
"""Find the newest version of binary that meets the min version."""
min_version = (min_major_version, min_minor_version)
parsed_versions = {}
# When calling Bash builtins like this the command and arguments must be
# passed in as a single string instead of as separate list members.
potential_binaries = self.check_output(
['bash', '-c', 'compgen -abck %s' % binary_name]).splitlines()
for binary in potential_binaries:
if self.verbose:
print 'Testing llvm-cov binary, %s' % binary
# Assuming that scripts that don't respond to --version correctly are not
# valid binaries and just happened to get globbed in. This is true for
# lcov and llvm-cov
try:
version_output = self.check_output([binary, '--version']).splitlines()
except subprocess.CalledProcessError:
if self.verbose:
print '--version returned failure status 1, so ignoring'
continue
for line in version_output:
matcher = re.match(version_regex, line)
if matcher:
parsed_version = (int(matcher.group(1)), int(matcher.group(2)))
if parsed_version >= min_version:
parsed_versions[parsed_version] = binary
break
if not parsed_versions:
return None
return parsed_versions[max(parsed_versions)]
def determine_proper_llvm_cov(self):
"""Find a version of llvm_cov that will work with the script."""
version_regex = re.compile('.*LLVM version ([\d]+)\.([\d]+).*')
return self.find_acceptable_binary('llvm-cov', version_regex, 3, 5)
def determine_proper_lcov(self):
"""Find a version of lcov that will work with the script."""
version_regex = re.compile('.*LCOV version ([\d]+)\.([\d]+).*')
return self.find_acceptable_binary('lcov', version_regex, 1, 11)
def build_binaries(self):
"""Build all the binaries that are going to be needed for coverage
generation."""
call_args = ['ninja']
if self.use_goma:
call_args.extend(['-j', '250'])
call_args.extend(['-C', self.build_directory])
return self.call(call_args, dry_run=self.dry_run) == 0
def generate_coverage(self, name, spec):
"""Generate the coverage data for a test
Args:
name: Name associated with the test to be run. This is used as a label
in the coverage data, so should be unique across all of the tests
being run.
spec: Tuple containing the path to the binary to run, and if this test
uses test_runner.py.
"""
if self.verbose:
print "Generating coverage for test '%s', using data '%s'" % (name, spec)
if not os.path.exists(spec.binary):
print('Unable to generate coverage for %s, since it appears to not exist'
' @ %s') % (name, spec.binary)
return False
if self.call_lcov(['--zerocounters'], dry_run=self.dry_run):
print 'Unable to clear counters for %s' % name
return False
binary_args = [spec.binary]
if spec.use_test_runner:
# Test runner performs multi-threading in the wrapper script, not the test
# binary, so need -j 1, otherwise multiple processes will be writing to
# the code coverage files, invalidating results.
# TODO(pdfium:811): Rewrite how test runner tests work, so that they can
# be run in multi-threaded mode.
binary_args.extend(['-j', '1', '--build-dir', self.build_directory])
if self.call(binary_args, dry_run=self.dry_run) and self.verbose:
print('Running %s appears to have failed, which might affect '
'results') % spec.binary
output_raw_path = os.path.join(self.output_directory, '%s_raw.info' % name)
if self.call_lcov(
['--capture', '--test-name', name, '--output-file', output_raw_path],
dry_run=self.dry_run):
print 'Unable to capture coverage data for %s' % name
return False
output_filtered_path = os.path.join(self.output_directory,
'%s_filtered.info' % name)
output_filters = [
'/usr/include/*', '*third_party*', '*testing*', '*_unittest.cpp',
'*_embeddertest.cpp'
]
if self.call_lcov(
['--remove', output_raw_path] + output_filters +
['--output-file', output_filtered_path],
dry_run=self.dry_run,
needs_directory=False):
print 'Unable to filter coverage data for %s' % name
return False
self.coverage_files.add(output_filtered_path)
return True
def merge_coverage(self):
"""Merge all of the coverage data sets into one for report generation."""
merge_args = []
for coverage_file in self.coverage_files:
merge_args.extend(['--add-tracefile', coverage_file])
merge_args.extend(['--output-file', self.coverage_totals_path])
return self.call_lcov(
merge_args, dry_run=self.dry_run, needs_directory=False) == 0
def generate_report(self):
"""Produce HTML coverage report based on combined coverage data set."""
config_file = os.path.join(
self.source_directory, 'tools', 'coverage', 'lcovrc')
lcov_args = ['genhtml',
'--config-file', config_file,
'--legend',
'--demangle-cpp',
'--show-details',
'--prefix', self.source_directory,
'--ignore-errors',
'source', self.coverage_totals_path,
'--output-directory', self.output_directory]
return self.call(lcov_args, dry_run=self.dry_run) == 0
def run(self):
"""Setup environment, execute the tests and generate coverage report"""
if not self.build_binaries():
print 'Failed to successfully build binaries'
return False
for name in self.coverage_tests.keys():
if not self.generate_coverage(name, self.coverage_tests[name]):
print 'Failed to successfully generate coverage data'
return False
if not self.merge_coverage():
print 'Failed to successfully merge generated coverage data'
return False
if not self.generate_report():
print 'Failed to successfully generated coverage report'
return False
return True
def main():
parser = argparse.ArgumentParser()
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.description = ('Generates a coverage report for given binaries using '
'llvm-cov & lcov.\n\n'
'Requires llvm-cov 3.5 or later.\n'
'Requires lcov 1.11 or later.\n\n'
'By default runs pdfium_unittests and '
'pdfium_embeddertests. If --slow is passed in then all '
'tests will be run. If any of the tests are specified '
'on the command line, then only those will be run.')
parser.add_argument(
'-s',
'--source_directory',
help='Location of PDFium source directory, defaults to CWD',
default=os.getcwd())
build_default = os.path.join('out', 'Coverage')
parser.add_argument(
'-b',
'--build_directory',
help=
'Location of PDFium build directory with coverage enabled, defaults to '
'%s under CWD' % build_default,
default=os.path.join(os.getcwd(), build_default))
output_default = 'coverage_report'
parser.add_argument(
'-o',
'--output_directory',
help='Location to write out coverage report to, defaults to %s under CWD '
% output_default,
default=os.path.join(os.getcwd(), output_default))
parser.add_argument(
'-n',
'--dry-run',
help='Output commands instead of executing them',
action='store_true')
parser.add_argument(
'-v',
'--verbose',
help='Output additional diagnostic information',
action='store_true')
parser.add_argument(
'--slow',
help='Run all tests, even those known to take a long time. Ignored if '
'specific tests are passed in.',
action='store_true')
parser.add_argument(
'tests',
help='Tests to be run, defaults to all. Valid entries are %s' %
COVERAGE_TESTS.keys(),
nargs='*')
args = vars(parser.parse_args())
if args['verbose']:
pprint.pprint(args)
executor = CoverageExecutor(parser, args)
if executor.run():
return 0
return 1
if __name__ == '__main__':
sys.exit(main())