| # Copyright 2015 The PDFium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| |
| import json |
| import os |
| import shlex |
| import shutil |
| import ssl |
| import urllib2 |
| |
| |
| def _ParseKeyValuePairs(kv_str): |
| """ |
| Parses a string of the type 'key1 value1 key2 value2' into a dict. |
| """ |
| kv_pairs = shlex.split(kv_str) |
| if len(kv_pairs) % 2: |
| raise ValueError('Uneven number of key/value pairs. Got %s' % kv_str) |
| return { kv_pairs[i]:kv_pairs[i + 1] for i in xrange(0, len(kv_pairs), 2) } |
| |
| |
| # This module downloads a json provided by Skia Gold with the expected baselines |
| # for each test file. |
| # |
| # The expected format for the json is: |
| # { |
| # "commit": { |
| # "author": "John Doe (jdoe@chromium.org)", |
| # "commit_time": 1510598123, |
| # "hash": "cee39e6e90c219cc91f2c94a912a06977f4461a0" |
| # }, |
| # "master": { |
| # "abc.pdf.1": { |
| # "0ec3d86f545052acd7c9a16fde8ca9d4": 1, |
| # "80455b71673becc9fbc100d6da56ca65": 1, |
| # "b68e2ecb80090b4502ec89ad1be2322c": 1 |
| # }, |
| # "defgh.pdf.0": { |
| # "01e020cd4cd05c6738e479a46a506044": 1, |
| # "b68e2ecb80090b4502ec89ad1be2322c": 1 |
| # } |
| # }, |
| # "changeLists": { |
| # "18499" : { |
| # "abc.pdf.1": { |
| # "d5dd649124cf1779152253dc8fb239c5": 1, |
| # "42a270581930579cdb0f28674972fb1a": 1, |
| # } |
| # } |
| # } |
| # } |
| class GoldBaseline(object): |
| |
| def __init__(self, properties_str): |
| """ |
| properties_str is a string with space separated key/value pairs that |
| is used to find the cl number for which to baseline |
| """ |
| self._properties = _ParseKeyValuePairs(properties_str) |
| self._baselines = self._LoadSkiaGoldBaselines() |
| |
| def _LoadSkiaGoldBaselines(self): |
| """ |
| Download the baseline json and return a list of the two baselines that |
| should be used to match hashes (master and cl#). |
| """ |
| GOLD_BASELINE_URL = 'https://pdfium-gold.skia.org/json/baseline' |
| |
| # If we have an issue number add it to the baseline URL |
| cl_number_str = self._properties.get('issue', None) |
| url = GOLD_BASELINE_URL + ('/' + cl_number_str if cl_number_str else '') |
| |
| json_data = '' |
| MAX_TIMEOUT = 33 # 5 tries. (2, 4, 8, 16, 32) |
| timeout = 2 |
| while True: |
| try: |
| response = urllib2.urlopen(url, timeout=timeout) |
| c_type = response.headers.get('Content-type', '') |
| EXPECTED_CONTENT_TYPE = 'application/json' |
| if c_type != EXPECTED_CONTENT_TYPE: |
| raise ValueError('Invalid content type. Got %s instead of %s' % ( |
| c_type, EXPECTED_CONTENT_TYPE)) |
| json_data = response.read() |
| break # If this line is reached, then no exception occurred. |
| except (ssl.SSLError, urllib2.HTTPError, urllib2.URLError) as e: |
| timeout *= 2 |
| if timeout < MAX_TIMEOUT: |
| continue |
| print ('Error: Unable to read skia gold json from %s: %s' % (url, e)) |
| return None |
| |
| try: |
| data = json.loads(json_data) |
| except ValueError as e: |
| print 'Error: Malformed json read from %s: %s' % (url, e) |
| return None |
| |
| return data.get('master', {}) |
| |
| # Return values for MatchLocalResult(). |
| MATCH = 'match' |
| MISMATCH = 'mismatch' |
| NO_BASELINE = 'no_baseline' |
| BASELINE_DOWNLOAD_FAILED = 'baseline_download_failed' |
| |
| def MatchLocalResult(self, test_name, md5_hash): |
| """ |
| Match a locally generated hash of a test cases rendered image with the |
| expected hashes downloaded in the baselines json. |
| |
| Each baseline is a dict mapping the test case name to a dict with the |
| expected hashes as keys. Therefore, this list of baselines should be |
| searched until the test case name is found, then the hash should be matched |
| with the options in that dict. If the hashes don't match, it should be |
| considered a failure and we should not continue searching the baseline list. |
| |
| Returns MATCH if the md5 provided matches the ones in the baseline json, |
| MISMATCH if it does not, NO_BASELINE if the test case has no baseline, or |
| BASELINE_DOWNLOAD_FAILED if the baseline could not be downloaded and parsed. |
| """ |
| if self._baselines is None: |
| return GoldBaseline.BASELINE_DOWNLOAD_FAILED |
| |
| found_test_case = False |
| if test_name in self._baselines: |
| found_test_case = True |
| if md5_hash in self._baselines[test_name]: |
| return GoldBaseline.MATCH |
| |
| return (GoldBaseline.MISMATCH if found_test_case |
| else GoldBaseline.NO_BASELINE) |
| |
| |
| # This module collects and writes output in a format expected by the |
| # Gold baseline tool. Based on meta data provided explicitly and by |
| # adding a series of test results it can be used to produce |
| # a JSON file that is uploaded to Google Storage and ingested by Gold. |
| # |
| # The output will look similar this: |
| # |
| # { |
| # "build_number" : "2", |
| # "gitHash" : "a4a338179013b029d6dd55e737b5bd648a9fb68c", |
| # "key" : { |
| # "arch" : "arm64", |
| # "compiler" : "Clang", |
| # }, |
| # "results" : [ |
| # { |
| # "key" : { |
| # "config" : "vk", |
| # "name" : "yuv_nv12_to_rgb_effect", |
| # "source_type" : "gm" |
| # }, |
| # "md5" : "7db34da246868d50ab9ddd776ce6d779", |
| # "options" : { |
| # "ext" : "png", |
| # "gamma_correct" : "no" |
| # } |
| # }, |
| # { |
| # "key" : { |
| # "config" : "vk", |
| # "name" : "yuv_to_rgb_effect", |
| # "source_type" : "gm" |
| # }, |
| # "md5" : "0b955f387740c66eb23bf0e253c80d64", |
| # "options" : { |
| # "ext" : "png", |
| # "gamma_correct" : "no" |
| # } |
| # } |
| # ], |
| # } |
| # |
| class GoldResults(object): |
| def __init__(self, source_type, output_dir, properties_str, key_str, |
| ignore_hashes_file): |
| """ |
| source_type is the source_type (=corpus) field used for all results. |
| output_dir is the directory where the resulting images are copied and |
| the dm.json file is written. If the directory exists it will |
| be removed and recreated. |
| properties_str is a string with space separated key/value pairs that |
| is used to set the top level fields in the output JSON file. |
| key_str is a string with space separated key/value pairs that |
| is used to set the 'key' field in the output JSON file. |
| ignore_hashes_file is a file that contains a list of image hashes |
| that should be ignored. |
| """ |
| self._source_type = source_type |
| self._properties = _ParseKeyValuePairs(properties_str) |
| self._properties['key'] = _ParseKeyValuePairs(key_str) |
| self._results = [] |
| self._passfail = [] |
| self._output_dir = output_dir |
| |
| # make sure the output directory exists and is empty. |
| if os.path.exists(output_dir): |
| shutil.rmtree(output_dir, ignore_errors=True) |
| os.makedirs(output_dir) |
| |
| self._ignore_hashes = set() |
| if ignore_hashes_file: |
| with open(ignore_hashes_file, 'r') as ig_file: |
| hashes=[x.strip() for x in ig_file.readlines() if x.strip()] |
| self._ignore_hashes = set(hashes) |
| |
| def AddTestResult(self, testName, md5Hash, outputImagePath, matchResult): |
| # If the hash is in the list of hashes to ignore then we don'try |
| # make a copy, but add it to the result. |
| imgExt = os.path.splitext(outputImagePath)[1].lstrip('.') |
| if md5Hash not in self._ignore_hashes: |
| # Copy the image to <output_dir>/<md5Hash>.<image_extension> |
| if not imgExt: |
| raise ValueError('File %s does not have an extension' % outputImagePath) |
| newFilePath = os.path.join(self._output_dir, md5Hash + '.' + imgExt) |
| shutil.copy2(outputImagePath, newFilePath) |
| |
| # Add an entry to the list of test results |
| self._results.append({ |
| 'key': { |
| 'name': testName, |
| 'source_type': self._source_type, |
| }, |
| 'md5': md5Hash, |
| 'options': { |
| 'ext': imgExt, |
| 'gamma_correct': 'no' |
| } |
| }) |
| |
| self._passfail.append((testName, matchResult)) |
| |
| def WriteResults(self): |
| self._properties.update({ |
| 'results': self._results |
| }) |
| |
| output_file_name = os.path.join(self._output_dir, 'dm.json') |
| with open(output_file_name, 'wb') as outfile: |
| json.dump(self._properties, outfile, indent=1) |
| outfile.write('\n') |
| |
| output_file_name = os.path.join(self._output_dir, 'passfail.json') |
| with open(output_file_name, 'wb') as outfile: |
| json.dump(self._passfail, outfile, indent=1) |
| outfile.write('\n') |
| |
| # Produce example output for manual testing. |
| if __name__ == '__main__': |
| # Create a test directory with three empty 'image' files. |
| test_dir = './testdirectory' |
| if not os.path.exists(test_dir): |
| os.makedirs(test_dir) |
| open(os.path.join(test_dir, 'image1.png'), 'wb').close() |
| open(os.path.join(test_dir, 'image2.png'), 'wb').close() |
| open(os.path.join(test_dir, 'image3.png'), 'wb').close() |
| |
| # Create an instance and add results. |
| prop_str = 'build_number 2 "builder name" Builder-Name gitHash ' \ |
| 'a4a338179013b029d6dd55e737b5bd648a9fb68c' |
| |
| key_str = 'arch arm64 compiler Clang configuration Debug' |
| |
| hash_file = os.path.join(test_dir, 'ignore_hashes.txt') |
| with open(hash_file, 'wb') as f: |
| f.write('\n'.join(['hash-1', 'hash-4']) + '\n') |
| |
| output_dir = './output_directory' |
| gr = GoldResults('pdfium', output_dir, prop_str, key_str, hash_file) |
| gr.AddTestResult('test-1', 'hash-1', os.path.join(test_dir, 'image1.png'), |
| GoldBaseline.MATCH) |
| gr.AddTestResult('test-2', 'hash-2', os.path.join(test_dir, 'image2.png'), |
| GoldBaseline.MATCH) |
| gr.AddTestResult('test-3', 'hash-3', os.path.join(test_dir, 'image3.png'), |
| GoldBaseline.MISMATCH) |
| gr.WriteResults() |
