Add Chromium perf result processing script.
We will override this script in a follow-up CL to meet ANGLE's
specific needs.
Bug: angleproject:6090
Change-Id: I8d3a326645367e90ae79f49ef15ace700d6af106
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/3042213
Reviewed-by: Jamie Madill <jmadill@chromium.org>
Commit-Queue: Jamie Madill <jmadill@chromium.org>
diff --git a/scripts/process_angle_perf_results.py b/scripts/process_angle_perf_results.py
new file mode 100755
index 0000000..912828c
--- /dev/null
+++ b/scripts/process_angle_perf_results.py
@@ -0,0 +1,757 @@
+#!/usr/bin/env vpython
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from __future__ import print_function
+
+import argparse
+import collections
+import json
+import logging
+import multiprocessing
+import os
+import shutil
+import sys
+import tempfile
+import time
+import uuid
+
+logging.basicConfig(
+ level=logging.INFO,
+ format='(%(levelname)s) %(asctime)s pid=%(process)d'
+ ' %(module)s.%(funcName)s:%(lineno)d %(message)s')
+
+import cross_device_test_config
+
+from core import path_util
+path_util.AddTelemetryToPath()
+
+from core import upload_results_to_perf_dashboard
+from core import results_merger
+from core import bot_platforms
+
+path_util.AddAndroidPylibToPath()
+
+try:
+ from pylib.utils import logdog_helper
+except ImportError:
+ pass
+
+RESULTS_URL = 'https://chromeperf.appspot.com'
+
+# Until we are migrated to LUCI, we will be utilizing a hard
+# coded master name based on what is passed in in the build properties.
+# See crbug.com/801289 for more details.
+MACHINE_GROUP_JSON_FILE = os.path.join(path_util.GetChromiumSrcDir(), 'tools', 'perf', 'core',
+ 'perf_dashboard_machine_group_mapping.json')
+
+JSON_CONTENT_TYPE = 'application/json'
+
+# Cache of what data format (ChartJSON, Histograms, etc.) each results file is
+# in so that only one disk read is required when checking the format multiple
+# times.
+_data_format_cache = {}
+DATA_FORMAT_GTEST = 'gtest'
+DATA_FORMAT_CHARTJSON = 'chartjson'
+DATA_FORMAT_HISTOGRAMS = 'histograms'
+DATA_FORMAT_UNKNOWN = 'unknown'
+
+
+def _GetMachineGroup(build_properties):
+ machine_group = None
+ if build_properties.get('perf_dashboard_machine_group', False):
+ # Once luci migration is complete this will exist as a property
+ # in the build properties
+ machine_group = build_properties['perf_dashboard_machine_group']
+ else:
+ builder_group_mapping = {}
+ with open(MACHINE_GROUP_JSON_FILE) as fp:
+ builder_group_mapping = json.load(fp)
+ if build_properties.get('builder_group', False):
+ legacy_builder_group = build_properties['builder_group']
+ else:
+ # TODO(crbug.com/1153958): remove reference to mastername.
+ legacy_builder_group = build_properties['mastername']
+ if builder_group_mapping.get(legacy_builder_group):
+ machine_group = builder_group_mapping[legacy_builder_group]
+ if not machine_group:
+ raise ValueError('Must set perf_dashboard_machine_group or have a valid '
+ 'mapping in '
+ 'src/tools/perf/core/perf_dashboard_machine_group_mapping.json'
+ 'See bit.ly/perf-dashboard-machine-group for more details')
+ return machine_group
+
+
+def _upload_perf_results(json_to_upload, name, configuration_name, build_properties,
+ output_json_file):
+ """Upload the contents of result JSON(s) to the perf dashboard."""
+ args = [
+ '--buildername', build_properties['buildername'], '--buildnumber',
+ build_properties['buildnumber'], '--name', name, '--configuration-name',
+ configuration_name, '--results-file', json_to_upload, '--results-url', RESULTS_URL,
+ '--got-revision-cp', build_properties['got_revision_cp'], '--got-v8-revision',
+ build_properties['got_v8_revision'], '--got-webrtc-revision',
+ build_properties['got_webrtc_revision'], '--output-json-file', output_json_file,
+ '--perf-dashboard-machine-group',
+ _GetMachineGroup(build_properties)
+ ]
+ buildbucket = build_properties.get('buildbucket', {})
+ if isinstance(buildbucket, basestring):
+ buildbucket = json.loads(buildbucket)
+
+ if 'build' in buildbucket:
+ args += [
+ '--project',
+ buildbucket['build'].get('project'),
+ '--buildbucket',
+ buildbucket['build'].get('bucket'),
+ ]
+
+ if build_properties.get('git_revision'):
+ args.append('--git-revision')
+ args.append(build_properties['git_revision'])
+ if _is_histogram(json_to_upload):
+ args.append('--send-as-histograms')
+
+ #TODO(crbug.com/1072729): log this in top level
+ logging.info('upload_results_to_perf_dashboard: %s.' % args)
+
+ return upload_results_to_perf_dashboard.main(args)
+
+
+def _is_histogram(json_file):
+ return _determine_data_format(json_file) == DATA_FORMAT_HISTOGRAMS
+
+
+def _is_gtest(json_file):
+ return _determine_data_format(json_file) == DATA_FORMAT_GTEST
+
+
+def _determine_data_format(json_file):
+ if json_file not in _data_format_cache:
+ with open(json_file) as f:
+ data = json.load(f)
+ if isinstance(data, list):
+ _data_format_cache[json_file] = DATA_FORMAT_HISTOGRAMS
+ elif isinstance(data, dict):
+ if 'charts' in data:
+ _data_format_cache[json_file] = DATA_FORMAT_CHARTJSON
+ else:
+ _data_format_cache[json_file] = DATA_FORMAT_GTEST
+ else:
+ _data_format_cache[json_file] = DATA_FORMAT_UNKNOWN
+ return _data_format_cache[json_file]
+ _data_format_cache[json_file] = DATA_FORMAT_UNKNOWN
+ return _data_format_cache[json_file]
+
+
+def _merge_json_output(output_json, jsons_to_merge, extra_links, test_cross_device=False):
+ """Merges the contents of one or more results JSONs.
+
+ Args:
+ output_json: A path to a JSON file to which the merged results should be
+ written.
+ jsons_to_merge: A list of JSON files that should be merged.
+ extra_links: a (key, value) map in which keys are the human-readable strings
+ which describe the data, and value is logdog url that contain the data.
+ """
+ begin_time = time.time()
+ merged_results = results_merger.merge_test_results(jsons_to_merge, test_cross_device)
+
+ # Only append the perf results links if present
+ if extra_links:
+ merged_results['links'] = extra_links
+
+ with open(output_json, 'w') as f:
+ json.dump(merged_results, f)
+
+ end_time = time.time()
+ print_duration('Merging json test results', begin_time, end_time)
+ return 0
+
+
+def _handle_perf_json_test_results(benchmark_directory_map, test_results_list):
+ """Checks the test_results.json under each folder:
+
+ 1. mark the benchmark 'enabled' if tests results are found
+ 2. add the json content to a list for non-ref.
+ """
+ begin_time = time.time()
+ benchmark_enabled_map = {}
+ for benchmark_name, directories in benchmark_directory_map.items():
+ for directory in directories:
+ # Obtain the test name we are running
+ is_ref = '.reference' in benchmark_name
+ enabled = True
+ try:
+ with open(os.path.join(directory, 'test_results.json')) as json_data:
+ json_results = json.load(json_data)
+ if not json_results:
+ # Output is null meaning the test didn't produce any results.
+ # Want to output an error and continue loading the rest of the
+ # test results.
+ logging.warning('No results produced for %s, skipping upload' % directory)
+ continue
+ if json_results.get('version') == 3:
+ # Non-telemetry tests don't have written json results but
+ # if they are executing then they are enabled and will generate
+ # chartjson results.
+ if not bool(json_results.get('tests')):
+ enabled = False
+ if not is_ref:
+ # We don't need to upload reference build data to the
+ # flakiness dashboard since we don't monitor the ref build
+ test_results_list.append(json_results)
+ except IOError as e:
+ # TODO(crbug.com/936602): Figure out how to surface these errors. Should
+ # we have a non-zero exit code if we error out?
+ logging.error('Failed to obtain test results for %s: %s', benchmark_name, e)
+ continue
+ if not enabled:
+ # We don't upload disabled benchmarks or tests that are run
+ # as a smoke test
+ logging.info('Benchmark %s ran no tests on at least one shard' % benchmark_name)
+ continue
+ benchmark_enabled_map[benchmark_name] = True
+
+ end_time = time.time()
+ print_duration('Analyzing perf json test results', begin_time, end_time)
+ return benchmark_enabled_map
+
+
+def _generate_unique_logdog_filename(name_prefix):
+ return name_prefix + '_' + str(uuid.uuid4())
+
+
+def _handle_perf_logs(benchmark_directory_map, extra_links):
+ """ Upload benchmark logs to logdog and add a page entry for them. """
+ begin_time = time.time()
+ benchmark_logs_links = collections.defaultdict(list)
+
+ for benchmark_name, directories in benchmark_directory_map.items():
+ for directory in directories:
+ benchmark_log_file = os.path.join(directory, 'benchmark_log.txt')
+ if os.path.exists(benchmark_log_file):
+ with open(benchmark_log_file) as f:
+ uploaded_link = logdog_helper.text(
+ name=_generate_unique_logdog_filename(benchmark_name), data=f.read())
+ benchmark_logs_links[benchmark_name].append(uploaded_link)
+
+ logdog_file_name = _generate_unique_logdog_filename('Benchmarks_Logs')
+ logdog_stream = logdog_helper.text(
+ logdog_file_name,
+ json.dumps(benchmark_logs_links, sort_keys=True, indent=4, separators=(',', ': ')),
+ content_type=JSON_CONTENT_TYPE)
+ extra_links['Benchmarks logs'] = logdog_stream
+ end_time = time.time()
+ print_duration('Generating perf log streams', begin_time, end_time)
+
+
+def _handle_benchmarks_shard_map(benchmarks_shard_map_file, extra_links):
+ begin_time = time.time()
+ with open(benchmarks_shard_map_file) as f:
+ benchmarks_shard_data = f.read()
+ logdog_file_name = _generate_unique_logdog_filename('Benchmarks_Shard_Map')
+ logdog_stream = logdog_helper.text(
+ logdog_file_name, benchmarks_shard_data, content_type=JSON_CONTENT_TYPE)
+ extra_links['Benchmarks shard map'] = logdog_stream
+ end_time = time.time()
+ print_duration('Generating benchmark shard map stream', begin_time, end_time)
+
+
+def _get_benchmark_name(directory):
+ return os.path.basename(directory).replace(" benchmark", "")
+
+
+def _scan_output_dir(task_output_dir):
+ benchmark_directory_map = {}
+ benchmarks_shard_map_file = None
+
+ directory_list = [
+ f for f in os.listdir(task_output_dir)
+ if not os.path.isfile(os.path.join(task_output_dir, f))
+ ]
+ benchmark_directory_list = []
+ for directory in directory_list:
+ for f in os.listdir(os.path.join(task_output_dir, directory)):
+ path = os.path.join(task_output_dir, directory, f)
+ if os.path.isdir(path):
+ benchmark_directory_list.append(path)
+ elif path.endswith('benchmarks_shard_map.json'):
+ benchmarks_shard_map_file = path
+ # Now create a map of benchmark name to the list of directories
+ # the lists were written to.
+ for directory in benchmark_directory_list:
+ benchmark_name = _get_benchmark_name(directory)
+ if benchmark_name in benchmark_directory_map.keys():
+ benchmark_directory_map[benchmark_name].append(directory)
+ else:
+ benchmark_directory_map[benchmark_name] = [directory]
+
+ return benchmark_directory_map, benchmarks_shard_map_file
+
+
+def process_perf_results(output_json,
+ configuration_name,
+ build_properties,
+ task_output_dir,
+ smoke_test_mode,
+ output_results_dir,
+ lightweight=False,
+ skip_perf=False):
+ """Process perf results.
+
+ Consists of merging the json-test-format output, uploading the perf test
+ output (chartjson and histogram), and store the benchmark logs in logdog.
+
+ Each directory in the task_output_dir represents one benchmark
+ that was run. Within this directory, there is a subdirectory with the name
+ of the benchmark that was run. In that subdirectory, there is a
+ perftest-output.json file containing the performance results in histogram
+ or dashboard json format and an output.json file containing the json test
+ results for the benchmark.
+
+ Returns:
+ (return_code, upload_results_map):
+ return_code is 0 if the whole operation is successful, non zero otherwise.
+ benchmark_upload_result_map: the dictionary that describe which benchmarks
+ were successfully uploaded.
+ """
+ handle_perf = not lightweight or not skip_perf
+ handle_non_perf = not lightweight or skip_perf
+ logging.info('lightweight mode: %r; handle_perf: %r; handle_non_perf: %r' %
+ (lightweight, handle_perf, handle_non_perf))
+
+ begin_time = time.time()
+ return_code = 0
+ benchmark_upload_result_map = {}
+
+ benchmark_directory_map, benchmarks_shard_map_file = _scan_output_dir(task_output_dir)
+
+ test_results_list = []
+ extra_links = {}
+
+ if handle_non_perf:
+ # First, upload benchmarks shard map to logdog and add a page
+ # entry for it in extra_links.
+ if benchmarks_shard_map_file:
+ _handle_benchmarks_shard_map(benchmarks_shard_map_file, extra_links)
+
+ # Second, upload all the benchmark logs to logdog and add a page entry for
+ # those links in extra_links.
+ _handle_perf_logs(benchmark_directory_map, extra_links)
+
+ # Then try to obtain the list of json test results to merge
+ # and determine the status of each benchmark.
+ benchmark_enabled_map = _handle_perf_json_test_results(benchmark_directory_map,
+ test_results_list)
+
+ build_properties_map = json.loads(build_properties)
+ if not configuration_name:
+ # we are deprecating perf-id crbug.com/817823
+ configuration_name = build_properties_map['buildername']
+
+ _update_perf_results_for_calibration(benchmarks_shard_map_file, benchmark_enabled_map,
+ benchmark_directory_map, configuration_name)
+ if not smoke_test_mode and handle_perf:
+ try:
+ return_code, benchmark_upload_result_map = _handle_perf_results(
+ benchmark_enabled_map, benchmark_directory_map, configuration_name,
+ build_properties_map, extra_links, output_results_dir)
+ except Exception:
+ logging.exception('Error handling perf results jsons')
+ return_code = 1
+
+ if handle_non_perf:
+ # Finally, merge all test results json, add the extra links and write out to
+ # output location
+ try:
+ _merge_json_output(output_json, test_results_list, extra_links,
+ configuration_name in cross_device_test_config.TARGET_DEVICES)
+ except Exception:
+ logging.exception('Error handling test results jsons.')
+
+ end_time = time.time()
+ print_duration('Total process_perf_results', begin_time, end_time)
+ return return_code, benchmark_upload_result_map
+
+
+def _merge_chartjson_results(chartjson_dicts):
+ merged_results = chartjson_dicts[0]
+ for chartjson_dict in chartjson_dicts[1:]:
+ for key in chartjson_dict:
+ if key == 'charts':
+ for add_key in chartjson_dict[key]:
+ merged_results[key][add_key] = chartjson_dict[key][add_key]
+ return merged_results
+
+
+def _merge_histogram_results(histogram_lists):
+ merged_results = []
+ for histogram_list in histogram_lists:
+ merged_results += histogram_list
+
+ return merged_results
+
+
+def _merge_perf_results(benchmark_name, results_filename, directories):
+ begin_time = time.time()
+ collected_results = []
+ for directory in directories:
+ filename = os.path.join(directory, 'perf_results.json')
+ try:
+ with open(filename) as pf:
+ collected_results.append(json.load(pf))
+ except IOError as e:
+ # TODO(crbug.com/936602): Figure out how to surface these errors. Should
+ # we have a non-zero exit code if we error out?
+ logging.error('Failed to obtain perf results from %s: %s', directory, e)
+ if not collected_results:
+ logging.error('Failed to obtain any perf results from %s.', benchmark_name)
+ return
+
+ # Assuming that multiple shards will only be chartjson or histogram set
+ # Non-telemetry benchmarks only ever run on one shard
+ merged_results = []
+ if isinstance(collected_results[0], dict):
+ merged_results = _merge_chartjson_results(collected_results)
+ elif isinstance(collected_results[0], list):
+ merged_results = _merge_histogram_results(collected_results)
+
+ with open(results_filename, 'w') as rf:
+ json.dump(merged_results, rf)
+
+ end_time = time.time()
+ print_duration(('%s results merging' % (benchmark_name)), begin_time, end_time)
+
+
+def _upload_individual(benchmark_name, directories, configuration_name, build_properties,
+ output_json_file):
+ tmpfile_dir = tempfile.mkdtemp()
+ try:
+ upload_begin_time = time.time()
+ # There are potentially multiple directores with results, re-write and
+ # merge them if necessary
+ results_filename = None
+ if len(directories) > 1:
+ merge_perf_dir = os.path.join(os.path.abspath(tmpfile_dir), benchmark_name)
+ if not os.path.exists(merge_perf_dir):
+ os.makedirs(merge_perf_dir)
+ results_filename = os.path.join(merge_perf_dir, 'merged_perf_results.json')
+ _merge_perf_results(benchmark_name, results_filename, directories)
+ else:
+ # It was only written to one shard, use that shards data
+ results_filename = os.path.join(directories[0], 'perf_results.json')
+
+ results_size_in_mib = os.path.getsize(results_filename) / (2**20)
+ logging.info('Uploading perf results from %s benchmark (size %s Mib)' %
+ (benchmark_name, results_size_in_mib))
+ with open(output_json_file, 'w') as oj:
+ upload_return_code = _upload_perf_results(results_filename, benchmark_name,
+ configuration_name, build_properties, oj)
+ upload_end_time = time.time()
+ print_duration(('%s upload time' % (benchmark_name)), upload_begin_time,
+ upload_end_time)
+ return (benchmark_name, upload_return_code == 0)
+ finally:
+ shutil.rmtree(tmpfile_dir)
+
+
+def _upload_individual_benchmark(params):
+ try:
+ return _upload_individual(*params)
+ except Exception:
+ benchmark_name = params[0]
+ upload_succeed = False
+ logging.exception('Error uploading perf result of %s' % benchmark_name)
+ return benchmark_name, upload_succeed
+
+
+def _GetCpuCount(log=True):
+ try:
+ cpu_count = multiprocessing.cpu_count()
+ if sys.platform == 'win32':
+ # TODO(crbug.com/1190269) - we can't use more than 56
+ # cores on Windows or Python3 may hang.
+ cpu_count = min(cpu_count, 56)
+ return cpu_count
+ except NotImplementedError:
+ if log:
+ logging.warn('Failed to get a CPU count for this bot. See crbug.com/947035.')
+ # TODO(crbug.com/948281): This is currently set to 4 since the mac masters
+ # only have 4 cores. Once we move to all-linux, this can be increased or
+ # we can even delete this whole function and use multiprocessing.cpu_count()
+ # directly.
+ return 4
+
+
+def _load_shard_id_from_test_results(directory):
+ shard_id = None
+ test_json_path = os.path.join(directory, 'test_results.json')
+ try:
+ with open(test_json_path) as f:
+ test_json = json.load(f)
+ all_results = test_json['tests']
+ for _, benchmark_results in all_results.items():
+ for _, measurement_result in benchmark_results.items():
+ shard_id = measurement_result['shard']
+ break
+ except IOError as e:
+ logging.error('Failed to open test_results.json from %s: %s', test_json_path, e)
+ except KeyError as e:
+ logging.error('Failed to locate results in test_results.json: %s', e)
+ return shard_id
+
+
+def _find_device_id_by_shard_id(benchmarks_shard_map_file, shard_id):
+ try:
+ with open(benchmarks_shard_map_file) as f:
+ shard_map_json = json.load(f)
+ device_id = shard_map_json['extra_infos']['bot #%s' % shard_id]
+ except KeyError as e:
+ logging.error('Failed to locate device name in shard map: %s', e)
+ return device_id
+
+
+def _update_perf_json_with_summary_on_device_id(directory, device_id):
+ perf_json_path = os.path.join(directory, 'perf_results.json')
+ try:
+ with open(perf_json_path, 'r') as f:
+ perf_json = json.load(f)
+ except IOError as e:
+ logging.error('Failed to open perf_results.json from %s: %s', perf_json_path, e)
+ summary_key_guid = str(uuid.uuid4())
+ summary_key_generic_set = {
+ 'values': ['device_id'],
+ 'guid': summary_key_guid,
+ 'type': 'GenericSet'
+ }
+ perf_json.insert(0, summary_key_generic_set)
+ logging.info('Inserted summary key generic set for perf result in %s: %s', directory,
+ summary_key_generic_set)
+ stories_guids = set()
+ for entry in perf_json:
+ if 'diagnostics' in entry:
+ entry['diagnostics']['summaryKeys'] = summary_key_guid
+ stories_guids.add(entry['diagnostics']['stories'])
+ for entry in perf_json:
+ if 'guid' in entry and entry['guid'] in stories_guids:
+ entry['values'].append(device_id)
+ try:
+ with open(perf_json_path, 'w') as f:
+ json.dump(perf_json, f)
+ except IOError as e:
+ logging.error('Failed to writing perf_results.json to %s: %s', perf_json_path, e)
+ logging.info('Finished adding device id %s in perf result.', device_id)
+
+
+def _should_add_device_id_in_perf_result(builder_name):
+ # We should always add device id in calibration builders.
+ # For testing purpose, adding fyi as well for faster turnaround, because
+ # calibration builders run every 24 hours.
+ return any([builder_name == p.name for p in bot_platforms.CALIBRATION_PLATFORMS
+ ]) or (builder_name == 'android-pixel2-perf-fyi')
+
+
+def _update_perf_results_for_calibration(benchmarks_shard_map_file, benchmark_enabled_map,
+ benchmark_directory_map, configuration_name):
+ if not _should_add_device_id_in_perf_result(configuration_name):
+ return
+ logging.info('Updating perf results for %s.', configuration_name)
+ for benchmark_name, directories in benchmark_directory_map.items():
+ if not benchmark_enabled_map.get(benchmark_name, False):
+ continue
+ for directory in directories:
+ shard_id = _load_shard_id_from_test_results(directory)
+ device_id = _find_device_id_by_shard_id(benchmarks_shard_map_file, shard_id)
+ _update_perf_json_with_summary_on_device_id(directory, device_id)
+
+
+def _handle_perf_results(benchmark_enabled_map, benchmark_directory_map, configuration_name,
+ build_properties, extra_links, output_results_dir):
+ """
+ Upload perf results to the perf dashboard.
+
+ This method also upload the perf results to logdog and augment it to
+ |extra_links|.
+
+ Returns:
+ (return_code, benchmark_upload_result_map)
+ return_code is 0 if this upload to perf dashboard successfully, 1
+ otherwise.
+ benchmark_upload_result_map is a dictionary describes which benchmark
+ was successfully uploaded.
+ """
+ begin_time = time.time()
+ # Upload all eligible benchmarks to the perf dashboard
+ results_dict = {}
+
+ invocations = []
+ for benchmark_name, directories in benchmark_directory_map.items():
+ if not benchmark_enabled_map.get(benchmark_name, False):
+ continue
+ # Create a place to write the perf results that you will write out to
+ # logdog.
+ output_json_file = os.path.join(output_results_dir, (str(uuid.uuid4()) + benchmark_name))
+ results_dict[benchmark_name] = output_json_file
+ #TODO(crbug.com/1072729): pass final arguments instead of build properties
+ # and configuration_name
+ invocations.append(
+ (benchmark_name, directories, configuration_name, build_properties, output_json_file))
+
+ # Kick off the uploads in multiple processes
+ # crbug.com/1035930: We are hitting HTTP Response 429. Limit ourselves
+ # to 2 processes to avoid this error. Uncomment the following code once
+ # the problem is fixed on the dashboard side.
+ # pool = multiprocessing.Pool(_GetCpuCount())
+ pool = multiprocessing.Pool(2)
+ upload_result_timeout = False
+ try:
+ async_result = pool.map_async(_upload_individual_benchmark, invocations)
+ # TODO(crbug.com/947035): What timeout is reasonable?
+ results = async_result.get(timeout=4000)
+ except multiprocessing.TimeoutError:
+ upload_result_timeout = True
+ logging.error('Timeout uploading benchmarks to perf dashboard in parallel')
+ results = []
+ for benchmark_name in benchmark_directory_map:
+ results.append((benchmark_name, False))
+ finally:
+ pool.terminate()
+
+ # Keep a mapping of benchmarks to their upload results
+ benchmark_upload_result_map = {}
+ for r in results:
+ benchmark_upload_result_map[r[0]] = r[1]
+
+ logdog_dict = {}
+ upload_failures_counter = 0
+ logdog_stream = None
+ logdog_label = 'Results Dashboard'
+ for benchmark_name, output_file in results_dict.items():
+ upload_succeed = benchmark_upload_result_map[benchmark_name]
+ if not upload_succeed:
+ upload_failures_counter += 1
+ is_reference = '.reference' in benchmark_name
+ _write_perf_data_to_logfile(
+ benchmark_name,
+ output_file,
+ configuration_name,
+ build_properties,
+ logdog_dict,
+ is_reference,
+ upload_failure=not upload_succeed)
+
+ logdog_file_name = _generate_unique_logdog_filename('Results_Dashboard_')
+ logdog_stream = logdog_helper.text(
+ logdog_file_name,
+ json.dumps(dict(logdog_dict), sort_keys=True, indent=4, separators=(',', ': ')),
+ content_type=JSON_CONTENT_TYPE)
+ if upload_failures_counter > 0:
+ logdog_label += (' %s merge script perf data upload failures' % upload_failures_counter)
+ extra_links[logdog_label] = logdog_stream
+ end_time = time.time()
+ print_duration('Uploading results to perf dashboard', begin_time, end_time)
+ if upload_result_timeout or upload_failures_counter > 0:
+ return 1, benchmark_upload_result_map
+ return 0, benchmark_upload_result_map
+
+
+def _write_perf_data_to_logfile(benchmark_name, output_file, configuration_name, build_properties,
+ logdog_dict, is_ref, upload_failure):
+ viewer_url = None
+ # logdog file to write perf results to
+ if os.path.exists(output_file):
+ results = None
+ with open(output_file) as f:
+ try:
+ results = json.load(f)
+ except ValueError:
+ logging.error('Error parsing perf results JSON for benchmark %s' % benchmark_name)
+ if results:
+ try:
+ output_json_file = logdog_helper.open_text(benchmark_name)
+ json.dump(results, output_json_file, indent=4, separators=(',', ': '))
+ except ValueError as e:
+ logging.error('ValueError: "%s" while dumping output to logdog' % e)
+ finally:
+ output_json_file.close()
+ viewer_url = output_json_file.get_viewer_url()
+ else:
+ logging.warning("Perf results JSON file doesn't exist for benchmark %s" % benchmark_name)
+
+ base_benchmark_name = benchmark_name.replace('.reference', '')
+
+ if base_benchmark_name not in logdog_dict:
+ logdog_dict[base_benchmark_name] = {}
+
+ # add links for the perf results and the dashboard url to
+ # the logs section of buildbot
+ if is_ref:
+ if viewer_url:
+ logdog_dict[base_benchmark_name]['perf_results_ref'] = viewer_url
+ if upload_failure:
+ logdog_dict[base_benchmark_name]['ref_upload_failed'] = 'True'
+ else:
+ logdog_dict[base_benchmark_name]['dashboard_url'] = (
+ upload_results_to_perf_dashboard.GetDashboardUrl(benchmark_name, configuration_name,
+ RESULTS_URL,
+ build_properties['got_revision_cp'],
+ _GetMachineGroup(build_properties)))
+ if viewer_url:
+ logdog_dict[base_benchmark_name]['perf_results'] = viewer_url
+ if upload_failure:
+ logdog_dict[base_benchmark_name]['upload_failed'] = 'True'
+
+
+def print_duration(step, start, end):
+ logging.info('Duration of %s: %d seconds' % (step, end - start))
+
+
+def main():
+ """ See collect_task.collect_task for more on the merge script API. """
+ logging.info(sys.argv)
+ parser = argparse.ArgumentParser()
+ # configuration-name (previously perf-id) is the name of bot the tests run on
+ # For example, buildbot-test is the name of the android-go-perf bot
+ # configuration-name and results-url are set in the json file which is going
+ # away tools/perf/core/chromium.perf.fyi.extras.json
+ parser.add_argument('--configuration-name', help=argparse.SUPPRESS)
+
+ parser.add_argument('--build-properties', help=argparse.SUPPRESS)
+ parser.add_argument('--summary-json', help=argparse.SUPPRESS)
+ parser.add_argument('--task-output-dir', help=argparse.SUPPRESS)
+ parser.add_argument('-o', '--output-json', required=True, help=argparse.SUPPRESS)
+ parser.add_argument(
+ '--skip-perf',
+ action='store_true',
+ help='In lightweight mode, using --skip-perf will skip the performance'
+ ' data handling.')
+ parser.add_argument(
+ '--lightweight',
+ action='store_true',
+ help='Choose the lightweight mode in which the perf result handling'
+ ' is performed on a separate VM.')
+ parser.add_argument('json_files', nargs='*', help=argparse.SUPPRESS)
+ parser.add_argument(
+ '--smoke-test-mode',
+ action='store_true',
+ help='This test should be run in smoke test mode'
+ ' meaning it does not upload to the perf dashboard')
+
+ args = parser.parse_args()
+
+ output_results_dir = tempfile.mkdtemp('outputresults')
+ try:
+ return_code, _ = process_perf_results(args.output_json, args.configuration_name,
+ args.build_properties, args.task_output_dir,
+ args.smoke_test_mode, output_results_dir,
+ args.lightweight, args.skip_perf)
+ return return_code
+ finally:
+ shutil.rmtree(output_results_dir)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
