Add a new script to upload perf tests.
Also add a script to do the bridge between a python 2 and a python 3 interpreter.
This should be removed when the merge scripts will be using python 3 (https://crbug.com/webrtc/13835).
Note that webrtc_dashboard_upload.py will be removed when the new script is stabilized.
Bug: webrtc:13806
Change-Id: I806fa11f417ef37674bdaeb5126c71570e3697d7
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/255560
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Christoffer Jansson <jansson@google.com>
Reviewed-by: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Christoffer Jansson <jansson@webrtc.org>
Commit-Queue: Jeremy Leconte <jleconte@google.com>
Cr-Commit-Position: refs/heads/main@{#36252}
diff --git a/PRESUBMIT.py b/PRESUBMIT.py
index 5c7db2e..f01338f 100755
--- a/PRESUBMIT.py
+++ b/PRESUBMIT.py
@@ -820,9 +820,10 @@
return input_api.os_path.join(input_api.PresubmitLocalPath(), *args)
excluded_files = [
- # This test should be run manually after webrtc_dashboard_upload target
+ # These tests should be run manually after webrtc_dashboard_upload target
# has been built.
- 'catapult_uploader_test.py'
+ 'catapult_uploader_test.py',
+ 'process_perf_results_test.py',
]
test_directories = [
diff --git a/tools_webrtc/perf/catapult_uploader.py b/tools_webrtc/perf/catapult_uploader.py
index c33ac89..d07c287 100644
--- a/tools_webrtc/perf/catapult_uploader.py
+++ b/tools_webrtc/perf/catapult_uploader.py
@@ -14,6 +14,7 @@
import time
import zlib
+from typing import Optional
import dataclasses
import httplib2
@@ -53,7 +54,7 @@
build_page_url: str
dashboard_url: str
input_results_file: str
- output_json_file: str
+ output_json_file: Optional[str] = None
wait_timeout_sec: datetime.timedelta = datetime.timedelta(seconds=1200)
wait_polling_period_sec: datetime.timedelta = datetime.timedelta(seconds=120)
@@ -305,5 +306,5 @@
exit_code = UploadToDashboardImpl(options)
except RuntimeError as e:
print(e)
- return 2
+ return 1
return exit_code
diff --git a/tools_webrtc/perf/process_perf_results.py b/tools_webrtc/perf/process_perf_results.py
new file mode 100644
index 0000000..e91b1f6
--- /dev/null
+++ b/tools_webrtc/perf/process_perf_results.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env vpython3
+
+# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+"""Adds build info to perf results and uploads them.
+
+The tests don't know which bot executed the tests or at what revision, so we
+need to take their output and enrich it with this information. We load the proto
+from the tests, add the build information as shared diagnostics and then
+upload it to the dashboard.
+
+This script can't be in recipes, because we can't access the catapult APIs from
+there. It needs to be here source-side.
+"""
+
+import argparse
+import json
+import os
+import sys
+
+from pathlib import Path
+
+# Even if protobuf is not used directly, this allows transitive imports
+# of the protobuf library to use the vpython wheel specified in the root
+# level .vpython (see bugs.webrtc.org/12211 for context).
+import google.protobuf # pylint: disable=unused-import
+
+
+def _ConfigurePythonPath(outdir):
+ # We just yank the python scripts we require into the PYTHONPATH. You could
+ # also imagine a solution where we use for instance
+ # protobuf:py_proto_runtime to copy catapult and protobuf code to out/.
+ # This is the convention in Chromium and WebRTC python scripts. We do need
+ # to build histogram_pb2 however, so that's why we add out/ to sys.path
+ # below.
+ #
+ # It would be better if there was an equivalent to py_binary in GN, but
+ # there's not.
+ script_dir = os.path.dirname(os.path.realpath(__file__))
+ checkout_root = os.path.abspath(os.path.join(script_dir, os.pardir,
+ os.pardir))
+
+ sys.path.insert(
+ 0, os.path.join(checkout_root, 'third_party', 'catapult', 'tracing'))
+ sys.path.insert(
+ 0, os.path.join(checkout_root, 'third_party', 'protobuf', 'python'))
+
+ # The webrtc_dashboard_upload gn rule will build the protobuf stub for
+ # python, so put it in the path for this script before we attempt to import
+ # it.
+ histogram_proto_path = os.path.join(outdir, 'pyproto', 'tracing', 'tracing',
+ 'proto')
+ sys.path.insert(0, histogram_proto_path)
+
+ # Fail early in case the proto hasn't been built.
+ from tracing.proto import histogram_proto
+ if not histogram_proto.HAS_PROTO:
+ print('Could not find histogram_pb2. You need to build the '
+ 'webrtc_dashboard_upload target before invoking this '
+ 'script. Expected to find '
+ 'histogram_pb2.py in %s.' % histogram_proto_path)
+ return 1
+ return 0
+
+
+def _UploadToDasboard(args):
+ build_properties = json.loads(args.build_properties)
+ exit_code = _ConfigurePythonPath(build_properties['outdir'])
+ if exit_code != 0:
+ return exit_code
+
+ import catapult_uploader
+
+ perftest_outputs = [
+ f.absolute() for f in Path(args.task_output_dir).rglob('perftest-output*')
+ if f.is_file()
+ ]
+ for perftest_output in perftest_outputs:
+ uploader_options = catapult_uploader.UploaderOptions(
+ perf_dashboard_machine_group=(
+ build_properties['perf_dashboard_machine_group']),
+ bot=build_properties['bot'],
+ webrtc_git_hash=build_properties['webrtc_git_hash'],
+ commit_position=build_properties['commit_position'],
+ build_page_url=build_properties['build_page_url'],
+ dashboard_url=build_properties['dashboard_url'],
+ test_suite=args.test_suite,
+ input_results_file=perftest_output,
+ )
+ exit_code = catapult_uploader.UploadToDashboard(uploader_options)
+ if exit_code != 0:
+ return exit_code
+ return 0
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--build-properties', help=argparse.SUPPRESS)
+ parser.add_argument('--summary-json', help=argparse.SUPPRESS)
+ parser.add_argument('--task-output-dir', help=argparse.SUPPRESS)
+ parser.add_argument('--test-suite', help=argparse.SUPPRESS)
+ parser.add_argument('-o', '--output-json', help=argparse.SUPPRESS)
+ parser.add_argument('json_files', nargs='*', help=argparse.SUPPRESS)
+ args = parser.parse_args()
+
+ exit_code = _UploadToDasboard(args)
+ if exit_code != 0:
+ with open(args.output_json, 'w') as f:
+ json.dump({
+ "global_tags": ["UNRELIABLE_RESULTS"],
+ "missing_shards": [0]
+ }, f)
+ return exit_code
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/tools_webrtc/perf/process_perf_results_py2.py b/tools_webrtc/perf/process_perf_results_py2.py
new file mode 100644
index 0000000..14b6858
--- /dev/null
+++ b/tools_webrtc/perf/process_perf_results_py2.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env vpython3
+
+# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+"""Calls process_perf_results.py with a python 3 interpreter."""
+
+import sys
+import subprocess
+
+
+# TODO(crbug.com/webrtc/13835): Delete this file and use
+# process_perf_results.py instead.
+def main():
+ cmd = sys.argv[0].replace('_py2', '')
+ print('Calling "%s" with py3 in case this script was called with py2.' % cmd)
+ return subprocess.call(['vpython3', cmd] + sys.argv[1:])
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/tools_webrtc/perf/process_perf_results_test.py b/tools_webrtc/perf/process_perf_results_test.py
new file mode 100644
index 0000000..3aa5afd
--- /dev/null
+++ b/tools_webrtc/perf/process_perf_results_test.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env vpython3
+
+# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import os
+import sys
+
+import unittest
+from unittest import mock
+
+_SCRIPT_DIR = os.path.dirname(__file__)
+_SRC_DIR = os.path.normpath(os.path.join(_SCRIPT_DIR, '..', '..'))
+
+sys.path.insert(0, os.path.join(_SRC_DIR, 'third_party', 'protobuf', 'python'))
+import process_perf_results
+
+
+class ProcessPerfResultsTest(unittest.TestCase):
+ def testConfigurePythonPath(self):
+ # pylint: disable=protected-access
+ self.assertEqual(
+ 0,
+ process_perf_results._ConfigurePythonPath(
+ os.path.join(_SRC_DIR, 'out/Default')))
+
+ def testUploadToDasboard(self):
+ outdir = os.path.join(_SRC_DIR, 'out/Default')
+ args = mock.Mock(
+ build_properties='{' + '"outdir":"' + outdir + '", ' +
+ '"perf_dashboard_machine_group":"mock_machine_group", ' +
+ '"bot":"mock_bot", ' + '"webrtc_git_hash":"mock_webrtc_git_hash", ' +
+ '"commit_position":"123456", ' +
+ '"build_page_url":"mock_build_page_url", ' +
+ '"dashboard_url":"mock_dashboard_url"' + '}',
+ summary_json='mock_sumary_json',
+ task_output_dir='mock_task_output_dir',
+ test_suite='mock_test_suite',
+ )
+ perftest_output = mock.Mock(
+ absolute=lambda: 'dummy_path/perftest-output.pb',
+ is_file=lambda: True,
+ )
+ with mock.patch('pathlib.Path.rglob') as mocked_rglob:
+ with mock.patch('catapult_uploader.UploadToDashboard') as mocked_upload:
+ mocked_rglob.return_value = [perftest_output]
+ mocked_upload.return_value = 0
+ # pylint: disable=protected-access
+ self.assertEqual(0, process_perf_results._UploadToDasboard(args))
+
+ import catapult_uploader
+ mocked_upload.assert_called_once_with(
+ catapult_uploader.UploaderOptions(
+ perf_dashboard_machine_group='mock_machine_group',
+ bot='mock_bot',
+ test_suite='mock_test_suite',
+ webrtc_git_hash='mock_webrtc_git_hash',
+ commit_position='123456',
+ build_page_url='mock_build_page_url',
+ dashboard_url='mock_dashboard_url',
+ input_results_file=perftest_output.absolute()))
+
+
+if (__name__) == '__main__':
+ unittest.main()
