commit 130a8b6366ee67bc632b8cbe244e954d11eb520e
author Jamie Madill <jmadill@chromium.org> Mon Aug 16 17:09:10 2021 -0400
committer Angle LUCI CQ <angle-scoped@luci-project-accounts.iam.gserviceaccount.com> Tue Aug 17 11:25:26 2021 +0000
tree ad6465c3b33936b3a733461d2ba6d13fdb5b7da8
parent 0be285c6ee727a2fd7bb0e239e46693bf1958c7d

Perf Runner: Properly merge histograms.

The prior method was emitting 5 histograms per test. The new script
method merges the 5 histogram samples together and recomputes the
running statistics.

Should improve reporting on the dashboard.

Bug: angleproject:6090
Change-Id: I58a5e020f3a3257a8545df2785bc3eaa811ee38e
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/3096550
Reviewed-by: Yuly Novikov <ynovikov@chromium.org>
Commit-Queue: Jamie Madill <jmadill@chromium.org>

src/tests/run_perf_tests.py

diff --git a/src/tests/run_perf_tests.py b/src/tests/run_perf_tests.py
index 3a79e6d..14ed4ac 100755
--- a/src/tests/run_perf_tests.py
+++ b/src/tests/run_perf_tests.py
@@ -31,6 +31,7 @@
 sys.path.append(os.path.join(ANGLE_DIR, 'third_party', 'catapult', 'tracing'))
 from tracing.value import histogram
 from tracing.value import histogram_set
+from tracing.value import merge_histograms
 
 DEFAULT_TEST_SUITE = 'angle_perftests'
 DEFAULT_LOG = 'info'
@@ -293,6 +294,7 @@
         logging.info('Running %s %d times with %d trials and %d steps per trial.' %
                      (test, args.samples_per_test, args.trials_per_sample, steps_per_trial))
         wall_times = []
+        test_histogram_set = histogram_set.HistogramSet()
         for sample in range(args.samples_per_test):
             if total_errors >= args.max_errors:
                 logging.error('Error count exceeded max errors (%d). Aborting.' % args.max_errors)
@@ -321,7 +323,7 @@
                         sample_json = json.load(histogram_file)
                         sample_histogram = histogram_set.HistogramSet()
                         sample_histogram.ImportDicts(sample_json)
-                        histograms.Merge(sample_histogram)
+                        test_histogram_set.Merge(sample_histogram)
                 else:
                     logging.error('Failed to get sample for test %s' % test)
                     total_errors += 1
@@ -340,10 +342,22 @@
 
             if len(wall_times) > 1:
                 logging.info(
-                    "Mean wall_time for %s is %.2f, with coefficient of variation %.2f%%" %
+                    'Mean wall_time for %s is %.2f, with coefficient of variation %.2f%%' %
                     (test, _mean(wall_times), (_coefficient_of_variation(wall_times) * 100.0)))
             test_results[test] = {'expected': PASS, 'actual': PASS}
             results['num_failures_by_type'][PASS] += 1
+
+            # Merge the histogram set into one histogram
+            with common.temporary_file() as merge_histogram_path:
+                logging.info('Writing merged histograms to %s.' % merge_histogram_path)
+                with open(merge_histogram_path, 'w') as merge_histogram_file:
+                    json.dump(test_histogram_set.AsDicts(), merge_histogram_file)
+                    merge_histogram_file.close()
+                merged_dicts = merge_histograms.MergeHistograms(
+                    merge_histogram_path, groupby=['name'])
+                merged_histogram = histogram_set.HistogramSet()
+                merged_histogram.ImportDicts(merged_dicts)
+                histograms.Merge(merged_histogram)
         else:
             logging.error('Test %s failed to record some samples' % test)
             test_results[test] = {'expected': PASS, 'actual': FAIL, 'is_unexpected': True}