| #!/usr/bin/env python |
| # Copyright 2016 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Tests for results_stats.""" |
| |
| import os |
| import sys |
| |
| import unittest |
| |
| try: |
| import numpy as np |
| except ImportError: |
| np = None |
| |
| sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), |
| '..'))) |
| from statistical_analysis import results_stats |
| |
| |
| class StatisticalBenchmarkResultsAnalysisTest(unittest.TestCase): |
| """Unit testing of several functions in results_stats.""" |
| |
| def testGetChartsFromBenchmarkResultJson(self): |
| """Unit test for errors raised when getting the charts element. |
| |
| Also makes sure that the 'trace' element is deleted if it exists. |
| """ |
| input_json_wrong_format = {'charts_wrong': {}} |
| input_json_empty = {'charts': {}} |
| with self.assertRaises(ValueError): |
| (results_stats.GetChartsFromBenchmarkResultJson(input_json_wrong_format)) |
| with self.assertRaises(ValueError): |
| (results_stats.GetChartsFromBenchmarkResultJson(input_json_empty)) |
| |
| input_json_with_trace = {'charts': |
| {'trace': {}, |
| 'Ex_metric_1': |
| {'Ex_page_1': {'type': 'list_of_scalar_values', |
| 'values': [1, 2]}, |
| 'Ex_page_2': {'type': 'histogram', |
| 'values': [1, 2]}}, |
| 'Ex_metric_2': |
| {'Ex_page_1': {'type': 'list_of_scalar_values'}, |
| 'Ex_page_2': {'type': 'list_of_scalar_values', |
| 'values': [1, 2]}}}} |
| |
| output = (results_stats. |
| GetChartsFromBenchmarkResultJson(input_json_with_trace)) |
| expected_output = {'Ex_metric_1': |
| {'Ex_page_1': {'type': 'list_of_scalar_values', |
| 'values': [1, 2]}}, |
| 'Ex_metric_2': |
| {'Ex_page_2': {'type': 'list_of_scalar_values', |
| 'values': [1, 2]}}} |
| self.assertEqual(output, expected_output) |
| |
| def testCreateBenchmarkResultDict(self): |
| """Unit test for benchmark result dict created from a benchmark json. |
| |
| Creates a json of the format created by tools/perf/run_benchmark and then |
| compares the output dict against an expected predefined output dict. |
| """ |
| metric_names = ['messageloop_start_time', |
| 'open_tabs_time', |
| 'window_display_time'] |
| metric_values = [[55, 72, 60], [54, 42, 65], [44, 89]] |
| |
| input_json = {'charts': {}} |
| for metric, metric_vals in zip(metric_names, metric_values): |
| input_json['charts'][metric] = {'summary': |
| {'values': metric_vals, |
| 'type': 'list_of_scalar_values'}} |
| |
| output = results_stats.CreateBenchmarkResultDict(input_json) |
| expected_output = {'messageloop_start_time': [55, 72, 60], |
| 'open_tabs_time': [54, 42, 65], |
| 'window_display_time': [44, 89]} |
| |
| self.assertEqual(output, expected_output) |
| |
| def testCreatePagesetBenchmarkResultDict(self): |
| """Unit test for pageset benchmark result dict created from benchmark json. |
| |
| Creates a json of the format created by tools/perf/run_benchmark when it |
| includes a pageset and then compares the output dict against an expected |
| predefined output dict. |
| """ |
| metric_names = ['messageloop_start_time', |
| 'open_tabs_time', |
| 'window_display_time'] |
| metric_values = [[55, 72, 60], [54, 42, 65], [44, 89]] |
| page_names = ['Ex_page_1', 'Ex_page_2'] |
| |
| input_json = {'charts': {}} |
| for metric, metric_vals in zip(metric_names, metric_values): |
| input_json['charts'][metric] = {'summary': |
| {'values': [0, 1, 2, 3], |
| 'type': 'list_of_scalar_values'}} |
| for page in page_names: |
| input_json['charts'][metric][page] = {'values': metric_vals, |
| 'type': 'list_of_scalar_values'} |
| |
| output = results_stats.CreatePagesetBenchmarkResultDict(input_json) |
| expected_output = {'messageloop_start_time': {'Ex_page_1': [55, 72, 60], |
| 'Ex_page_2': [55, 72, 60]}, |
| 'open_tabs_time': {'Ex_page_1': [54, 42, 65], |
| 'Ex_page_2': [54, 42, 65]}, |
| 'window_display_time': {'Ex_page_1': [44, 89], |
| 'Ex_page_2': [44, 89]}} |
| |
| self.assertEqual(output, expected_output) |
| |
| def testCombinePValues(self): |
| """Unit test for Fisher's Method that combines multiple p-values.""" |
| test_p_values = [0.05, 0.04, 0.10, 0.07, 0.01] |
| |
| expected_output = 0.00047334256271885721 |
| output = results_stats.CombinePValues(test_p_values) |
| |
| self.assertEqual(output, expected_output) |
| |
| def CreateRandomNormalDistribution(self, mean=0, size=30): |
| """Creates two pseudo random samples for testing in multiple methods.""" |
| if not np: |
| raise ImportError('This function requires Numpy.') |
| |
| np.random.seed(0) |
| sample = np.random.normal(loc=mean, scale=1, size=size) |
| |
| return sample |
| |
| def testIsNormallyDistributed(self): |
| """Unit test for values returned when testing for normality.""" |
| if not np: |
| self.skipTest("Numpy is not installed.") |
| |
| test_samples = [self.CreateRandomNormalDistribution(0), |
| self.CreateRandomNormalDistribution(1)] |
| |
| expected_outputs = [(True, 0.5253966450691223), |
| (True, 0.5253913402557373)] |
| for sample, expected_output in zip(test_samples, expected_outputs): |
| output = results_stats.IsNormallyDistributed(sample) |
| |
| self.assertEqual(output, expected_output) |
| |
| def testAreSamplesDifferent(self): |
| """Unit test for values returned after running the statistical tests. |
| |
| Creates two pseudo-random normally distributed samples to run the |
| statistical tests and compares the resulting answer and p-value against |
| their pre-calculated values. |
| """ |
| test_samples = [3 * [0, 0, 2, 4, 4], 3 * [5, 5, 7, 9, 9]] |
| with self.assertRaises(results_stats.SampleSizeError): |
| results_stats.AreSamplesDifferent(test_samples[0], test_samples[1], |
| test=results_stats.MANN) |
| with self.assertRaises(results_stats.NonNormalSampleError): |
| results_stats.AreSamplesDifferent(test_samples[0], test_samples[1], |
| test=results_stats.WELCH) |
| |
| test_samples_equal = (20 * [1], 20 * [1]) |
| expected_output_equal = (False, 1.0) |
| output_equal = results_stats.AreSamplesDifferent(test_samples_equal[0], |
| test_samples_equal[1], |
| test=results_stats.MANN) |
| self.assertEqual(output_equal, expected_output_equal) |
| |
| if not np: |
| self.skipTest("Numpy is not installed.") |
| |
| test_samples = [self.CreateRandomNormalDistribution(0), |
| self.CreateRandomNormalDistribution(1)] |
| test_options = results_stats.ALL_TEST_OPTIONS |
| |
| expected_outputs = [(True, 2 * 0.00068516628052438266), |
| (True, 0.0017459498829507842), |
| (True, 0.00084765230478226514)] |
| |
| for test, expected_output in zip(test_options, expected_outputs): |
| output = results_stats.AreSamplesDifferent(test_samples[0], |
| test_samples[1], |
| test=test) |
| self.assertEqual(output, expected_output) |
| |
| def testAssertThatKeysMatch(self): |
| """Unit test for exception raised when input dicts' metrics don't match.""" |
| differing_input_dicts = [{'messageloop_start_time': [55, 72, 60], |
| 'display_time': [44, 89]}, |
| {'messageloop_start_time': [55, 72, 60]}] |
| with self.assertRaises(results_stats.DictMismatchError): |
| results_stats.AssertThatKeysMatch(differing_input_dicts[0], |
| differing_input_dicts[1]) |
| |
| def testAreBenchmarkResultsDifferent(self): |
| """Unit test for statistical test outcome dict.""" |
| test_input_dicts = [{'open_tabs_time': |
| self.CreateRandomNormalDistribution(0), |
| 'display_time': |
| self.CreateRandomNormalDistribution(0)}, |
| {'open_tabs_time': |
| self.CreateRandomNormalDistribution(0), |
| 'display_time': |
| self.CreateRandomNormalDistribution(1)}] |
| test_options = results_stats.ALL_TEST_OPTIONS |
| |
| expected_outputs = [{'open_tabs_time': (False, 2 * 0.49704973080841425), |
| 'display_time': (True, 2 * 0.00068516628052438266)}, |
| {'open_tabs_time': (False, 1.0), |
| 'display_time': (True, 0.0017459498829507842)}, |
| {'open_tabs_time': (False, 1.0), |
| 'display_time': (True, 0.00084765230478226514)}] |
| |
| for test, expected_output in zip(test_options, expected_outputs): |
| output = results_stats.AreBenchmarkResultsDifferent(test_input_dicts[0], |
| test_input_dicts[1], |
| test=test) |
| self.assertEqual(output, expected_output) |
| |
| def testArePagesetBenchmarkResultsDifferent(self): |
| """Unit test for statistical test outcome dict.""" |
| distributions = (self.CreateRandomNormalDistribution(0), |
| self.CreateRandomNormalDistribution(1)) |
| test_input_dicts = ({'open_tabs_time': {'Ex_page_1': distributions[0], |
| 'Ex_page_2': distributions[0]}, |
| 'display_time': {'Ex_page_1': distributions[1], |
| 'Ex_page_2': distributions[1]}}, |
| {'open_tabs_time': {'Ex_page_1': distributions[0], |
| 'Ex_page_2': distributions[1]}, |
| 'display_time': {'Ex_page_1': distributions[1], |
| 'Ex_page_2': distributions[0]}}) |
| test_options = results_stats.ALL_TEST_OPTIONS |
| |
| expected_outputs = ({'open_tabs_time': # Mann. |
| {'Ex_page_1': (False, 2 * 0.49704973080841425), |
| 'Ex_page_2': (True, 2 * 0.00068516628052438266)}, |
| 'display_time': |
| {'Ex_page_1': (False, 2 * 0.49704973080841425), |
| 'Ex_page_2': (True, 2 * 0.00068516628052438266)}}, |
| {'open_tabs_time': # Kolmogorov. |
| {'Ex_page_1': (False, 1.0), |
| 'Ex_page_2': (True, 0.0017459498829507842)}, |
| 'display_time': |
| {'Ex_page_1': (False, 1.0), |
| 'Ex_page_2': (True, 0.0017459498829507842)}}, |
| {'open_tabs_time': # Welch. |
| {'Ex_page_1': (False, 1.0), |
| 'Ex_page_2': (True, 0.00084765230478226514)}, |
| 'display_time': |
| {'Ex_page_1': (False, 1.0), |
| 'Ex_page_2': (True, 0.00084765230478226514)}}) |
| |
| for test, expected_output in zip(test_options, expected_outputs): |
| output = (results_stats. |
| ArePagesetBenchmarkResultsDifferent(test_input_dicts[0], |
| test_input_dicts[1], |
| test=test)) |
| self.assertEqual(output, expected_output) |
| |
| |
| if __name__ == '__main__': |
| sys.exit(unittest.main()) |
