| #!/usr/bin/env python |
| # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Prepare tests that require re-baselining for input to make_expectations.py. |
| |
| The regularly running perf-AV tests require re-baselineing of expectations |
| about once a week. The steps involved in rebaselining are: |
| |
| 1.) Identify the tests to update, based off reported e-mail results. |
| 2.) Figure out reva and revb values, which is the starting and ending revision |
| numbers for the range that we should use to obtain new thresholds. |
| 3.) Modify lines in perf_expectations.json referring to the tests to be updated, |
| so that they may be used as input to make_expectations.py. |
| |
| This script automates the last step above. |
| |
| Here's a sample line from perf_expectations.json: |
| |
| "win-release/media_tests_av_perf/fps/tulip2.m4a": {"reva": 163299, \ |
| "revb": 164141, "type": "absolute", "better": "higher", "improve": 0, \ |
| "regress": 0, "sha1": "54d94538"}, |
| |
| To get the above test ready for input to make_expectations.py, it should become: |
| |
| "win-release/media_tests_av_perf/fps/tulip2.m4a": {"reva": <new reva>, \ |
| "revb": <new revb>, "type": "absolute", "better": "higher", "improve": 0, \ |
| "regress": 0}, |
| |
| Examples: |
| |
| 1.) To update the test specified above and get baseline |
| values using the revision range 12345 and 23456, run this script with a command |
| line like this: |
| python update_perf_expectations.py -f \ |
| win-release/media_tests_av_perf/fps/tulip2.m4a --reva 12345 --revb 23456 |
| Or, using an input file, |
| where the input file contains a single line with text |
| win-release/media_tests_av_perf/fps/tulip2.m4a |
| run with this command line: |
| python update_perf_expectations.py -i input.txt --reva 12345 --revb 23456 |
| |
| 2.) Let's say you want to update all seek tests on windows, and get baseline |
| values using the revision range 12345 and 23456. |
| Run this script with this command line: |
| python update_perf_expectations.py -f win-release/media_tests_av_perf/seek/ \ |
| --reva 12345 --revb 23456 |
| Or: |
| python update_perf_expectations.py -f win-release/.*/seek/ --reva 12345 \ |
| --revb 23456 |
| |
| Or, using an input file, |
| where the input file contains a single line with text win-release/.*/seek/: |
| python update_perf_expectations.py -i input.txt --reva 12345 --revb 23456 |
| |
| 3.) Similarly, if you want to update seek tests on all platforms |
| python update_perf_expectations.py -f .*-release/.*/seek/ --reva 12345 \ |
| --revb 23456 |
| |
| """ |
| |
| import logging |
| from optparse import OptionParser |
| import os |
| import re |
| |
| import make_expectations as perf_ex_lib |
| |
| # Default logging is INFO. Use --verbose to enable DEBUG logging. |
| _DEFAULT_LOG_LEVEL = logging.INFO |
| |
| |
| def GetTestsToUpdate(contents, all_test_keys): |
| """Parses input contents and obtains tests to be re-baselined. |
| |
| Args: |
| contents: string containing contents of input file. |
| all_test_keys: list of keys of test dictionary. |
| Returns: |
| A list of keys for tests that should be updated. |
| """ |
| # Each line of the input file specifies a test case to update. |
| tests_list = [] |
| for test_case_filter in contents.splitlines(): |
| # Skip any empty lines. |
| if test_case_filter: |
| # Sample expected line: |
| # win-release/media_tests_av_perf/seek/\ |
| # CACHED_BUFFERED_SEEK_NoConstraints_crowd1080.ogv |
| # Or, if reg-ex, then sample line: |
| # win-release/media-tests_av_perf/seek* |
| # Skip any leading spaces if they exist in the input file. |
| logging.debug('Trying to match %s', test_case_filter) |
| tests_list.extend(GetMatchingTests(test_case_filter.strip(), |
| all_test_keys)) |
| return tests_list |
| |
| |
| def GetMatchingTests(tests_to_update, all_test_keys): |
| """Parses input reg-ex filter and obtains tests to be re-baselined. |
| |
| Args: |
| tests_to_update: reg-ex string specifying tests to be updated. |
| all_test_keys: list of keys of tests dictionary. |
| Returns: |
| A list of keys for tests that should be updated. |
| """ |
| tests_list = [] |
| search_string = re.compile(tests_to_update) |
| # Get matching tests from the dictionary of tests |
| for test_key in all_test_keys: |
| if search_string.match(test_key): |
| tests_list.append(test_key) |
| logging.debug('%s will be updated', test_key) |
| logging.info('%s tests found matching reg-ex: %s', len(tests_list), |
| tests_to_update) |
| return tests_list |
| |
| |
| def PrepareTestsForUpdate(tests_to_update, all_tests, reva, revb): |
| """Modifies value of tests that are to re-baselined: |
| Set reva and revb values to specified new values. Remove sha1. |
| |
| Args: |
| tests_to_update: list of tests to be updated. |
| all_tests: dictionary of all tests. |
| reva: oldest revision in range to use for new values. |
| revb: newest revision in range to use for new values. |
| Raises: |
| ValueError: If reva or revb are not valid ints, or if either |
| of them are negative. |
| """ |
| reva = int(reva) |
| revb = int(revb) |
| |
| if reva < 0 or revb < 0: |
| raise ValueError('Revision values should be positive.') |
| # Ensure reva is less than revb. |
| # (this is similar to the check done in make_expectations.py) |
| if revb < reva: |
| temp = revb |
| revb = reva |
| reva = temp |
| for test_key in tests_to_update: |
| # Get original test from the dictionary of tests |
| test_value = all_tests[test_key] |
| if test_value: |
| # Sample line in perf_expectations.json: |
| # "linux-release/media_tests _av_perf/dropped_frames/crowd360.webm":\ |
| # {"reva": 155180, "revb": 155280, "type": "absolute", \ |
| # "better": "lower", "improve": 0, "regress": 3, "sha1": "276ba29c"}, |
| # Set new revision range |
| test_value['reva'] = reva |
| test_value['revb'] = revb |
| # Remove sha1 to indicate this test requires an update |
| # Check first to make sure it exist. |
| if 'sha1' in test_value: |
| del test_value['sha1'] |
| else: |
| logging.warning('%s does not exist.', test_key) |
| logging.info('Done preparing tests for update.') |
| |
| |
| def GetCommandLineOptions(): |
| """Parse command line arguments. |
| |
| Returns: |
| An options object containing command line arguments and their values. |
| """ |
| parser = OptionParser() |
| |
| parser.add_option('--reva', dest='reva', type='int', |
| help='Starting revision of new range.', |
| metavar='START_REVISION') |
| parser.add_option('--revb', dest='revb', type='int', |
| help='Ending revision of new range.', |
| metavar='END_REVISION') |
| parser.add_option('-f', dest='tests_filter', |
| help='Regex to use for filtering tests to be updated. ' |
| 'At least one of -filter or -input_file must be provided. ' |
| 'If both are provided, then input-file is used.', |
| metavar='FILTER', default='') |
| parser.add_option('-i', dest='input_file', |
| help='Optional path to file with reg-exes for tests to' |
| ' update. If provided, it overrides the filter argument.', |
| metavar='INPUT_FILE', default='') |
| parser.add_option('--config', dest='config_file', |
| default=perf_ex_lib.DEFAULT_CONFIG_FILE, |
| help='Set the config file to FILE.', metavar='FILE') |
| parser.add_option('-v', dest='verbose', action='store_true', default=False, |
| help='Enable verbose output.') |
| options = parser.parse_args()[0] |
| return options |
| |
| |
| def Main(): |
| """Main driver function.""" |
| options = GetCommandLineOptions() |
| |
| _SetLogger(options.verbose) |
| # Do some command-line validation |
| if not options.input_file and not options.tests_filter: |
| logging.error('At least one of input-file or test-filter must be provided.') |
| exit(1) |
| if options.input_file and options.tests_filter: |
| logging.error('Specify only one of input file or test-filter.') |
| exit(1) |
| if not options.reva or not options.revb: |
| logging.error('Start and end revision of range must be specified.') |
| exit(1) |
| |
| # Load config. |
| config = perf_ex_lib.ConvertJsonIntoDict( |
| perf_ex_lib.ReadFile(options.config_file)) |
| |
| # Obtain the perf expectations file from the config file. |
| perf_file = os.path.join( |
| os.path.dirname(options.config_file), config['perf_file']) |
| |
| # We should have all the information we require now. |
| # On to the real thang. |
| # First, get all the existing tests from the original perf_expectations file. |
| all_tests = perf_ex_lib.ConvertJsonIntoDict( |
| perf_ex_lib.ReadFile(perf_file)) |
| all_test_keys = all_tests.keys() |
| # Remove the load key, because we don't want to modify it. |
| all_test_keys.remove('load') |
| # Keep tests sorted, like in the original file. |
| all_test_keys.sort() |
| |
| # Next, get all tests that have been identified for an update. |
| tests_to_update = [] |
| if options.input_file: |
| # Tests to update have been specified in an input_file. |
| # Get contents of file. |
| tests_filter = perf_ex_lib.ReadFile(options.input_file) |
| elif options.tests_filter: |
| # Tests to update have been specified as a reg-ex filter. |
| tests_filter = options.tests_filter |
| |
| # Get tests to update based on filter specified. |
| tests_to_update = GetTestsToUpdate(tests_filter, all_test_keys) |
| logging.info('Done obtaining matching tests.') |
| |
| # Now, prepare tests for update. |
| PrepareTestsForUpdate(tests_to_update, all_tests, options.reva, options.revb) |
| |
| # Finally, write modified tests back to perf_expectations file. |
| perf_ex_lib.WriteJson(perf_file, all_tests, all_test_keys, |
| calculate_sha1=False) |
| logging.info('Done writing tests for update to %s.', perf_file) |
| |
| |
| def _SetLogger(verbose): |
| log_level = _DEFAULT_LOG_LEVEL |
| if verbose: |
| log_level = logging.DEBUG |
| logging.basicConfig(level=log_level, format='%(message)s') |
| |
| |
| if __name__ == '__main__': |
| Main() |