| #!/usr/bin/env python |
| # |
| # Copyright 2017, The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """ |
| Command line utility for running Android tests through TradeFederation. |
| |
| atest helps automate the flow of building test modules across the Android |
| code base and executing the tests via the TradeFederation test harness. |
| |
| atest is designed to support any test types that can be ran by TradeFederation. |
| """ |
| |
| import logging |
| import os |
| import subprocess |
| import sys |
| import tempfile |
| import time |
| |
| import atest_error |
| import atest_utils |
| import cli_translator |
| # pylint: disable=import-error |
| import constants |
| import module_info |
| import test_runner_handler |
| from test_runners import regression_test_runner |
| |
| EXPECTED_VARS = frozenset([ |
| constants.ANDROID_BUILD_TOP, |
| 'ANDROID_TARGET_OUT_TESTCASES', |
| constants.ANDROID_OUT]) |
| BUILD_STEP = 'build' |
| INSTALL_STEP = 'install' |
| TEST_STEP = 'test' |
| ALL_STEPS = [BUILD_STEP, INSTALL_STEP, TEST_STEP] |
| TEST_RUN_DIR_PREFIX = 'atest_run_%s_' |
| HELP_DESC = '''Build, install and run Android tests locally.''' |
| REBUILD_MODULE_INFO_FLAG = '--rebuild-module-info' |
| CUSTOM_ARG_FLAG = '--' |
| |
| EPILOG_TEXT = ''' |
| |
| |
| - - - - - - - - - |
| IDENTIFYING TESTS |
| - - - - - - - - - |
| |
| The positional argument <tests> should be a reference to one or more |
| of the tests you'd like to run. Multiple tests can be run in one command by |
| separating test references with spaces. |
| |
| Usage template: atest <reference_to_test_1> <reference_to_test_2> |
| |
| A <reference_to_test> can be satisfied by the test's MODULE NAME, |
| MODULE:CLASS, CLASS NAME, TF INTEGRATION TEST, FILE PATH or PACKAGE NAME. |
| Explanations and examples of each follow. |
| |
| |
| < MODULE NAME > |
| |
| Identifying a test by its module name will run the entire module. Input |
| the name as it appears in the LOCAL_MODULE or LOCAL_PACKAGE_NAME |
| variables in that test's Android.mk or Android.bp file. |
| |
| Note: Use < TF INTEGRATION TEST > to run non-module tests integrated |
| directly into TradeFed. |
| |
| Examples: |
| atest FrameworksServicesTests |
| atest CtsJankDeviceTestCases |
| |
| |
| < MODULE:CLASS > |
| |
| Identifying a test by its class name will run just the tests in that |
| class and not the whole module. MODULE:CLASS is the preferred way to run |
| a single class. MODULE is the same as described above. CLASS is the |
| name of the test class in the .java file. It can either be the fully |
| qualified class name or just the basic name. |
| |
| Examples: |
| atest FrameworksServicesTests:ScreenDecorWindowTests |
| atest FrameworksServicesTests:com.android.server.wm.ScreenDecorWindowTests |
| atest CtsJankDeviceTestCases:CtsDeviceJankUi |
| |
| |
| < CLASS NAME > |
| |
| A single class can also be run by referencing the class name without |
| the module name. |
| |
| Examples: |
| atest ScreenDecorWindowTests |
| atest CtsDeviceJankUi |
| |
| However, this will take more time than the equivalent MODULE:CLASS |
| reference, so we suggest using a MODULE:CLASS reference whenever |
| possible. Examples below are ordered by performance from the fastest |
| to the slowest: |
| |
| Examples: |
| atest FrameworksServicesTests:com.android.server.wm.ScreenDecorWindowTests |
| atest FrameworksServicesTests:ScreenDecorWindowTests |
| atest ScreenDecorWindowTests |
| |
| < TF INTEGRATION TEST > |
| |
| To run tests that are integrated directly into TradeFed (non-modules), |
| input the name as it appears in the output of the "tradefed.sh list |
| configs" cmd. |
| |
| Examples: |
| atest example/reboot |
| atest native-benchmark |
| |
| |
| < FILE PATH > |
| |
| Both module-based tests and integration-based tests can be run by |
| inputting the path to their test file or dir as appropriate. A single |
| class can also be run by inputting the path to the class's java file. |
| Both relative and absolute paths are supported. |
| |
| Example - 2 ways to run the `CtsJankDeviceTestCases` module via path: |
| 1. run module from android <repo root>: |
| atest cts/tests/jank/jank |
| |
| 2. from <android root>/cts/tests/jank: |
| atest . |
| |
| Example - run a specific class within CtsJankDeviceTestCases module |
| from <android repo> root via path: |
| atest cts/tests/jank/src/android/jank/cts/ui/CtsDeviceJankUi.java |
| |
| Example - run an integration test from <android repo> root via path: |
| atest tools/tradefederation/contrib/res/config/example/reboot.xml |
| |
| |
| < PACKAGE NAME > |
| |
| Atest supports searching tests from package name as well. |
| |
| Examples: |
| atest com.android.server.wm |
| atest android.jank.cts |
| |
| |
| - - - - - - - - - - - - - - - - - - - - - - - - - - |
| SPECIFYING INDIVIDUAL STEPS: BUILD, INSTALL OR RUN |
| - - - - - - - - - - - - - - - - - - - - - - - - - - |
| |
| The -b, -i and -t options allow you to specify which steps you want to run. |
| If none of those options are given, then all steps are run. If any of these |
| options are provided then only the listed steps are run. |
| |
| Note: -i alone is not currently support and can only be included with -t. |
| Both -b and -t can be run alone. |
| |
| Examples: |
| atest -b <test> (just build targets) |
| atest -t <test> (run tests only) |
| atest -it <test> (install apk and run tests) |
| atest -bt <test> (build targets, run tests, but skip installing apk) |
| |
| |
| Atest now has the ability to force a test to skip its cleanup/teardown step. |
| Many tests, e.g. CTS, cleanup the device after the test is run, so trying to |
| rerun your test with -t will fail without having the --disable-teardown |
| parameter. Use -d before -t to skip the test clean up step and test iteratively. |
| |
| atest -d <test> (disable installing apk and cleanning up device) |
| atest -t <test> |
| |
| Note that -t disables both setup/install and teardown/cleanup of the |
| device. So you can continue to rerun your test with just |
| |
| atest -t <test> |
| |
| as many times as you want. |
| |
| |
| - - - - - - - - - - - - - |
| RUNNING SPECIFIC METHODS |
| - - - - - - - - - - - - - |
| |
| It is possible to run only specific methods within a test class. To run |
| only specific methods, identify the class in any of the ways supported for |
| identifying a class (MODULE:CLASS, FILE PATH, etc) and then append the |
| name of the method or method using the following template: |
| |
| <reference_to_class>#<method1> |
| |
| Multiple methods can be specified with commas: |
| |
| <reference_to_class>#<method1>,<method2>,<method3>... |
| |
| Examples: |
| atest com.android.server.wm.ScreenDecorWindowTests#testMultipleDecors |
| |
| atest FrameworksServicesTests:ScreenDecorWindowTests#testFlagChange,testRemoval |
| |
| |
| - - - - - - - - - - - - - |
| RUNNING MULTIPLE CLASSES |
| - - - - - - - - - - - - - |
| |
| To run multiple classes, deliminate them with spaces just like you would |
| when running multiple tests. Atest will handle building and running |
| classes in the most efficient way possible, so specifying a subset of |
| classes in a module will improve performance over running the whole module. |
| |
| |
| Examples: |
| - two classes in same module: |
| atest FrameworksServicesTests:ScreenDecorWindowTests FrameworksServicesTests:DimmerTests |
| |
| - two classes, different modules: |
| atest FrameworksServicesTests:ScreenDecorWindowTests CtsJankDeviceTestCases:CtsDeviceJankUi |
| |
| |
| - - - - - - - - - - - |
| REGRESSION DETECTION |
| - - - - - - - - - - - |
| |
| Generate pre-patch or post-patch metrics without running regression detection: |
| |
| Example: |
| atest <test> --generate-baseline <optional iter> |
| atest <test> --generate-new-metrics <optional iter> |
| |
| Local regression detection can be run in three options: |
| |
| 1) Provide a folder containing baseline (pre-patch) metrics (generated |
| previously). Atest will run the tests n (default 5) iterations, generate |
| a new set of post-patch metrics, and compare those against existing metrics. |
| |
| Example: |
| atest <test> --detect-regression </path/to/baseline> --generate-new-metrics <optional iter> |
| |
| 2) Provide a folder containing post-patch metrics (generated previously). |
| Atest will run the tests n (default 5) iterations, generate a new set of |
| pre-patch metrics, and compare those against those provided. Note: the |
| developer needs to revert the device/tests to pre-patch state to generate |
| baseline metrics. |
| |
| Example: |
| atest <test> --detect-regression </path/to/new> --generate-baseline <optional iter> |
| |
| 3) Provide 2 folders containing both pre-patch and post-patch metrics. Atest |
| will run no tests but the regression detection algorithm. |
| |
| Example: |
| atest --detect-regression </path/to/baseline> </path/to/new> |
| |
| |
| ''' |
| |
| |
| def _parse_args(argv): |
| """Parse command line arguments. |
| |
| Args: |
| argv: A list of arguments. |
| |
| Returns: |
| An argspace.Namespace class instance holding parsed args. |
| """ |
| import argparse |
| parser = argparse.ArgumentParser( |
| description=HELP_DESC, |
| epilog=EPILOG_TEXT, |
| formatter_class=argparse.RawTextHelpFormatter) |
| parser.add_argument('tests', nargs='*', help='Tests to build and/or run.') |
| parser.add_argument('-b', '--build', action='append_const', dest='steps', |
| const=BUILD_STEP, help='Run a build.') |
| parser.add_argument('-i', '--install', action='append_const', dest='steps', |
| const=INSTALL_STEP, help='Install an APK.') |
| parser.add_argument('-t', '--test', action='append_const', dest='steps', |
| const=TEST_STEP, |
| help='Run the tests. WARNING: Many test configs force cleanup ' |
| 'of device after test run. In this case, -d must be used in previous ' |
| 'test run to disable cleanup, for -t to work. Otherwise, ' |
| 'device will need to be setup again with -i.') |
| parser.add_argument('-s', '--serial', |
| help='The device to run the test on.') |
| parser.add_argument('-d', '--disable-teardown', action='store_true', |
| help='Disables test teardown and cleanup.') |
| parser.add_argument('-m', REBUILD_MODULE_INFO_FLAG, action='store_true', |
| help='Forces a rebuild of the module-info.json file. ' |
| 'This may be necessary following a repo sync or ' |
| 'when writing a new test.') |
| parser.add_argument('-w', '--wait-for-debugger', action='store_true', |
| help='Only for instrumentation tests. Waits for ' |
| 'debugger prior to execution.') |
| parser.add_argument('-v', '--verbose', action='store_true', |
| help='Display DEBUG level logging.') |
| parser.add_argument('--generate-baseline', nargs='?', type=int, const=5, default=0, |
| help='Generate baseline metrics, run 5 iterations by default. ' |
| 'Provide an int argument to specify # iterations.') |
| parser.add_argument('--generate-new-metrics', nargs='?', type=int, const=5, default=0, |
| help='Generate new metrics, run 5 iterations by default. ' |
| 'Provide an int argument to specify # iterations.') |
| parser.add_argument('--detect-regression', nargs='*', |
| help='Run regression detection algorithm. Supply ' |
| 'path to baseline and/or new metrics folders.') |
| # This arg actually doesn't consume anything, it's primarily used for the |
| # help description and creating custom_args in the NameSpace object. |
| parser.add_argument('--', dest='custom_args', nargs='*', |
| help='Specify custom args for the test runners. ' |
| 'Everything after -- will be consumed as custom ' |
| 'args.') |
| # Store everything after '--' in custom_args. |
| pruned_argv = argv |
| custom_args_index = None |
| if CUSTOM_ARG_FLAG in argv: |
| custom_args_index = argv.index(CUSTOM_ARG_FLAG) |
| pruned_argv = argv[:custom_args_index] |
| args = parser.parse_args(pruned_argv) |
| args.custom_args = [] |
| if custom_args_index is not None: |
| args.custom_args = argv[custom_args_index+1:] |
| return args |
| |
| |
| def _configure_logging(verbose): |
| """Configure the logger. |
| |
| Args: |
| verbose: A boolean. If true display DEBUG level logs. |
| """ |
| if verbose: |
| logging.basicConfig(level=logging.DEBUG) |
| else: |
| logging.basicConfig(level=logging.INFO) |
| |
| |
| def _missing_environment_variables(): |
| """Verify the local environment has been set up to run atest. |
| |
| Returns: |
| List of strings of any missing environment variables. |
| """ |
| missing = filter(None, [x for x in EXPECTED_VARS if not os.environ.get(x)]) |
| if missing: |
| logging.error('Local environment doesn\'t appear to have been ' |
| 'initialized. Did you remember to run lunch? Expected ' |
| 'Environment Variables: %s.', missing) |
| return missing |
| |
| |
| def make_test_run_dir(): |
| """Make the test run dir in tmp. |
| |
| Returns: |
| A string of the dir path. |
| """ |
| utc_epoch_time = int(time.time()) |
| prefix = TEST_RUN_DIR_PREFIX % utc_epoch_time |
| return tempfile.mkdtemp(prefix=prefix) |
| |
| |
| def run_tests(run_commands): |
| """Shell out and execute tradefed run commands. |
| |
| Args: |
| run_commands: A list of strings of Tradefed run commands. |
| """ |
| logging.info('Running tests') |
| # TODO: Build result parser for run command. Until then display raw stdout. |
| for run_command in run_commands: |
| logging.debug('Executing command: %s', run_command) |
| subprocess.check_call(run_command, shell=True, stderr=subprocess.STDOUT) |
| |
| |
| def get_extra_args(args): |
| """Get extra args for test runners. |
| |
| Args: |
| args: arg parsed object. |
| |
| Returns: |
| Dict of extra args for test runners to utilize. |
| """ |
| extra_args = {} |
| if args.wait_for_debugger: |
| extra_args[constants.WAIT_FOR_DEBUGGER] = None |
| steps = args.steps or ALL_STEPS |
| if INSTALL_STEP not in steps: |
| extra_args[constants.DISABLE_INSTALL] = None |
| if args.disable_teardown: |
| extra_args[constants.DISABLE_TEARDOWN] = args.disable_teardown |
| if args.generate_baseline: |
| extra_args[constants.PRE_PATCH_ITERATIONS] = args.generate_baseline |
| if args.serial: |
| extra_args[constants.SERIAL] = args.serial |
| if args.generate_new_metrics: |
| extra_args[constants.POST_PATCH_ITERATIONS] = args.generate_new_metrics |
| if args.custom_args: |
| extra_args[constants.CUSTOM_ARGS] = args.custom_args |
| return extra_args |
| |
| |
| def _get_regression_detection_args(args, results_dir): |
| """Get args for regression detection test runners. |
| |
| Args: |
| args: parsed args object. |
| results_dir: string directory to store atest results. |
| |
| Returns: |
| Dict of args for regression detection test runner to utilize. |
| """ |
| regression_args = {} |
| pre_patch_folder = (os.path.join(results_dir, 'baseline-metrics') if args.generate_baseline |
| else args.detect_regression.pop(0)) |
| post_patch_folder = (os.path.join(results_dir, 'new-metrics') if args.generate_new_metrics |
| else args.detect_regression.pop(0)) |
| regression_args[constants.PRE_PATCH_FOLDER] = pre_patch_folder |
| regression_args[constants.POST_PATCH_FOLDER] = post_patch_folder |
| return regression_args |
| |
| |
| def _will_run_tests(args): |
| """Determine if there are tests to run. |
| |
| Currently only used by detect_regression to skip the test if just running regression detection. |
| |
| Args: |
| args: parsed args object. |
| |
| Returns: |
| True if there are tests to run, false otherwise. |
| """ |
| return not (args.detect_regression and len(args.detect_regression) == 2) |
| |
| |
| def _has_valid_regression_detection_args(args): |
| """Validate regression detection args. |
| |
| Args: |
| args: parsed args object. |
| |
| Returns: |
| True if args are valid |
| """ |
| if args.generate_baseline and args.generate_new_metrics: |
| logging.error('Cannot collect both baseline and new metrics at the same time.') |
| return False |
| if args.detect_regression is not None: |
| if not args.detect_regression: |
| logging.error('Need to specify at least 1 arg for regression detection.') |
| return False |
| elif len(args.detect_regression) == 1: |
| if args.generate_baseline or args.generate_new_metrics: |
| return True |
| logging.error('Need to specify --generate-baseline or --generate-new-metrics.') |
| return False |
| elif len(args.detect_regression) == 2: |
| if args.generate_baseline: |
| logging.error('Specified 2 metric paths and --generate-baseline, ' |
| 'either drop --generate-baseline or drop a path') |
| return False |
| if args.generate_new_metrics: |
| logging.error('Specified 2 metric paths and --generate-new-metrics, ' |
| 'either drop --generate-new-metrics or drop a path') |
| return False |
| return True |
| else: |
| logging.error('Specified more than 2 metric paths.') |
| return False |
| return True |
| |
| |
| def main(argv): |
| """Entry point of atest script. |
| |
| Args: |
| argv: A list of arguments. |
| |
| Returns: |
| Exit code. |
| """ |
| args = _parse_args(argv) |
| _configure_logging(args.verbose) |
| if _missing_environment_variables(): |
| return constants.EXIT_CODE_ENV_NOT_SETUP |
| if args.generate_baseline and args.generate_new_metrics: |
| logging.error('Cannot collect both baseline and new metrics at the same time.') |
| return constants.EXIT_CODE_ERROR |
| if not _has_valid_regression_detection_args(args): |
| return constants.EXIT_CODE_ERROR |
| results_dir = make_test_run_dir() |
| mod_info = module_info.ModuleInfo(force_build=args.rebuild_module_info) |
| translator = cli_translator.CLITranslator(module_info=mod_info) |
| build_targets = set() |
| test_infos = set() |
| if _will_run_tests(args): |
| try: |
| build_targets, test_infos = translator.translate(args.tests) |
| except atest_error.TestDiscoveryException: |
| logging.exception('Error occured in test discovery:') |
| logging.info('This can happen after a repo sync or if the test is ' |
| 'new. Running: with "%s" may resolve the issue.', |
| REBUILD_MODULE_INFO_FLAG) |
| return constants.EXIT_CODE_TEST_NOT_FOUND |
| build_targets |= test_runner_handler.get_test_runner_reqs(mod_info, |
| test_infos) |
| extra_args = get_extra_args(args) |
| if args.detect_regression: |
| build_targets |= (regression_test_runner.RegressionTestRunner('') |
| .get_test_runner_build_reqs()) |
| # args.steps will be None if none of -bit set, else list of params set. |
| steps = args.steps if args.steps else ALL_STEPS |
| if build_targets and BUILD_STEP in steps: |
| # Add module-info.json target to the list of build targets to keep the |
| # file up to date. |
| build_targets.add(mod_info.module_info_target) |
| success = atest_utils.build(build_targets, args.verbose) |
| if not success: |
| return constants.EXIT_CODE_BUILD_FAILURE |
| elif TEST_STEP not in steps: |
| logging.warn('Install step without test step currently not ' |
| 'supported, installing AND testing instead.') |
| steps.append(TEST_STEP) |
| if TEST_STEP in steps: |
| test_runner_handler.run_all_tests(results_dir, test_infos, extra_args) |
| if args.detect_regression: |
| regression_args = _get_regression_detection_args(args, results_dir) |
| regression_test_runner.RegressionTestRunner('').run_tests(None, regression_args) |
| return constants.EXIT_CODE_SUCCESS |
| |
| if __name__ == '__main__': |
| sys.exit(main(sys.argv[1:])) |