atest/atest_arg_parser.py

#!/usr/bin/env python
#
# Copyright 2018, The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Atest Argument Parser class for atest.
"""

import argparse

import constants


class AtestArgParser(argparse.ArgumentParser):
    """Atest wrapper of ArgumentParser."""

    def __init__(self):
        """Initialise an ArgumentParser instance."""
        super(AtestArgParser, self).__init__(
            description=constants.HELP_DESC,
            epilog=self.EPILOG_TEXT,
            formatter_class=argparse.RawTextHelpFormatter)

    def add_atest_args(self):
        """A function that does ArgumentParser.add_argument()"""
        self.add_argument('tests', nargs='*', help='Tests to build and/or run.')
        self.add_argument('-b', '--build', action='append_const', dest='steps',
                          const=constants.BUILD_STEP, help='Run a build.')
        self.add_argument('-i', '--install', action='append_const', dest='steps',
                          const=constants.INSTALL_STEP, help='Install an APK.')
        self.add_argument('--info', action='store_true',
                          help='Show module information.')
        self.add_argument('--dry-run', action='store_true',
                          help='Dry run atest without building, installing and running '
                               'tests in real.')
        self.add_argument('-t', '--test', action='append_const', dest='steps',
                          const=constants.TEST_STEP,
                          help='Run the tests. WARNING: Many test configs force cleanup '
                               'of device after test run. In this case, -d must be used in '
                               'previous test run to disable cleanup, for -t to work. '
                               'Otherwise, device will need to be setup again with -i.')
        self.add_argument('-s', '--serial', help='The device to run the test on.')
        self.add_argument('-d', '--disable-teardown', action='store_true',
                          help='Disables test teardown and cleanup.')
        self.add_argument('-m', constants.REBUILD_MODULE_INFO_FLAG, action='store_true',
                          help='Forces a rebuild of the module-info.json file. '
                               'This may be necessary following a repo sync or '
                               'when writing a new test.')
        self.add_argument('-w', '--wait-for-debugger', action='store_true',
                          help='Only for instrumentation tests. Waits for '
                               'debugger prior to execution.')
        self.add_argument('-v', '--verbose', action='store_true',
                          help='Display DEBUG level logging.')
        self.add_argument('-a', '--all-abi', action='store_true',
                          help='Set to run tests for all abi.')
        self.add_argument('--generate-baseline', nargs='?', type=int, const=5, default=0,
                          help='Generate baseline metrics, run 5 iterations by default. '
                               'Provide an int argument to specify # iterations.')
        self.add_argument('--generate-new-metrics', nargs='?', type=int, const=5, default=0,
                          help='Generate new metrics, run 5 iterations by default. '
                               'Provide an int argument to specify # iterations.')
        self.add_argument('--detect-regression', nargs='*',
                          help='Run regression detection algorithm. Supply '
                               'path to baseline and/or new metrics folders.')
        # Options related to Test Mapping
        self.add_argument('-p', '--test-mapping', action='store_true',
                          help='Run tests in TEST_MAPPING files.')
        self.add_argument('--include-subdirs', action='store_true',
                          help='Include tests in TEST_MAPPING files in sub directories.')
        # Options related to deviceless testing.
        self.add_argument('--host', action='store_true',
                          help='Run the test completely on the host without '
                               'a device. (Note: running a host test that '
                               'requires a device with --host will fail.)')
        # This arg actually doesn't consume anything, it's primarily used for the
        # help description and creating custom_args in the NameSpace object.
        self.add_argument('--', dest='custom_args', nargs='*',
                          help='Specify custom args for the test runners. '
                               'Everything after -- will be consumed as custom args.')

    def get_args(self):
        """This method is to get args from actions and return optional args.

        Returns:
            A list of optional arguments.
        """
        argument_list = []
        # The output of _get_optional_actions(): [['-t', '--test'], [--info]]
        # return an argument list: ['-t', '--test', '--info']
        for arg in self._get_optional_actions():
            argument_list.extend(arg.option_strings)
        return argument_list


    EPILOG_TEXT = '''

- - - - - - - - -
IDENTIFYING TESTS
- - - - - - - - -

    The positional argument <tests> should be a reference to one or more
    of the tests you'd like to run. Multiple tests can be run in one command by
    separating test references with spaces.

    Usage template: atest <reference_to_test_1> <reference_to_test_2>

    A <reference_to_test> can be satisfied by the test's MODULE NAME,
    MODULE:CLASS, CLASS NAME, TF INTEGRATION TEST, FILE PATH or PACKAGE NAME.
    Explanations and examples of each follow.


    < MODULE NAME >

        Identifying a test by its module name will run the entire module. Input
        the name as it appears in the LOCAL_MODULE or LOCAL_PACKAGE_NAME
        variables in that test's Android.mk or Android.bp file.

        Note: Use < TF INTEGRATION TEST > to run non-module tests integrated
        directly into TradeFed.

        Examples:
            atest FrameworksServicesTests
            atest CtsJankDeviceTestCases


    < MODULE:CLASS >

        Identifying a test by its class name will run just the tests in that
        class and not the whole module. MODULE:CLASS is the preferred way to run
        a single class. MODULE is the same as described above. CLASS is the
        name of the test class in the .java file. It can either be the fully
        qualified class name or just the basic name.

        Examples:
            atest FrameworksServicesTests:ScreenDecorWindowTests
            atest FrameworksServicesTests:com.android.server.wm.ScreenDecorWindowTests
            atest CtsJankDeviceTestCases:CtsDeviceJankUi


    < CLASS NAME >

        A single class can also be run by referencing the class name without
        the module name.

        Examples:
            atest ScreenDecorWindowTests
            atest CtsDeviceJankUi

        However, this will take more time than the equivalent MODULE:CLASS
        reference, so we suggest using a MODULE:CLASS reference whenever
        possible. Examples below are ordered by performance from the fastest
        to the slowest:

        Examples:
            atest FrameworksServicesTests:com.android.server.wm.ScreenDecorWindowTests
            atest FrameworksServicesTests:ScreenDecorWindowTests
            atest ScreenDecorWindowTests

    < TF INTEGRATION TEST >

        To run tests that are integrated directly into TradeFed (non-modules),
        input the name as it appears in the output of the "tradefed.sh list
        configs" cmd.

        Examples:
           atest example/reboot
           atest native-benchmark


    < FILE PATH >

        Both module-based tests and integration-based tests can be run by
        inputting the path to their test file or dir as appropriate. A single
        class can also be run by inputting the path to the class's java file.
        Both relative and absolute paths are supported.

        Example - 2 ways to run the `CtsJankDeviceTestCases` module via path:
        1. run module from android <repo root>:
            atest cts/tests/jank/jank

        2. from <android root>/cts/tests/jank:
            atest .

        Example - run a specific class within CtsJankDeviceTestCases module
        from <android repo> root via path:
           atest cts/tests/jank/src/android/jank/cts/ui/CtsDeviceJankUi.java

        Example - run an integration test from <android repo> root via path:
           atest tools/tradefederation/contrib/res/config/example/reboot.xml


    < PACKAGE NAME >

        Atest supports searching tests from package name as well.

        Examples:
           atest com.android.server.wm
           atest android.jank.cts


- - - - - - - - - - - - - - - - - - - - - - - - - -
SPECIFYING INDIVIDUAL STEPS: BUILD, INSTALL OR RUN
- - - - - - - - - - - - - - - - - - - - - - - - - -

    The -b, -i and -t options allow you to specify which steps you want to run.
    If none of those options are given, then all steps are run. If any of these
    options are provided then only the listed steps are run.

    Note: -i alone is not currently support and can only be included with -t.
    Both -b and -t can be run alone.

    Examples:
        atest -b <test>    (just build targets)
        atest -t <test>    (run tests only)
        atest -it <test>   (install apk and run tests)
        atest -bt <test>   (build targets, run tests, but skip installing apk)


    Atest now has the ability to force a test to skip its cleanup/teardown step.
    Many tests, e.g. CTS, cleanup the device after the test is run, so trying to
    rerun your test with -t will fail without having the --disable-teardown
    parameter. Use -d before -t to skip the test clean up step and test iteratively.

        atest -d <test>    (disable installing apk and cleanning up device)
        atest -t <test>

    Note that -t disables both setup/install and teardown/cleanup of the
    device. So you can continue to rerun your test with just

        atest -t <test>

    as many times as you want.


- - - - - - - - - - - - -
RUNNING SPECIFIC METHODS
- - - - - - - - - - - - -

    It is possible to run only specific methods within a test class. To run
    only specific methods, identify the class in any of the ways supported for
    identifying a class (MODULE:CLASS, FILE PATH, etc) and then append the
    name of the method or method using the following template:

      <reference_to_class>#<method1>

    Multiple methods can be specified with commas:

      <reference_to_class>#<method1>,<method2>,<method3>...

    Examples:
      atest com.android.server.wm.ScreenDecorWindowTests#testMultipleDecors

      atest FrameworksServicesTests:ScreenDecorWindowTests#testFlagChange,testRemoval


- - - - - - - - - - - - -
RUNNING MULTIPLE CLASSES
- - - - - - - - - - - - -

    To run multiple classes, deliminate them with spaces just like you would
    when running multiple tests.  Atest will handle building and running
    classes in the most efficient way possible, so specifying a subset of
    classes in a module will improve performance over running the whole module.


    Examples:
    - two classes in same module:
      atest FrameworksServicesTests:ScreenDecorWindowTests FrameworksServicesTests:DimmerTests

    - two classes, different modules:
      atest FrameworksServicesTests:ScreenDecorWindowTests CtsJankDeviceTestCases:CtsDeviceJankUi


- - - - - - - - - - -
REGRESSION DETECTION
- - - - - - - - - - -

    Generate pre-patch or post-patch metrics without running regression detection:

    Example:
        atest <test> --generate-baseline <optional iter>
        atest <test> --generate-new-metrics <optional iter>

    Local regression detection can be run in three options:

    1) Provide a folder containing baseline (pre-patch) metrics (generated
       previously). Atest will run the tests n (default 5) iterations, generate
       a new set of post-patch metrics, and compare those against existing metrics.

    Example:
        atest <test> --detect-regression </path/to/baseline> --generate-new-metrics <optional iter>

    2) Provide a folder containing post-patch metrics (generated previously).
       Atest will run the tests n (default 5) iterations, generate a new set of
       pre-patch metrics, and compare those against those provided. Note: the
       developer needs to revert the device/tests to pre-patch state to generate
       baseline metrics.

    Example:
        atest <test> --detect-regression </path/to/new> --generate-baseline <optional iter>

    3) Provide 2 folders containing both pre-patch and post-patch metrics. Atest
       will run no tests but the regression detection algorithm.

    Example:
        atest --detect-regression </path/to/baseline> </path/to/new>


- - - - - - - - - - - -
TESTS IN TEST MAPPING
- - - - - - - - - - - -

    Atest can run tests in TEST_MAPPING files:

    1) Run presubmit tests in TEST_MAPPING files in current and parent
       directories. You can also specify a target directory.

    Example:
        atest  (run presubmit tests in TEST_MAPPING files in current and parent directories)
        atest --test-mapping </path/to/project>
               (run presubmit tests in TEST_MAPPING files in </path/to/project> and its parent directories)

    2) Run a specified test group in TEST_MAPPING files.

    Example:
        atest :postsubmit
              (run postsubmit tests in TEST_MAPPING files in current and parent directories)
        atest :all
              (Run tests from all groups in TEST_MAPPING files)
        atest --test-mapping </path/to/project>:postsubmit
              (run postsubmit tests in TEST_MAPPING files in </path/to/project> and its parent directories)

    3) Run tests in TEST_MAPPING files including sub directories

    By default, atest will only search for tests in TEST_MAPPING files in
    current (or given directory) and its parent directories. If you want to run
    tests in TEST_MAPPING files in the sub-directories, you can use option
    --include-subdirs to force atest to include those tests too.

    Example:
        atest --include-subdirs [optional </path/to/project>:<test_group_name>]
              (run presubmit tests in TEST_MAPPING files in current, sub and parent directories)
    A path can be provided optionally if you want to search for tests in a give
    directory, with optional test group name. By default, the test group is
    presubmit.

'''