| # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import datetime |
| import logging |
| |
| import common |
| |
| from autotest_lib.client.common_lib import base_job |
| from autotest_lib.client.common_lib import error |
| from autotest_lib.client.common_lib import priorities |
| from autotest_lib.client.common_lib import time_utils |
| from autotest_lib.client.common_lib import utils |
| from autotest_lib.client.common_lib.cros import dev_server |
| from autotest_lib.server.cros import provision |
| from autotest_lib.server.cros.dynamic_suite import constants |
| from autotest_lib.server.cros.dynamic_suite import frontend_wrappers |
| from autotest_lib.server.cros.dynamic_suite import tools |
| from autotest_lib.server.cros.dynamic_suite.suite import Suite |
| from autotest_lib.tko import utils as tko_utils |
| |
| |
| |
| """CrOS dynamic test suite generation and execution module. |
| |
| This module implements runtime-generated test suites for CrOS. |
| Design doc: http://goto.google.com/suitesv2 |
| |
| Individual tests can declare themselves as a part of one or more |
| suites, and the code here enables control files to be written |
| that can refer to these "dynamic suites" by name. We also provide |
| support for reimaging devices with a given build and running a |
| dynamic suite across all reimaged devices. |
| |
| The public API for defining a suite includes one method: reimage_and_run(). |
| A suite control file can be written by importing this module and making |
| an appropriate call to this single method. In normal usage, this control |
| file will be run in a 'hostless' server-side autotest job, scheduling |
| sub-jobs to do the needed reimaging and test running. |
| |
| Example control file: |
| |
| import common |
| from autotest_lib.server.cros import provision |
| from autotest_lib.server.cros.dynamic_suite import dynamic_suite |
| |
| dynamic_suite.reimage_and_run( |
| build=build, board=board, name='bvt', job=job, pool=pool, |
| check_hosts=check_hosts, add_experimental=True, num=num, |
| devserver_url=devserver_url, version_prefix=provision.CROS_VERSION_PREFIX) |
| |
| This will -- at runtime -- find all control files that contain "bvt" in their |
| "SUITE=" clause, schedule jobs to reimage |num| or less devices in the |
| specified pool of the specified board with the specified build and, upon |
| completion of those jobs, schedule and wait for jobs that run all the tests it |
| discovered. |
| |
| Suites can be run by using the atest command-line tool: |
| atest suite create -b <board> -i <build/name> <suite> |
| e.g. |
| atest suite create -b x86-mario -i x86-mario/R20-2203.0.0 bvt |
| |
| ------------------------------------------------------------------------- |
| Implementation details |
| |
| A Suite instance represents a single test suite, defined by some predicate |
| run over all known control files. The simplest example is creating a Suite |
| by 'name'. |
| |
| create_suite_job() takes the parameters needed to define a suite run (board, |
| build to test, machine pool, and which suite to run), ensures important |
| preconditions are met, finds the appropraite suite control file, and then |
| schedules the hostless job that will do the rest of the work. |
| |
| Note that we have more than one Dev server in our test lab architecture. |
| We currently load balance per-build being tested, so one and only one dev |
| server is used by any given run through the reimaging/testing flow. |
| |
| - create_suite_job() |
| The primary role of create_suite_job() is to ensure that the required |
| artifacts for the build to be tested are staged on the dev server. This |
| includes payloads required to autoupdate machines to the desired build, as |
| well as the autotest control files appropriate for that build. Then, the |
| RPC pulls the control file for the suite to be run from the dev server and |
| uses it to create the suite job with the autotest frontend. |
| |
| +----------------+ |
| | Google Storage | Client |
| +----------------+ | |
| | ^ | create_suite_job() |
| payloads/ | | | |
| control files | | request | |
| V | V |
| +-------------+ download request +--------------------------+ |
| | |<----------------------| | |
| | Dev Server | | Autotest Frontend (AFE) | |
| | |---------------------->| | |
| +-------------+ suite control file +--------------------------+ |
| | |
| V |
| Suite Job (hostless) |
| |
| - Reimage and Run |
| The overall process is to schedule all the tests, and then wait for the tests |
| to complete. |
| |
| - The Reimaging Process |
| |
| As an artifact of an old implementation, the number of machines to use |
| is called the 'sharding_factor', and the default is defined in the [CROS] |
| section of global_config.ini. This can be overridden by passing a 'num=N' |
| parameter to create_suite_job(), which is piped through to reimage_and_run() |
| just like the 'build' and 'board' parameters are. However, with provisioning, |
| this machine accounting hasn't been implemented nor removed. However, 'num' is |
| still passed around, as it might be used one day. |
| |
| A test control file can specify a list of DEPENDENCIES, which are really just |
| the set of labels a host needs to have in order for that test to be scheduled |
| on it. In the case of a dynamic_suite, many tests in the suite may have |
| DEPENDENCIES specified. All tests are scheduled with the DEPENDENCIES that |
| they specify, along with any suite dependencies that were specified, and the |
| scheduler will find and provision a host capable of running the test. |
| |
| - Scheduling Suites |
| A Suite instance uses the labels specified in the suite dependencies to |
| schedule tests across all the hosts in the pool. It then waits for all these |
| jobs. As an optimization, the Dev server stages the payloads necessary to |
| run a suite in the background _after_ it has completed all the things |
| necessary for reimaging. Before running a suite, reimage_and_run() calls out |
| to the Dev server and blocks until it's completed staging all build artifacts |
| needed to run test suites. |
| |
| Step by step: |
| 0) At instantiation time, find all appropriate control files for this suite |
| that were included in the build to be tested. To do this, we consult the |
| Dev Server, where all these control files are staged. |
| |
| +------------+ control files? +--------------------------+ |
| | |<----------------------| | |
| | Dev Server | | Autotest Frontend (AFE) | |
| | |---------------------->| [Suite Job] | |
| +------------+ control files! +--------------------------+ |
| |
| 1) Now that the Suite instance exists, it schedules jobs for every control |
| file it deemed appropriate, to be run on the hosts that were labeled |
| by the provisioning. We stuff keyvals into these jobs, indicating what |
| build they were testing and which suite they were for. |
| |
| +--------------------------+ Job for VersLabel +--------+ |
| | |------------------------>| Host 1 | VersLabel |
| | Autotest Frontend (AFE) | +--------+ +--------+ |
| | [Suite Job] |----------->| Host 2 | |
| +--------------------------+ Job for +--------+ |
| | ^ VersLabel VersLabel |
| | | |
| +----------------+ |
| One job per test |
| {'build': build/name, |
| 'suite': suite_name} |
| |
| 2) Now that all jobs are scheduled, they'll be doled out as labeled hosts |
| finish their assigned work and become available again. |
| |
| - Waiting on Suites |
| 0) As we clean up each test job, we check to see if any crashes occurred. If |
| they did, we look at the 'build' keyval in the job to see which build's debug |
| symbols we'll need to symbolicate the crash dump we just found. |
| |
| 1) Using this info, we tell a special Crash Server to stage the required debug |
| symbols. Once that's done, we ask the Crash Server to use those symbols to |
| symbolicate the crash dump in question. |
| |
| +----------------+ |
| | Google Storage | |
| +----------------+ |
| | ^ |
| symbols! | | symbols? |
| V | |
| +------------+ stage symbols for build +--------------------------+ |
| | |<--------------------------| | |
| | Crash | | | |
| | Server | dump to symbolicate | Autotest Frontend (AFE) | |
| | |<--------------------------| [Suite Job] | |
| | |-------------------------->| | |
| +------------+ symbolicated dump +--------------------------+ |
| |
| 2) As jobs finish, we record their success or failure in the status of the suite |
| job. We also record a 'job keyval' in the suite job for each test, noting |
| the job ID and job owner. This can be used to refer to test logs later. |
| 3) Once all jobs are complete, status is recorded for the suite job, and the |
| job_repo_url host attribute is removed from all hosts used by the suite. |
| |
| """ |
| |
| |
| DEFAULT_TRY_JOB_TIMEOUT_MINS = tools.try_job_timeout_mins() |
| |
| # Relevant CrosDynamicSuiteExceptions are defined in client/common_lib/error.py. |
| |
| class SuiteSpec(object): |
| """ |
| This class contains the info that defines a suite run. |
| |
| Currently required: |
| @var build: the build to install e.g. |
| x86-alex-release/R18-1655.0.0-a1-b1584. |
| @var board: which kind of devices to reimage. |
| @var devserver: An instance of the devserver to use with this suite. |
| @var name: a value of the SUITE control file variable to search for. |
| @var job: an instance of client.common_lib.base_job representing the |
| currently running suite job. |
| |
| Currently supported optional fields: |
| @var pool: specify the pool of machines to use for scheduling purposes. |
| Default: None |
| @var num: the maximum number of devices to reimage. |
| Default in global_config |
| @var check_hosts: require appropriate hosts to be available now. |
| @var add_experimental: schedule experimental tests as well, or not. |
| Default: True |
| @var dependencies: map of test names to dependency lists. |
| Initially {'': []}. |
| @param suite_dependencies: A string with a comma separated list of suite |
| level dependencies, which act just like test |
| dependencies and are appended to each test's |
| set of dependencies at job creation time. |
| @param predicate: Optional argument. If present, should be a function |
| mapping ControlData objects to True if they should be |
| included in suite. If argument is absent, suite |
| behavior will default to creating a suite of based |
| on the SUITE field of control files. |
| """ |
| |
| def _verify_builds(self, build, builds): |
| """Verify the value of build and builds passed in to create a suite. |
| |
| TODO(crbug.com/496782): This method should be removed after R45 falls |
| off stable channel. Add `builds` to required_keywords in __init__, and |
| remove `build` in __init__. |
| |
| @param build: the build to install e.g. |
| x86-alex-release/R18-1655.0.0-a1-b1584. |
| @param builds: the builds to install e.g. |
| {'cros-version:': 'x86-alex-release/R18-1655.0.0', |
| 'fw-version:': 'x86-alex-firmware/R36-5771.50.0'} |
| |
| @raise: SuiteArgumentException if value for build or builds is invalid. |
| |
| """ |
| if not builds and not build: |
| raise error.SuiteArgumentException( |
| 'reimage_and_run() needs at least one of builds or build ' |
| 'being specified.') |
| if build and builds and not build in builds.values(): |
| raise error.SuiteArgumentException( |
| 'Arguments build and builds for reimage_and_run() is ' |
| 'inconsistent. `build` must be one of the values of ' |
| '`builds`. build="%s". builds="%s"' % (build, builds)) |
| build_arg_check = {'build': str, 'builds': dict} |
| for key, expected in build_arg_check.iteritems(): |
| value = locals().get(key) |
| if value and not isinstance(value, expected): |
| raise error.SuiteArgumentException( |
| 'reimage_and_run() needs %s=<%r>' % (key, expected)) |
| |
| |
| def __init__(self, build=None, builds=None, board=None, name=None, job=None, |
| pool=None, num=None, check_hosts=True, |
| add_experimental=True, file_bugs=False, |
| file_experimental_bugs=False, max_runtime_mins=24*60, |
| timeout=24, timeout_mins=None, firmware_reimage=False, |
| suite_dependencies=[], version_prefix=None, |
| bug_template={}, devserver_url=None, |
| priority=priorities.Priority.DEFAULT, predicate=None, |
| wait_for_results=True, job_retry=False, max_retries=None, |
| offload_failures_only=False, test_source_build=None, |
| run_prod_code=False, **dargs): |
| """ |
| Vets arguments for reimage_and_run() and populates self with supplied |
| values. |
| |
| TODO(dshi): crbug.com/496782 once R45 falls off stable channel, we |
| should remove option build, firmware_reimage and version_prefix, as they |
| will be all merged into option builds. |
| |
| Currently required args: |
| @param board: which kind of devices to reimage. |
| @param name: a value of the SUITE control file variable to search for. |
| @param job: an instance of client.common_lib.base_job representing the |
| currently running suite job. |
| @param devserver_url: url to the selected devserver. |
| |
| Currently supported optional args: |
| @param build: the build to install e.g. |
| x86-alex-release/R18-1655.0.0-a1-b1584. |
| @param builds: the builds to install e.g. |
| {'cros-version:': 'x86-alex-release/R18-1655.0.0', |
| 'fw-version:': 'x86-alex-firmware/R36-5771.50.0'} |
| @param test_source_build: Build that contains the server-side test code, |
| e.g., it can be the value of builds['cros-version:'] or |
| builds['fw-version:']. Default is None, that is, use |
| the server-side test code from builds['cros-version:'] |
| @param pool: specify the pool of machines to use for scheduling purposes |
| Default: None |
| @param num: the maximum number of devices to reimage. |
| Default in global_config |
| @param check_hosts: require appropriate hosts to be available now. |
| @param add_experimental: schedule experimental tests as well, or not. |
| Default: True |
| @param file_bugs: File bugs when tests in this suite fail. |
| Default: False |
| @param file_experimental_bugs: File bugs when experimental tests in |
| this suite fail. |
| Default: False |
| @param max_runtime_mins: Max runtime in mins for each of the sub-jobs |
| this suite will run. |
| @param timeout: Max lifetime in hours for each of the sub-jobs that |
| this suite run. |
| @param firmware_reimage: True if we should use FW_RW_VERSION_PREFIX as |
| the version_prefix. |
| False if we should use CROS_VERSION_PREFIX as |
| the version_prefix. |
| (This flag has now been deprecated in favor of |
| version_prefix.) |
| @param suite_dependencies: A list of strings of suite level |
| dependencies, which act just like test |
| dependencies and are appended to each test's |
| set of dependencies at job creation time. |
| A string of comma seperated labels is |
| accepted for backwards compatibility. |
| @param bug_template: A template dictionary specifying the default bug |
| filing options for failures in this suite. |
| @param version_prefix: A version prefix from provision.py that the |
| tests should be scheduled with. |
| @param priority: Integer priority level. Higher is more important. |
| @param predicate: Optional argument. If present, should be a function |
| mapping ControlData objects to True if they should be |
| included in suite. If argument is absent, suite |
| behavior will default to creating a suite of based |
| on the SUITE field of control files. |
| @param wait_for_results: Set to False to run the suite job without |
| waiting for test jobs to finish. Default is |
| True. |
| @param job_retry: Set to True to enable job-level retry. Default is |
| False. |
| @param max_retries: Maximum retry limit at suite level. |
| Regardless how many times each individual test |
| has been retried, the total number of retries |
| happening in the suite can't exceed _max_retries. |
| Default to None, no max. |
| @param offload_failures_only: Only enable gs_offloading for failed |
| jobs. |
| @param run_prod_code: If true, the suite will run the test code that |
| lives in prod aka the test code currently on the |
| lab servers. |
| @param **dargs: these arguments will be ignored. This allows us to |
| deprecate and remove arguments in ToT while not |
| breaking branch builds. |
| """ |
| # TODO(dshi): crbug.com/496782 Following should be added to |
| # required_keywords after R45 falls off stable channel: |
| # 'builds': dict, |
| # To allow the transition, build is removed from the list, but the code |
| # will check either build or builds should exist. |
| required_keywords = {'board': str, |
| 'name': str, |
| 'job': base_job.base_job, |
| 'devserver_url': str} |
| for key, expected in required_keywords.iteritems(): |
| value = locals().get(key) |
| if not value or not isinstance(value, expected): |
| raise error.SuiteArgumentException( |
| 'reimage_and_run() needs %s=<%r>' % (key, expected)) |
| self._verify_builds(build, builds) |
| |
| self.board = 'board:%s' % board |
| self.devserver = dev_server.ImageServer(devserver_url) |
| |
| if builds: |
| self.builds = builds |
| else: |
| # TODO(dshi): crbug.com/496782 This warning can be removed after R45 |
| # falls off stable channel. |
| logging.warning('reimage_and_run arguments firmware_reimage and ' |
| 'version_prefix have been deprecated. Please use ' |
| 'a dictionary builds to specify images, e.g., ' |
| '{\'cros-version:\':\'peppy-release/R38-5655.0.0\',' |
| ' \'fw-version:\':\'peppy-firmware/R36-5371.0.0\'}') |
| |
| if version_prefix: |
| prefix = version_prefix |
| else: |
| prefix = (provision.FW_RW_VERSION_PREFIX if firmware_reimage |
| else provision.CROS_VERSION_PREFIX) |
| self.builds = {prefix: build} |
| |
| if provision.CROS_VERSION_PREFIX in self.builds: |
| translated_build = self.devserver.translate( |
| self.builds[provision.CROS_VERSION_PREFIX]) |
| self.builds[provision.CROS_VERSION_PREFIX] = translated_build |
| |
| if test_source_build: |
| test_source_build = self.devserver.translate(test_source_build) |
| |
| self.test_source_build = Suite.get_test_source_build( |
| self.builds, test_source_build=test_source_build) |
| |
| self.name = name |
| self.job = job |
| if pool: |
| self.pool = 'pool:%s' % pool |
| else: |
| self.pool = pool |
| self.num = num |
| self.check_hosts = check_hosts |
| self.skip_reimage = skip_reimage |
| self.add_experimental = add_experimental |
| self.file_bugs = file_bugs |
| self.file_experimental_bugs = file_experimental_bugs |
| self.dependencies = {'': []} |
| self.max_runtime_mins = max_runtime_mins |
| self.timeout = timeout |
| self.timeout_mins = timeout_mins or timeout * 60 |
| if isinstance(suite_dependencies, str): |
| self.suite_dependencies = [dep.strip(' ') for dep |
| in suite_dependencies.split(',')] |
| else: |
| self.suite_dependencies = suite_dependencies |
| self.bug_template = bug_template |
| self.priority = priority |
| self.predicate = predicate |
| self.wait_for_results = wait_for_results |
| self.job_retry = job_retry |
| self.max_retries = max_retries |
| self.offload_failures_only = offload_failures_only |
| self.run_prod_code = run_prod_code |
| |
| |
| def skip_reimage(g): |
| """ |
| Pulls the SKIP_IMAGE value out of a global variables dictionary. |
| @param g: The global variables dictionary. |
| @return: Value associated with SKIP-IMAGE |
| """ |
| return False |
| |
| |
| def reimage_and_run(**dargs): |
| """ |
| Backward-compatible API for dynamic_suite. |
| |
| Will re-image a number of devices (of the specified board) with the |
| provided build, and then run the indicated test suite on them. |
| Guaranteed to be compatible with any build from stable to dev. |
| |
| @param dargs: Dictionary containing the arguments listed below. |
| |
| Currently required args: |
| @param board: which kind of devices to reimage. |
| @param name: a value of the SUITE control file variable to search for. |
| @param job: an instance of client.common_lib.base_job representing the |
| currently running suite job. |
| |
| Currently supported optional args: |
| @param build: the build to install e.g. |
| x86-alex-release/R18-1655.0.0-a1-b1584. |
| @param builds: the builds to install e.g. |
| {'cros-version:': 'x86-alex-release/R18-1655.0.0', |
| 'fw-version:': 'x86-alex-firmware/R36-5771.50.0'} |
| @param pool: specify the pool of machines to use for scheduling purposes. |
| Default: None |
| @param num: the maximum number of devices to reimage. |
| Default in global_config |
| @param check_hosts: require appropriate hosts to be available now. |
| @param add_experimental: schedule experimental tests as well, or not. |
| Default: True |
| @param file_bugs: automatically file bugs on test failures. |
| Default: False |
| @param suite_dependencies: A string with a comma separated list of suite |
| level dependencies, which act just like test |
| dependencies and are appended to each test's |
| set of dependencies at job creation time. |
| @param devserver_url: url to the selected devserver. |
| @param predicate: Optional argument. If present, should be a function |
| mapping ControlData objects to True if they should be |
| included in suite. If argument is absent, suite |
| behavior will default to creating a suite of based |
| on the SUITE field of control files. |
| @param job_retry: A bool value indicating whether jobs should be retired |
| on failure. If True, the field 'JOB_RETRIES' in control |
| files will be respected. If False, do not retry. |
| @param max_retries: Maximum retry limit at suite level. |
| Regardless how many times each individual test |
| has been retried, the total number of retries |
| happening in the suite can't exceed _max_retries. |
| Default to None, no max. |
| @param offload_failures_only: Only enable gs_offloading for failed jobs. |
| @raises AsynchronousBuildFailure: if there was an issue finishing staging |
| from the devserver. |
| @raises MalformedDependenciesException: if the dependency_info file for |
| the required build fails to parse. |
| """ |
| suite_spec = SuiteSpec(**dargs) |
| |
| # To support provision both CrOS and firmware, option builds is added to |
| # SuiteSpec, e.g., |
| # builds = {'cros-version:': 'x86-alex-release/R18-1655.0.0', |
| # 'fw-version:': 'x86-alex-firmware/R36-5771.50.0'} |
| # Option build, version_prefix and firmware_reimage will all be obsoleted. |
| # For backwards compatibility, these option will be default to |
| # firmware_reimage = False |
| # version_prefix = provision.CROS_VERSION_PREFIX |
| # build will be used as CrOS build |
| suite_spec.firmware_reimage = False |
| # </backwards_compatibility_hacks> |
| |
| # version_prefix+build should make it into each test as a DEPENDENCY. The |
| # easiest way to do this is to tack it onto the suite_dependencies. |
| suite_spec.suite_dependencies.extend( |
| provision.join(version_prefix, build) |
| for version_prefix, build in suite_spec.builds.items()) |
| |
| afe = frontend_wrappers.RetryingAFE(timeout_min=30, delay_sec=10, |
| user=suite_spec.job.user, debug=False) |
| tko = frontend_wrappers.RetryingTKO(timeout_min=30, delay_sec=10, |
| user=suite_spec.job.user, debug=False) |
| |
| try: |
| my_job_id = int(tko_utils.get_afe_job_id(dargs['job'].tag)) |
| logging.debug('Determined own job id: %d', my_job_id) |
| except ValueError: |
| my_job_id = None |
| logging.warning('Could not determine own job id.') |
| |
| if suite_spec.predicate is None: |
| predicate = Suite.name_in_tag_predicate(suite_spec.name) |
| else: |
| predicate = suite_spec.predicate |
| |
| _perform_reimage_and_run(suite_spec, afe, tko, |
| predicate, suite_job_id=my_job_id) |
| |
| logging.debug('Returning from dynamic_suite.reimage_and_run.') |
| |
| |
| def _perform_reimage_and_run(spec, afe, tko, predicate, suite_job_id=None): |
| """ |
| Do the work of reimaging hosts and running tests. |
| |
| @param spec: a populated SuiteSpec object. |
| @param afe: an instance of AFE as defined in server/frontend.py. |
| @param tko: an instance of TKO as defined in server/frontend.py. |
| @param predicate: A function mapping ControlData objects to True if they |
| should be included in the suite. |
| @param suite_job_id: Job id that will act as parent id to all sub jobs. |
| Default: None |
| """ |
| # We can't do anything else until the devserver has finished downloading |
| # control_files and test_suites packages so that we can get the control |
| # files we should schedule. |
| try: |
| if not spec.run_prod_code: |
| spec.devserver.stage_artifacts(spec.test_source_build, |
| ['control_files', 'test_suites']) |
| except dev_server.DevServerException as e: |
| # If we can't get the control files, there's nothing to run. |
| raise error.AsynchronousBuildFailure(e) |
| |
| timestamp = datetime.datetime.now().strftime(time_utils.TIME_FMT) |
| utils.write_keyval( |
| spec.job.resultdir, |
| {constants.ARTIFACT_FINISHED_TIME: timestamp}) |
| |
| suite = Suite.create_from_predicates( |
| predicates=[predicate], name=spec.name, |
| builds=spec.builds, board=spec.board, devserver=spec.devserver, |
| afe=afe, tko=tko, pool=spec.pool, |
| results_dir=spec.job.resultdir, |
| max_runtime_mins=spec.max_runtime_mins, timeout_mins=spec.timeout_mins, |
| file_bugs=spec.file_bugs, |
| file_experimental_bugs=spec.file_experimental_bugs, |
| suite_job_id=suite_job_id, extra_deps=spec.suite_dependencies, |
| priority=spec.priority, wait_for_results=spec.wait_for_results, |
| job_retry=spec.job_retry, max_retries=spec.max_retries, |
| offload_failures_only=spec.offload_failures_only, |
| test_source_build=spec.test_source_build, |
| run_prod_code=spec.run_prod_code) |
| |
| # Now we get to asychronously schedule tests. |
| suite.schedule(spec.job.record_entry, spec.add_experimental) |
| |
| if suite.wait_for_results: |
| logging.debug('Waiting on suite.') |
| suite.wait(spec.job.record_entry, spec.bug_template) |
| logging.debug('Finished waiting on suite. ' |
| 'Returning from _perform_reimage_and_run.') |
| else: |
| logging.info('wait_for_results is set to False, suite job will exit ' |
| 'without waiting for test jobs to finish.') |