blob: 17189898d7da79466b6f6754c60f9c4e9dc98153 [file] [log] [blame]
# Copyright 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import logging
import os
import sys
from telemetry.core import util
from telemetry.core import platform as platform_module
from telemetry import decorators
from telemetry.internal.browser import browser_finder
from telemetry.internal.browser import browser_finder_exceptions
from telemetry.internal.browser import browser_options
from telemetry.internal.platform import android_device
from telemetry.internal.util import binary_manager
from telemetry.internal.util import command_line
from telemetry.internal.util import ps_util
from telemetry.testing import browser_test_case
from telemetry.testing import options_for_unittests
from py_utils import cloud_storage
from py_utils import xvfb
import typ
class RunTestsCommand(command_line.OptparseCommand):
"""Run unit tests"""
usage = '[test_name ...] [<options>]'
xvfb_process = None
def __init__(self):
super(RunTestsCommand, self).__init__()
self.stream = sys.stdout
@classmethod
def CreateParser(cls):
options = browser_options.BrowserFinderOptions()
options.browser_type = 'any'
parser = options.CreateParser('%%prog %s' % cls.usage)
return parser
@classmethod
def AddCommandLineArgs(cls, parser, _):
parser.add_option('--start-xvfb', action='store_true',
default=False, help='Start Xvfb display if needed.')
parser.add_option('--disable-cloud-storage-io', action='store_true',
default=False, help=('Disable cloud storage IO when '
'tests are run in parallel.'))
parser.add_option('--repeat-count', type='int', default=1,
help='Repeats each a provided number of times.')
parser.add_option('--no-browser', action='store_true', default=False,
help='Don\'t require an actual browser to run the tests.')
parser.add_option('-d', '--also-run-disabled-tests',
dest='run_disabled_tests',
action='store_true', default=False,
help='Ignore @Disabled and @Enabled restrictions.')
parser.add_option('--exact-test-filter', action='store_true', default=False,
help='Treat test filter as exact matches (default is '
'substring matches).')
parser.add_option('--client-config', dest='client_configs',
action='append', default=[])
parser.add_option('--disable-logging-config', action='store_true',
default=False, help='Configure logging (default on)')
typ.ArgumentParser.add_option_group(parser,
"Options for running the tests",
running=True,
skip=['-d', '-v', '--verbose'])
typ.ArgumentParser.add_option_group(parser,
"Options for reporting the results",
reporting=True)
@classmethod
def ProcessCommandLineArgs(cls, parser, args, _):
# We retry failures by default unless we're running a list of tests
# explicitly.
if not args.retry_limit and not args.positional_args:
args.retry_limit = 3
if args.no_browser:
return
if args.start_xvfb and xvfb.ShouldStartXvfb():
cls.xvfb_process = xvfb.StartXvfb()
try:
possible_browser = browser_finder.FindBrowser(args)
except browser_finder_exceptions.BrowserFinderException, ex:
parser.error(ex)
if not possible_browser:
parser.error('No browser found of type %s. Cannot run tests.\n'
'Re-run with --browser=list to see '
'available browser types.' % args.browser_type)
@classmethod
def main(cls, args=None, stream=None): # pylint: disable=arguments-differ
# We override the superclass so that we can hook in the 'stream' arg.
parser = cls.CreateParser()
cls.AddCommandLineArgs(parser, None)
options, positional_args = parser.parse_args(args)
options.positional_args = positional_args
try:
# Must initialize the DependencyManager before calling
# browser_finder.FindBrowser(args)
binary_manager.InitDependencyManager(options.client_configs)
cls.ProcessCommandLineArgs(parser, options, None)
obj = cls()
if stream is not None:
obj.stream = stream
return obj.Run(options)
finally:
if cls.xvfb_process:
cls.xvfb_process.kill()
def Run(self, args):
runner = typ.Runner()
if self.stream:
runner.host.stdout = self.stream
if args.no_browser:
possible_browser = None
platform = platform_module.GetHostPlatform()
else:
possible_browser = browser_finder.FindBrowser(args)
platform = possible_browser.platform
fetch_reference_chrome_binary = False
# Fetch all binaries needed by telemetry before we run the benchmark.
if possible_browser and possible_browser.browser_type == 'reference':
fetch_reference_chrome_binary = True
binary_manager.FetchBinaryDepdencies(
platform, args.client_configs, fetch_reference_chrome_binary)
# Telemetry seems to overload the system if we run one test per core,
# so we scale things back a fair amount. Many of the telemetry tests
# are long-running, so there's a limit to how much parallelism we
# can effectively use for now anyway.
#
# It should be possible to handle multiple devices if we adjust the
# browser_finder code properly, but for now we only handle one on ChromeOS.
if platform.GetOSName() == 'chromeos':
runner.args.jobs = 1
elif platform.GetOSName() == 'android':
android_devs = android_device.FindAllAvailableDevices(args)
runner.args.jobs = len(android_devs)
if runner.args.jobs == 0:
raise RuntimeError("No Android device found")
print 'Running tests with %d Android device(s).' % runner.args.jobs
elif platform.GetOSVersionName() == 'xp':
# For an undiagnosed reason, XP falls over with more parallelism.
# See crbug.com/388256
runner.args.jobs = max(int(args.jobs) // 4, 1)
else:
runner.args.jobs = max(int(args.jobs) // 2, 1)
runner.args.metadata = args.metadata
runner.args.passthrough = args.passthrough
runner.args.path = args.path
runner.args.retry_limit = args.retry_limit
runner.args.test_results_server = args.test_results_server
runner.args.test_type = args.test_type
runner.args.top_level_dir = args.top_level_dir
runner.args.write_full_results_to = args.write_full_results_to
runner.args.write_trace_to = args.write_trace_to
runner.args.list_only = args.list_only
runner.args.shard_index = args.shard_index
runner.args.total_shards = args.total_shards
runner.args.path.append(util.GetUnittestDataDir())
# Always print out these info for the ease of debugging.
runner.args.timing = True
runner.args.verbose = 3
runner.classifier = GetClassifier(args, possible_browser)
runner.context = args
runner.setup_fn = _SetUpProcess
runner.teardown_fn = _TearDownProcess
runner.win_multiprocessing = typ.WinMultiprocessing.importable
try:
ret, _, _ = runner.run()
except KeyboardInterrupt:
print >> sys.stderr, "interrupted, exiting"
ret = 130
return ret
def GetClassifier(args, possible_browser):
def ClassifyTestWithoutBrowser(test_set, test):
name = test.id()
if (not args.positional_args
or _MatchesSelectedTest(name, args.positional_args,
args.exact_test_filter)):
# TODO(telemetry-team): Make sure that all telemetry unittest that invokes
# actual browser are subclasses of browser_test_case.BrowserTestCase
# (crbug.com/537428)
if issubclass(test.__class__, browser_test_case.BrowserTestCase):
test_set.tests_to_skip.append(typ.TestInput(
name, msg='Skip the test because it requires a browser.'))
else:
test_set.parallel_tests.append(typ.TestInput(name))
def ClassifyTestWithBrowser(test_set, test):
name = test.id()
if (not args.positional_args
or _MatchesSelectedTest(name, args.positional_args,
args.exact_test_filter)):
assert hasattr(test, '_testMethodName')
method = getattr(
test, test._testMethodName) # pylint: disable=protected-access
should_skip, reason = decorators.ShouldSkip(method, possible_browser)
if should_skip and not args.run_disabled_tests:
test_set.tests_to_skip.append(typ.TestInput(name, msg=reason))
elif decorators.ShouldBeIsolated(method, possible_browser):
test_set.isolated_tests.append(typ.TestInput(name))
else:
test_set.parallel_tests.append(typ.TestInput(name))
if possible_browser:
return ClassifyTestWithBrowser
else:
return ClassifyTestWithoutBrowser
def _MatchesSelectedTest(name, selected_tests, selected_tests_are_exact):
if not selected_tests:
return False
if selected_tests_are_exact:
return any(name in selected_tests)
else:
return any(test in name for test in selected_tests)
def _SetUpProcess(child, context): # pylint: disable=unused-argument
ps_util.EnableListingStrayProcessesUponExitHook()
# Make sure that we don't invokes cloud storage I/Os when we run the tests in
# parallel.
# TODO(nednguyen): always do this once telemetry tests in Chromium is updated
# to prefetch files.
# (https://github.com/catapult-project/catapult/issues/2192)
args = context
if args.disable_cloud_storage_io:
os.environ[cloud_storage.DISABLE_CLOUD_STORAGE_IO] = '1'
if binary_manager.NeedsInit():
# Typ doesn't keep the DependencyManager initialization in the child
# processes.
binary_manager.InitDependencyManager(context.client_configs)
# We need to reset the handlers in case some other parts of telemetry already
# set it to make this work.
if not args.disable_logging_config:
logging.getLogger().handlers = []
logging.basicConfig(
level=logging.INFO,
format='(%(levelname)s) %(asctime)s pid=%(process)d'
' %(module)s.%(funcName)s:%(lineno)d'
' %(message)s')
if args.remote_platform_options.device == 'android':
android_devices = android_device.FindAllAvailableDevices(args)
if not android_devices:
raise RuntimeError("No Android device found")
android_devices.sort(key=lambda device: device.name)
args.remote_platform_options.device = (
android_devices[child.worker_num-1].guid)
options_for_unittests.Push(args)
def _TearDownProcess(child, context): # pylint: disable=unused-argument
# It's safe to call teardown_browser even if we did not start any browser
# in any of the tests.
browser_test_case.teardown_browser()
options_for_unittests.Pop()
if __name__ == '__main__':
ret_code = RunTestsCommand.main()
sys.exit(ret_code)