run_tests.py cleanup and simplification (#28808)
* only support cmake for CLanguage in run_tests.py
* add support for run_tests.py build step environ
* switch C/C++ run_tests.py build to build_cxx script
* CLanguage cleanup
* build C# entirely with build_csharp script
* move entire PHP build to build_php.sh
* fixup C# build on linux and mac
* run_dep_checks Makefile target is deprecated
* get rid of the "makefile" logic in run_tests.py
* fixup C# build on linux and mac
* XML_REPORT env variable is useless for --use_docker runs
* add a TODO
* move "main" functionality towards end of run_tests.py
* use self.args instead of global
* yapf format
* remove the no longer useful --update_submodules features of run_tests.py
* fix check_epollexclusive check in run_tests.py
diff --git a/tools/run_tests/helper_scripts/build_csharp.bat b/tools/run_tests/helper_scripts/build_csharp.bat
index 7721d80..6972ef0 100644
--- a/tools/run_tests/helper_scripts/build_csharp.bat
+++ b/tools/run_tests/helper_scripts/build_csharp.bat
@@ -14,7 +14,19 @@
setlocal
-cd /d %~dp0\..\..\..\src\csharp
+cd /d %~dp0\..\..\..
+
+mkdir cmake
+cd cmake
+mkdir build
+cd build
+mkdir %ARCHITECTURE%
+cd %ARCHITECTURE%
+
+cmake -G "Visual Studio 14 2015" -A %ARCHITECTURE% -DgRPC_BUILD_TESTS=OFF -DgRPC_MSVC_STATIC_RUNTIME=ON -DgRPC_XDS_USER_AGENT_IS_CSHARP=ON -DgRPC_BUILD_MSVC_MP_COUNT=%GRPC_RUN_TESTS_JOBS% ../../.. || goto :error
+cmake --build . --target grpc_csharp_ext --config %MSBUILD_CONFIG% || goto :error
+
+cd ..\..\..\src\csharp
dotnet build --configuration %MSBUILD_CONFIG% Grpc.sln || goto :error
diff --git a/tools/run_tests/helper_scripts/build_csharp.sh b/tools/run_tests/helper_scripts/build_csharp.sh
index c6bee82..3e0a2fc 100755
--- a/tools/run_tests/helper_scripts/build_csharp.sh
+++ b/tools/run_tests/helper_scripts/build_csharp.sh
@@ -15,7 +15,16 @@
set -ex
-cd "$(dirname "$0")/../../../src/csharp"
+cd "$(dirname "$0")/../../.."
+
+mkdir -p cmake/build
+pushd cmake/build
+
+cmake -DgRPC_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE="${MSBUILD_CONFIG}" -DgRPC_XDS_USER_AGENT_IS_CSHARP=ON ../..
+make -j"${GRPC_RUN_TESTS_JOBS}" grpc_csharp_ext
+
+popd
+pushd src/csharp
if [ "$CONFIG" == "gcov" ]
then
diff --git a/tools/run_tests/helper_scripts/pre_build_cmake.bat b/tools/run_tests/helper_scripts/build_cxx.bat
similarity index 77%
rename from tools/run_tests/helper_scripts/pre_build_cmake.bat
rename to tools/run_tests/helper_scripts/build_cxx.bat
index aea18c9..d695dd1 100644
--- a/tools/run_tests/helper_scripts/pre_build_cmake.bat
+++ b/tools/run_tests/helper_scripts/build_cxx.bat
@@ -1,4 +1,4 @@
-@rem Copyright 2017 gRPC authors.
+@rem Copyright 2022 The gRPC Authors
@rem
@rem Licensed under the Apache License, Version 2.0 (the "License");
@rem you may not use this file except in compliance with the License.
@@ -23,6 +23,9 @@
cmake -DgRPC_BUILD_TESTS=ON %* ../.. || goto :error
+@rem GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX will be set to either "c" or "cxx"
+cmake --build . --target buildtests_%GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX% --config %MSBUILD_CONFIG% || goto :error
+
endlocal
goto :EOF
diff --git a/tools/run_tests/helper_scripts/pre_build_cmake.sh b/tools/run_tests/helper_scripts/build_cxx.sh
similarity index 74%
rename from tools/run_tests/helper_scripts/pre_build_cmake.sh
rename to tools/run_tests/helper_scripts/build_cxx.sh
index 62eaf1f..af47959 100755
--- a/tools/run_tests/helper_scripts/pre_build_cmake.sh
+++ b/tools/run_tests/helper_scripts/build_cxx.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# Copyright 2015 gRPC authors.
+# Copyright 2022 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -22,3 +22,6 @@
# MSBUILD_CONFIG's values are suitable for cmake as well
cmake -DgRPC_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE="${MSBUILD_CONFIG}" "$@" ../..
+
+# GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX will be set to either "c" or "cxx"
+make -j"${GRPC_RUN_TESTS_JOBS}" "buildtests_${GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX}" "tools_${GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX}" "check_epollexclusive"
diff --git a/tools/run_tests/helper_scripts/build_php.sh b/tools/run_tests/helper_scripts/build_php.sh
index 4add036..782a6c1 100755
--- a/tools/run_tests/helper_scripts/build_php.sh
+++ b/tools/run_tests/helper_scripts/build_php.sh
@@ -20,7 +20,10 @@
# change to grpc repo root
cd "$(dirname "$0")/../../.."
-root=$(pwd)
+# build C core first
+make -j"${GRPC_RUN_TESTS_JOBS}" EMBED_OPENSSL=true EMBED_ZLIB=true static_c shared_c
+
+repo_root="$(pwd)"
export GRPC_LIB_SUBDIR=libs/$CONFIG
export CFLAGS="-Wno-parentheses-equality"
@@ -30,8 +33,8 @@
cd ext/grpc
phpize
if [ "$CONFIG" != "gcov" ] ; then
- ./configure --enable-grpc="$root" --enable-tests
+ ./configure --enable-grpc="${repo_root}" --enable-tests
else
- ./configure --enable-grpc="$root" --enable-coverage --enable-tests
+ ./configure --enable-grpc="${repo_root}" --enable-coverage --enable-tests
fi
-make
+make -j"${GRPC_RUN_TESTS_JOBS}"
diff --git a/tools/run_tests/helper_scripts/pre_build_csharp.bat b/tools/run_tests/helper_scripts/pre_build_csharp.bat
index 886fee5..d0b0ab4 100644
--- a/tools/run_tests/helper_scripts/pre_build_csharp.bat
+++ b/tools/run_tests/helper_scripts/pre_build_csharp.bat
@@ -16,21 +16,10 @@
setlocal
-set ARCHITECTURE=%1
-
@rem enter repo root
cd /d %~dp0\..\..\..
-mkdir cmake
-cd cmake
-mkdir build
-cd build
-mkdir %ARCHITECTURE%
-cd %ARCHITECTURE%
-
-cmake -G "Visual Studio 14 2015" -A %ARCHITECTURE% -DgRPC_BUILD_TESTS=OFF -DgRPC_MSVC_STATIC_RUNTIME=ON -DgRPC_XDS_USER_AGENT_IS_CSHARP=ON -DgRPC_BUILD_MSVC_MP_COUNT=4 ../../.. || goto :error
-
-cd ..\..\..\src\csharp
+cd src\csharp
dotnet restore Grpc.sln || goto :error
diff --git a/tools/run_tests/helper_scripts/pre_build_csharp.sh b/tools/run_tests/helper_scripts/pre_build_csharp.sh
index e334d7e..eb98db2 100755
--- a/tools/run_tests/helper_scripts/pre_build_csharp.sh
+++ b/tools/run_tests/helper_scripts/pre_build_csharp.sh
@@ -18,11 +18,6 @@
# cd to repository root
cd "$(dirname "$0")/../../.."
-mkdir -p cmake/build
-cd cmake/build
-
-cmake -DgRPC_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE="${MSBUILD_CONFIG}" -DgRPC_XDS_USER_AGENT_IS_CSHARP=ON ../..
-
-cd ../../src/csharp
+cd src/csharp
dotnet restore Grpc.sln
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index ebb8900..98a53a3 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -240,16 +240,14 @@
class CLanguage(object):
- def __init__(self, make_target, test_lang):
- self.make_target = make_target
+ def __init__(self, lang_suffix, test_lang):
+ self.lang_suffix = lang_suffix
self.platform = platform_string()
self.test_lang = test_lang
def configure(self, config, args):
self.config = config
self.args = args
- self._make_options = []
- self._use_cmake = True
if self.platform == 'windows':
_check_compiler(self.args.compiler, [
'default', 'cmake', 'cmake_vs2015', 'cmake_vs2017',
@@ -286,7 +284,7 @@
out = []
binaries = get_c_tests(self.args.travis, self.test_lang)
for target in binaries:
- if self._use_cmake and target.get('boringssl', False):
+ if target.get('boringssl', False):
# cmake doesn't build boringssl tests
continue
auto_timeout_scaling = target.get('auto_timeout_scaling', True)
@@ -327,11 +325,8 @@
binary = 'cmake/build/%s/%s.exe' % (_MSBUILD_CONFIG[
self.config.build_config], target['name'])
else:
- if self._use_cmake:
- binary = 'cmake/build/%s' % target['name']
- else:
- binary = 'bins/%s/%s' % (self.config.build_config,
- target['name'])
+ binary = 'cmake/build/%s' % target['name']
+
cpu_cost = target['cpu_cost']
if cpu_cost == 'capacity':
cpu_cost = multiprocessing.cpu_count()
@@ -419,32 +414,22 @@
print('\nWARNING: binary not found, skipping', binary)
return sorted(out)
- def make_targets(self):
- if self.platform == 'windows':
- # don't build tools on windows just yet
- return ['buildtests_%s' % self.make_target]
- return [
- 'buildtests_%s' % self.make_target,
- 'tools_%s' % self.make_target, 'check_epollexclusive'
- ]
-
- def make_options(self):
- return self._make_options
-
def pre_build_steps(self):
- if self.platform == 'windows':
- return [[
- 'tools\\run_tests\\helper_scripts\\pre_build_cmake.bat',
- '-DgRPC_BUILD_MSVC_MP_COUNT=%d' % args.jobs
- ] + self._cmake_configure_extra_args]
- elif self._use_cmake:
- return [['tools/run_tests/helper_scripts/pre_build_cmake.sh'] +
- self._cmake_configure_extra_args]
- else:
- return []
+ return []
def build_steps(self):
- return []
+ if self.platform == 'windows':
+ return [[
+ 'tools\\run_tests\\helper_scripts\\build_cxx.bat',
+ '-DgRPC_BUILD_MSVC_MP_COUNT=%d' % self.args.jobs
+ ] + self._cmake_configure_extra_args]
+ else:
+ return [['tools/run_tests/helper_scripts/build_cxx.sh'] +
+ self._cmake_configure_extra_args]
+
+ def build_steps_environ(self):
+ """Extra environment variables set for pre_build_steps and build_steps jobs."""
+ return {'GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX': self.lang_suffix}
def post_tests_steps(self):
if self.platform == 'windows':
@@ -452,12 +437,6 @@
else:
return [['tools/run_tests/helper_scripts/post_tests_c.sh']]
- def makefile_name(self):
- if self._use_cmake:
- return 'cmake/build/Makefile'
- else:
- return 'Makefile'
-
def _clang_cmake_configure_extra_args(self, version_suffix=''):
return [
'-DCMAKE_C_COMPILER=clang%s' % version_suffix,
@@ -497,7 +476,7 @@
self._docker_distro, _docker_arch_suffix(self.args.arch))
def __str__(self):
- return self.make_target
+ return self.lang_suffix
# This tests Node on grpc/grpc-node and will become the standard for Node testing
@@ -545,21 +524,16 @@
def pre_build_steps(self):
return []
- def make_targets(self):
- return []
-
- def make_options(self):
- return []
-
def build_steps(self):
return []
+ def build_steps_environ(self):
+ """Extra environment variables set for pre_build_steps and build_steps jobs."""
+ return {}
+
def post_tests_steps(self):
return []
- def makefile_name(self):
- return 'Makefile'
-
def dockerfile_dir(self):
return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(
self.args.arch)
@@ -574,7 +548,6 @@
self.config = config
self.args = args
_check_compiler(self.args.compiler, ['default'])
- self._make_options = ['EMBED_OPENSSL=true', 'EMBED_ZLIB=true']
def test_specs(self):
return [
@@ -585,21 +558,16 @@
def pre_build_steps(self):
return []
- def make_targets(self):
- return ['static_c', 'shared_c']
-
- def make_options(self):
- return self._make_options
-
def build_steps(self):
return [['tools/run_tests/helper_scripts/build_php.sh']]
+ def build_steps_environ(self):
+ """Extra environment variables set for pre_build_steps and build_steps jobs."""
+ return {}
+
def post_tests_steps(self):
return [['tools/run_tests/helper_scripts/post_tests_php.sh']]
- def makefile_name(self):
- return 'Makefile'
-
def dockerfile_dir(self):
return 'tools/dockerfile/test/php7_debian11_%s' % _docker_arch_suffix(
self.args.arch)
@@ -671,24 +639,19 @@
def pre_build_steps(self):
return []
- def make_targets(self):
- return []
-
- def make_options(self):
- return []
-
def build_steps(self):
return [config.build for config in self.pythons]
+ def build_steps_environ(self):
+ """Extra environment variables set for pre_build_steps and build_steps jobs."""
+ return {}
+
def post_tests_steps(self):
if self.config.build_config != 'gcov':
return []
else:
return [['tools/run_tests/helper_scripts/post_tests_python.sh']]
- def makefile_name(self):
- return 'Makefile'
-
def dockerfile_dir(self):
return 'tools/dockerfile/test/python_%s_%s' % (
self._python_docker_distro_name(),
@@ -858,21 +821,16 @@
def pre_build_steps(self):
return [['tools/run_tests/helper_scripts/pre_build_ruby.sh']]
- def make_targets(self):
- return []
-
- def make_options(self):
- return []
-
def build_steps(self):
return [['tools/run_tests/helper_scripts/build_ruby.sh']]
+ def build_steps_environ(self):
+ """Extra environment variables set for pre_build_steps and build_steps jobs."""
+ return {}
+
def post_tests_steps(self):
return [['tools/run_tests/helper_scripts/post_tests_ruby.sh']]
- def makefile_name(self):
- return 'Makefile'
-
def dockerfile_dir(self):
return 'tools/dockerfile/test/ruby_debian11_%s' % _docker_arch_suffix(
self.args.arch)
@@ -944,39 +902,29 @@
def pre_build_steps(self):
if self.platform == 'windows':
- return [[
- 'tools\\run_tests\\helper_scripts\\pre_build_csharp.bat',
- self._cmake_arch_option
- ]]
+ return [['tools\\run_tests\\helper_scripts\\pre_build_csharp.bat']]
else:
return [['tools/run_tests/helper_scripts/pre_build_csharp.sh']]
- def make_targets(self):
- return ['grpc_csharp_ext']
-
- def make_options(self):
- return []
-
def build_steps(self):
if self.platform == 'windows':
return [['tools\\run_tests\\helper_scripts\\build_csharp.bat']]
else:
return [['tools/run_tests/helper_scripts/build_csharp.sh']]
+ def build_steps_environ(self):
+ """Extra environment variables set for pre_build_steps and build_steps jobs."""
+ if self.platform == 'windows':
+ return {'ARCHITECTURE': self._cmake_arch_option}
+ else:
+ return {}
+
def post_tests_steps(self):
if self.platform == 'windows':
return [['tools\\run_tests\\helper_scripts\\post_tests_csharp.bat']]
else:
return [['tools/run_tests/helper_scripts/post_tests_csharp.sh']]
- def makefile_name(self):
- if self.platform == 'windows':
- return 'cmake/build/%s/Makefile' % self._cmake_arch_option
- else:
- # no need to set x86 specific flags as run_tests.py
- # currently forbids x86 C# builds on both Linux and MacOS.
- return 'cmake/build/Makefile'
-
def dockerfile_dir(self):
return 'tools/dockerfile/test/csharp_%s_%s' % (
self._docker_distro, _docker_arch_suffix(self.args.arch))
@@ -1139,21 +1087,16 @@
def pre_build_steps(self):
return []
- def make_targets(self):
- return []
-
- def make_options(self):
- return []
-
def build_steps(self):
return []
+ def build_steps_environ(self):
+ """Extra environment variables set for pre_build_steps and build_steps jobs."""
+ return {}
+
def post_tests_steps(self):
return []
- def makefile_name(self):
- return 'Makefile'
-
def dockerfile_dir(self):
return None
@@ -1191,21 +1134,16 @@
def pre_build_steps(self):
return []
- def make_targets(self):
- return ['run_dep_checks']
-
- def make_options(self):
- return []
-
def build_steps(self):
return []
+ def build_steps_environ(self):
+ """Extra environment variables set for pre_build_steps and build_steps jobs."""
+ return {}
+
def post_tests_steps(self):
return []
- def makefile_name(self):
- return 'Makefile'
-
def dockerfile_dir(self):
return 'tools/dockerfile/test/sanity'
@@ -1237,6 +1175,16 @@
}
+def _build_step_environ(cfg, extra_env={}):
+ """Environment variables set for each build step."""
+ environ = {'CONFIG': cfg, 'GRPC_RUN_TESTS_JOBS': str(args.jobs)}
+ msbuild_cfg = _MSBUILD_CONFIG.get(cfg)
+ if msbuild_cfg:
+ environ['MSBUILD_CONFIG'] = msbuild_cfg
+ environ.update(extra_env)
+ return environ
+
+
def _windows_arch_option(arch):
"""Returns msbuild cmdline option for selected architecture."""
if arch == 'default' or arch == 'x86':
@@ -1319,369 +1267,8 @@
return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)
-# parse command line
-argp = argparse.ArgumentParser(description='Run grpc tests.')
-argp.add_argument('-c',
- '--config',
- choices=sorted(_CONFIGS.keys()),
- default='opt')
-argp.add_argument(
- '-n',
- '--runs_per_test',
- default=1,
- type=runs_per_test_type,
- help='A positive integer or "inf". If "inf", all tests will run in an '
- 'infinite loop. Especially useful in combination with "-f"')
-argp.add_argument('-r', '--regex', default='.*', type=str)
-argp.add_argument('--regex_exclude', default='', type=str)
-argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int)
-argp.add_argument('-s', '--slowdown', default=1.0, type=float)
-argp.add_argument('-p',
- '--sample_percent',
- default=100.0,
- type=percent_type,
- help='Run a random sample with that percentage of tests')
-argp.add_argument(
- '-t',
- '--travis',
- default=False,
- action='store_const',
- const=True,
- help='When set, indicates that the script is running on CI (= not locally).'
-)
-argp.add_argument('--newline_on_success',
- default=False,
- action='store_const',
- const=True)
-argp.add_argument('-l',
- '--language',
- choices=sorted(_LANGUAGES.keys()),
- nargs='+',
- required=True)
-argp.add_argument('-S',
- '--stop_on_failure',
- default=False,
- action='store_const',
- const=True)
-argp.add_argument('--use_docker',
- default=False,
- action='store_const',
- const=True,
- help='Run all the tests under docker. That provides ' +
- 'additional isolation and prevents the need to install ' +
- 'language specific prerequisites. Only available on Linux.')
-argp.add_argument(
- '--allow_flakes',
- default=False,
- action='store_const',
- const=True,
- help=
- 'Allow flaky tests to show as passing (re-runs failed tests up to five times)'
-)
-argp.add_argument(
- '--arch',
- choices=['default', 'x86', 'x64'],
- default='default',
- help=
- 'Selects architecture to target. For some platforms "default" is the only supported choice.'
-)
-argp.add_argument(
- '--compiler',
- choices=[
- 'default',
- 'gcc4.9',
- 'gcc10.2',
- 'gcc10.2_openssl102',
- 'gcc11',
- 'gcc_musl',
- 'clang4',
- 'clang13',
- 'python2.7',
- 'python3.5',
- 'python3.6',
- 'python3.7',
- 'python3.8',
- 'python3.9',
- 'pypy',
- 'pypy3',
- 'python_alpine',
- 'all_the_cpythons',
- 'electron1.3',
- 'electron1.6',
- 'coreclr',
- 'cmake',
- 'cmake_vs2015',
- 'cmake_vs2017',
- 'cmake_vs2019',
- 'mono',
- ],
- default='default',
- help=
- 'Selects compiler to use. Allowed values depend on the platform and language.'
-)
-argp.add_argument('--iomgr_platform',
- choices=['native', 'gevent', 'asyncio'],
- default='native',
- help='Selects iomgr platform to build on')
-argp.add_argument('--build_only',
- default=False,
- action='store_const',
- const=True,
- help='Perform all the build steps but don\'t run any tests.')
-argp.add_argument('--measure_cpu_costs',
- default=False,
- action='store_const',
- const=True,
- help='Measure the cpu costs of tests')
-argp.add_argument(
- '--update_submodules',
- default=[],
- nargs='*',
- help=
- 'Update some submodules before building. If any are updated, also run generate_projects. '
- +
- 'Submodules are specified as SUBMODULE_NAME:BRANCH; if BRANCH is omitted, master is assumed.'
-)
-argp.add_argument('-a', '--antagonists', default=0, type=int)
-argp.add_argument('-x',
- '--xml_report',
- default=None,
- type=str,
- help='Generates a JUnit-compatible XML report')
-argp.add_argument('--report_suite_name',
- default='tests',
- type=str,
- help='Test suite name to use in generated JUnit XML report')
-argp.add_argument(
- '--report_multi_target',
- default=False,
- const=True,
- action='store_const',
- help='Generate separate XML report for each test job (Looks better in UIs).'
-)
-argp.add_argument(
- '--quiet_success',
- default=False,
- action='store_const',
- const=True,
- help=
- 'Don\'t print anything when a test passes. Passing tests also will not be reported in XML report. '
- + 'Useful when running many iterations of each test (argument -n).')
-argp.add_argument(
- '--force_default_poller',
- default=False,
- action='store_const',
- const=True,
- help='Don\'t try to iterate over many polling strategies when they exist')
-argp.add_argument(
- '--force_use_pollers',
- default=None,
- type=str,
- help='Only use the specified comma-delimited list of polling engines. '
- 'Example: --force_use_pollers epoll1,poll '
- ' (This flag has no effect if --force_default_poller flag is also used)')
-argp.add_argument('--max_time',
- default=-1,
- type=int,
- help='Maximum test runtime in seconds')
-argp.add_argument('--bq_result_table',
- default='',
- type=str,
- nargs='?',
- help='Upload test results to a specified BQ table.')
-args = argp.parse_args()
-
-flaky_tests = set()
-shortname_to_cpu = {}
-
-if args.force_default_poller:
- _POLLING_STRATEGIES = {}
-elif args.force_use_pollers:
- _POLLING_STRATEGIES[platform_string()] = args.force_use_pollers.split(',')
-
-jobset.measure_cpu_costs = args.measure_cpu_costs
-
-# update submodules if necessary
-need_to_regenerate_projects = False
-for spec in args.update_submodules:
- spec = spec.split(':', 1)
- if len(spec) == 1:
- submodule = spec[0]
- branch = 'master'
- elif len(spec) == 2:
- submodule = spec[0]
- branch = spec[1]
- cwd = 'third_party/%s' % submodule
-
- def git(cmd, cwd=cwd):
- print('in %s: git %s' % (cwd, cmd))
- run_shell_command('git %s' % cmd, cwd=cwd)
-
- git('fetch')
- git('checkout %s' % branch)
- git('pull origin %s' % branch)
- if os.path.exists('src/%s/gen_build_yaml.py' % submodule):
- need_to_regenerate_projects = True
-if need_to_regenerate_projects:
- if jobset.platform_string() == 'linux':
- run_shell_command('tools/buildgen/generate_projects.sh')
- else:
- print(
- 'WARNING: may need to regenerate projects, but since we are not on')
- print(
- ' Linux this step is being skipped. Compilation MAY fail.')
-
-# grab config
-run_config = _CONFIGS[args.config]
-build_config = run_config.build_config
-
-if args.travis:
- _FORCE_ENVIRON_FOR_WRAPPERS = {'GRPC_TRACE': 'api'}
-
-languages = set(_LANGUAGES[l] for l in args.language)
-for l in languages:
- l.configure(run_config, args)
-
-language_make_options = []
-if any(language.make_options() for language in languages):
- if not 'gcov' in args.config and len(languages) != 1:
- print(
- 'languages with custom make options cannot be built simultaneously with other languages'
- )
- sys.exit(1)
- else:
- # Combining make options is not clean and just happens to work. It allows C & C++ to build
- # together, and is only used under gcov. All other configs should build languages individually.
- language_make_options = list(
- set([
- make_option for lang in languages
- for make_option in lang.make_options()
- ]))
-
-if args.use_docker:
- if not args.travis:
- print('Seen --use_docker flag, will run tests under docker.')
- print('')
- print(
- 'IMPORTANT: The changes you are testing need to be locally committed'
- )
- print(
- 'because only the committed changes in the current branch will be')
- print('copied to the docker environment.')
- time.sleep(5)
-
- dockerfile_dirs = set([l.dockerfile_dir() for l in languages])
- if len(dockerfile_dirs) > 1:
- print('Languages to be tested require running under different docker '
- 'images.')
- sys.exit(1)
- else:
- dockerfile_dir = next(iter(dockerfile_dirs))
-
- child_argv = [arg for arg in sys.argv if not arg == '--use_docker']
- run_tests_cmd = 'python3 tools/run_tests/run_tests.py %s' % ' '.join(
- child_argv[1:])
-
- env = os.environ.copy()
- env['DOCKERFILE_DIR'] = dockerfile_dir
- env['DOCKER_RUN_SCRIPT'] = 'tools/run_tests/dockerize/docker_run.sh'
- env['DOCKER_RUN_SCRIPT_COMMAND'] = run_tests_cmd
- # TODO(jtattermusch): is the XML_REPORT env variable any useful?
- if args.xml_report:
- env['XML_REPORT'] = args.xml_report
-
- retcode = subprocess.call(
- 'tools/run_tests/dockerize/build_and_run_docker.sh',
- shell=True,
- env=env)
- _print_debug_info_epilogue(dockerfile_dir=dockerfile_dir)
- sys.exit(retcode)
-
-_check_arch_option(args.arch)
-
-
-def make_jobspec(cfg, targets, makefile='Makefile'):
- if platform_string() == 'windows':
- return [
- jobset.JobSpec([
- 'cmake', '--build', '.', '--target',
- '%s' % target, '--config', _MSBUILD_CONFIG[cfg]
- ],
- cwd=os.path.dirname(makefile),
- timeout_seconds=None) for target in targets
- ]
- else:
- if targets and makefile.startswith('cmake/build/'):
- # With cmake, we've passed all the build configuration in the pre-build step already
- return [
- jobset.JobSpec(
- [os.getenv('MAKE', 'make'), '-j',
- '%d' % args.jobs] + targets,
- cwd='cmake/build',
- timeout_seconds=None)
- ]
- if targets:
- return [
- jobset.JobSpec(
- [
- os.getenv('MAKE', 'make'), '-f', makefile, '-j',
- '%d' % args.jobs,
- 'EXTRA_DEFINES=GRPC_TEST_SLOWDOWN_MACHINE_FACTOR=%f' %
- args.slowdown,
- 'CONFIG=%s' % cfg, 'Q='
- ] + language_make_options +
- ([] if not args.travis else ['JENKINS_BUILD=1']) + targets,
- timeout_seconds=None)
- ]
- else:
- return []
-
-
-make_targets = {}
-for l in languages:
- makefile = l.makefile_name()
- make_targets[makefile] = make_targets.get(makefile, set()).union(
- set(l.make_targets()))
-
-
-def build_step_environ(cfg):
- environ = {'CONFIG': cfg}
- msbuild_cfg = _MSBUILD_CONFIG.get(cfg)
- if msbuild_cfg:
- environ['MSBUILD_CONFIG'] = msbuild_cfg
- return environ
-
-
-build_steps = list(
- set(
- jobset.JobSpec(cmdline,
- environ=build_step_environ(build_config),
- timeout_seconds=_PRE_BUILD_STEP_TIMEOUT_SECONDS,
- flake_retries=2)
- for l in languages
- for cmdline in l.pre_build_steps()))
-if make_targets:
- make_commands = itertools.chain.from_iterable(
- make_jobspec(build_config, list(targets), makefile)
- for (makefile, targets) in make_targets.items())
- build_steps.extend(set(make_commands))
-build_steps.extend(
- set(
- jobset.JobSpec(cmdline,
- environ=build_step_environ(build_config),
- timeout_seconds=None)
- for l in languages
- for cmdline in l.build_steps()))
-
-post_tests_steps = list(
- set(
- jobset.JobSpec(cmdline, environ=build_step_environ(build_config))
- for l in languages
- for cmdline in l.post_tests_steps()))
-runs_per_test = args.runs_per_test
-
-
def _shut_down_legacy_server(legacy_server_port):
+ """Shut down legacy version of port server."""
try:
version = int(
urllib.request.urlopen('http://localhost:%d/version_number' %
@@ -1712,16 +1299,8 @@
return num_runs, num_failures
-# _build_and_run results
-class BuildAndRunError(object):
-
- BUILD = object()
- TEST = object()
- POST_TEST = object()
-
-
def _has_epollexclusive():
- binary = 'bins/%s/check_epollexclusive' % args.config
+ binary = 'cmake/build/check_epollexclusive'
if not os.path.exists(binary):
return False
try:
@@ -1734,6 +1313,14 @@
return False
+class BuildAndRunError(object):
+ """Represents error type in _build_and_run."""
+
+ BUILD = object()
+ TEST = object()
+ POST_TEST = object()
+
+
# returns a list of things that failed (or an empty list on success)
def _build_and_run(check_cancelled,
newline_on_success,
@@ -1869,6 +1456,267 @@
return out
+# parse command line
+argp = argparse.ArgumentParser(description='Run grpc tests.')
+argp.add_argument('-c',
+ '--config',
+ choices=sorted(_CONFIGS.keys()),
+ default='opt')
+argp.add_argument(
+ '-n',
+ '--runs_per_test',
+ default=1,
+ type=runs_per_test_type,
+ help='A positive integer or "inf". If "inf", all tests will run in an '
+ 'infinite loop. Especially useful in combination with "-f"')
+argp.add_argument('-r', '--regex', default='.*', type=str)
+argp.add_argument('--regex_exclude', default='', type=str)
+argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int)
+argp.add_argument('-s', '--slowdown', default=1.0, type=float)
+argp.add_argument('-p',
+ '--sample_percent',
+ default=100.0,
+ type=percent_type,
+ help='Run a random sample with that percentage of tests')
+argp.add_argument(
+ '-t',
+ '--travis',
+ default=False,
+ action='store_const',
+ const=True,
+ help='When set, indicates that the script is running on CI (= not locally).'
+)
+argp.add_argument('--newline_on_success',
+ default=False,
+ action='store_const',
+ const=True)
+argp.add_argument('-l',
+ '--language',
+ choices=sorted(_LANGUAGES.keys()),
+ nargs='+',
+ required=True)
+argp.add_argument('-S',
+ '--stop_on_failure',
+ default=False,
+ action='store_const',
+ const=True)
+argp.add_argument('--use_docker',
+ default=False,
+ action='store_const',
+ const=True,
+ help='Run all the tests under docker. That provides ' +
+ 'additional isolation and prevents the need to install ' +
+ 'language specific prerequisites. Only available on Linux.')
+argp.add_argument(
+ '--allow_flakes',
+ default=False,
+ action='store_const',
+ const=True,
+ help=
+ 'Allow flaky tests to show as passing (re-runs failed tests up to five times)'
+)
+argp.add_argument(
+ '--arch',
+ choices=['default', 'x86', 'x64'],
+ default='default',
+ help=
+ 'Selects architecture to target. For some platforms "default" is the only supported choice.'
+)
+argp.add_argument(
+ '--compiler',
+ choices=[
+ 'default',
+ 'gcc4.9',
+ 'gcc10.2',
+ 'gcc10.2_openssl102',
+ 'gcc11',
+ 'gcc_musl',
+ 'clang4',
+ 'clang13',
+ 'python2.7',
+ 'python3.5',
+ 'python3.6',
+ 'python3.7',
+ 'python3.8',
+ 'python3.9',
+ 'pypy',
+ 'pypy3',
+ 'python_alpine',
+ 'all_the_cpythons',
+ 'electron1.3',
+ 'electron1.6',
+ 'coreclr',
+ 'cmake',
+ 'cmake_vs2015',
+ 'cmake_vs2017',
+ 'cmake_vs2019',
+ 'mono',
+ ],
+ default='default',
+ help=
+ 'Selects compiler to use. Allowed values depend on the platform and language.'
+)
+argp.add_argument('--iomgr_platform',
+ choices=['native', 'gevent', 'asyncio'],
+ default='native',
+ help='Selects iomgr platform to build on')
+argp.add_argument('--build_only',
+ default=False,
+ action='store_const',
+ const=True,
+ help='Perform all the build steps but don\'t run any tests.')
+argp.add_argument('--measure_cpu_costs',
+ default=False,
+ action='store_const',
+ const=True,
+ help='Measure the cpu costs of tests')
+argp.add_argument('-a', '--antagonists', default=0, type=int)
+argp.add_argument('-x',
+ '--xml_report',
+ default=None,
+ type=str,
+ help='Generates a JUnit-compatible XML report')
+argp.add_argument('--report_suite_name',
+ default='tests',
+ type=str,
+ help='Test suite name to use in generated JUnit XML report')
+argp.add_argument(
+ '--report_multi_target',
+ default=False,
+ const=True,
+ action='store_const',
+ help='Generate separate XML report for each test job (Looks better in UIs).'
+)
+argp.add_argument(
+ '--quiet_success',
+ default=False,
+ action='store_const',
+ const=True,
+ help=
+ 'Don\'t print anything when a test passes. Passing tests also will not be reported in XML report. '
+ + 'Useful when running many iterations of each test (argument -n).')
+argp.add_argument(
+ '--force_default_poller',
+ default=False,
+ action='store_const',
+ const=True,
+ help='Don\'t try to iterate over many polling strategies when they exist')
+argp.add_argument(
+ '--force_use_pollers',
+ default=None,
+ type=str,
+ help='Only use the specified comma-delimited list of polling engines. '
+ 'Example: --force_use_pollers epoll1,poll '
+ ' (This flag has no effect if --force_default_poller flag is also used)')
+argp.add_argument('--max_time',
+ default=-1,
+ type=int,
+ help='Maximum test runtime in seconds')
+argp.add_argument('--bq_result_table',
+ default='',
+ type=str,
+ nargs='?',
+ help='Upload test results to a specified BQ table.')
+args = argp.parse_args()
+
+flaky_tests = set()
+shortname_to_cpu = {}
+
+if args.force_default_poller:
+ _POLLING_STRATEGIES = {}
+elif args.force_use_pollers:
+ _POLLING_STRATEGIES[platform_string()] = args.force_use_pollers.split(',')
+
+jobset.measure_cpu_costs = args.measure_cpu_costs
+
+# grab config
+run_config = _CONFIGS[args.config]
+build_config = run_config.build_config
+
+# TODO(jtattermusch): is this setting applied/being used?
+if args.travis:
+ _FORCE_ENVIRON_FOR_WRAPPERS = {'GRPC_TRACE': 'api'}
+
+languages = set(_LANGUAGES[l] for l in args.language)
+for l in languages:
+ l.configure(run_config, args)
+
+if len(languages) != 1:
+ print('Building multiple languages simultaneously is not supported!')
+ sys.exit(1)
+
+# If --use_docker was used, respawn the run_tests.py script under a docker container
+# instead of continuing.
+if args.use_docker:
+ if not args.travis:
+ print('Seen --use_docker flag, will run tests under docker.')
+ print('')
+ print(
+ 'IMPORTANT: The changes you are testing need to be locally committed'
+ )
+ print(
+ 'because only the committed changes in the current branch will be')
+ print('copied to the docker environment.')
+ time.sleep(5)
+
+ dockerfile_dirs = set([l.dockerfile_dir() for l in languages])
+ if len(dockerfile_dirs) > 1:
+ print('Languages to be tested require running under different docker '
+ 'images.')
+ sys.exit(1)
+ else:
+ dockerfile_dir = next(iter(dockerfile_dirs))
+
+ child_argv = [arg for arg in sys.argv if not arg == '--use_docker']
+ run_tests_cmd = 'python3 tools/run_tests/run_tests.py %s' % ' '.join(
+ child_argv[1:])
+
+ env = os.environ.copy()
+ env['DOCKERFILE_DIR'] = dockerfile_dir
+ env['DOCKER_RUN_SCRIPT'] = 'tools/run_tests/dockerize/docker_run.sh'
+ env['DOCKER_RUN_SCRIPT_COMMAND'] = run_tests_cmd
+
+ retcode = subprocess.call(
+ 'tools/run_tests/dockerize/build_and_run_docker.sh',
+ shell=True,
+ env=env)
+ _print_debug_info_epilogue(dockerfile_dir=dockerfile_dir)
+ sys.exit(retcode)
+
+_check_arch_option(args.arch)
+
+# collect pre-build steps (which get retried if they fail, e.g. to avoid
+# flakes on downloading dependencies etc.)
+build_steps = list(
+ set(
+ jobset.JobSpec(cmdline,
+ environ=_build_step_environ(
+ build_config, extra_env=l.build_steps_environ()),
+ timeout_seconds=_PRE_BUILD_STEP_TIMEOUT_SECONDS,
+ flake_retries=2)
+ for l in languages
+ for cmdline in l.pre_build_steps()))
+
+# collect build steps
+build_steps.extend(
+ set(
+ jobset.JobSpec(cmdline,
+ environ=_build_step_environ(
+ build_config, extra_env=l.build_steps_environ()),
+ timeout_seconds=None)
+ for l in languages
+ for cmdline in l.build_steps()))
+
+# collect post test steps
+post_tests_steps = list(
+ set(
+ jobset.JobSpec(cmdline,
+ environ=_build_step_environ(
+ build_config, extra_env=l.build_steps_environ()))
+ for l in languages
+ for cmdline in l.post_tests_steps()))
+runs_per_test = args.runs_per_test
+
errors = _build_and_run(check_cancelled=lambda: False,
newline_on_success=args.newline_on_success,
xml_report=args.xml_report,
