ci: add bazel_integration_test (#338)
This runs a py_test with a copy of bazel as a data dep.
It glob()s up the sources for each example and runs nested bazel test on them.
This detects whether the examples are fully working and self-contained.
Follow-up step is to replace the rules_python.tgz with a HEAD version so we detect
breakages.
diff --git a/.bazelrc b/.bazelrc
index e8874a9..2b60f35 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -1,8 +1,9 @@
# For bazel-in-bazel testing
-# Trick bazel into treating BUILD files under examples/* and e2e/* as being regular files
+# Trick bazel into treating BUILD files under examples/* as being regular files
# This lets us glob() up all the files inside the examples to make them inputs to tests
# (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
-# To update these lines, run this command:
-# sed -i.bak "/^[^#].*--deleted_packages/s#=.*#=$(find examples/*/* \( -name BUILD -or -name BUILD.bazel \) | xargs -n 1 dirname | paste -sd, -)#" .bazelrc && rm .bazelrc.bak
+# To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh
build --deleted_packages=examples/pip/boto,examples/pip/extras,examples/pip/helloworld
query --deleted_packages=examples/pip/boto,examples/pip/extras,examples/pip/helloworld
+
+test --test_output=errors
diff --git a/examples/BUILD b/examples/BUILD
index 426ddc0..18377eb 100644
--- a/examples/BUILD
+++ b/examples/BUILD
@@ -11,6 +11,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+load("//tools/bazel_integration_test:bazel_integration_test.bzl", "bazel_integration_test")
+
package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # Apache 2.0
+
+bazel_integration_test(
+ name = "pip_example",
+ timeout = "long",
+)
diff --git a/examples/pip/.bazelrc b/examples/pip/.bazelrc
new file mode 100644
index 0000000..0dca511
--- /dev/null
+++ b/examples/pip/.bazelrc
@@ -0,0 +1,2 @@
+# Watch Bazel run, and only run one at a time
+test --test_output=streamed
diff --git a/examples/pip/boto/BUILD b/examples/pip/boto/BUILD
index c5ac4fc..7bd91d9 100644
--- a/examples/pip/boto/BUILD
+++ b/examples/pip/boto/BUILD
@@ -22,8 +22,10 @@
py_test(
name = "boto_test",
srcs = ["boto_test.py"],
+ python_version = "PY2",
deps = [
requirement("boto3"),
+ requirement("pip"),
# six is a transitive dependency via python-dateutil. Explicitly depend
# on it to work around issue #70; see issue #98.
requirement("six"),
diff --git a/examples/pip/helloworld/helloworld_test.py b/examples/pip/helloworld/helloworld_test.py
index ca9f6b1..da6ac77 100644
--- a/examples/pip/helloworld/helloworld_test.py
+++ b/examples/pip/helloworld/helloworld_test.py
@@ -12,10 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import helloworld
import unittest
-from examples.helloworld import helloworld
-
class HelloWorldTest(unittest.TestCase):
diff --git a/internal_deps.bzl b/internal_deps.bzl
index 281efd8..15fe0a5 100644
--- a/internal_deps.bzl
+++ b/internal_deps.bzl
@@ -110,3 +110,13 @@
name = "piptool_deps",
requirements = "@rules_python//python:requirements.txt",
)
+
+ maybe(
+ http_archive,
+ name = "build_bazel_integration_testing",
+ urls = [
+ "https://github.com/bazelbuild/bazel-integration-testing/archive/165440b2dbda885f8d1ccb8d0f417e6cf8c54f17.zip",
+ ],
+ strip_prefix = "bazel-integration-testing-165440b2dbda885f8d1ccb8d0f417e6cf8c54f17",
+ sha256 = "2401b1369ef44cc42f91dc94443ef491208dbd06da1e1e10b702d8c189f098e3",
+ )
diff --git a/internal_setup.bzl b/internal_setup.bzl
index 90c74f2..646b86c 100644
--- a/internal_setup.bzl
+++ b/internal_setup.bzl
@@ -1,13 +1,20 @@
"""Setup for rules_python tests and tools."""
+load("@build_bazel_integration_testing//tools:repositories.bzl", "bazel_binaries")
+
# Requirements for building our piptool.
load(
"@piptool_deps//:requirements.bzl",
_piptool_install = "pip_install",
)
+load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS")
+
def rules_python_internal_setup():
"""Setup for rules_python tests and tools."""
# Requirements for building our piptool.
_piptool_install()
+
+ # Depend on the Bazel binaries for running bazel-in-bazel tests
+ bazel_binaries(versions = SUPPORTED_BAZEL_VERSIONS)
diff --git a/tools/bazel_integration_test/BUILD b/tools/bazel_integration_test/BUILD
new file mode 100644
index 0000000..10566c4
--- /dev/null
+++ b/tools/bazel_integration_test/BUILD
@@ -0,0 +1 @@
+exports_files(["test_runner.py"])
diff --git a/tools/bazel_integration_test/bazel_integration_test.bzl b/tools/bazel_integration_test/bazel_integration_test.bzl
new file mode 100644
index 0000000..b23360b
--- /dev/null
+++ b/tools/bazel_integration_test/bazel_integration_test.bzl
@@ -0,0 +1,111 @@
+"Define a rule for running bazel test under Bazel"
+
+load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS")
+load("//python:defs.bzl", "py_test")
+
+BAZEL_BINARY = "@build_bazel_bazel_%s//:bazel_binary" % SUPPORTED_BAZEL_VERSIONS[0].replace(".", "_")
+
+_ATTRS = {
+ "bazel_binary": attr.label(
+ default = BAZEL_BINARY,
+ doc = """The bazel binary files to test against.
+
+It is assumed by the test runner that the bazel binary is found at label_workspace/bazel (wksp/bazel.exe on Windows)""",
+ ),
+ "bazel_commands": attr.string_list(
+ default = ["info", "test ..."],
+ doc = """The list of bazel commands to run. Defaults to `["info", "test ..."]`.
+
+Note that if a command contains a bare `--` argument, the --test_arg passed to Bazel will appear before it.
+""",
+ ),
+ "workspace_files": attr.label(
+ doc = """A filegroup of all files in the workspace-under-test necessary to run the test.""",
+ ),
+}
+
+# Avoid using non-normalized paths (workspace/../other_workspace/path)
+def _to_manifest_path(ctx, file):
+ if file.short_path.startswith("../"):
+ return file.short_path[3:]
+ else:
+ return ctx.workspace_name + "/" + file.short_path
+
+def _config_impl(ctx):
+ if len(SUPPORTED_BAZEL_VERSIONS) > 1:
+ fail("""
+ bazel_integration_test doesn't support multiple Bazel versions to test against yet.
+ """)
+ if len(ctx.files.workspace_files) == 0:
+ fail("""
+No files were found to run under integration testing. See comment in /.bazelrc.
+You probably need to run
+ tools/bazel_integration_test/update_deleted_packages.sh
+""")
+
+ # Serialize configuration file for test runner
+ config = ctx.actions.declare_file("%s.json" % ctx.attr.name)
+ ctx.actions.write(
+ output = config,
+ content = """
+{{
+ "workspaceRoot": "{TMPL_workspace_root}",
+ "bazelBinaryWorkspace": "{TMPL_bazel_binary_workspace}",
+ "bazelCommands": [ {TMPL_bazel_commands} ]
+}}
+""".format(
+ TMPL_workspace_root = ctx.files.workspace_files[0].dirname,
+ TMPL_bazel_binary_workspace = ctx.attr.bazel_binary.label.workspace_name,
+ TMPL_bazel_commands = ", ".join(["\"%s\"" % s for s in ctx.attr.bazel_commands]),
+ ),
+ )
+
+ return [DefaultInfo(
+ files = depset([config]),
+ runfiles = ctx.runfiles(files = [config]),
+ )]
+
+_config = rule(
+ implementation = _config_impl,
+ doc = "Configures an integration test that runs a specified version of bazel against an external workspace.",
+ attrs = _ATTRS,
+)
+
+def bazel_integration_test(name, **kwargs):
+ """Wrapper macro to set default srcs and run a py_test with config
+
+ Args:
+ name: name of the resulting py_test
+ **kwargs: additional attributes like timeout and visibility
+ """
+ # By default, we assume sources for "pip_example" are in examples/pip/**/*
+ dirname = name[:-len("_example")]
+ native.filegroup(
+ name = "_%s_sources" % name,
+ srcs = native.glob(
+ ["%s/**/*" % dirname],
+ exclude = ["%s/bazel-*/**" % dirname],
+ ),
+ )
+ workspace_files = kwargs.pop("workspace_files", "_%s_sources" % name)
+
+ _config(
+ name = "_%s_config" % name,
+ workspace_files = workspace_files,
+ )
+
+ py_test(
+ name = name,
+ srcs = [Label("//tools/bazel_integration_test:test_runner.py")],
+ main = "test_runner.py",
+ args = [native.package_name() + "/_%s_config.json" % name],
+ deps = [Label("//python/runfiles")],
+ data = [
+ BAZEL_BINARY,
+ "_%s_config" % name,
+ workspace_files,
+ ],
+ **kwargs,
+ )
+
+
\ No newline at end of file
diff --git a/tools/bazel_integration_test/test_runner.py b/tools/bazel_integration_test/test_runner.py
new file mode 100644
index 0000000..46ba734
--- /dev/null
+++ b/tools/bazel_integration_test/test_runner.py
@@ -0,0 +1,45 @@
+from pathlib import Path
+import json
+import os
+import platform
+from subprocess import Popen
+import sys
+
+from rules_python.python.runfiles import runfiles
+
+def main(conf_file):
+ with open(conf_file) as j:
+ config = json.load(j)
+ r = runfiles.Create()
+
+ isWindows = platform.system() == 'Windows'
+ bazelBinary = r.Rlocation(os.path.join(config['bazelBinaryWorkspace'], 'bazel.exe' if isWindows else 'bazel'))
+
+ workspacePath = config['workspaceRoot']
+ # Canonicalize bazel external/some_repo/foo
+ if workspacePath.startswith('external/'):
+ workspacePath = '..' + workspacePath[len('external'):]
+
+ for command in config['bazelCommands']:
+ bazel_args = command.split(' ')
+ try:
+ doubleHyphenPos = bazel_args.index('--')
+ print("patch that in ", doubleHyphenPos)
+ except ValueError:
+ pass
+
+
+ # Bazel's wrapper script needs this or you get
+ # 2020/07/13 21:58:11 could not get the user's cache directory: $HOME is not defined
+ os.environ['HOME'] = str(Path.home())
+
+ bazel_args.insert(0, bazelBinary)
+ bazel_process = Popen(bazel_args, cwd = workspacePath)
+ bazel_process.wait()
+ if bazel_process.returncode != 0:
+ # Test failure in Bazel is exit 3
+ # https://github.com/bazelbuild/bazel/blob/486206012a664ecb20bdb196a681efc9a9825049/src/main/java/com/google/devtools/build/lib/util/ExitCode.java#L44
+ sys.exit(3)
+
+if __name__ == '__main__':
+ main(sys.argv[1])
diff --git a/tools/bazel_integration_test/update_deleted_packages.sh b/tools/bazel_integration_test/update_deleted_packages.sh
new file mode 100755
index 0000000..95a8500
--- /dev/null
+++ b/tools/bazel_integration_test/update_deleted_packages.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# For integration tests, we want to be able to glob() up the sources inside a nested package
+# See explanation in .bazelrc
+
+set -eux
+
+DIR="$(dirname $0)/../.."
+# The sed -i.bak pattern is compatible between macos and linux
+sed -i.bak "/^[^#].*--deleted_packages/s#=.*#=$(\
+ find examples/*/* \( -name BUILD -or -name BUILD.bazel \) | xargs -n 1 dirname | paste -sd, -\
+)#" $DIR/.bazelrc && rm .bazelrc.bak
diff --git a/version.bzl b/version.bzl
index b9e2216..00d16b5 100644
--- a/version.bzl
+++ b/version.bzl
@@ -14,3 +14,20 @@
"""The version of rules_python."""
version = "0.0.2"
+
+# Currently used Bazel version. This version is what the rules here are tested
+# against.
+# This version should be updated together with the version of the Bazel
+# in .bazelversion.
+# TODO(alexeagle): assert this is the case in a test
+BAZEL_VERSION = "3.3.1"
+
+# Versions of Bazel which users should be able to use.
+# Ensures we don't break backwards-compatibility,
+# accidentally forcing users to update their LTS-supported bazel.
+# These are the versions used when testing nested workspaces with
+# bazel_integration_test.
+SUPPORTED_BAZEL_VERSIONS = [
+ # TODO: add LTS versions of bazel like 1.0.0, 2.0.0
+ BAZEL_VERSION,
+]
