feat(bzlmod)!: Move each bzlmod extension into its own file (#1226)
This commit refactors the files that contain the bzlmod
extensions.
- All extensions are moved under the new extensions folder
- Private extensions are moved under extensions/private
- All extension files are renamed to remove the _extension suffix
- pip and internal_deps extensions are moved to their own file
This commit organizes the extensions better and also follows the
best practice of having a single extension per file. Having each
extension in its own file allows them to use some additional features
while helping avoid backwards incompatible changes.
## BREAKING CHANGES
This splits `//python:extensions.bzl`, which previously held the
`python`
and `pip` extensions, into separate files (`python.bzl` and `pip.bzl`,
respectively). Unfortunately, moving the location of the extensions is a
breaking change due to how bzlmod extension identity works (see
https://bazel.build/external/extension#extension_identity). Fortunately,
by moving to one extension per file, we shouldn't have to ever do this
again.
Users must update the file path in their `use_repo()` statements as
follows:
* `use_extension("@rules_python//python:extensions.bzl", "python")` ->
`use_extension("@rules_python//python/extensions:python.bzl", "python")`
* `use_extension("@rules_python//python:extensions.bzl", "pip")` ->
`use_extension("@rules_python//python/extensions:pip.bzl", "pip")`
The following `sed` commands should approximate the necessary changes:
```
sed 'sXuse_extension("@rules_python//python:extensions.bzl", "python")Xuse_extension("@rules_python//python/extensions:python.bzl", "python")X'`
sed 'sXuse_extension("@rules_python//python:extensions.bzl", "pip")Xuse_extension("@rules_python//python/extensions:pip.bzl", "pip")X'`
```
See `examples/bzlmod_build_file_generation/MODULE.bazel` for an example
of the new paths.diff --git a/MODULE.bazel b/MODULE.bazel
index e490beb..ddd946c 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -11,7 +11,7 @@
bazel_dep(name = "rules_proto", version = "5.3.0-21.7")
bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf")
-internal_deps = use_extension("@rules_python//python:extensions.bzl", "internal_deps")
+internal_deps = use_extension("@rules_python//python/extensions/private:internal_deps.bzl", "internal_deps")
internal_deps.install()
use_repo(
internal_deps,
@@ -47,5 +47,5 @@
"pypi__coverage_cp39_x86_64-unknown-linux-gnu",
)
-python = use_extension("@rules_python//python:extensions.bzl", "python")
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
use_repo(python, "pythons_hub")
diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel
index ce91228..61d7967 100644
--- a/examples/bzlmod/MODULE.bazel
+++ b/examples/bzlmod/MODULE.bazel
@@ -10,7 +10,7 @@
path = "../..",
)
-python = use_extension("@rules_python//python:extensions.bzl", "python")
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
python.toolchain(
name = "python3_9",
configure_coverage_tool = True,
@@ -23,7 +23,7 @@
"@python3_9_toolchains//:all",
)
-pip = use_extension("@rules_python//python:extensions.bzl", "pip")
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
pip.parse(
name = "pip",
requirements_lock = "//:requirements_lock.txt",
diff --git a/examples/bzlmod_build_file_generation/MODULE.bazel b/examples/bzlmod_build_file_generation/MODULE.bazel
index d59fbb3..179fe1b 100644
--- a/examples/bzlmod_build_file_generation/MODULE.bazel
+++ b/examples/bzlmod_build_file_generation/MODULE.bazel
@@ -42,7 +42,7 @@
# The following stanze returns a proxy object representing a module extension;
# its methods can be invoked to create module extension tags.
-python = use_extension("@rules_python//python:extensions.bzl", "python")
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
# This name is passed into python.toolchain and it's use_repo statement.
# We also use the same name for python.host_python_interpreter.
@@ -74,7 +74,7 @@
# The interpreter extension discovers the platform specific Python binary.
# It creates a symlink to the binary, and we pass the label to the following
# pip.parse call.
-interpreter = use_extension("@rules_python//python:interpreter_extension.bzl", "interpreter")
+interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter")
interpreter.install(
name = "interpreter_python3",
python_name = PYTHON_NAME,
@@ -88,7 +88,7 @@
# You can instead check this `requirements.bzl` file into your repo.
# Because this project has different requirements for windows vs other
# operating systems, we have requirements for each.
-pip = use_extension("@rules_python//python:extensions.bzl", "pip")
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
pip.parse(
name = "pip",
# When using gazelle you must use set the following flag
diff --git a/examples/py_proto_library/MODULE.bazel b/examples/py_proto_library/MODULE.bazel
index 5ce0924..6fb1a05 100644
--- a/examples/py_proto_library/MODULE.bazel
+++ b/examples/py_proto_library/MODULE.bazel
@@ -12,7 +12,7 @@
path = "../..",
)
-python = use_extension("@rules_python//python:extensions.bzl", "python")
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
python.toolchain(
name = "python3_9",
configure_coverage_tool = True,
diff --git a/python/extensions.bzl b/python/extensions.bzl
deleted file mode 100644
index ce11069..0000000
--- a/python/extensions.bzl
+++ /dev/null
@@ -1,141 +0,0 @@
-# Copyright 2023 The Bazel Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"Module extensions for use with bzlmod"
-
-load("@rules_python//python:repositories.bzl", "python_register_toolchains")
-load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "pip_repository_bzlmod", "use_isolated", "whl_library")
-load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
-load("@rules_python//python/pip_install:requirements_parser.bzl", parse_requirements = "parse")
-load("@rules_python//python/private:coverage_deps.bzl", "install_coverage_deps")
-load("@rules_python//python/private:interpreter_hub.bzl", "hub_repo")
-
-def _python_impl(module_ctx):
- toolchains = []
- for mod in module_ctx.modules:
- for toolchain_attr in mod.tags.toolchain:
- python_register_toolchains(
- name = toolchain_attr.name,
- python_version = toolchain_attr.python_version,
- bzlmod = True,
- # Toolchain registration in bzlmod is done in MODULE file
- register_toolchains = False,
- register_coverage_tool = toolchain_attr.configure_coverage_tool,
- ignore_root_user_error = toolchain_attr.ignore_root_user_error,
- )
-
- # We collect all of the toolchain names to create
- # the INTERPRETER_LABELS map. This is used
- # by interpreter_extensions.bzl
- toolchains.append(toolchain_attr.name)
-
- hub_repo(
- name = "pythons_hub",
- toolchains = toolchains,
- )
-
-python = module_extension(
- implementation = _python_impl,
- tag_classes = {
- "toolchain": tag_class(
- attrs = {
- "configure_coverage_tool": attr.bool(
- mandatory = False,
- doc = "Whether or not to configure the default coverage tool for the toolchains.",
- ),
- "ignore_root_user_error": attr.bool(
- default = False,
- doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.",
- mandatory = False,
- ),
- "name": attr.string(mandatory = True),
- "python_version": attr.string(mandatory = True),
- },
- ),
- },
-)
-
-# buildifier: disable=unused-variable
-def _internal_deps_impl(module_ctx):
- pip_install_dependencies()
- install_coverage_deps()
-
-internal_deps = module_extension(
- implementation = _internal_deps_impl,
- tag_classes = {
- "install": tag_class(attrs = dict()),
- },
-)
-
-def _pip_impl(module_ctx):
- for mod in module_ctx.modules:
- for attr in mod.tags.parse:
- requrements_lock = locked_requirements_label(module_ctx, attr)
-
- # Parse the requirements file directly in starlark to get the information
- # needed for the whl_libary declarations below. This is needed to contain
- # the pip_repository logic to a single module extension.
- requirements_lock_content = module_ctx.read(requrements_lock)
- parse_result = parse_requirements(requirements_lock_content)
- requirements = parse_result.requirements
- extra_pip_args = attr.extra_pip_args + parse_result.options
-
- # Create the repository where users load the `requirement` macro. Under bzlmod
- # this does not create the install_deps() macro.
- pip_repository_bzlmod(
- name = attr.name,
- requirements_lock = attr.requirements_lock,
- incompatible_generate_aliases = attr.incompatible_generate_aliases,
- )
-
- for name, requirement_line in requirements:
- whl_library(
- name = "%s_%s" % (attr.name, _sanitize_name(name)),
- requirement = requirement_line,
- repo = attr.name,
- repo_prefix = attr.name + "_",
- annotation = attr.annotations.get(name),
- python_interpreter = attr.python_interpreter,
- python_interpreter_target = attr.python_interpreter_target,
- quiet = attr.quiet,
- timeout = attr.timeout,
- isolated = use_isolated(module_ctx, attr),
- extra_pip_args = extra_pip_args,
- download_only = attr.download_only,
- pip_data_exclude = attr.pip_data_exclude,
- enable_implicit_namespace_pkgs = attr.enable_implicit_namespace_pkgs,
- environment = attr.environment,
- )
-
-# Keep in sync with python/pip_install/tools/bazel.py
-def _sanitize_name(name):
- return name.replace("-", "_").replace(".", "_").lower()
-
-def _pip_parse_ext_attrs():
- attrs = dict({
- "name": attr.string(mandatory = True),
- }, **pip_repository_attrs)
-
- # Like the pip_repository rule, we end up setting this manually so
- # don't allow users to override it.
- attrs.pop("repo_prefix")
-
- return attrs
-
-pip = module_extension(
- implementation = _pip_impl,
- tag_classes = {
- "parse": tag_class(attrs = _pip_parse_ext_attrs()),
- },
-)
diff --git a/python/extensions/BUILD.bazel b/python/extensions/BUILD.bazel
new file mode 100644
index 0000000..7f6873d
--- /dev/null
+++ b/python/extensions/BUILD.bazel
@@ -0,0 +1,23 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+filegroup(
+ name = "distribution",
+ srcs = glob(["**"]),
+ visibility = ["//extensions:__pkg__"],
+)
diff --git a/python/interpreter_extension.bzl b/python/extensions/interpreter.bzl
similarity index 100%
rename from python/interpreter_extension.bzl
rename to python/extensions/interpreter.bzl
diff --git a/python/extensions/pip.bzl b/python/extensions/pip.bzl
new file mode 100644
index 0000000..2ec2bbf
--- /dev/null
+++ b/python/extensions/pip.bzl
@@ -0,0 +1,84 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"pip module extension for use with bzlmod"
+
+load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "pip_repository_bzlmod", "use_isolated", "whl_library")
+load("@rules_python//python/pip_install:requirements_parser.bzl", parse_requirements = "parse")
+
+def _pip_impl(module_ctx):
+ for mod in module_ctx.modules:
+ for attr in mod.tags.parse:
+ requrements_lock = locked_requirements_label(module_ctx, attr)
+
+ # Parse the requirements file directly in starlark to get the information
+ # needed for the whl_libary declarations below. This is needed to contain
+ # the pip_repository logic to a single module extension.
+ requirements_lock_content = module_ctx.read(requrements_lock)
+ parse_result = parse_requirements(requirements_lock_content)
+ requirements = parse_result.requirements
+ extra_pip_args = attr.extra_pip_args + parse_result.options
+
+ # Create the repository where users load the `requirement` macro. Under bzlmod
+ # this does not create the install_deps() macro.
+ pip_repository_bzlmod(
+ name = attr.name,
+ requirements_lock = attr.requirements_lock,
+ incompatible_generate_aliases = attr.incompatible_generate_aliases,
+ )
+
+ for name, requirement_line in requirements:
+ whl_library(
+ name = "%s_%s" % (attr.name, _sanitize_name(name)),
+ requirement = requirement_line,
+ repo = attr.name,
+ repo_prefix = attr.name + "_",
+ annotation = attr.annotations.get(name),
+ python_interpreter = attr.python_interpreter,
+ python_interpreter_target = attr.python_interpreter_target,
+ quiet = attr.quiet,
+ timeout = attr.timeout,
+ isolated = use_isolated(module_ctx, attr),
+ extra_pip_args = extra_pip_args,
+ download_only = attr.download_only,
+ pip_data_exclude = attr.pip_data_exclude,
+ enable_implicit_namespace_pkgs = attr.enable_implicit_namespace_pkgs,
+ environment = attr.environment,
+ )
+
+# Keep in sync with python/pip_install/tools/bazel.py
+def _sanitize_name(name):
+ return name.replace("-", "_").replace(".", "_").lower()
+
+def _pip_parse_ext_attrs():
+ attrs = dict({
+ "name": attr.string(mandatory = True),
+ }, **pip_repository_attrs)
+
+ # Like the pip_repository rule, we end up setting this manually so
+ # don't allow users to override it.
+ attrs.pop("repo_prefix")
+
+ return attrs
+
+pip = module_extension(
+ doc = """\
+This extension is used to create a pip respository and create the various wheel libaries if
+provided in a requirements file.
+""",
+ implementation = _pip_impl,
+ tag_classes = {
+ "parse": tag_class(attrs = _pip_parse_ext_attrs()),
+ },
+)
diff --git a/python/extensions/private/BUILD.bazel b/python/extensions/private/BUILD.bazel
new file mode 100644
index 0000000..f367b71
--- /dev/null
+++ b/python/extensions/private/BUILD.bazel
@@ -0,0 +1,23 @@
+# Copyright 2022 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//visibility:private"])
+
+licenses(["notice"])
+
+filegroup(
+ name = "distribution",
+ srcs = glob(["**"]),
+ visibility = ["//python/extensions/private:__pkg__"],
+)
diff --git a/python/extensions/private/internal_deps.bzl b/python/extensions/private/internal_deps.bzl
new file mode 100644
index 0000000..dfa3e26
--- /dev/null
+++ b/python/extensions/private/internal_deps.bzl
@@ -0,0 +1,25 @@
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"Python toolchain module extension for internal rule use"
+
+load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
+load("@rules_python//python/private:coverage_deps.bzl", "install_coverage_deps")
+
+# buildifier: disable=unused-variable
+def _internal_deps_impl(module_ctx):
+ pip_install_dependencies()
+ install_coverage_deps()
+
+internal_deps = module_extension(
+ doc = "This extension to register internal rules_python dependecies.",
+ implementation = _internal_deps_impl,
+ tag_classes = {
+ "install": tag_class(attrs = dict()),
+ },
+)
diff --git a/python/private/interpreter_hub.bzl b/python/extensions/private/interpreter_hub.bzl
similarity index 100%
rename from python/private/interpreter_hub.bzl
rename to python/extensions/private/interpreter_hub.bzl
diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl
new file mode 100644
index 0000000..9a3d9ed
--- /dev/null
+++ b/python/extensions/python.bzl
@@ -0,0 +1,64 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"Python toolchain module extensions for use with bzlmod"
+
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+load("@rules_python//python/extensions/private:interpreter_hub.bzl", "hub_repo")
+
+def _python_impl(module_ctx):
+ toolchains = []
+ for mod in module_ctx.modules:
+ for toolchain_attr in mod.tags.toolchain:
+ python_register_toolchains(
+ name = toolchain_attr.name,
+ python_version = toolchain_attr.python_version,
+ bzlmod = True,
+ # Toolchain registration in bzlmod is done in MODULE file
+ register_toolchains = False,
+ register_coverage_tool = toolchain_attr.configure_coverage_tool,
+ ignore_root_user_error = toolchain_attr.ignore_root_user_error,
+ )
+
+ # We collect all of the toolchain names to create
+ # the INTERPRETER_LABELS map. This is used
+ # by interpreter_extensions.bzl
+ toolchains.append(toolchain_attr.name)
+
+ hub_repo(
+ name = "pythons_hub",
+ toolchains = toolchains,
+ )
+
+python = module_extension(
+ doc = "Bzlmod extension that is used to register a Python toolchain.",
+ implementation = _python_impl,
+ tag_classes = {
+ "toolchain": tag_class(
+ attrs = {
+ "configure_coverage_tool": attr.bool(
+ mandatory = False,
+ doc = "Whether or not to configure the default coverage tool for the toolchains.",
+ ),
+ "ignore_root_user_error": attr.bool(
+ default = False,
+ doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.",
+ mandatory = False,
+ ),
+ "name": attr.string(mandatory = True),
+ "python_version": attr.string(mandatory = True),
+ },
+ ),
+ },
+)
