fix(pip.parse): make the pip extension reproducible if PyPI is not called (#1937)
With this PR we can finally have fewer lock file entries in setups which
do not use the `experimental_index_url` feature yet. This is fully
backwards compatible change as it relies on `bazel` doing the right
thing and regenerating the lock file.
Fixes #1643.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ca62e38..0720a36 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -57,6 +57,12 @@
replaced by whl_modifications.
* (pip) Correctly select wheels when the python tag includes minor versions.
See ([#1930](https://github.com/bazelbuild/rules_python/issues/1930))
+* (pip.parse): The lock file is now reproducible on any host platform if the
+ `experimental_index_url` is not used by any of the modules in the dependency
+ chain. To make the lock file identical on each `os` and `arch`, please use
+ the `experimental_index_url` feature which will fetch metadata from PyPI or a
+ different private index and write the contents to the lock file. Fixes
+ [#1643](https://github.com/bazelbuild/rules_python/issues/1643).
### Added
* (rules) Precompiling Python source at build time is available. but is
diff --git a/MODULE.bazel b/MODULE.bazel
index 04d8fb2..38ee678 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -56,9 +56,8 @@
#####################
# Install twine for our own runfiles wheel publishing and allow bzlmod users to use it.
-pip = use_extension("//python/extensions:pip.bzl", "pip")
+pip = use_extension("//python/private/bzlmod:pip.bzl", "pip_internal")
pip.parse(
- experimental_index_url = "https://pypi.org/simple",
hub_name = "rules_python_publish_deps",
python_version = "3.11",
requirements_by_platform = {
@@ -80,13 +79,11 @@
bazel_dep(name = "gazelle", version = "0.33.0", dev_dependency = True, repo_name = "bazel_gazelle")
dev_pip = use_extension(
- "//python/extensions:pip.bzl",
- "pip",
+ "//python/private/bzlmod:pip.bzl",
+ "pip_internal",
dev_dependency = True,
)
dev_pip.parse(
- envsubst = ["PIP_INDEX_URL"],
- experimental_index_url = "${PIP_INDEX_URL:-https://pypi.org/simple}",
experimental_requirement_cycles = {
"sphinx": [
"sphinx",
@@ -102,8 +99,6 @@
requirements_lock = "//docs/sphinx:requirements.txt",
)
dev_pip.parse(
- envsubst = ["PIP_INDEX_URL"],
- experimental_index_url = "${PIP_INDEX_URL:-https://pypi.org/simple}",
hub_name = "pypiserver",
python_version = "3.11",
requirements_lock = "//examples/wheel:requirements_server.txt",
diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl
index 9e29877..8702f1f 100644
--- a/python/private/bzlmod/pip.bzl
+++ b/python/private/bzlmod/pip.bzl
@@ -103,6 +103,7 @@
def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache):
logger = repo_utils.logger(module_ctx)
python_interpreter_target = pip_attr.python_interpreter_target
+ is_hub_reproducible = True
# if we do not have the python_interpreter set in the attributes
# we programmatically find it.
@@ -274,6 +275,7 @@
logger.debug(lambda: "Selected: {}".format(distribution))
if distribution:
+ is_hub_reproducible = False
whl_library_args["requirement"] = requirement.srcs.requirement
whl_library_args["urls"] = [distribution.url]
whl_library_args["sha256"] = distribution.sha256
@@ -303,6 +305,8 @@
),
)
+ return is_hub_reproducible
+
def _pip_impl(module_ctx):
"""Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories.
@@ -412,6 +416,7 @@
hub_group_map = {}
simpleapi_cache = {}
+ is_extension_reproducible = True
for mod in module_ctx.modules:
for pip_attr in mod.tags.parse:
@@ -448,7 +453,8 @@
else:
pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version)
- _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache)
+ is_hub_reproducible = _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache)
+ is_extension_reproducible = is_extension_reproducible and is_hub_reproducible
for hub_name, whl_map in hub_whl_map.items():
pip_repository(
@@ -462,7 +468,34 @@
groups = hub_group_map.get(hub_name),
)
-def _pip_parse_ext_attrs():
+ if bazel_features.external_deps.extension_metadata_has_reproducible:
+ # If we are not using the `experimental_index_url feature, the extension is fully
+ # deterministic and we don't need to create a lock entry for it.
+ #
+ # In order to be able to dogfood the `experimental_index_url` feature before it gets
+ # stabilized, we have created the `_pip_non_reproducible` function, that will result
+ # in extra entries in the lock file.
+ return module_ctx.extension_metadata(reproducible = is_extension_reproducible)
+ else:
+ return None
+
+def _pip_non_reproducible(module_ctx):
+ _pip_impl(module_ctx)
+
+ # We default to calling the PyPI index and that will go into the
+ # MODULE.bazel.lock file, hence return nothing here.
+ return None
+
+def _pip_parse_ext_attrs(**kwargs):
+ """Get the attributes for the pip extension.
+
+ Args:
+ **kwargs: A kwarg for setting defaults for the specific attributes. The
+ key is expected to be the same as the attribute key.
+
+ Returns:
+ A dict of attributes.
+ """
attrs = dict({
"experimental_extra_index_urls": attr.string_list(
doc = """\
@@ -477,6 +510,7 @@
default = [],
),
"experimental_index_url": attr.string(
+ default = kwargs.get("experimental_index_url", ""),
doc = """\
The index URL to use for downloading wheels using bazel downloader. This value is going
to be subject to `envsubst` substitutions if necessary.
@@ -661,17 +695,6 @@
other tags in this extension.""",
)
-def _extension_extra_args():
- args = {}
-
- if bazel_features.external_deps.module_extension_has_os_arch_dependent:
- args = args | {
- "arch_dependent": True,
- "os_dependent": True,
- }
-
- return args
-
pip = module_extension(
doc = """\
This extension is used to make dependencies from pip available.
@@ -714,7 +737,56 @@
""",
),
},
- **_extension_extra_args()
+)
+
+pip_internal = module_extension(
+ doc = """\
+This extension is used to make dependencies from pypi available.
+
+For now this is intended to be used internally so that usage of the `pip`
+extension in `rules_python` does not affect the evaluations of the extension
+for the consumers.
+
+pip.parse:
+To use, call `pip.parse()` and specify `hub_name` and your requirements file.
+Dependencies will be downloaded and made available in a repo named after the
+`hub_name` argument.
+
+Each `pip.parse()` call configures a particular Python version. Multiple calls
+can be made to configure different Python versions, and will be grouped by
+the `hub_name` argument. This allows the same logical name, e.g. `@pypi//numpy`
+to automatically resolve to different, Python version-specific, libraries.
+
+pip.whl_mods:
+This tag class is used to help create JSON files to describe modifications to
+the BUILD files for wheels.
+""",
+ implementation = _pip_non_reproducible,
+ tag_classes = {
+ "override": _override_tag,
+ "parse": tag_class(
+ attrs = _pip_parse_ext_attrs(
+ experimental_index_url = "https://pypi.org/simple",
+ ),
+ doc = """\
+This tag class is used to create a pypi hub and all of the spokes that are part of that hub.
+This tag class reuses most of the pypi attributes that are found in
+@rules_python//python/pip_install:pip_repository.bzl.
+The exception is it does not use the arg 'repo_prefix'. We set the repository
+prefix for the user and the alias arg is always True in bzlmod.
+""",
+ ),
+ "whl_mods": tag_class(
+ attrs = _whl_mod_attrs(),
+ doc = """\
+This tag class is used to create JSON file that are used when calling wheel_builder.py. These
+JSON files contain instructions on how to modify a wheel's project. Each of the attributes
+create different modifications based on the type of attribute. Previously to bzlmod these
+JSON files where referred to as annotations, and were renamed to whl_modifications in this
+extension.
+""",
+ ),
+ },
)
def _whl_mods_repo_impl(rctx):
