feat(pip): provide pypi -> whl target mapping in requirements.bzl (#1532)
Currently a BUILD file can load `all_whl_requirements` but then can't
determine which one is associated with a given package installed by the
user.
This makes it impossible to build rules where the user can choose a
given package and then override the wheel used, for example. @mattem and
I are working at a client where we need this ability.
This PR makes a small, non-breaking refactoring to the generated
`requirements.bzl` file so that this information is available in a new
`all_whl_requirements_by_package` symbol.
Users can then do something like this:
```
load("@pip//:requirements.bzl", "all_whl_requirements_by_package")
some_rule(
wheels = dict(all_whl_requirements_by_package, **{
"charset-normalizer": "@charset_1_2_3//:file"
}),
)
```diff --git a/CHANGELOG.md b/CHANGELOG.md
index 02aca34..185ac37 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,6 +32,10 @@
* (pip_parse) The installation of `pip_parse` repository rule toolchain
dependencies is now done as part of `py_repositories` call.
+* (pip_parse) The generated `requirements.bzl` file now has an additional symbol
+ `all_whl_requirements_by_package` which provides a map from the original package name
+ (as it appears in requirements.txt) to the target that provides the built wheel file.
+
* (pip_parse) The flag `incompatible_generate_aliases` has been flipped to
`True` by default on `non-bzlmod` setups allowing users to use the same label
strings during the transition period. For example, instead of
diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl
index 53208f5..48371ed 100644
--- a/examples/pip_parse_vendored/requirements.bzl
+++ b/examples/pip_parse_vendored/requirements.bzl
@@ -9,7 +9,9 @@
all_requirements = ["@pip//certifi:pkg", "@pip//charset_normalizer:pkg", "@pip//idna:pkg", "@pip//requests:pkg", "@pip//urllib3:pkg"]
-all_whl_requirements = ["@pip//certifi:whl", "@pip//charset_normalizer:whl", "@pip//idna:whl", "@pip//requests:whl", "@pip//urllib3:whl"]
+all_whl_requirements_by_package = {"certifi": "@pip//certifi:whl", "charset-normalizer": "@pip//charset_normalizer:whl", "idna": "@pip//idna:whl", "requests": "@pip//requests:whl", "urllib3": "@pip//urllib3:whl"}
+
+all_whl_requirements = all_whl_requirements_by_package.values()
all_data_requirements = ["@pip//certifi:data", "@pip//charset_normalizer:data", "@pip//idna:data", "@pip//requests:data", "@pip//urllib3:data"]
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index f018f65..b841772 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -276,7 +276,7 @@
packages = [(normalize_name(name), requirement) for name, requirement in parsed_requirements_txt.requirements]
- bzl_packages = sorted([name for name, _ in packages])
+ bzl_packages = dict(sorted([[name, normalize_name(name)] for name, _ in parsed_requirements_txt.requirements]))
imports = [
'load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library")',
@@ -314,7 +314,7 @@
if rctx.attr.incompatible_generate_aliases:
macro_tmpl = "@%s//{}:{}" % rctx.attr.name
- aliases = render_pkg_aliases(repo_name = rctx.attr.name, bzl_packages = bzl_packages)
+ aliases = render_pkg_aliases(repo_name = rctx.attr.name, bzl_packages = bzl_packages.values())
for path, contents in aliases.items():
rctx.file(path, contents)
else:
@@ -324,16 +324,16 @@
rctx.template("requirements.bzl", rctx.attr._template, substitutions = {
"%%ALL_DATA_REQUIREMENTS%%": _format_repr_list([
macro_tmpl.format(p, "data")
- for p in bzl_packages
+ for p in bzl_packages.values()
]),
"%%ALL_REQUIREMENTS%%": _format_repr_list([
macro_tmpl.format(p, "pkg")
- for p in bzl_packages
+ for p in bzl_packages.values()
]),
- "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([
- macro_tmpl.format(p, "whl")
- for p in bzl_packages
- ]),
+ "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": _format_dict(_repr_dict({
+ name: macro_tmpl.format(p, "whl")
+ for name, p in bzl_packages.items()
+ })),
"%%ANNOTATIONS%%": _format_dict(_repr_dict(annotations)),
"%%CONFIG%%": _format_dict(_repr_dict(config)),
"%%EXTRA_PIP_ARGS%%": json.encode(options),
diff --git a/python/pip_install/pip_repository_requirements.bzl.tmpl b/python/pip_install/pip_repository_requirements.bzl.tmpl
index 92ea5be..23c8311 100644
--- a/python/pip_install/pip_repository_requirements.bzl.tmpl
+++ b/python/pip_install/pip_repository_requirements.bzl.tmpl
@@ -8,7 +8,9 @@
all_requirements = %%ALL_REQUIREMENTS%%
-all_whl_requirements = %%ALL_WHL_REQUIREMENTS%%
+all_whl_requirements_by_package = %%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%
+
+all_whl_requirements = all_whl_requirements_by_package.values()
all_data_requirements = %%ALL_DATA_REQUIREMENTS%%
diff --git a/python/private/bzlmod/pip_repository.bzl b/python/private/bzlmod/pip_repository.bzl
index f5bb46f..e4e59b5 100644
--- a/python/private/bzlmod/pip_repository.bzl
+++ b/python/private/bzlmod/pip_repository.bzl
@@ -51,10 +51,10 @@
macro_tmpl.format(p, p)
for p in bzl_packages
]),
- "%%ALL_WHL_REQUIREMENTS%%": render.list([
- macro_tmpl.format(p, "whl")
+ "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": render.dict({
+ p: macro_tmpl.format(p, "whl")
for p in bzl_packages
- ]),
+ }),
"%%MACRO_TMPL%%": macro_tmpl,
"%%NAME%%": rctx.attr.name,
})
diff --git a/python/private/bzlmod/requirements.bzl.tmpl b/python/private/bzlmod/requirements.bzl.tmpl
index c72187c..5ed1e49 100644
--- a/python/private/bzlmod/requirements.bzl.tmpl
+++ b/python/private/bzlmod/requirements.bzl.tmpl
@@ -5,7 +5,9 @@
all_requirements = %%ALL_REQUIREMENTS%%
-all_whl_requirements = %%ALL_WHL_REQUIREMENTS%%
+all_whl_requirements_by_package = %%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%
+
+all_whl_requirements = all_whl_requirements_by_package.values()
all_data_requirements = %%ALL_DATA_REQUIREMENTS%%
