Add option to use "pip download" instead of "pip wheel" to download wheels for other platforms (#773)
diff --git a/docs/pip_repository.md b/docs/pip_repository.md
index ef7f72b..c66d8bf 100644
--- a/docs/pip_repository.md
+++ b/docs/pip_repository.md
@@ -5,10 +5,10 @@
## pip_repository
<pre>
-pip_repository(<a href="#pip_repository-name">name</a>, <a href="#pip_repository-annotations">annotations</a>, <a href="#pip_repository-enable_implicit_namespace_pkgs">enable_implicit_namespace_pkgs</a>, <a href="#pip_repository-environment">environment</a>, <a href="#pip_repository-extra_pip_args">extra_pip_args</a>,
- <a href="#pip_repository-incremental">incremental</a>, <a href="#pip_repository-isolated">isolated</a>, <a href="#pip_repository-pip_data_exclude">pip_data_exclude</a>, <a href="#pip_repository-python_interpreter">python_interpreter</a>, <a href="#pip_repository-python_interpreter_target">python_interpreter_target</a>,
- <a href="#pip_repository-quiet">quiet</a>, <a href="#pip_repository-repo_prefix">repo_prefix</a>, <a href="#pip_repository-requirements">requirements</a>, <a href="#pip_repository-requirements_darwin">requirements_darwin</a>, <a href="#pip_repository-requirements_linux">requirements_linux</a>,
- <a href="#pip_repository-requirements_lock">requirements_lock</a>, <a href="#pip_repository-requirements_windows">requirements_windows</a>, <a href="#pip_repository-timeout">timeout</a>)
+pip_repository(<a href="#pip_repository-name">name</a>, <a href="#pip_repository-annotations">annotations</a>, <a href="#pip_repository-download_only">download_only</a>, <a href="#pip_repository-enable_implicit_namespace_pkgs">enable_implicit_namespace_pkgs</a>, <a href="#pip_repository-environment">environment</a>,
+ <a href="#pip_repository-extra_pip_args">extra_pip_args</a>, <a href="#pip_repository-incremental">incremental</a>, <a href="#pip_repository-isolated">isolated</a>, <a href="#pip_repository-pip_data_exclude">pip_data_exclude</a>, <a href="#pip_repository-python_interpreter">python_interpreter</a>,
+ <a href="#pip_repository-python_interpreter_target">python_interpreter_target</a>, <a href="#pip_repository-quiet">quiet</a>, <a href="#pip_repository-repo_prefix">repo_prefix</a>, <a href="#pip_repository-requirements">requirements</a>, <a href="#pip_repository-requirements_darwin">requirements_darwin</a>,
+ <a href="#pip_repository-requirements_linux">requirements_linux</a>, <a href="#pip_repository-requirements_lock">requirements_lock</a>, <a href="#pip_repository-requirements_windows">requirements_windows</a>, <a href="#pip_repository-timeout">timeout</a>)
</pre>
A rule for importing `requirements.txt` dependencies into Bazel.
@@ -58,6 +58,7 @@
| :-------------: | :-------------: | :-------------: | :-------------: | :-------------: |
| name | A unique name for this repository. | <a href="https://bazel.build/docs/build-ref.html#name">Name</a> | required | |
| annotations | Optional annotations to apply to packages | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {} |
+| download_only | Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of --platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different platform from the host platform. | Boolean | optional | False |
| enable_implicit_namespace_pkgs | If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary and py_test targets must specify either <code>legacy_create_init=False</code> or the global Bazel option <code>--incompatible_default_to_explicit_init_py</code> to prevent <code>__init__.py</code> being automatically generated in every directory.<br><br>This option is required to support some packages which cannot handle the conversion to pkg-util style. | Boolean | optional | False |
| environment | Environment variables to set in the pip subprocess. Can be used to set common variables such as <code>http_proxy</code>, <code>https_proxy</code> and <code>no_proxy</code> Note that pip is run with "--isolated" on the CLI so PIP_<VAR>_<NAME> style env vars are ignored, but env vars that control requests and urllib3 can be passed. | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {} |
| extra_pip_args | Extra arguments to pass on to pip. Must not contain spaces. | List of strings | optional | [] |
@@ -81,9 +82,9 @@
## whl_library
<pre>
-whl_library(<a href="#whl_library-name">name</a>, <a href="#whl_library-annotation">annotation</a>, <a href="#whl_library-enable_implicit_namespace_pkgs">enable_implicit_namespace_pkgs</a>, <a href="#whl_library-environment">environment</a>, <a href="#whl_library-extra_pip_args">extra_pip_args</a>, <a href="#whl_library-isolated">isolated</a>,
- <a href="#whl_library-pip_data_exclude">pip_data_exclude</a>, <a href="#whl_library-python_interpreter">python_interpreter</a>, <a href="#whl_library-python_interpreter_target">python_interpreter_target</a>, <a href="#whl_library-quiet">quiet</a>, <a href="#whl_library-repo">repo</a>, <a href="#whl_library-repo_prefix">repo_prefix</a>,
- <a href="#whl_library-requirement">requirement</a>, <a href="#whl_library-timeout">timeout</a>)
+whl_library(<a href="#whl_library-name">name</a>, <a href="#whl_library-annotation">annotation</a>, <a href="#whl_library-download_only">download_only</a>, <a href="#whl_library-enable_implicit_namespace_pkgs">enable_implicit_namespace_pkgs</a>, <a href="#whl_library-environment">environment</a>,
+ <a href="#whl_library-extra_pip_args">extra_pip_args</a>, <a href="#whl_library-isolated">isolated</a>, <a href="#whl_library-pip_data_exclude">pip_data_exclude</a>, <a href="#whl_library-python_interpreter">python_interpreter</a>, <a href="#whl_library-python_interpreter_target">python_interpreter_target</a>,
+ <a href="#whl_library-quiet">quiet</a>, <a href="#whl_library-repo">repo</a>, <a href="#whl_library-repo_prefix">repo_prefix</a>, <a href="#whl_library-requirement">requirement</a>, <a href="#whl_library-timeout">timeout</a>)
</pre>
@@ -97,6 +98,7 @@
| :-------------: | :-------------: | :-------------: | :-------------: | :-------------: |
| name | A unique name for this repository. | <a href="https://bazel.build/docs/build-ref.html#name">Name</a> | required | |
| annotation | Optional json encoded file containing annotation to apply to the extracted wheel. See <code>package_annotation</code> | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
+| download_only | Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of --platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different platform from the host platform. | Boolean | optional | False |
| enable_implicit_namespace_pkgs | If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary and py_test targets must specify either <code>legacy_create_init=False</code> or the global Bazel option <code>--incompatible_default_to_explicit_init_py</code> to prevent <code>__init__.py</code> being automatically generated in every directory.<br><br>This option is required to support some packages which cannot handle the conversion to pkg-util style. | Boolean | optional | False |
| environment | Environment variables to set in the pip subprocess. Can be used to set common variables such as <code>http_proxy</code>, <code>https_proxy</code> and <code>no_proxy</code> Note that pip is run with "--isolated" on the CLI so PIP_<VAR>_<NAME> style env vars are ignored, but env vars that control requests and urllib3 can be passed. | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {} |
| extra_pip_args | Extra arguments to pass on to pip. Must not contain spaces. | List of strings | optional | [] |
diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl
index 038b612..58c6e7b 100644
--- a/examples/pip_parse_vendored/requirements.bzl
+++ b/examples/pip_parse_vendored/requirements.bzl
@@ -12,7 +12,7 @@
all_whl_requirements = ["@pip_certifi//:whl", "@pip_charset_normalizer//:whl", "@pip_idna//:whl", "@pip_requests//:whl", "@pip_urllib3//:whl"]
_packages = [("pip_certifi", "certifi==2021.10.8 --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"), ("pip_charset_normalizer", "charset-normalizer==2.0.12 --hash=sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597 --hash=sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"), ("pip_idna", "idna==3.3 --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"), ("pip_requests", "requests==2.27.1 --hash=sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61 --hash=sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"), ("pip_urllib3", "urllib3==1.26.9 --hash=sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14 --hash=sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e")]
-_config = {"enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": interpreter, "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600}
+_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": interpreter, "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600}
_annotations = {}
def _clean_name(name):
diff --git a/python/pip_install/extract_wheels/arguments.py b/python/pip_install/extract_wheels/arguments.py
index d7d3452..ce77bb0 100644
--- a/python/pip_install/extract_wheels/arguments.py
+++ b/python/pip_install/extract_wheels/arguments.py
@@ -39,6 +39,12 @@
required=True,
help="Prefix to prepend to packages",
)
+ parser.add_argument(
+ "--download_only",
+ action="store_true",
+ help="Use 'pip download' instead of 'pip wheel'. Disables building wheels from source, but allows use of "
+ "--platform, --python-version, --implementation, and --abi in --extra_pip_args.",
+ )
return parser
diff --git a/python/pip_install/extract_wheels/extract_single_wheel.py b/python/pip_install/extract_wheels/extract_single_wheel.py
index 27f29ac..a7cc672 100644
--- a/python/pip_install/extract_wheels/extract_single_wheel.py
+++ b/python/pip_install/extract_wheels/extract_single_wheel.py
@@ -38,7 +38,7 @@
pip_args = (
[sys.executable, "-m", "pip"]
+ (["--isolated"] if args.isolated else [])
- + ["wheel", "--no-deps"]
+ + ["download" if args.download_only else "wheel", "--no-deps"]
+ deserialized_args["extra_pip_args"]
)
diff --git a/python/pip_install/extract_wheels/extract_wheels.py b/python/pip_install/extract_wheels/extract_wheels.py
index 7e583eb..2addaf8 100644
--- a/python/pip_install/extract_wheels/extract_wheels.py
+++ b/python/pip_install/extract_wheels/extract_wheels.py
@@ -80,7 +80,7 @@
pip_args = (
[sys.executable, "-m", "pip"]
+ (["--isolated"] if args.isolated else [])
- + ["wheel", "-r", args.requirements]
+ + ["download" if args.download_only else "wheel", "-r", args.requirements]
+ ["--wheel-dir", os.getcwd()]
+ deserialized_args["extra_pip_args"]
)
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index 706edef..eab59f3 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -130,6 +130,9 @@
struct(arg = rctx.attr.extra_pip_args).to_json(),
]
+ if rctx.attr.download_only:
+ args.append("--download_only")
+
if rctx.attr.pip_data_exclude != None:
args += [
"--pip_data_exclude",
@@ -250,6 +253,13 @@
]
common_attrs = {
+ "download_only": attr.bool(
+ doc = """
+Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of
+--platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different
+platform from the host platform.
+ """,
+ ),
"enable_implicit_namespace_pkgs": attr.bool(
default = False,
doc = """
