Expose resolved .whl files in filegroup target of pypi__ packages. (#364)
diff --git a/python/pip_install/README.md b/python/pip_install/README.md
index c21dda0..4db1852 100644
--- a/python/pip_install/README.md
+++ b/python/pip_install/README.md
@@ -64,18 +64,27 @@
#### Example `BUILD` file.
```python
-load("@py_deps//:requirements.bzl", "requirement")
+load("@py_deps//:requirements.bzl", "requirement", "whl_requirement")
py_binary(
name = "main",
srcs = ["main.py"],
deps = [
requirement("boto3"),
- ],
+ ]
+)
+
+# If you need to depend on the wheel dists themselves, for instance to pass them
+# to some other packaging tool, you can get a handle to them with the whl_requirement macro.
+filegroup(
+ name = "whl_files",
+ data = [
+ whl_requirement("boto3"),
+ ]
)
```
-Note that above you do not need to add transitively required packages to `deps = [ ... ]`
+Note that above you do not need to add transitively required packages to `deps = [ ... ]` or `data = [ ... ]`
#### Setup `requirements.txt`
diff --git a/python/pip_install/extract_wheels/lib/BUILD b/python/pip_install/extract_wheels/lib/BUILD
index 001e333..4493bd1 100644
--- a/python/pip_install/extract_wheels/lib/BUILD
+++ b/python/pip_install/extract_wheels/lib/BUILD
@@ -41,6 +41,19 @@
],
)
+py_test(
+ name = "whl_filegroup_test",
+ size = "small",
+ srcs = [
+ "whl_filegroup_test.py",
+ ],
+ tags = ["unit"],
+ deps = [
+ ":lib",
+ ],
+ data = ["//experimental/examples/wheel:minimal_with_py_package"]
+)
+
filegroup(
name = "distribution",
srcs = glob(
diff --git a/python/pip_install/extract_wheels/lib/bazel.py b/python/pip_install/extract_wheels/lib/bazel.py
index 978f3a9..057c546 100644
--- a/python/pip_install/extract_wheels/lib/bazel.py
+++ b/python/pip_install/extract_wheels/lib/bazel.py
@@ -3,18 +3,22 @@
import textwrap
import json
from typing import Iterable, List, Dict, Set
+import shutil
from python.pip_install.extract_wheels.lib import namespace_pkgs, wheel, purelib
+WHEEL_FILE_LABEL = "whl"
+
def generate_build_file_contents(
- name: str, dependencies: List[str], pip_data_exclude: List[str]
+ name: str, dependencies: List[str], whl_file_deps: List[str], pip_data_exclude: List[str],
) -> str:
"""Generate a BUILD file for an unzipped Wheel
Args:
name: the target name of the py_library
dependencies: a list of Bazel labels pointing to dependencies of the library
+ whl_file_deps: a list of Bazel labels pointing to wheel file dependencies of this wheel.
Returns:
A complete BUILD file as a string
@@ -23,7 +27,7 @@
there may be no Python sources whatsoever (e.g. packages written in Cython: like `pymssql`).
"""
- data_exclude = ["**/*.py", "**/* *", "BUILD", "WORKSPACE"] + pip_data_exclude
+ data_exclude = ["*.whl", "**/*.py", "**/* *", "BUILD", "WORKSPACE"] + pip_data_exclude
return textwrap.dedent(
"""\
@@ -31,6 +35,12 @@
load("@rules_python//python:defs.bzl", "py_library")
+ filegroup(
+ name="{whl_file_label}",
+ srcs=glob(["*.whl"]),
+ data=[{whl_file_deps}]
+ )
+
py_library(
name = "{name}",
srcs = glob(["**/*.py"], allow_empty = True),
@@ -44,6 +54,8 @@
name=name,
dependencies=",".join(dependencies),
data_exclude=json.dumps(data_exclude),
+ whl_file_label=WHEEL_FILE_LABEL,
+ whl_file_deps=",".join(whl_file_deps),
)
)
@@ -70,6 +82,9 @@
def requirement(name):
name_key = name.replace("-", "_").replace(".", "_").lower()
return "{repo}//pypi__" + name_key
+
+ def whl_requirement(name):
+ return requirement(name) + ":whl"
""".format(
repo=repo_name, requirement_labels=",".join(sorted(targets))
)
@@ -125,7 +140,7 @@
pip_data_exclude: List[str],
enable_implicit_namespace_pkgs: bool,
) -> str:
- """Extracts wheel into given directory and creates a py_library target.
+ """Extracts wheel into given directory and creates py_library and filegroup targets.
Args:
wheel_file: the filepath of the .whl
@@ -141,6 +156,8 @@
directory = sanitise_name(whl.name)
os.mkdir(directory)
+ # copy the original wheel
+ shutil.copy(whl.path, directory)
whl.unzip(directory)
# Note: Order of operations matters here
@@ -150,14 +167,18 @@
setup_namespace_pkg_compatibility(directory)
extras_requested = extras[whl.name] if whl.name in extras else set()
+ whl_deps = sorted(whl.dependencies(extras_requested))
sanitised_dependencies = [
- '"//%s"' % sanitise_name(d) for d in sorted(whl.dependencies(extras_requested))
+ '"//%s"' % sanitise_name(d) for d in whl_deps
+ ]
+ sanitised_wheel_file_dependencies = [
+ '"//%s:%s"' % (sanitise_name(d), WHEEL_FILE_LABEL) for d in whl_deps
]
with open(os.path.join(directory, "BUILD"), "w") as build_file:
contents = generate_build_file_contents(
- sanitise_name(whl.name), sanitised_dependencies, pip_data_exclude,
+ sanitise_name(whl.name), sanitised_dependencies, sanitised_wheel_file_dependencies, pip_data_exclude
)
build_file.write(contents)
diff --git a/python/pip_install/extract_wheels/lib/whl_filegroup_test.py b/python/pip_install/extract_wheels/lib/whl_filegroup_test.py
new file mode 100644
index 0000000..39589c1
--- /dev/null
+++ b/python/pip_install/extract_wheels/lib/whl_filegroup_test.py
@@ -0,0 +1,26 @@
+import os
+import unittest
+
+from python.pip_install.extract_wheels.lib import bazel
+
+
+class TestExtractWheel(unittest.TestCase):
+ def test_generated_build_file_has_filegroup_target(self) -> None:
+ wheel_name = "example_minimal_package-0.0.1-py3-none-any.whl"
+ wheel_dir = "experimental/examples/wheel/"
+ wheel_path = wheel_dir + wheel_name
+ generated_bazel_dir = bazel.extract_wheel(
+ wheel_path,
+ extras={},
+ pip_data_exclude=[],
+ enable_implicit_namespace_pkgs=False,
+ )[2:] # Take off the leading // from the returned label.
+ # Assert that the raw wheel ends up in the package.
+ self.assertIn(wheel_name, os.listdir(generated_bazel_dir))
+ with open("{}/BUILD".format(generated_bazel_dir)) as build_file:
+ build_file_content = build_file.read()
+ self.assertIn('filegroup', build_file_content)
+
+
+if __name__ == "__main__":
+ unittest.main()
