feat: support parsing whl METADATA on any python version (#1693)
With this PR we can deterministically parse the METADATA and generate a
`BUILD.bazel` file using the config settings introduced in #1555. Let's
imagine we had a `requirements.txt` file that used only wheels, we could
use the host interpreter to parse the wheel metadata for all the target
platforms and use the version aware toolchain at runtime. This
potentially
unlocks more clever layouts of the `bzlmod` hub repos explored in #1625
where we could have a single `whl_library` instance for all versions
within
a single hub repo.
Work towards #1643.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 821053b..1e67ca8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -35,6 +35,15 @@
* (py_wheel) Added `requires_file` and `extra_requires_files` attributes.
+* (whl_library) *experimental_target_platforms* now supports specifying the
+ Python version explicitly and the output `BUILD.bazel` file will be correct
+ irrespective of the python interpreter that is generating the file and
+ extracting the `whl` distribution. Multiple python target version can be
+ specified and the code generation will generate version specific dependency
+ closures but that is not yet ready to be used and may break the build if
+ the default python version is not selected using
+ `common --@rules_python//python/config_settings:python_version=X.Y.Z`.
+
## 0.29.0 - 2024-01-22
[0.29.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.29.0
@@ -59,6 +68,7 @@
platform-specific content in `MODULE.bazel.lock` files; Follow
[#1643](https://github.com/bazelbuild/rules_python/issues/1643) for removing
platform-specific content in `MODULE.bazel.lock` files.
+
* (wheel) The stamp variables inside the distribution name are no longer
lower-cased when normalizing under PEP440 conventions.
diff --git a/examples/bzlmod/.bazelrc b/examples/bzlmod/.bazelrc
index 6f557e6..e9a73c5 100644
--- a/examples/bzlmod/.bazelrc
+++ b/examples/bzlmod/.bazelrc
@@ -1,4 +1,4 @@
-common --experimental_enable_bzlmod
+common --enable_bzlmod
coverage --java_runtime_version=remotejdk_11
diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel
index e49b586..ceb0010 100644
--- a/examples/bzlmod/MODULE.bazel
+++ b/examples/bzlmod/MODULE.bazel
@@ -107,9 +107,10 @@
# You can use one of the values below to specify the target platform
# to generate the dependency graph for.
experimental_target_platforms = [
- "all",
- "linux_*",
- "host",
+ # Specifying the target platforms explicitly
+ "cp39_linux_x86_64",
+ "cp39_linux_*",
+ "cp39_*",
],
hub_name = "pip",
python_version = "3.9",
@@ -137,8 +138,13 @@
# You can use one of the values below to specify the target platform
# to generate the dependency graph for.
experimental_target_platforms = [
- "all",
+ # Using host python version
"linux_*",
+ "osx_*",
+ "windows_*",
+ # Or specifying an exact platform
+ "linux_x86_64",
+ # Or the following to get the `host` platform only
"host",
],
hub_name = "pip",
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index fe58472..542e312 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -493,13 +493,19 @@
implementation that is being used at runtime is different between different platforms.
This has been tested for CPython only.
-Special values: `all` (for generating deps for all platforms), `host` (for
-generating deps for the host platform only). `linux_*` and other `<os>_*` values.
-In the future we plan to set `all` as the default to this attribute.
-
For specific target platforms use values of the form `<os>_<arch>` where `<os>`
is one of `linux`, `osx`, `windows` and arch is one of `x86_64`, `x86_32`,
`aarch64`, `s390x` and `ppc64le`.
+
+You can also target a specific Python version by using `cp3<minor_version>_<os>_<arch>`.
+If multiple python versions are specified as target platforms, then select statements
+of the `lib` and `whl` targets will include usage of version aware toolchain config
+settings like `@rules_python//python/config_settings:is_python_3.y`.
+
+Special values: `host` (for generating deps for the host platform only) and
+`<prefix>_*` values. For example, `cp39_*`, `linux_*`, `cp39_linux_*`.
+
+NOTE: this is not for cross-compiling Python wheels but rather for parsing the `whl` METADATA correctly.
""",
),
"extra_pip_args": attr.string_list(
@@ -749,7 +755,7 @@
# NOTE @aignas 2023-12-04: if the wheel is a platform specific
# wheel, we only include deps for that target platform
target_platforms = [
- "{}_{}".format(p.os, p.cpu)
+ "{}_{}_{}".format(parsed_whl.abi_tag, p.os, p.cpu)
for p in whl_target_platforms(parsed_whl.platform_tag)
]
diff --git a/python/pip_install/private/generate_whl_library_build_bazel.bzl b/python/pip_install/private/generate_whl_library_build_bazel.bzl
index 568b00e..19650d1 100644
--- a/python/pip_install/private/generate_whl_library_build_bazel.bzl
+++ b/python/pip_install/private/generate_whl_library_build_bazel.bzl
@@ -48,8 +48,7 @@
"""
_BUILD_TEMPLATE = """\
-load("@rules_python//python:defs.bzl", "py_library", "py_binary")
-load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
+{loads}
package(default_visibility = ["//visibility:public"])
@@ -102,22 +101,37 @@
)
"""
+def _plat_label(plat):
+ if plat.startswith("@//"):
+ return "@@" + str(Label("//:BUILD.bazel")).partition("//")[0].strip("@") + plat.strip("@")
+ elif plat.startswith("@"):
+ return str(Label(plat))
+ else:
+ return ":is_" + plat
+
def _render_list_and_select(deps, deps_by_platform, tmpl):
- deps = render.list([tmpl.format(d) for d in deps])
+ deps = render.list([tmpl.format(d) for d in sorted(deps)])
if not deps_by_platform:
return deps
deps_by_platform = {
- p if p.startswith("@") else ":is_" + p: [
+ _plat_label(p): [
tmpl.format(d)
- for d in deps
+ for d in sorted(deps)
]
- for p, deps in deps_by_platform.items()
+ for p, deps in sorted(deps_by_platform.items())
}
# Add the default, which means that we will be just using the dependencies in
# `deps` for platforms that are not handled in a special way by the packages
+ #
+ # FIXME @aignas 2024-01-24: This currently works as expected only if the default
+ # value of the @rules_python//python/config_settings:python_version is set in
+ # the `.bazelrc`. If it is unset, then the we don't get the expected behaviour
+ # in cases where we are using a simple `py_binary` using the default toolchain
+ # without forcing any transitions. If the `python_version` config setting is set
+ # via .bazelrc, then everything works correctly.
deps_by_platform["//conditions:default"] = []
deps_by_platform = render.select(deps_by_platform, value_repr = render.list)
@@ -126,6 +140,87 @@
else:
return "{} + {}".format(deps, deps_by_platform)
+def _render_config_settings(dependencies_by_platform):
+ py_version_by_os_arch = {}
+ for p in dependencies_by_platform:
+ # p can be one of the following formats:
+ # * @platforms//os:{value}
+ # * @platforms//cpu:{value}
+ # * @//python/config_settings:is_python_3.{minor_version}
+ # * {os}_{cpu}
+ # * cp3{minor_version}_{os}_{cpu}
+ if p.startswith("@"):
+ continue
+
+ abi, _, tail = p.partition("_")
+ if not abi.startswith("cp"):
+ tail = p
+ abi = ""
+ os, _, arch = tail.partition("_")
+ os = "" if os == "anyos" else os
+ arch = "" if arch == "anyarch" else arch
+
+ py_version_by_os_arch.setdefault((os, arch), []).append(abi)
+
+ if not py_version_by_os_arch:
+ return None, None
+
+ loads = []
+ additional_content = []
+ for (os, arch), abis in py_version_by_os_arch.items():
+ constraint_values = []
+ if os:
+ constraint_values.append("@platforms//os:{}".format(os))
+ if arch:
+ constraint_values.append("@platforms//cpu:{}".format(arch))
+
+ os_arch = (os or "anyos") + "_" + (arch or "anyarch")
+ additional_content.append(
+ """\
+config_setting(
+ name = "is_{name}",
+ constraint_values = {values},
+ visibility = ["//visibility:private"],
+)""".format(
+ name = os_arch,
+ values = render.indent(render.list(sorted([str(Label(c)) for c in constraint_values]))).strip(),
+ ),
+ )
+
+ if abis == [""]:
+ if not os or not arch:
+ fail("BUG: both os and arch should be set in this case")
+ continue
+
+ for abi in abis:
+ if not loads:
+ loads.append("""load("@bazel_skylib//lib:selects.bzl", "selects")""")
+ minor_version = int(abi[len("cp3"):])
+ setting = "@@{rules_python}//python/config_settings:is_python_3.{version}".format(
+ rules_python = str(Label("//:BUILD.bazel")).partition("//")[0].strip("@"),
+ version = minor_version,
+ )
+ settings = [
+ ":is_" + os_arch,
+ setting,
+ ]
+
+ plat = "{}_{}".format(abi, os_arch)
+
+ additional_content.append(
+ """\
+selects.config_setting_group(
+ name = "{name}",
+ match_all = {values},
+ visibility = ["//visibility:private"],
+)""".format(
+ name = _plat_label(plat).lstrip(":"),
+ values = render.indent(render.list(sorted(settings))).strip(),
+ ),
+ )
+
+ return loads, "\n\n".join(additional_content)
+
def generate_whl_library_build_bazel(
*,
repo_prefix,
@@ -228,24 +323,17 @@
if deps
}
- for p in dependencies_by_platform:
- if p.startswith("@"):
- continue
+ loads = [
+ """load("@rules_python//python:defs.bzl", "py_library", "py_binary")""",
+ """load("@bazel_skylib//rules:copy_file.bzl", "copy_file")""",
+ ]
- os, _, cpu = p.partition("_")
-
- additional_content.append(
- """\
-config_setting(
- name = "is_{os}_{cpu}",
- constraint_values = [
- "@platforms//cpu:{cpu}",
- "@platforms//os:{os}",
- ],
- visibility = ["//visibility:private"],
-)
-""".format(os = os, cpu = cpu),
- )
+ loads_, config_settings_content = _render_config_settings(dependencies_by_platform)
+ if config_settings_content:
+ for line in loads_:
+ if line not in loads:
+ loads.append(line)
+ additional_content.append(config_settings_content)
lib_dependencies = _render_list_and_select(
deps = dependencies,
@@ -277,6 +365,7 @@
contents = "\n".join(
[
_BUILD_TEMPLATE.format(
+ loads = "\n".join(loads),
py_library_public_label = PY_LIBRARY_PUBLIC_LABEL,
py_library_impl_label = PY_LIBRARY_IMPL_LABEL,
py_library_actual_label = library_impl_label,
diff --git a/python/pip_install/tools/wheel_installer/arguments_test.py b/python/pip_install/tools/wheel_installer/arguments_test.py
index 840c2fa..cafb85f 100644
--- a/python/pip_install/tools/wheel_installer/arguments_test.py
+++ b/python/pip_install/tools/wheel_installer/arguments_test.py
@@ -58,7 +58,8 @@
args=[
"--platform=host",
"--platform=linux_*",
- "--platform=all",
+ "--platform=osx_*",
+ "--platform=windows_*",
"--requirement=foo",
]
)
diff --git a/python/pip_install/tools/wheel_installer/wheel.py b/python/pip_install/tools/wheel_installer/wheel.py
index 2275f77..750ebfc 100644
--- a/python/pip_install/tools/wheel_installer/wheel.py
+++ b/python/pip_install/tools/wheel_installer/wheel.py
@@ -84,16 +84,31 @@
return int(value.value)
+def host_interpreter_minor_version() -> int:
+ return sys.version_info.minor
+
+
@dataclass(frozen=True)
class Platform:
- os: OS
+ os: Optional[OS] = None
arch: Optional[Arch] = None
+ minor_version: Optional[int] = None
+
+ def __post_init__(self):
+ if not self.os and not self.arch and not self.minor_version:
+ raise ValueError(
+ "At least one of os, arch, minor_version must be specified"
+ )
@classmethod
- def all(cls, want_os: Optional[OS] = None) -> List["Platform"]:
+ def all(
+ cls,
+ want_os: Optional[OS] = None,
+ minor_version: Optional[int] = None,
+ ) -> List["Platform"]:
return sorted(
[
- cls(os=os, arch=arch)
+ cls(os=os, arch=arch, minor_version=minor_version)
for os in OS
for arch in Arch
if not want_os or want_os == os
@@ -121,7 +136,14 @@
yield self
if self.arch is None:
for arch in Arch:
- yield Platform(os=self.os, arch=arch)
+ yield Platform(os=self.os, arch=arch, minor_version=self.minor_version)
+ if self.os is None:
+ for os in OS:
+ yield Platform(os=os, arch=self.arch, minor_version=self.minor_version)
+ if self.arch is None and self.os is None:
+ for os in OS:
+ for arch in Arch:
+ yield Platform(os=os, arch=arch, minor_version=self.minor_version)
def __lt__(self, other: Any) -> bool:
"""Add a comparison method, so that `sorted` returns the most specialized platforms first."""
@@ -137,10 +159,25 @@
return self_os < other_os
def __str__(self) -> str:
- if self.arch is None:
- return f"@platforms//os:{self.os}"
+ if self.minor_version is None:
+ assert (
+ self.os is not None
+ ), f"if minor_version is None, OS must be specified, got {repr(self)}"
+ if self.arch is None:
+ return f"@platforms//os:{self.os}"
+ else:
+ return f"{self.os}_{self.arch}"
- return f"{self.os}_{self.arch}"
+ if self.arch is None and self.os is None:
+ return f"@//python/config_settings:is_python_3.{self.minor_version}"
+
+ if self.arch is None:
+ return f"cp3{self.minor_version}_{self.os}_anyarch"
+
+ if self.os is None:
+ return f"cp3{self.minor_version}_anyos_{self.arch}"
+
+ return f"cp3{self.minor_version}_{self.os}_{self.arch}"
@classmethod
def from_string(cls, platform: Union[str, List[str]]) -> List["Platform"]:
@@ -150,14 +187,33 @@
for p in platform:
if p == "host":
ret.update(cls.host())
- elif p == "all":
- ret.update(cls.all())
- elif p.endswith("*"):
- os, _, _ = p.partition("_")
- ret.update(cls.all(OS[os]))
+ continue
+
+ abi, _, tail = p.partition("_")
+ if not abi.startswith("cp"):
+ # The first item is not an abi
+ tail = p
+ abi = ""
+ os, _, arch = tail.partition("_")
+ arch = arch or "*"
+
+ minor_version = int(abi[len("cp3") :]) if abi else None
+
+ if arch != "*":
+ ret.add(
+ cls(
+ os=OS[os] if os != "*" else None,
+ arch=Arch[arch],
+ minor_version=minor_version,
+ )
+ )
else:
- os, _, arch = p.partition("_")
- ret.add(cls(os=OS[os], arch=Arch[arch]))
+ ret.update(
+ cls.all(
+ want_os=OS[os] if os != "*" else None,
+ minor_version=minor_version,
+ )
+ )
return sorted(ret)
@@ -227,6 +283,9 @@
return ""
def env_markers(self, extra: str) -> Dict[str, str]:
+ # If it is None, use the host version
+ minor_version = self.minor_version or host_interpreter_minor_version()
+
return {
"extra": extra,
"os_name": self.os_name,
@@ -235,11 +294,14 @@
"platform_system": self.platform_system,
"platform_release": "", # unset
"platform_version": "", # unset
+ "python_version": f"3.{minor_version}",
+ # FIXME @aignas 2024-01-14: is putting zero last a good idea? Maybe we should
+ # use `20` or something else to avoid having weird issues where the full version is used for
+ # matching and the author decides to only support 3.y.5 upwards.
+ "implementation_version": f"3.{minor_version}.0",
+ "python_full_version": f"3.{minor_version}.0",
# we assume that the following are the same as the interpreter used to setup the deps:
- # "implementation_version": "X.Y.Z",
# "implementation_name": "cpython"
- # "python_version": "X.Y",
- # "python_full_version": "X.Y.Z",
# "platform_python_implementation: "CPython",
}
@@ -251,16 +313,36 @@
class Deps:
+ """Deps is a dependency builder that has a build() method to return FrozenDeps."""
+
def __init__(
self,
name: str,
+ requires_dist: List[str],
*,
- requires_dist: Optional[List[str]],
extras: Optional[Set[str]] = None,
platforms: Optional[Set[Platform]] = None,
):
+ """Create a new instance and parse the requires_dist
+
+ Args:
+ name (str): The name of the whl distribution
+ requires_dist (list[Str]): The Requires-Dist from the METADATA of the whl
+ distribution.
+ extras (set[str], optional): The list of requested extras, defaults to None.
+ platforms (set[Platform], optional): The list of target platforms, defaults to
+ None. If the list of platforms has multiple `minor_version` values, it
+ will change the code to generate the select statements using
+ `@rules_python//python/config_settings:is_python_3.y` conditions.
+ """
self.name: str = Deps._normalize(name)
self._platforms: Set[Platform] = platforms or set()
+ self._target_versions = {p.minor_version for p in platforms or {}}
+ self._add_version_select = platforms and len(self._target_versions) > 2
+ if None in self._target_versions and len(self._target_versions) > 2:
+ raise ValueError(
+ f"all python versions need to be specified explicitly, got: {platforms}"
+ )
# Sort so that the dictionary order in the FrozenDeps is deterministic
# without the final sort because Python retains insertion order. That way
@@ -301,18 +383,39 @@
self._select[p].add(dep)
- if len(self._select[platform]) != 1:
+ if len(self._select[platform]) == 1:
+ # We are adding a new item to the select and we need to ensure that
+ # existing dependencies from less specialized platforms are propagated
+ # to the newly added dependency set.
+ for p, deps in self._select.items():
+ # Check if the existing platform overlaps with the given platform
+ if p == platform or platform not in p.all_specializations():
+ continue
+
+ self._select[platform].update(self._select[p])
+
+ def _maybe_add_common_dep(self, dep):
+ if len(self._target_versions) < 2:
return
- # We are adding a new item to the select and we need to ensure that
- # existing dependencies from less specialized platforms are propagated
- # to the newly added dependency set.
- for p, deps in self._select.items():
- # Check if the existing platform overlaps with the given platform
- if p == platform or platform not in p.all_specializations():
- continue
+ platforms = [Platform(minor_version=v) for v in self._target_versions]
- self._select[platform].update(self._select[p])
+ # If the dep is targeting all target python versions, lets add it to
+ # the common dependency list to simplify the select statements.
+ for p in platforms:
+ if p not in self._select:
+ return
+
+ if dep not in self._select[p]:
+ return
+
+ # All of the python version-specific branches have the dep, so lets add
+ # it to the common deps.
+ self._deps.add(dep)
+ for p in platforms:
+ self._select[p].remove(dep)
+ if not self._select[p]:
+ self._select.pop(p)
@staticmethod
def _normalize(name: str) -> str:
@@ -400,8 +503,9 @@
]
)
match_arch = "platform_machine" in marker_str
+ match_version = "version" in marker_str
- if not (match_os or match_arch):
+ if not (match_os or match_arch or match_version):
if any(req.marker.evaluate({"extra": extra}) for extra in extras):
self._add(req.name, None)
return
@@ -414,8 +518,17 @@
if match_arch:
self._add(req.name, plat)
- else:
+ elif match_os and self._add_version_select:
+ self._add(req.name, Platform(plat.os, minor_version=plat.minor_version))
+ elif match_os:
self._add(req.name, Platform(plat.os))
+ elif match_version and self._add_version_select:
+ self._add(req.name, Platform(minor_version=plat.minor_version))
+ elif match_version:
+ self._add(req.name, None)
+
+ # Merge to common if possible after processing all platforms
+ self._maybe_add_common_dep(req.name)
def build(self) -> FrozenDeps:
return FrozenDeps(
diff --git a/python/pip_install/tools/wheel_installer/wheel_test.py b/python/pip_install/tools/wheel_installer/wheel_test.py
index 5e95ee3..f7c847f 100644
--- a/python/pip_install/tools/wheel_installer/wheel_test.py
+++ b/python/pip_install/tools/wheel_installer/wheel_test.py
@@ -15,12 +15,6 @@
self.assertEqual({}, got.deps_select)
def test_can_add_os_specific_deps(self):
- platforms = {
- "linux_x86_64",
- "osx_x86_64",
- "osx_aarch64",
- "windows_x86_64",
- }
deps = wheel.Deps(
"foo",
requires_dist=[
@@ -29,7 +23,49 @@
"posix_dep; os_name=='posix'",
"win_dep; os_name=='nt'",
],
- platforms=set(wheel.Platform.from_string(platforms)),
+ platforms={
+ wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.x86_64),
+ wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.x86_64),
+ wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.aarch64),
+ wheel.Platform(os=wheel.OS.windows, arch=wheel.Arch.x86_64),
+ },
+ )
+
+ got = deps.build()
+
+ self.assertEqual(["bar"], got.deps)
+ self.assertEqual(
+ {
+ "@platforms//os:linux": ["posix_dep"],
+ "@platforms//os:osx": ["an_osx_dep", "posix_dep"],
+ "@platforms//os:windows": ["win_dep"],
+ },
+ got.deps_select,
+ )
+
+ def test_can_add_os_specific_deps_with_specific_python_version(self):
+ deps = wheel.Deps(
+ "foo",
+ requires_dist=[
+ "bar",
+ "an_osx_dep; sys_platform=='darwin'",
+ "posix_dep; os_name=='posix'",
+ "win_dep; os_name=='nt'",
+ ],
+ platforms={
+ wheel.Platform(
+ os=wheel.OS.linux, arch=wheel.Arch.x86_64, minor_version=8
+ ),
+ wheel.Platform(
+ os=wheel.OS.osx, arch=wheel.Arch.x86_64, minor_version=8
+ ),
+ wheel.Platform(
+ os=wheel.OS.osx, arch=wheel.Arch.aarch64, minor_version=8
+ ),
+ wheel.Platform(
+ os=wheel.OS.windows, arch=wheel.Arch.x86_64, minor_version=8
+ ),
+ },
)
got = deps.build()
@@ -45,17 +81,16 @@
)
def test_deps_are_added_to_more_specialized_platforms(self):
- platforms = {
- "osx_x86_64",
- "osx_aarch64",
- }
got = wheel.Deps(
"foo",
requires_dist=[
"m1_dep; sys_platform=='darwin' and platform_machine=='arm64'",
"mac_dep; sys_platform=='darwin'",
],
- platforms=set(wheel.Platform.from_string(platforms)),
+ platforms={
+ wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.x86_64),
+ wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.aarch64),
+ },
).build()
self.assertEqual(
@@ -70,17 +105,16 @@
)
def test_deps_from_more_specialized_platforms_are_propagated(self):
- platforms = {
- "osx_x86_64",
- "osx_aarch64",
- }
got = wheel.Deps(
"foo",
requires_dist=[
"a_mac_dep; sys_platform=='darwin'",
"m1_dep; sys_platform=='darwin' and platform_machine=='arm64'",
],
- platforms=set(wheel.Platform.from_string(platforms)),
+ platforms={
+ wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.x86_64),
+ wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.aarch64),
+ },
).build()
self.assertEqual([], got.deps)
@@ -93,12 +127,6 @@
)
def test_non_platform_markers_are_added_to_common_deps(self):
- platforms = {
- "linux_x86_64",
- "osx_x86_64",
- "osx_aarch64",
- "windows_x86_64",
- }
got = wheel.Deps(
"foo",
requires_dist=[
@@ -106,7 +134,12 @@
"baz; implementation_name=='cpython'",
"m1_dep; sys_platform=='darwin' and platform_machine=='arm64'",
],
- platforms=set(wheel.Platform.from_string(platforms)),
+ platforms={
+ wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.x86_64),
+ wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.x86_64),
+ wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.aarch64),
+ wheel.Platform(os=wheel.OS.windows, arch=wheel.Arch.x86_64),
+ },
).build()
self.assertEqual(["bar", "baz"], got.deps)
@@ -152,6 +185,91 @@
self.assertEqual(["bar", "baz", "zdep"], got.deps)
self.assertEqual({}, got.deps_select)
+ def test_can_get_deps_based_on_specific_python_version(self):
+ requires_dist = [
+ "bar",
+ "baz; python_version < '3.8'",
+ "posix_dep; os_name=='posix' and python_version >= '3.8'",
+ ]
+
+ py38_deps = wheel.Deps(
+ "foo",
+ requires_dist=requires_dist,
+ platforms=[
+ wheel.Platform(
+ os=wheel.OS.linux, arch=wheel.Arch.x86_64, minor_version=8
+ ),
+ ],
+ ).build()
+ py37_deps = wheel.Deps(
+ "foo",
+ requires_dist=requires_dist,
+ platforms=[
+ wheel.Platform(
+ os=wheel.OS.linux, arch=wheel.Arch.x86_64, minor_version=7
+ ),
+ ],
+ ).build()
+
+ self.assertEqual(["bar", "baz"], py37_deps.deps)
+ self.assertEqual({}, py37_deps.deps_select)
+ self.assertEqual(["bar"], py38_deps.deps)
+ self.assertEqual({"@platforms//os:linux": ["posix_dep"]}, py38_deps.deps_select)
+
+ def test_can_get_version_select(self):
+ requires_dist = [
+ "bar",
+ "baz; python_version < '3.8'",
+ "posix_dep; os_name=='posix'",
+ "posix_dep_with_version; os_name=='posix' and python_version >= '3.8'",
+ ]
+
+ deps = wheel.Deps(
+ "foo",
+ requires_dist=requires_dist,
+ platforms=[
+ wheel.Platform(
+ os=wheel.OS.linux, arch=wheel.Arch.x86_64, minor_version=minor
+ )
+ for minor in [7, 8, 9]
+ ],
+ )
+ got = deps.build()
+
+ self.assertEqual(["bar"], got.deps)
+ self.assertEqual(
+ {
+ "@//python/config_settings:is_python_3.7": ["baz"],
+ "cp37_linux_anyarch": ["baz", "posix_dep"],
+ "cp38_linux_anyarch": ["posix_dep", "posix_dep_with_version"],
+ "cp39_linux_anyarch": ["posix_dep", "posix_dep_with_version"],
+ },
+ got.deps_select,
+ )
+
+ def test_deps_spanning_all_target_py_versions_are_added_to_common(self):
+ requires_dist = [
+ "bar",
+ "baz (<2,>=1.11) ; python_version < '3.8'",
+ "baz (<2,>=1.14) ; python_version >= '3.8'",
+ ]
+
+ deps = wheel.Deps(
+ "foo",
+ requires_dist=requires_dist,
+ platforms=wheel.Platform.from_string(["cp37_*", "cp38_*", "cp39_*"]),
+ )
+ got = deps.build()
+
+ self.assertEqual(["bar", "baz"], got.deps)
+ self.assertEqual({}, got.deps_select)
+
+
+class MinorVersionTest(unittest.TestCase):
+ def test_host(self):
+ host = wheel.host_interpreter_minor_version()
+ self.assertIsNotNone(host)
+
class PlatformTest(unittest.TestCase):
def test_can_get_host(self):
@@ -160,16 +278,64 @@
self.assertEqual(1, len(wheel.Platform.from_string("host")))
self.assertEqual(host, wheel.Platform.from_string("host"))
- def test_can_get_all(self):
- all_platforms = wheel.Platform.all()
- self.assertEqual(15, len(all_platforms))
- self.assertEqual(all_platforms, wheel.Platform.from_string("all"))
+ def test_can_get_linux_x86_64_without_py_version(self):
+ got = wheel.Platform.from_string("linux_x86_64")
+ want = wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.x86_64)
+ self.assertEqual(want, got[0])
+
+ def test_can_get_specific_from_string(self):
+ got = wheel.Platform.from_string("cp33_linux_x86_64")
+ want = wheel.Platform(
+ os=wheel.OS.linux, arch=wheel.Arch.x86_64, minor_version=3
+ )
+ self.assertEqual(want, got[0])
+
+ def test_can_get_all_for_py_version(self):
+ cp39 = wheel.Platform.all(minor_version=9)
+ self.assertEqual(15, len(cp39), f"Got {cp39}")
+ self.assertEqual(cp39, wheel.Platform.from_string("cp39_*"))
def test_can_get_all_for_os(self):
+ linuxes = wheel.Platform.all(wheel.OS.linux, minor_version=9)
+ self.assertEqual(5, len(linuxes))
+ self.assertEqual(linuxes, wheel.Platform.from_string("cp39_linux_*"))
+
+ def test_can_get_all_for_os_for_host_python(self):
linuxes = wheel.Platform.all(wheel.OS.linux)
self.assertEqual(5, len(linuxes))
self.assertEqual(linuxes, wheel.Platform.from_string("linux_*"))
+ def test_specific_version_specializations(self):
+ any_py33 = wheel.Platform(minor_version=3)
+
+ # When
+ all_specializations = list(any_py33.all_specializations())
+
+ want = (
+ [any_py33]
+ + [
+ wheel.Platform(arch=arch, minor_version=any_py33.minor_version)
+ for arch in wheel.Arch
+ ]
+ + [
+ wheel.Platform(os=os, minor_version=any_py33.minor_version)
+ for os in wheel.OS
+ ]
+ + wheel.Platform.all(minor_version=any_py33.minor_version)
+ )
+ self.assertEqual(want, all_specializations)
+
+ def test_aarch64_specializations(self):
+ any_aarch64 = wheel.Platform(arch=wheel.Arch.aarch64)
+ all_specializations = list(any_aarch64.all_specializations())
+ want = [
+ wheel.Platform(os=None, arch=wheel.Arch.aarch64),
+ wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.aarch64),
+ wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.aarch64),
+ wheel.Platform(os=wheel.OS.windows, arch=wheel.Arch.aarch64),
+ ]
+ self.assertEqual(want, all_specializations)
+
def test_linux_specializations(self):
any_linux = wheel.Platform(os=wheel.OS.linux)
all_specializations = list(any_linux.all_specializations())
diff --git a/tests/pip_install/whl_library/generate_build_bazel_tests.bzl b/tests/pip_install/whl_library/generate_build_bazel_tests.bzl
index b89477f..72423aa 100644
--- a/tests/pip_install/whl_library/generate_build_bazel_tests.bzl
+++ b/tests/pip_install/whl_library/generate_build_bazel_tests.bzl
@@ -42,9 +42,91 @@
data = [
"@pypi_bar_baz//:whl",
"@pypi_foo//:whl",
+ ],
+ visibility = ["//visibility:private"],
+)
+
+py_library(
+ name = "_pkg",
+ srcs = glob(
+ ["site-packages/**/*.py"],
+ exclude=[],
+ # Empty sources are allowed to support wheels that don't have any
+ # pure-Python code, e.g. pymssql, which is written in Cython.
+ allow_empty = True,
+ ),
+ data = [] + glob(
+ ["site-packages/**/*"],
+ exclude=["**/* *", "**/*.py", "**/*.pyc", "**/*.pyc.*", "**/*.dist-info/RECORD"],
+ ),
+ # This makes this directory a top-level in the python import
+ # search path for anything that depends on this.
+ imports = ["site-packages"],
+ deps = [
+ "@pypi_bar_baz//:pkg",
+ "@pypi_foo//:pkg",
+ ],
+ tags = ["tag1", "tag2"],
+ visibility = ["//visibility:private"],
+)
+
+alias(
+ name = "pkg",
+ actual = "_pkg",
+)
+
+alias(
+ name = "whl",
+ actual = "_whl",
+)
+"""
+ actual = generate_whl_library_build_bazel(
+ repo_prefix = "pypi_",
+ whl_name = "foo.whl",
+ dependencies = ["foo", "bar-baz"],
+ dependencies_by_platform = {},
+ data_exclude = [],
+ tags = ["tag1", "tag2"],
+ entry_points = {},
+ annotation = None,
+ )
+ env.expect.that_str(actual).equals(want)
+
+_tests.append(_test_simple)
+
+def _test_dep_selects(env):
+ want = """\
+load("@rules_python//python:defs.bzl", "py_library", "py_binary")
+load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+ name = "dist_info",
+ srcs = glob(["site-packages/*.dist-info/**"], allow_empty = True),
+)
+
+filegroup(
+ name = "data",
+ srcs = glob(["data/**"], allow_empty = True),
+)
+
+filegroup(
+ name = "_whl",
+ srcs = ["foo.whl"],
+ data = [
+ "@pypi_bar_baz//:whl",
+ "@pypi_foo//:whl",
] + select(
{
- "@platforms//os:windows": ["@pypi_colorama//:whl"],
+ "@//python/config_settings:is_python_3.9": ["@pypi_py39_dep//:whl"],
+ "@platforms//cpu:aarch64": ["@pypi_arm_dep//:whl"],
+ "@platforms//os:windows": ["@pypi_win_dep//:whl"],
+ ":is_cp310_linux_ppc": ["@pypi_py310_linux_ppc_dep//:whl"],
+ ":is_cp39_anyos_aarch64": ["@pypi_py39_arm_dep//:whl"],
+ ":is_cp39_linux_anyarch": ["@pypi_py39_linux_dep//:whl"],
+ ":is_linux_x86_64": ["@pypi_linux_intel_dep//:whl"],
"//conditions:default": [],
},
),
@@ -72,7 +154,13 @@
"@pypi_foo//:pkg",
] + select(
{
- "@platforms//os:windows": ["@pypi_colorama//:pkg"],
+ "@//python/config_settings:is_python_3.9": ["@pypi_py39_dep//:pkg"],
+ "@platforms//cpu:aarch64": ["@pypi_arm_dep//:pkg"],
+ "@platforms//os:windows": ["@pypi_win_dep//:pkg"],
+ ":is_cp310_linux_ppc": ["@pypi_py310_linux_ppc_dep//:pkg"],
+ ":is_cp39_anyos_aarch64": ["@pypi_py39_arm_dep//:pkg"],
+ ":is_cp39_linux_anyarch": ["@pypi_py39_linux_dep//:pkg"],
+ ":is_linux_x86_64": ["@pypi_linux_intel_dep//:pkg"],
"//conditions:default": [],
},
),
@@ -89,20 +177,85 @@
name = "whl",
actual = "_whl",
)
+
+config_setting(
+ name = "is_linux_ppc",
+ constraint_values = [
+ "@platforms//cpu:ppc",
+ "@platforms//os:linux",
+ ],
+ visibility = ["//visibility:private"],
+)
+
+selects.config_setting_group(
+ name = "is_cp310_linux_ppc",
+ match_all = [
+ ":is_linux_ppc",
+ "@//python/config_settings:is_python_3.10",
+ ],
+ visibility = ["//visibility:private"],
+)
+
+config_setting(
+ name = "is_anyos_aarch64",
+ constraint_values = ["@platforms//cpu:aarch64"],
+ visibility = ["//visibility:private"],
+)
+
+selects.config_setting_group(
+ name = "is_cp39_anyos_aarch64",
+ match_all = [
+ ":is_anyos_aarch64",
+ "@//python/config_settings:is_python_3.9",
+ ],
+ visibility = ["//visibility:private"],
+)
+
+config_setting(
+ name = "is_linux_anyarch",
+ constraint_values = ["@platforms//os:linux"],
+ visibility = ["//visibility:private"],
+)
+
+selects.config_setting_group(
+ name = "is_cp39_linux_anyarch",
+ match_all = [
+ ":is_linux_anyarch",
+ "@//python/config_settings:is_python_3.9",
+ ],
+ visibility = ["//visibility:private"],
+)
+
+config_setting(
+ name = "is_linux_x86_64",
+ constraint_values = [
+ "@platforms//cpu:x86_64",
+ "@platforms//os:linux",
+ ],
+ visibility = ["//visibility:private"],
+)
"""
actual = generate_whl_library_build_bazel(
repo_prefix = "pypi_",
whl_name = "foo.whl",
dependencies = ["foo", "bar-baz"],
- dependencies_by_platform = {"@platforms//os:windows": ["colorama"]},
+ dependencies_by_platform = {
+ "@//python/config_settings:is_python_3.9": ["py39_dep"],
+ "@platforms//cpu:aarch64": ["arm_dep"],
+ "@platforms//os:windows": ["win_dep"],
+ "cp310_linux_ppc": ["py310_linux_ppc_dep"],
+ "cp39_anyos_aarch64": ["py39_arm_dep"],
+ "cp39_linux_anyarch": ["py39_linux_dep"],
+ "linux_x86_64": ["linux_intel_dep"],
+ },
data_exclude = [],
tags = ["tag1", "tag2"],
entry_points = {},
annotation = None,
)
- env.expect.that_str(actual).equals(want)
+ env.expect.that_str(actual.replace("@@", "@")).equals(want)
-_tests.append(_test_simple)
+_tests.append(_test_dep_selects)
def _test_with_annotation(env):
want = """\
@@ -308,11 +461,11 @@
srcs = ["foo.whl"],
data = ["@pypi_bar_baz//:whl"] + select(
{
+ "@platforms//os:linux": ["@pypi_box//:whl"],
":is_linux_x86_64": [
"@pypi_box//:whl",
"@pypi_box_amd64//:whl",
],
- "@platforms//os:linux": ["@pypi_box//:whl"],
"//conditions:default": [],
},
),
@@ -337,11 +490,11 @@
imports = ["site-packages"],
deps = ["@pypi_bar_baz//:pkg"] + select(
{
+ "@platforms//os:linux": ["@pypi_box//:pkg"],
":is_linux_x86_64": [
"@pypi_box//:pkg",
"@pypi_box_amd64//:pkg",
],
- "@platforms//os:linux": ["@pypi_box//:pkg"],
"//conditions:default": [],
},
),
@@ -375,7 +528,7 @@
dependencies_by_platform = {
"linux_x86_64": ["box", "box-amd64"],
"windows_x86_64": ["fox"],
- "@platforms//os:linux": ["box"], # buildifier: disable=unsorted-dict-items
+ "@platforms//os:linux": ["box"], # buildifier: disable=unsorted-dict-items to check that we sort inside the test
},
tags = [],
entry_points = {},
@@ -384,7 +537,7 @@
group_name = "qux",
group_deps = ["foo", "fox", "qux"],
)
- env.expect.that_str(actual).equals(want)
+ env.expect.that_str(actual.replace("@@", "@")).equals(want)
_tests.append(_test_group_member)
