changes to pip_repository source files now re-trigger the repo rule (#601)
* changes to pip_repository source files now re-trigger the repo rule
* update diff test message.
* added note about auto-gen.
* Updated docs
* remove unnecessary snippet
* installer -> updater
Co-authored-by: Jonathon Belotti <jonathon@canva.com>
diff --git a/docs/BUILD b/docs/BUILD
index 6ef0d6e..e08b751 100644
--- a/docs/BUILD
+++ b/docs/BUILD
@@ -61,9 +61,7 @@
bzl_library(
name = "pip_install_bzl",
srcs = [
- "//python/pip_install:pip_repository.bzl",
- "//python/pip_install:repositories.bzl",
- "//python/pip_install:requirements.bzl",
+ "//python/pip_install:bzl",
],
deps = [
":defs",
diff --git a/python/BUILD b/python/BUILD
index 96df383..7dbbec3 100644
--- a/python/BUILD
+++ b/python/BUILD
@@ -46,6 +46,7 @@
"defs.bzl",
"packaging.bzl",
"pip.bzl",
+ "//python/pip_install:bzl",
"//python/private:bzl",
],
visibility = ["//:__pkg__"],
diff --git a/python/pip_install/BUILD b/python/pip_install/BUILD
index 90be44c..41cac99 100644
--- a/python/pip_install/BUILD
+++ b/python/pip_install/BUILD
@@ -7,14 +7,26 @@
"pip_compile.py",
"//python/pip_install/extract_wheels:distribution",
"//python/pip_install/parse_requirements_to_bzl:distribution",
+ "//python/pip_install/private:distribution",
],
visibility = ["//:__pkg__"],
)
filegroup(
name = "bzl",
- srcs = glob(["*.bzl"]),
- visibility = ["//:__pkg__"],
+ srcs = glob(["*.bzl"]) + [
+ "//python/pip_install/private:bzl_srcs",
+ ],
+ visibility = ["//:__subpackages__"],
+)
+
+filegroup(
+ name = "py_srcs",
+ srcs = [
+ "//python/pip_install/extract_wheels:py_srcs",
+ "//python/pip_install/parse_requirements_to_bzl:py_srcs",
+ ],
+ visibility = ["//visibility:public"],
)
exports_files(
diff --git a/python/pip_install/extract_wheels/BUILD b/python/pip_install/extract_wheels/BUILD
index 92a0c7a..a92c562 100644
--- a/python/pip_install/extract_wheels/BUILD
+++ b/python/pip_install/extract_wheels/BUILD
@@ -17,3 +17,14 @@
],
visibility = ["//python/pip_install:__subpackages__"],
)
+
+filegroup(
+ name = "py_srcs",
+ srcs = glob(
+ include = ["**/*.py"],
+ exclude = ["**/*_test.py"],
+ ) + [
+ "//python/pip_install/extract_wheels/lib:py_srcs",
+ ],
+ visibility = ["//python/pip_install:__subpackages__"],
+)
diff --git a/python/pip_install/extract_wheels/lib/BUILD b/python/pip_install/extract_wheels/lib/BUILD
index 2b0a91f..4758f15 100644
--- a/python/pip_install/extract_wheels/lib/BUILD
+++ b/python/pip_install/extract_wheels/lib/BUILD
@@ -110,3 +110,12 @@
),
visibility = ["//python/pip_install:__subpackages__"],
)
+
+filegroup(
+ name = "py_srcs",
+ srcs = glob(
+ include = ["**/*.py"],
+ exclude = ["**/*_test.py"],
+ ),
+ visibility = ["//python/pip_install:__subpackages__"],
+)
diff --git a/python/pip_install/parse_requirements_to_bzl/BUILD b/python/pip_install/parse_requirements_to_bzl/BUILD
index bb60323..8a876ab 100644
--- a/python/pip_install/parse_requirements_to_bzl/BUILD
+++ b/python/pip_install/parse_requirements_to_bzl/BUILD
@@ -41,3 +41,14 @@
],
visibility = ["//python/pip_install:__subpackages__"],
)
+
+filegroup(
+ name = "py_srcs",
+ srcs = glob(
+ include = ["**/*.py"],
+ exclude = ["**/*_test.py"],
+ ) + [
+ "//python/pip_install/parse_requirements_to_bzl/extract_single_wheel:py_srcs",
+ ],
+ visibility = ["//python/pip_install:__subpackages__"],
+)
diff --git a/python/pip_install/parse_requirements_to_bzl/extract_single_wheel/BUILD b/python/pip_install/parse_requirements_to_bzl/extract_single_wheel/BUILD
index 17bdfe7..bc0f640 100644
--- a/python/pip_install/parse_requirements_to_bzl/extract_single_wheel/BUILD
+++ b/python/pip_install/parse_requirements_to_bzl/extract_single_wheel/BUILD
@@ -6,3 +6,12 @@
),
visibility = ["//python/pip_install:__subpackages__"],
)
+
+filegroup(
+ name = "py_srcs",
+ srcs = glob(
+ include = ["**/*.py"],
+ exclude = ["**/*_test.py"],
+ ),
+ visibility = ["//python/pip_install:__subpackages__"],
+)
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index 1dc49c7..3d0710a 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -1,6 +1,7 @@
""
load("//python/pip_install:repositories.bzl", "all_requirements")
+load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS")
def _construct_pypath(rctx):
"""Helper function to construct a PYTHONPATH.
@@ -250,6 +251,11 @@
default = 600,
doc = "Timeout (in seconds) on the rule's execution duration.",
),
+ "_py_srcs": attr.label_list(
+ doc = "Python sources used in the repository rule",
+ allow_files = True,
+ default = PIP_INSTALL_PY_SRCS,
+ ),
}
pip_repository_attrs = {
diff --git a/python/pip_install/private/BUILD b/python/pip_install/private/BUILD
new file mode 100644
index 0000000..86b4b3d
--- /dev/null
+++ b/python/pip_install/private/BUILD
@@ -0,0 +1,24 @@
+load(":pip_install_utils.bzl", "srcs_module")
+
+package(default_visibility = ["//:__subpackages__"])
+
+exports_files([
+ "srcs.bzl",
+])
+
+filegroup(
+ name = "distribution",
+ srcs = glob(["*"]),
+ visibility = ["//python/pip_install:__subpackages__"],
+)
+
+filegroup(
+ name = "bzl_srcs",
+ srcs = glob(["*.bzl"]),
+)
+
+srcs_module(
+ name = "srcs_module",
+ srcs = "//python/pip_install:py_srcs",
+ dest = ":srcs.bzl",
+)
diff --git a/python/pip_install/private/pip_install_utils.bzl b/python/pip_install/private/pip_install_utils.bzl
new file mode 100644
index 0000000..038ee0e
--- /dev/null
+++ b/python/pip_install/private/pip_install_utils.bzl
@@ -0,0 +1,118 @@
+"""Utilities for `rules_python` pip rules"""
+
+_SRCS_TEMPLATE = """\
+\"\"\"A generate file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
+
+This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.install` target. Please
+`bazel run` this target to apply any updates. Note that doing so will discard any local modifications.
+"\"\"
+
+# Each source file is tracked as a target so `pip_repository` rules will know to automatically rebuild if any of the
+# sources changed.
+PIP_INSTALL_PY_SRCS = [
+ {srcs}
+]
+"""
+
+def _src_label(file):
+ dir_path, file_name = file.short_path.rsplit("/", 1)
+
+ return "@rules_python//{}:{}".format(
+ dir_path,
+ file_name,
+ )
+
+def _srcs_module_impl(ctx):
+ srcs = [_src_label(src) for src in ctx.files.srcs]
+ if not srcs:
+ fail("`srcs` cannot be empty")
+ output = ctx.actions.declare_file(ctx.label.name)
+
+ ctx.actions.write(
+ output = output,
+ content = _SRCS_TEMPLATE.format(
+ srcs = "\n ".join(["\"{}\",".format(src) for src in srcs]),
+ ),
+ )
+
+ return DefaultInfo(
+ files = depset([output]),
+ )
+
+_srcs_module = rule(
+ doc = "A rule for writing a list of sources to a templated file",
+ implementation = _srcs_module_impl,
+ attrs = {
+ "srcs": attr.label(
+ doc = "A filegroup of source files",
+ allow_files = True,
+ ),
+ },
+)
+
+_INSTALLER_TEMPLATE = """\
+#!/bin/bash
+set -euo pipefail
+cp -f "{path}" "${{BUILD_WORKSPACE_DIRECTORY}}/{dest}"
+"""
+
+def _srcs_updater_impl(ctx):
+ output = ctx.actions.declare_file(ctx.label.name + ".sh")
+ target_file = ctx.file.input
+ dest = ctx.file.dest.short_path
+
+ ctx.actions.write(
+ output = output,
+ content = _INSTALLER_TEMPLATE.format(
+ path = target_file.short_path,
+ dest = dest,
+ ),
+ is_executable = True,
+ )
+
+ return DefaultInfo(
+ files = depset([output]),
+ runfiles = ctx.runfiles(files = [target_file]),
+ executable = output,
+ )
+
+_srcs_updater = rule(
+ doc = "A rule for writing a `srcs.bzl` file back to the repository",
+ implementation = _srcs_updater_impl,
+ attrs = {
+ "dest": attr.label(
+ doc = "The target file to write the new `input` to.",
+ allow_single_file = ["srcs.bzl"],
+ mandatory = True,
+ ),
+ "input": attr.label(
+ doc = "The file to write back to the repository",
+ allow_single_file = True,
+ mandatory = True,
+ ),
+ },
+ executable = True,
+)
+
+def srcs_module(name, dest, **kwargs):
+ """A helper rule to ensure `pip_repository` rules are always up to date
+
+ Args:
+ name (str): The name of the sources module
+ dest (str): The filename the module should be written as in the current package.
+ **kwargs (dict): Additional keyword arguments
+ """
+ tags = kwargs.pop("tags", [])
+
+ _srcs_module(
+ name = name,
+ tags = tags,
+ **kwargs
+ )
+
+ _srcs_updater(
+ name = name + ".update",
+ input = name,
+ dest = dest,
+ tags = tags,
+ )
diff --git a/python/pip_install/private/srcs.bzl b/python/pip_install/private/srcs.bzl
new file mode 100644
index 0000000..3784fa8
--- /dev/null
+++ b/python/pip_install/private/srcs.bzl
@@ -0,0 +1,23 @@
+"""A generate file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
+
+This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.install` target. Please
+`bazel run` this target to apply any updates. Note that doing so will discard any local modifications.
+"""
+
+# Each source file is tracked as a target so `pip_repository` rules will know to automatically rebuild if any of the
+# sources changed.
+PIP_INSTALL_PY_SRCS = [
+ "@rules_python//python/pip_install/extract_wheels:__init__.py",
+ "@rules_python//python/pip_install/extract_wheels:__main__.py",
+ "@rules_python//python/pip_install/extract_wheels/lib:__init__.py",
+ "@rules_python//python/pip_install/extract_wheels/lib:arguments.py",
+ "@rules_python//python/pip_install/extract_wheels/lib:bazel.py",
+ "@rules_python//python/pip_install/extract_wheels/lib:namespace_pkgs.py",
+ "@rules_python//python/pip_install/extract_wheels/lib:purelib.py",
+ "@rules_python//python/pip_install/extract_wheels/lib:requirements.py",
+ "@rules_python//python/pip_install/extract_wheels/lib:wheel.py",
+ "@rules_python//python/pip_install/parse_requirements_to_bzl:__init__.py",
+ "@rules_python//python/pip_install/parse_requirements_to_bzl:__main__.py",
+ "@rules_python//python/pip_install/parse_requirements_to_bzl/extract_single_wheel:__init__.py",
+ "@rules_python//python/pip_install/parse_requirements_to_bzl/extract_single_wheel:__main__.py",
+]
diff --git a/python/pip_install/private/test/BUILD b/python/pip_install/private/test/BUILD
new file mode 100644
index 0000000..90d1846
--- /dev/null
+++ b/python/pip_install/private/test/BUILD
@@ -0,0 +1,17 @@
+load("@bazel_skylib//rules:diff_test.bzl", "diff_test")
+
+diff_test(
+ name = "srcs_diff_test",
+ failure_message = (
+ "Please run `bazel run //python/pip_install/private:srcs_module.update` " +
+ "to update the `srcs.bzl` module found in the same package."
+ ),
+ file1 = "//python/pip_install/private:srcs_module",
+ file2 = "//python/pip_install/private:srcs.bzl",
+ # TODO: The diff_test here fails on Windows. As does the
+ # install script. This should be fixed.
+ target_compatible_with = select({
+ "@platforms//os:windows": ["@platforms//:incompatible"],
+ "//conditions:default": [],
+ }),
+)
