Use decorator for build function registration.
It has the following benefits:
- prevents typos
- enables completion
- for the code to be ordered in topological order
- removes to logic
Change-Id: Ia3766d7459c172708a08e3921216e0406189efee
diff --git a/qemu/scripts/rebuild.py b/qemu/scripts/rebuild.py
index b76e9e6..f78e45e 100755
--- a/qemu/scripts/rebuild.py
+++ b/qemu/scripts/rebuild.py
@@ -15,7 +15,6 @@
"""A script to rebuild QEMU from scratch on Linux."""
import argparse
-import inspect
import os
from pathlib import Path
import shlex
@@ -28,8 +27,6 @@
from typing import List
from typing import Sequence
-sys.path.insert(0, os.path.dirname(__file__))
-
def copy_file(src: Path, dst: Path) -> None:
log(" COPY_FILE %s --> %s" % (src, dst))
@@ -287,12 +284,61 @@
##########################################################################
##########################################################################
#####
+##### B U I L D S E Q U E N C E R
+#####
+##### A |Project| can register build tasks and their dependencies.
+##### Then it can return a build plan to be executed in sequence.
+#####
+##########################################################################
+##########################################################################
+
+BuildTaskFn = Callable[[BuildConfig], None]
+
+
+class Project:
+ # Type of a build task function that takes a single |BuildConfig| argument.
+
+ def __init__(self):
+ self.tasks = {}
+
+ def task(self, deps: List[BuildTaskFn]):
+ """Decorator that registers a |BuildTaskFn| and its dependencies."""
+
+ def decorator(fn: BuildTaskFn) -> BuildTaskFn:
+ for dep in deps:
+ if dep not in self.tasks:
+ raise ValueError(
+ f"Task {fn} depends on {dep}, but {dep} is was not yet defined."
+ " Did you forgot to annotate it?"
+ )
+ if fn in self.tasks:
+ raise ValueError(f"Task {fn} already defined.")
+ self.tasks[fn] = deps
+ return fn
+
+ return decorator
+
+ def get_build_task_list(
+ self, task_function: BuildTaskFn
+ ) -> List[BuildTaskFn]:
+ """Returns the transitive dependency list of the current task."""
+ # Rely on the fact that:
+ # a - function are registered in topological order
+ # b - python dictionaries are iterated in insertion order.
+ task_list = list(self.tasks.keys())
+ return task_list[: task_list.index(task_function) + 1]
+
+
+project = Project()
+
+##########################################################################
+##########################################################################
+#####
##### I N D I V I D U A L T A S K S
#####
##### Each build_task_for_xxx() function below should only access a single
-##### BuildConfig argument, and its docstring may contain a line
-##### that looks like: "Requires: <depname>+" to list the other
-##### tasks it depends on.
+##### BuildConfig argument, be decorated with `project.task` and enumerate
+##### the tasks it depends on.
#####
##### These functions should also only use the BuildConfig methods
##### to do their work, i.e. they shall not directly modify the
@@ -302,6 +348,7 @@
##########################################################################
+@project.task([])
def build_task_for_sysroot(build: BuildConfig):
# populate_sysroot(build.build_dir / 'sysroot', build.prebuilts_dir),
dst_sysroot = build.build_dir / "sysroot"
@@ -331,14 +378,15 @@
build.copy_file(shared_libgcc_dir / lib, dst_sysroot_lib_dir / lib)
+@project.task([build_task_for_sysroot])
def build_task_for_ninja(build: BuildConfig):
- """Requires: sysroot"""
build.copy_file(
build.prebuilts_dir / "ninja" / "ninja",
build.install_dir / "usr" / "bin" / "ninja",
)
+@project.task([])
def build_task_for_python(build: BuildConfig):
src_python_dir = build.third_party_dir / "python"
dst_python_dir = build.install_dir / "usr"
@@ -346,8 +394,10 @@
build.copy_dir(src_python_dir / d, dst_python_dir / d)
+@project.task(
+ [build_task_for_sysroot, build_task_for_ninja, build_task_for_python]
+)
def build_task_for_meson(build: BuildConfig):
- """Requires: sysroot ninja python"""
meson_packager = (
build.third_party_dir / "meson" / "packaging" / "create_zipapp.py"
)
@@ -362,6 +412,7 @@
])
+@project.task([])
def build_task_for_rust(build: BuildConfig):
log("Install prebuilt rust.")
src_rust_dir = build.prebuilts_dir / "rust" / "linux-x86" / "1.65.0"
@@ -372,14 +423,15 @@
build.copy_dir(src_dir, dst_dir)
+@project.task([build_task_for_sysroot])
def build_task_for_make(build: BuildConfig) -> None:
- """Requires: sysroot"""
build.copy_file(
build.prebuilts_dir / "build-tools" / "linux-x86" / "bin" / "make",
build.install_dir / "usr" / "bin" / "make",
)
+@project.task([])
def build_task_for_cmake(build: BuildConfig):
log("Install Cmake prebuilt.")
build.copy_file(
@@ -392,8 +444,8 @@
)
+@project.task([build_task_for_make])
def build_task_for_bzip2(build: BuildConfig):
- """Requires: make"""
build_dir = build.make_subdir(Path("bzip2"))
build.copy_dir(build.third_party_dir / "bzip2", build_dir)
env = build.env_copy()
@@ -418,8 +470,8 @@
)
+@project.task([build_task_for_make])
def build_task_for_pkg_config(build: BuildConfig):
- """Requires: make"""
build_dir = build.make_subdir(Path("pkg-config"))
build.copy_dir(build.third_party_dir / "pkg-config", build_dir)
build.run(
@@ -447,8 +499,8 @@
build.run_make_install(build_dir)
+@project.task([build_task_for_make])
def build_task_for_patchelf(build: BuildConfig):
- """Requires: make"""
build_dir = build.make_subdir(Path("patchelf"))
build.copy_dir(build.third_party_dir / "patchelf", build_dir)
# Run configure separately so that we can pass "--with-internal-glib".
@@ -464,8 +516,8 @@
build.run_make_install(build_dir)
+@project.task([build_task_for_make, build_task_for_cmake])
def build_task_for_zlib(build: BuildConfig):
- """Requires: make cmake"""
lib_name = "zlib"
src_dir = build.third_party_dir / lib_name
build_dir = build.make_subdir(Path(lib_name))
@@ -494,8 +546,8 @@
build.run_make_install(build_dir, use_DESTDIR=False)
+@project.task([build_task_for_make, build_task_for_bzip2])
def build_task_for_libpcre2(build: BuildConfig):
- """Requires: make bzip2"""
build_dir = build.make_subdir(Path("pcre"))
build.copy_dir(build.third_party_dir / "pcre", build_dir)
@@ -509,8 +561,8 @@
build.run_make_install(build_dir, use_DESTDIR=True)
+@project.task([build_task_for_make])
def build_task_for_libffi(build: BuildConfig):
- """Requires: make"""
build_dir = build.make_subdir(Path("libffi"))
build.copy_dir(build.third_party_dir / "libffi", build_dir)
@@ -526,8 +578,15 @@
build.run_make_install(build_dir, use_DESTDIR=True)
+@project.task([
+ build_task_for_make,
+ build_task_for_meson,
+ build_task_for_libffi,
+ build_task_for_libpcre2,
+ build_task_for_zlib,
+ build_task_for_pkg_config,
+])
def build_task_for_glib(build: BuildConfig):
- """Requires: make meson libffi libpcre2 zlib pkg_config"""
src_dir = build.third_party_dir / "glib"
build_dir = build.make_subdir(Path("glib"))
@@ -555,8 +614,12 @@
build.run(["ninja", "install"], build_dir)
+@project.task([
+ build_task_for_make,
+ build_task_for_meson,
+ build_task_for_pkg_config,
+])
def build_task_for_pixman(build: BuildConfig):
- """Requires: meson make pkg_config"""
src_dir = build.third_party_dir / "pixman"
build_dir = build.make_subdir(Path("pixman"))
@@ -591,8 +654,11 @@
)
+@project.task([
+ build_task_for_make,
+ build_task_for_glib,
+])
def build_task_for_libslirp(build: BuildConfig):
- """Requires: make glib"""
src_dir = build.third_party_dir / "libslirp"
build_dir = build.make_subdir(Path("libslirp"))
@@ -611,8 +677,11 @@
build.run(["ninja", "install"], build_dir)
+@project.task([
+ build_task_for_make,
+ build_task_for_cmake,
+])
def build_task_for_googletest(build: BuildConfig):
- """Requires: cmake make"""
dir_name = Path("googletest")
build.make_subdir(dir_name)
cmd_args = [
@@ -625,8 +694,12 @@
build.run_make_install(dir_name, use_DESTDIR=False)
+@project.task([
+ build_task_for_make,
+ build_task_for_cmake,
+ build_task_for_googletest,
+])
def build_task_for_aemu_base(build: BuildConfig):
- """Requires: cmake make googletest"""
dir_name = Path("aemu")
build.make_subdir(dir_name)
# Options from third_party/aemu/rebuild.sh
@@ -643,8 +716,11 @@
build.run_make_install(dir_name, use_DESTDIR=False)
+@project.task([
+ build_task_for_make,
+ build_task_for_cmake,
+])
def build_task_for_flatbuffers(build: BuildConfig):
- """Requires: cmake make"""
dir_name = Path("flatbuffers")
build.make_subdir(dir_name)
cmd_args = [
@@ -657,8 +733,11 @@
build.run_make_install(dir_name, use_DESTDIR=False)
+@project.task([
+ build_task_for_make,
+ build_task_for_meson,
+])
def build_task_for_libpciaccess(build: BuildConfig):
- """Requires: make meson"""
dir_name = Path("libpciaccess")
src_dir = build.third_party_dir / dir_name
build_dir = build.make_subdir(dir_name)
@@ -688,8 +767,12 @@
)
+@project.task([
+ build_task_for_make,
+ build_task_for_meson,
+ build_task_for_libpciaccess,
+])
def build_task_for_libdrm(build: BuildConfig):
- """Requires: make meson libpciaccess"""
dir_name = Path("libdrm")
src_dir = build.third_party_dir / dir_name
build_dir = build.make_subdir(dir_name)
@@ -719,8 +802,14 @@
)
+@project.task([
+ build_task_for_make,
+ build_task_for_meson,
+ build_task_for_aemu_base,
+ build_task_for_flatbuffers,
+ build_task_for_libdrm,
+])
def build_task_for_gfxstream(build: BuildConfig):
- """Requires: cmake make aemu_base flatbuffers libdrm"""
dir_name = Path("gfxstream-build")
out_dir = build.make_subdir(dir_name)
cmd_args = [
@@ -740,8 +829,12 @@
)
+@project.task([
+ build_task_for_make,
+ build_task_for_rust,
+ build_task_for_gfxstream,
+])
def build_task_for_rutabaga(build: BuildConfig):
- """Requires: make rust gfxstream"""
out_dir = build.make_subdir(Path("rutabaga"))
cmd_args = [
build.install_dir / "usr/bin/cargo",
@@ -781,8 +874,17 @@
)
+@project.task([
+ build_task_for_make,
+ build_task_for_libslirp,
+ build_task_for_glib,
+ build_task_for_pixman,
+ build_task_for_zlib,
+ build_task_for_pkg_config,
+ build_task_for_rutabaga,
+ build_task_for_gfxstream,
+])
def build_task_for_qemu(build: BuildConfig):
- """Requires: make libslirp glib pixman zlib pkg_config rutabaga gfxstream"""
target_list = [
"riscv64-softmmu",
# "x86_64-softmmu",
@@ -803,8 +905,11 @@
build.run_make_build(build_dir)
+@project.task([
+ build_task_for_qemu,
+ build_task_for_patchelf,
+])
def build_task_for_qemu_portable(build: BuildConfig):
- """Requires: qemu patchelf"""
package_dir = build.make_subdir(Path("qemu-portable"))
# Install to a new directory rather than to the common taks install dir.
build.run_make_install(
@@ -823,8 +928,10 @@
build.run(["tar", "-czvf", "qemu-portable.tar.gz", package_dir])
+@project.task([
+ build_task_for_qemu_portable,
+])
def build_task_for_qemu_test(build: BuildConfig):
- """Requires: qemu_portable"""
build.run(["make", "test"], build.build_dir / "qemu")
@@ -837,117 +944,6 @@
##########################################################################
-class BuildTask(object):
- """Encapsulate a build task and its dependencies.
-
- Do not use directly, use a BuildTaskSet instead.
- """
-
- def __init__(self, name: str, build_func: Callable, deps: List[str] = []):
- """Create instance.
-
- Args:
- name: Unique name for this task.
- build_func: A lambda used to perform the task. It must take a single
- BuildConfig argument.
- deps: List of direct dependency names.
- """
- self._name = name
- self._builder = build_func
- self._deps = deps
-
- @property
- def name(self):
- return self._name
-
- @property
- def deps(self):
- return self._deps
-
- def build(self, bc: BuildConfig):
- self._builder(bc)
-
-
-class BuildTaskSet(object):
- # The prefix of all functions in this file that will be returned by
- # the get_build_functions() method below.
- BUILD_FUNCTION_PREFIX = "build_task_for_"
-
- """A complete set of BuildTasks and their dependencies.
-
- Usage is:
- 1) Create instance.
-
- 2) Call add() or add_build_function() to add new tasks.
-
- 3) Call get_build_task_list() to retrieve list of BuildTask instances
- in reverse topological order.
- """
-
- def __init__(self):
- self._tasks: Dict[str, BuildTask] = {}
-
- def add_task(
- self, name: str, build_func: Callable, deps: List[str] | None = None
- ):
- self._tasks[name] = BuildTask(name, build_func, deps)
-
- def add_build_function(self, f: Callable):
- name = f.__name__
- assert name.startswith(self.BUILD_FUNCTION_PREFIX)
- name = name[len(self.BUILD_FUNCTION_PREFIX) :]
- deps = []
- doc = f.__doc__
- if doc is not None:
- for line in doc.splitlines():
- line = line.strip()
- if line.startswith("Requires: "):
- deps.extend(line[10:].split(" "))
- self.add_task(name, f, deps)
-
- @staticmethod
- def get_build_functions():
- """Return the list of all build_task_for_xxx() functions in this file."""
- return [
- f
- for name, f in inspect.getmembers(sys.modules[__name__])
- if inspect.isfunction(f)
- and name.startswith(BuildTaskSet.BUILD_FUNCTION_PREFIX)
- ]
-
- def get_build_task_list(self, task_name: str):
- """Compute the transitive dependency list of the current task.
-
- Args:
- task_name: Top-level task name.
-
- Returns:
- A list of BuildTask instance, in reverse dependency order
- (i.e. dependencies appear before their dependents). Note that
- this always includes |self| as the last item.
- """
- result = []
- visited = set()
- result = []
- task = self._tasks[task_name]
- stack = [(task, 0)]
- visited.add(task)
- while stack:
- task, child_index = stack[-1]
- stack = stack[:-1]
- if child_index >= len(task.deps):
- result.append(task)
- else:
- dep_name = task.deps[child_index]
- child = self._tasks.get(dep_name)
- assert child is not None, f"Unknown {task.name} dependency: {dep_name}"
- stack.append((task, child_index + 1))
- if child not in visited:
- visited.add(child)
- stack.append((child, 0))
- return result
-
-
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--build-dir", required=True, help="Build directory.")
@@ -989,23 +985,17 @@
else:
os.makedirs(build_dir)
- # Create build task set
- all_tasks = BuildTaskSet()
-
- # Grab all build_task_for_xxx() functions in this file and add them as build tasks.
- # Their docstring will be parsed for `Requires: [dependency]+'
- for f in all_tasks.get_build_functions():
- all_tasks.add_build_function(f)
-
# Compute the build plan to get 'qemu'
- build_plan = all_tasks.get_build_task_list(
- "qemu_test" if args.run_tests else "qemu_portable"
+ build_tasks = project.get_build_task_list(
+ build_task_for_qemu_test
+ if args.run_tests
+ else build_task_for_qemu_portable
)
- print("BUILD PLAN: %s" % ", ".join([t.name for t in build_plan]))
+ print("BUILD PLAN: %s" % ", ".join([t.__name__ for t in build_tasks]))
- for t in build_plan:
- t.build(build_config)
+ for task in build_tasks:
+ task(build_config)
return 0
