Migrate code generation to typed python. am: 32e6044366
Original change: https://android-review.googlesource.com/c/device/google/cuttlefish_vmm/+/2818639
Change-Id: I35d26a80b2025b36c374d8a3e57379f5a76d6a5b
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
diff --git a/Android.bp b/Android.bp
index 9930817..f3da432 100644
--- a/Android.bp
+++ b/Android.bp
@@ -1,6 +1,6 @@
-// Autogenerated via gen_android_bp.sh
+// Autogenerated via gen_android_bp.py
//
-// Copyright (C) 2019 The Android Open Source Project
+// Copyright (C) 2023 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -26,14 +26,6 @@
check_elf_files: false,
}
cc_prebuilt_binary {
- name: "aarch64_linux_gnu_gfxstream_graphics_detector_for_crosvm",
- srcs: ["aarch64-linux-gnu/bin/gfxstream_graphics_detector"],
- stem: "gfxstream_graphics_detector",
- relative_install_path: "aarch64-linux-gnu",
- defaults: ["cuttlefish_host"],
- check_elf_files: false,
-}
-cc_prebuilt_binary {
name: "aarch64_linux_gnu_libdrm.so.2_for_crosvm",
srcs: ["aarch64-linux-gnu/bin/libdrm.so.2"],
stem: "libdrm.so.2",
@@ -97,6 +89,18 @@
defaults: ["cuttlefish_host"],
check_elf_files: false,
}
+prebuilt_usr_share_host {
+ name: "x86_64_efi-virtio.rom_resource_for_qemu",
+ src: "qemu/x86_64-linux-gnu/usr/share/qemu/efi-virtio.rom",
+ filename: "efi-virtio.rom",
+ sub_dir: "qemu",
+}
+prebuilt_usr_share_host {
+ name: "x86_64_en-us_resource_for_qemu",
+ src: "qemu/x86_64-linux-gnu/usr/share/qemu/keymaps/en-us",
+ filename: "en-us",
+ sub_dir: "qemu/keymaps",
+}
// Note: This is commented out to avoid a conflict with the binary built
// from external/crosvm. This should be uncommented out when backporting to
// older branches with just use the prebuilt and which do not build from
@@ -110,10 +114,9 @@
// check_elf_files: false,
// }
cc_prebuilt_binary {
- name: "x86_64_linux_gnu_gfxstream_graphics_detector_for_crosvm",
- srcs: ["x86_64-linux-gnu/bin/gfxstream_graphics_detector"],
- stem: "gfxstream_graphics_detector",
- relative_install_path: "x86_64-linux-gnu",
+ name: "x86_64_linux_gnu_libc++.so.1_binary_for_qemu",
+ srcs: ["qemu/x86_64-linux-gnu/bin/libc++.so.1"],
+ stem: "libc++.so.1",
defaults: ["cuttlefish_host"],
check_elf_files: false,
}
@@ -150,6 +153,13 @@
check_elf_files: false,
}
cc_prebuilt_binary {
+ name: "x86_64_linux_gnu_libgfxstream_backend.so.0_binary_for_qemu",
+ srcs: ["qemu/x86_64-linux-gnu/bin/libgfxstream_backend.so.0"],
+ stem: "libgfxstream_backend.so.0",
+ defaults: ["cuttlefish_host"],
+ check_elf_files: false,
+}
+cc_prebuilt_binary {
name: "x86_64_linux_gnu_libgfxstream_backend.so_for_crosvm",
srcs: ["x86_64-linux-gnu/bin/libgfxstream_backend.so"],
stem: "libgfxstream_backend.so",
@@ -166,6 +176,13 @@
check_elf_files: false,
}
cc_prebuilt_binary {
+ name: "x86_64_linux_gnu_librutabaga_gfx_ffi.so.0_binary_for_qemu",
+ srcs: ["qemu/x86_64-linux-gnu/bin/librutabaga_gfx_ffi.so.0"],
+ stem: "librutabaga_gfx_ffi.so.0",
+ defaults: ["cuttlefish_host"],
+ check_elf_files: false,
+}
+cc_prebuilt_binary {
name: "x86_64_linux_gnu_libvirglrenderer.so.1_for_crosvm",
srcs: ["x86_64-linux-gnu/bin/libvirglrenderer.so.1"],
stem: "libvirglrenderer.so.1",
@@ -182,27 +199,6 @@
check_elf_files: false,
}
cc_prebuilt_binary {
- name: "x86_64_linux_gnu_libc++.so.1_binary_for_qemu",
- srcs: ["qemu/x86_64-linux-gnu/bin/libc++.so.1"],
- stem: "libc++.so.1",
- defaults: ["cuttlefish_host"],
- check_elf_files: false,
-}
-cc_prebuilt_binary {
- name: "x86_64_linux_gnu_libgfxstream_backend.so.0_binary_for_qemu",
- srcs: ["qemu/x86_64-linux-gnu/bin/libgfxstream_backend.so.0"],
- stem: "libgfxstream_backend.so.0",
- defaults: ["cuttlefish_host"],
- check_elf_files: false,
-}
-cc_prebuilt_binary {
- name: "x86_64_linux_gnu_librutabaga_gfx_ffi.so.0_binary_for_qemu",
- srcs: ["qemu/x86_64-linux-gnu/bin/librutabaga_gfx_ffi.so.0"],
- stem: "librutabaga_gfx_ffi.so.0",
- defaults: ["cuttlefish_host"],
- check_elf_files: false,
-}
-cc_prebuilt_binary {
name: "x86_64_linux_gnu_libz.so.1_binary_for_qemu",
srcs: ["qemu/x86_64-linux-gnu/bin/libz.so.1"],
stem: "libz.so.1",
@@ -231,18 +227,6 @@
check_elf_files: false,
}
prebuilt_usr_share_host {
- name: "x86_64_efi-virtio.rom_resource_for_qemu",
- src: "qemu/x86_64-linux-gnu/usr/share/qemu/efi-virtio.rom",
- filename: "efi-virtio.rom",
- sub_dir: "qemu",
-}
-prebuilt_usr_share_host {
- name: "x86_64_en-us_resource_for_qemu",
- src: "qemu/x86_64-linux-gnu/usr/share/qemu/keymaps/en-us",
- filename: "en-us",
- sub_dir: "qemu/keymaps",
-}
-prebuilt_usr_share_host {
name: "x86_64_opensbi-riscv64-generic-fw_dynamic.bin_resource_for_qemu",
src: "qemu/x86_64-linux-gnu/usr/share/qemu/opensbi-riscv64-generic-fw_dynamic.bin",
filename: "opensbi-riscv64-generic-fw_dynamic.bin",
diff --git a/aarch64-linux-gnu/Android.bp b/aarch64-linux-gnu/Android.bp
index baaf57b..626a9ab 100644
--- a/aarch64-linux-gnu/Android.bp
+++ b/aarch64-linux-gnu/Android.bp
@@ -1,6 +1,6 @@
-// Autogenerated via gen_android_bp.sh
+// Autogenerated via gen_android_bp.py
//
-// Copyright (C) 2019 The Android Open Source Project
+// Copyright (C) 2023 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -117,18 +117,18 @@
sub_dir: "cuttlefish/aarch64-linux-gnu/seccomp",
}
prebuilt_usr_share_host {
- name: "serial_device.policy_at_aarch64",
- src: "etc/seccomp/serial_device.policy",
- filename: "serial_device.policy",
- sub_dir: "cuttlefish/aarch64-linux-gnu/seccomp",
-}
-prebuilt_usr_share_host {
name: "serial.policy_at_aarch64",
src: "etc/seccomp/serial.policy",
filename: "serial.policy",
sub_dir: "cuttlefish/aarch64-linux-gnu/seccomp",
}
prebuilt_usr_share_host {
+ name: "serial_device.policy_at_aarch64",
+ src: "etc/seccomp/serial_device.policy",
+ filename: "serial_device.policy",
+ sub_dir: "cuttlefish/aarch64-linux-gnu/seccomp",
+}
+prebuilt_usr_share_host {
name: "snd_cras_device.policy_at_aarch64",
src: "etc/seccomp/snd_cras_device.policy",
filename: "snd_cras_device.policy",
@@ -189,14 +189,14 @@
sub_dir: "cuttlefish/aarch64-linux-gnu/seccomp",
}
prebuilt_usr_share_host {
- name: "xhci_device.policy_at_aarch64",
- src: "etc/seccomp/xhci_device.policy",
- filename: "xhci_device.policy",
- sub_dir: "cuttlefish/aarch64-linux-gnu/seccomp",
-}
-prebuilt_usr_share_host {
name: "xhci.policy_at_aarch64",
src: "etc/seccomp/xhci.policy",
filename: "xhci.policy",
sub_dir: "cuttlefish/aarch64-linux-gnu/seccomp",
}
+prebuilt_usr_share_host {
+ name: "xhci_device.policy_at_aarch64",
+ src: "etc/seccomp/xhci_device.policy",
+ filename: "xhci_device.policy",
+ sub_dir: "cuttlefish/aarch64-linux-gnu/seccomp",
+}
diff --git a/gen_android_bp.py b/gen_android_bp.py
new file mode 100644
index 0000000..91aba12
--- /dev/null
+++ b/gen_android_bp.py
@@ -0,0 +1,322 @@
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This script enumerates pre-built resources and updates Android.bp files:
+
+ - $ANDROID_BUILD_TOP/device/google/cuttlefish_vmm/Android.bp
+ - $ANDROID_BUILD_TOP/device/google/cuttlefish_vmm/$ARCH/Android.bp
+ - $ANDROID_BUILD_TOP/device/google/cuttlefish/Android.bp
+
+It is intended to be run manually:
+
+ python3 $ANDROID_BUILD_TOP/device/google/cuttlefish_vmm/gen_android_bp.py
+
+"""
+import datetime
+import os
+import re
+import sys
+import textwrap
+from dataclasses import dataclass
+from enum import auto
+from enum import Enum
+from pathlib import Path
+from typing import Dict
+from typing import List
+from typing import Iterator
+from typing import TypeAlias
+
+
+def tool_name() -> str:
+ """Returns a short name for this generation tool."""
+ return Path(__file__).name
+
+
+class Architecture(Enum):
+ """Host instruction set architectures."""
+
+ AARCH64 = auto()
+ X86_64 = auto()
+
+ def dir(self) -> Path:
+ "Returns the relative directory path to the specified architecture."
+ return Path(f"{self.name.lower()}-linux-gnu")
+
+
+# Android.bp variant value type.
+Value: TypeAlias = str | Path | bool | list["Value"]
+
+
+def value_to_bp(value: Value) -> str:
+ """Returns `bp` expression for the specified value"""
+ if isinstance(value, list):
+ if len(value) == 1:
+ return "[%s]" % value_to_bp(value[0])
+ return "[\n %s,\n]" % ",\n ".join(value_to_bp(e) for e in value)
+ elif isinstance(value, bool):
+ return str(value).lower()
+ elif isinstance(value, Path):
+ return value_to_bp(str(value))
+ else:
+ return f'"{value}"'
+
+
+@dataclass
+class Module:
+ """Android bp build rule."""
+
+ module_type: str
+ parameters: Dict[str, Value]
+
+ @property
+ def name(self) -> str:
+ assert isinstance(self.parameters.get("name"), str)
+ return self.parameters["name"]
+
+ def __str__(self) -> str:
+ """Returns a `.bp` string for this modules with its parameters."""
+ body = "\n ".join(
+ f"{k}: {value_to_bp(v)}," for k, v in self.parameters.items()
+ )
+ return f"{self.module_type} {{\n {body}\n}}\n"
+
+
+def update_generated_section(file_path: Path, tag: str, content: str):
+ """Reads a text file, matches and replaces the content between
+ a start beacon and an end beacon with the specified one, and
+ modifies the file in place.
+
+ The generated content is delimited by `// Start of generated` and
+ `// End of generated`. The specified content is indented the same
+ way as the start beacon is.
+
+ Args:
+ file_path: path to the text file to be modified.
+ tag: marks the beginning aned end of the string generated content.
+ content: text to replace the content between the start and end beacons with.
+ """
+ # Read the contents of the text file.
+ with open(file_path, "rt", encoding="utf-8") as f:
+ file_contents = f.read()
+
+ # Find the start and end beacon positions in the file contents.
+ start = f"// Start of generated {tag}\n"
+ end = f"// End of generated {tag}\n"
+
+ match = re.match(
+ f"^(?P<head>.*)^(?P<indent>[ \t]*){re.escape(start)}.*{re.escape(end)}(?P<tail>.*)$",
+ file_contents,
+ re.DOTALL | re.MULTILINE,
+ )
+ if not match:
+ raise ValueError(
+ f"Generated content beacons {(start, end)} not matched in file {file_path}"
+ )
+
+ with open(file_path, "wt", encoding="utf-8") as f:
+ f.write(f"{match.group('head')}{match.group('indent')}{start}")
+ f.write(f"{match.group('indent')}// Generated by {tool_name()}\n")
+ f.write(textwrap.indent(content, match.group("indent")))
+ f.write(f"{match.group('indent')}{end}{match.group('tail')}")
+
+
+def license() -> str:
+ """Returns a license header at the current date, with generation warning."""
+ current_year = datetime.datetime.now().year
+ return textwrap.dedent(
+ f"""\
+ // Autogenerated via {tool_name()}
+ //
+ // Copyright (C) {current_year} The Android Open Source Project
+ //
+ // Licensed under the Apache License, Version 2.0 (the "License");
+ // you may not use this file except in compliance with the License.
+ // You may obtain a copy of the License at
+ //
+ // http://www.apache.org/licenses/LICENSE-2.0
+ //
+ // Unless required by applicable law or agreed to in writing, software
+ // distributed under the License is distributed on an "AS IS" BASIS,
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ // See the License for the specific language governing permissions and
+ // limitations under the License.
+
+ """
+ )
+
+
+def comment(text: str) -> str:
+ """Prefixes each lines with '/// ' and return the commented content."""
+ return re.sub("^(?!$)", "// ", text, flags=re.MULTILINE)
+
+
+def gen_android_bp4seccomp(path: Path, arch: Architecture):
+ """Regenerates the specified '.bp' file for the given architecture."""
+
+ with open(path, "wt") as out:
+ subdir = Path("etc/seccomp")
+ seccomp_dir = arch.dir() / subdir
+ where_in_etc_on_user_machine = "cuttlefish" / arch.dir() / "seccomp"
+ out.write(license())
+
+ for path in sorted(seccomp_dir.iterdir()):
+ module = Module(
+ "prebuilt_usr_share_host",
+ dict(
+ name=f"{path.name}_at_{arch.name.lower()}",
+ src=subdir / path.name,
+ filename=path.name,
+ sub_dir=where_in_etc_on_user_machine,
+ ),
+ )
+ out.write(str(module))
+
+
+def crosvm_binaries(arch: Architecture) -> Iterator[Path]:
+ """Lists crosvm binary paths."""
+ dir = arch.dir() / "bin"
+ yield dir / "crosvm"
+ yield dir / "libminijail.so"
+ yield dir / "libgfxstream_backend.so"
+ yield from dir.glob("*.so.?")
+
+
+def qemu_binaries(arch: Architecture) -> Iterator[Path]:
+ """Lists qemu binary paths."""
+ dir = Path("qemu/x86_64-linux-gnu/bin")
+ yield from dir.glob("*.so.?")
+ yield from dir.glob("qemu-system*")
+
+
+def gen_main_android_bp_modules() -> Iterator[Module]:
+ """Returns the Modules to write in the main 'Android.bp' file."""
+ for arch in Architecture:
+ for path in crosvm_binaries(arch):
+ name = re.sub("/bin/|/|-|_bin_", "_", str(path))
+ if path.stem != "crosvm":
+ name = f"{name}_for_crosvm"
+
+ yield Module(
+ "cc_prebuilt_binary",
+ dict(
+ name=name,
+ srcs=[path],
+ stem=path.name,
+ relative_install_path=arch.dir(),
+ defaults=["cuttlefish_host"],
+ check_elf_files=False,
+ ),
+ )
+
+ for binary_path in qemu_binaries(Architecture.X86_64):
+ yield Module(
+ "cc_prebuilt_binary",
+ dict(
+ name=f"x86_64_linux_gnu_{binary_path.name}_binary_for_qemu",
+ srcs=[binary_path],
+ stem=binary_path.name,
+ defaults=["cuttlefish_host"],
+ check_elf_files=False,
+ ),
+ )
+
+ resource_paths = [
+ Path("qemu/efi-virtio.rom"),
+ Path("qemu/keymaps/en-us"),
+ Path("qemu/opensbi-riscv64-generic-fw_dynamic.bin"),
+ ]
+
+ for resource_path in resource_paths:
+ base_name = resource_path.name
+ arch = "x86_64"
+ sub_dir = resource_path.parent
+ yield Module(
+ "prebuilt_usr_share_host",
+ dict(
+ name=f"{arch}_{base_name}_resource_for_qemu",
+ src=f"qemu/x86_64-linux-gnu/usr/share/{resource_path}",
+ filename=base_name,
+ sub_dir=sub_dir,
+ ),
+ )
+
+
+def gen_main_android_bp_file(android_bp_path: Path, modules: List[Module]):
+ """Writes the main 'Android.bp' file with the specified modules."""
+
+ disclamer = f"""\
+ // NOTE: Using cc_prebuilt_binary because cc_prebuilt_library will add
+ // unwanted .so file extensions when installing shared libraries
+
+ """
+ crosvm_comment = """\
+ // Note: This is commented out to avoid a conflict with the binary built
+ // from external/crosvm. This should be uncommented out when backporting to
+ // older branches with just use the prebuilt and which do not build from
+ // source.
+ """
+
+ with open(android_bp_path, "wt") as out:
+ out.write(license())
+ out.write(textwrap.dedent(disclamer))
+
+ sort_key = lambda m: (
+ m.name,
+ m.parameters["name"].rsplit("_")[-1],
+ m.parameters["name"],
+ )
+ for module in sorted(modules, key=sort_key):
+ module_text = str(module)
+ if module.parameters["name"] == "x86_64_linux_gnu_crosvm":
+ out.write(textwrap.dedent(crosvm_comment))
+ module_text = comment(module_text)
+ out.write(module_text)
+
+
+def generate_all_cuttlefish_host_package_android_bp(path: Path, modules: list[Module]):
+ """Updates the list of module in 'device/google/cuttlefish/Android.bp'."""
+
+ qemu_binaries = [m for m in modules if m.name.endswith("_binary_for_qemu")]
+ qemu_resources = [m for m in modules if m.name.endswith("_resource_for_qemu")]
+
+ def update_list(list_name: str, modules:list[Module]):
+ names = sorted(m.parameters["name"] for m in modules)
+ text = f"{list_name} = {value_to_bp(names)}\n"
+ update_generated_section(path, list_name, text)
+
+ update_list("qemu_x86_64_linux_gnu_binary", qemu_binaries)
+ update_list("qemu_x86_64_linux_gnu_resource", qemu_resources)
+
+
+def main(argv):
+ if len(argv) != 1:
+ print(f"Usage error: {argv[0]} does not accept any argument.")
+ return 1
+
+ # Set the current directory to the script location.
+ os.chdir(Path(__file__).parent)
+
+ modules = list(gen_main_android_bp_modules())
+ gen_main_android_bp_file(Path("Android.bp"), modules)
+
+ generate_all_cuttlefish_host_package_android_bp(
+ Path("../cuttlefish/build/Android.bp"), modules
+ )
+
+ for arch in Architecture:
+ gen_android_bp4seccomp(arch.dir() / "Android.bp", arch)
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))
diff --git a/x86_64-linux-gnu/Android.bp b/x86_64-linux-gnu/Android.bp
index 8de52b0..9968d58 100644
--- a/x86_64-linux-gnu/Android.bp
+++ b/x86_64-linux-gnu/Android.bp
@@ -1,6 +1,6 @@
-// Autogenerated via gen_android_bp.sh
+// Autogenerated via gen_android_bp.py
//
-// Copyright (C) 2019 The Android Open Source Project
+// Copyright (C) 2023 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -51,18 +51,18 @@
sub_dir: "cuttlefish/x86_64-linux-gnu/seccomp",
}
prebuilt_usr_share_host {
- name: "coiommu_device.policy_at_x86_64",
- src: "etc/seccomp/coiommu_device.policy",
- filename: "coiommu_device.policy",
- sub_dir: "cuttlefish/x86_64-linux-gnu/seccomp",
-}
-prebuilt_usr_share_host {
name: "coiommu.policy_at_x86_64",
src: "etc/seccomp/coiommu.policy",
filename: "coiommu.policy",
sub_dir: "cuttlefish/x86_64-linux-gnu/seccomp",
}
prebuilt_usr_share_host {
+ name: "coiommu_device.policy_at_x86_64",
+ src: "etc/seccomp/coiommu_device.policy",
+ filename: "coiommu_device.policy",
+ sub_dir: "cuttlefish/x86_64-linux-gnu/seccomp",
+}
+prebuilt_usr_share_host {
name: "common_device.policy_at_x86_64",
src: "etc/seccomp/common_device.policy",
filename: "common_device.policy",
@@ -159,6 +159,12 @@
sub_dir: "cuttlefish/x86_64-linux-gnu/seccomp",
}
prebuilt_usr_share_host {
+ name: "serial.policy_at_x86_64",
+ src: "etc/seccomp/serial.policy",
+ filename: "serial.policy",
+ sub_dir: "cuttlefish/x86_64-linux-gnu/seccomp",
+}
+prebuilt_usr_share_host {
name: "serial_device.policy_at_x86_64",
src: "etc/seccomp/serial_device.policy",
filename: "serial_device.policy",
@@ -177,12 +183,6 @@
sub_dir: "cuttlefish/x86_64-linux-gnu/seccomp",
}
prebuilt_usr_share_host {
- name: "serial.policy_at_x86_64",
- src: "etc/seccomp/serial.policy",
- filename: "serial.policy",
- sub_dir: "cuttlefish/x86_64-linux-gnu/seccomp",
-}
-prebuilt_usr_share_host {
name: "snd_cras_device.policy_at_x86_64",
src: "etc/seccomp/snd_cras_device.policy",
filename: "snd_cras_device.policy",
@@ -267,14 +267,14 @@
sub_dir: "cuttlefish/x86_64-linux-gnu/seccomp",
}
prebuilt_usr_share_host {
- name: "xhci_device.policy_at_x86_64",
- src: "etc/seccomp/xhci_device.policy",
- filename: "xhci_device.policy",
- sub_dir: "cuttlefish/x86_64-linux-gnu/seccomp",
-}
-prebuilt_usr_share_host {
name: "xhci.policy_at_x86_64",
src: "etc/seccomp/xhci.policy",
filename: "xhci.policy",
sub_dir: "cuttlefish/x86_64-linux-gnu/seccomp",
}
+prebuilt_usr_share_host {
+ name: "xhci_device.policy_at_x86_64",
+ src: "etc/seccomp/xhci_device.policy",
+ filename: "xhci_device.policy",
+ sub_dir: "cuttlefish/x86_64-linux-gnu/seccomp",
+}
