| # Copyright (C) 2023 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """Generates an xml manifest for the `repo` tool. |
| |
| It enumerates all git submodules recursively, and |
| creates a `project` entry in the manifest, pointing |
| at the repository's url and hash. |
| |
| Repository's url are remapped to point either at AOSP |
| or at experimental-qemu-build-internal when the AOSP |
| repository is not ready. |
| |
| Some submodules are not used by the build and are not mapped. |
| |
| Nested repositories are relocated into ./third_party and a |
| symlink is created to recreate the structure. |
| """ |
| |
| import argparse |
| import dataclasses |
| import os |
| import subprocess |
| import sys |
| import textwrap |
| import xml.etree.ElementTree as ET |
| |
| # Host name of AOSP Git on borg repositories. |
| # See rpc://android.googlesource.com/{repo name} |
| AOSP_HOST = "android.googlesource.com" |
| |
| # Submodules that are not used by the build. |
| REPO_TO_SKIP = { |
| "https://boringssl.googlesource.com/boringssl", |
| "https://chromium.googlesource.com/chromium/tools/depot_tools.git", |
| "https://chromium.googlesource.com/chromiumos/platform/minigbm", |
| "https://chromium.googlesource.com/chromiumos/platform/minijail", |
| "https://chromium.googlesource.com/chromiumos/third_party/tpm2", |
| "https://chromium.googlesource.com/crosvm/perfetto/", |
| "https://github.com/akheron/jansson", |
| "https://github.com/google/brotli", |
| "https://github.com/google/wycheproof", |
| "https://github.com/kkos/oniguruma", |
| "https://github.com/krb5/krb5", |
| "https://github.com/openssl/openssl", |
| "https://github.com/pyca/cryptography.git", |
| "https://github.com/tianocore/edk2-cmocka.git", |
| "https://github.com/ucb-bar/berkeley-softfloat-3.git", |
| "https://gitlab.com/libvirt/libvirt-ci.git", |
| "https://github.com/gost-engine/engine", |
| "https://github.com/provider-corner/libprov.git", |
| "https://github.com/MIPI-Alliance/public-mipi-sys-t.git", |
| "https://github.com/Zeex/subhook.git", |
| "https://github.com/zeux/pugixml.git", |
| "https://github.com/devicetree-org/pylibfdt.git", |
| "https://gitlab.com/qemu-project/edk2.git", |
| "https://gitlab.com/qemu-project/ipxe.git", |
| "https://gitlab.com/qemu-project/libvfio-user.git", |
| "https://gitlab.com/qemu-project/openbios.git", |
| "https://gitlab.com/qemu-project/opensbi.git", |
| "https://gitlab.com/qemu-project/qboot.git", |
| "https://gitlab.com/qemu-project/qemu-palcode.git", |
| "https://gitlab.com/qemu-project/QemuMacDrivers.git", |
| "https://gitlab.com/qemu-project/seabios-hppa.git", |
| "https://gitlab.com/qemu-project/seabios.git/", |
| "https://gitlab.com/qemu-project/skiboot.git", |
| "https://gitlab.com/qemu-project/SLOF.git", |
| "https://gitlab.com/qemu-project/u-boot-sam460ex.git", |
| "https://gitlab.com/qemu-project/vbootrom.git", |
| } |
| |
| # Replaces repositories URLs. |
| REPO_MAPPING = { |
| # Submodules remapped to android AOSP. |
| "git://anongit.freedesktop.org/git/pixman.git": ( |
| "https://android.googlesource.com/platform/external/pixman" |
| ), |
| "https://boringssl.googlesource.com/boringssl": ( |
| "https://android.googlesource.com/platform/external/boringssl" |
| ), |
| "https://chromium.googlesource.com/chromiumos/third_party/virglrenderer": ( |
| "https://android.googlesource.com/platform/external/virglrenderer" |
| ), |
| "https://chromium.googlesource.com/crosvm/crosvm": ( |
| "https://android.googlesource.com/platform/external/crosvm" |
| ), |
| "https://github.com/google/googletest.git": ( |
| "https://android.googlesource.com/platform/external/googletest" |
| ), |
| "https://github.com/mesonbuild/meson.git": ( |
| "https://android.googlesource.com/trusty/external/qemu-meson" |
| ), |
| "https://gitlab.com/qemu-project/dtc.git": ( |
| "https://android.googlesource.com/platform/external/dtc" |
| ), |
| "https://gitlab.com/qemu-project/meson.git": ( |
| "https://android.googlesource.com/trusty/external/qemu-meson" |
| ), |
| "https://gitlab.com/qemu-project/u-boot.git": ( # Can probably be removed. |
| "https://android.googlesource.com/platform/external/u-boot" |
| ), |
| "https://gitlab.freedesktop.org/slirp/libslirp.git": ( |
| "https://android.googlesource.com/trusty/external/qemu-libslirp" |
| ), |
| "https://gitlab.gnome.org/GNOME/glib.git": ( |
| "https://android.googlesource.com/platform/external/bluetooth/glib" |
| ), |
| "https://github.com/KhronosGroup/EGL-Registry.git": ( |
| "https://android.googlesource.com/platform/external/egl-registry" |
| ), |
| "https://gitlab.com/qemu-project/berkeley-softfloat-3.git": "https://android.googlesource.com/platform/external/berkeley-softfloat-3", |
| "https://gitlab.com/qemu-project/berkeley-testfloat-3.git": "https://android.googlesource.com/platform/external/berkeley-testfloat-3", |
| "https://salsa.debian.org/xorg-team/lib/libpciaccess.git": ( |
| "https://android.googlesource.com/platform/external/libpciaccess" |
| ), |
| "https://gitlab.com/qemu-project/keycodemapdb.git": ( |
| "https://android.googlesource.com/trusty/external/qemu-keycodemapdb" |
| ), |
| "https://gitlab.gnome.org/GNOME/gvdb.git": ( |
| "https://android.googlesource.com/platform/external/gvdb" |
| ), |
| } |
| |
| |
| @dataclasses.dataclass |
| class Submodule: |
| """Occurrence of a module in the tree.""" |
| |
| path: str |
| origin_url: str |
| hash: str |
| |
| |
| @dataclasses.dataclass |
| class Project: |
| """Project in a repo manifest.""" |
| |
| origin_url: str |
| gob_host: str |
| gob_path: str |
| revision: str |
| shallow: bool |
| linkat: list |
| |
| |
| def GetAllSubmodules(path): |
| """Yields a Submodule for each module recursively found in the specified repo.""" |
| |
| yield Submodule( |
| ".", |
| "https://android.googlesource.com/device/google/cuttlefish_vmm", |
| "main", |
| ) |
| |
| output = subprocess.check_output([ |
| "git", |
| "submodule", |
| "foreach", |
| "--recursive", |
| "-q", |
| "echo ${displaypath} $(git remote get-url origin) ${sha1}", |
| ]) |
| for line in output.decode().strip().split("\n"): |
| yield Submodule(*line.split(" ")) |
| |
| |
| def MatchProject(url): |
| prefix_to_remote = { |
| "https://android.googlesource.com/": AOSP_HOST, |
| "persistent-https://android.git.corp.google.com/": AOSP_HOST, |
| } |
| for prefix, remote in prefix_to_remote.items(): |
| if url.startswith(prefix): |
| return remote, url.removeprefix(prefix) |
| raise ValueError( |
| f"Url {url} could neither be mapped to AOSP to" |
| " experimental-qemu-build-internal. If this is a new submodule, verify" |
| " if it is used by the build. If it is, we should import it to AOSP. If" |
| " not, add it to the REPO_TO_SKIP." |
| ) |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument("input", help="git root directory") |
| parser.add_argument( |
| "--repo_manifest", |
| type=argparse.FileType("w"), |
| help="output repo tool manifest", |
| ) |
| |
| args = parser.parse_args() |
| |
| submodules = list(GetAllSubmodules(args.input)) |
| |
| # Build a list of rep project. |
| project_by_name = {} |
| for module in submodules: |
| repo_url = module.origin_url |
| # Skip unused code repositories. |
| if repo_url in REPO_TO_SKIP: |
| continue |
| # Remap repository to their new location or to None. |
| if repo_url in REPO_MAPPING: |
| repo_url = REPO_MAPPING[repo_url] |
| gob_host, gob_path = MatchProject(repo_url) |
| # Add the repository instance to the list merging by gob_path |
| if gob_path not in project_by_name: |
| project_by_name[gob_path] = Project( |
| origin_url=module.origin_url, |
| gob_host=gob_host, |
| gob_path=gob_path, |
| revision=module.hash, |
| shallow=module.path.startswith("qemu/prebuilts/"), |
| linkat=[], |
| ) |
| |
| assert project_by_name[gob_path].gob_host == gob_host |
| project_by_name[gob_path].linkat.append(module.path) |
| |
| GenerateRepoManifest(project_by_name.values(), args.repo_manifest) |
| |
| |
| def SimpleName(project): |
| return project.gob_path.replace("/", "_") |
| |
| |
| def GenerateRepoManifest(projects, out): |
| # Generate repo manifest xml. |
| manifest = ET.Element("manifest") |
| ET.SubElement( |
| manifest, |
| "remote", |
| name="aosp", |
| fetch="sso://android.googlesource.com", |
| review="sso://android/", |
| ) |
| ET.SubElement( |
| manifest, |
| "remote", |
| name="experimental-qemu-build-internal", |
| fetch="sso://experimental-qemu-build-internal", |
| ) |
| ET.SubElement( |
| manifest, |
| "default", |
| attrib={"sync-j": "16"}, |
| revision="main", |
| remote="aosp", |
| ) |
| |
| for project in projects: |
| # The project is instantiated only once. |
| elem = ET.SubElement( |
| manifest, |
| "project", |
| path=project.linkat[0], |
| name=project.gob_path, |
| revision=project.revision, |
| ) |
| if project.gob_host != AOSP_HOST: |
| elem.attrib["remote"] = project.gob_host |
| else: |
| pass # Use the default value which is AOSP. |
| |
| if project.shallow: |
| elem.attrib["clone-depth"] = "1" |
| |
| for linkat in project.linkat[1:]: |
| ET.SubElement(elem, "linkfile", src=".", dest=linkat) |
| |
| tree = ET.ElementTree(manifest) |
| ET.indent(tree) |
| tree.write(out, encoding="unicode") |
| |
| |
| if __name__ == "__main__": |
| sys.exit(main()) |