mainline/update.py

#!/usr/bin/env -S python3 -B
#
# Copyright (C) 2020 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Downloads prebuilts for ART Module dependencies and creates CLs in git."""

import argparse
import collections
import os
import subprocess
import sys
import tempfile


# Prebuilt description used in commit message
PREBUILT_DESCR = "ART dependencies"


# Branch and target tuple for a CI build
BuildSource = collections.namedtuple("BuildSource", [
    "branch",
    "target",
])


ARCHES = ["arm", "arm64", "x86", "x86_64"]

# CI build for SDK snapshots
SDK_SOURCE = BuildSource("aosp-master", "mainline_modules_sdks-userdebug")

# Architecture-specific CI builds for APEXes. These are only used in the chroot
# test setup (see art/tools/buildbot-build.sh). They never become part of any
# dist artifact.
#
# There are currently no CI builds except for x86_64, so APEX updates are
# skipped by default.
APEX_SOURCE = {
    "x86_64": BuildSource("aosp-master", "mainline_modules_x86_64-userdebug"),
}

# Architecture-specific CI builds for implementation libraries. These are only
# used in the chroot test setup (see art/tools/buildbot-build.sh). They never
# become part of any dist artifact.
#
# We'd prefer to take all these from the aosp-master branch, but unfortunately
# they're not available there, so we need to use aosp-master-throttled as well.
# As long as they're new enough to contain the required changes (see README.md
# instructions) and pass tests, it doesn't matter much which builds they come
# from. Also, x86 has no userdebug build available, so we use eng for that.
IMPL_LIB_SOURCE = {
    "arm": BuildSource("aosp-master-throttled", "aosp_arm-userdebug"),
    "arm64": BuildSource("aosp-master", "aosp_arm64-userdebug"),
    "x86": BuildSource("aosp-master", "aosp_x86-eng"),
    "x86_64": BuildSource("aosp-master", "aosp_x86_64-userdebug"),
}

# Paths to git projects to prepare CLs in
GIT_PROJECT_ROOTS = [
    "prebuilts/module_sdk/conscrypt",
    "prebuilts/module_sdk/StatsD",
    "prebuilts/runtime",
]

SDK_VERSION = "current"

SCRIPT_PATH = "prebuilts/runtime/mainline/update.py"


InstallEntry = collections.namedtuple(
    "InstallEntry",
    [
        # One of "apex", "module_sdk" and "impl_lib", for the --skip-* flags.
        "type",
        # Source CI build as a BuildSource tuple, or None if none exists.
        "source_build",
        # Artifact path in the build, passed to fetch_target. Should match a
        # single file. "{BUILD}" gets replaced with the build number.
        "source_path",
        # Local install path.
        "install_path",
        # True if the entry is a zip file that should be unzipped to
        # install_path.
        "install_unzipped",
        # If set, the entry is a zip from which this single file is extracted to
        # install_path.
        "unzip_single_file",
    ],
    defaults=(False, None), # Defaults for install_unzipped and unzip_single_file.
)


def install_apex_entries(module_name, apex_name):
  return [
      InstallEntry(
          type="apex",
          source_build=APEX_SOURCE.get(arch, None),
          source_path=os.path.join(
              "mainline_modules_" + arch,
              apex_name + ".apex"),
          install_path=os.path.join(
              "prebuilts/runtime/mainline",
              module_name,
              "apex",
              apex_name + "-" + arch + ".apex"))
      for arch in ARCHES]


def install_unbundled_sdk_entries(apex_name, mainline_sdk_name, sdk_type, install_path):
  return [
      InstallEntry(
          type="module_sdk",
          source_build=SDK_SOURCE,
          source_path=os.path.join(
              "mainline-sdks/for-latest-build",
              SDK_VERSION,
              apex_name,
              sdk_type,
              mainline_sdk_name + "-" + sdk_type + "-" + SDK_VERSION + ".zip"),
          install_path=install_path,
          install_unzipped=True)]


def install_bundled_sdk_entries(module_name, sdk_type):
  return [
      InstallEntry(
          type="module_sdk",
          source_build=SDK_SOURCE,
          source_path=os.path.join(
              "bundled-mainline-sdks",
              "com.android." + module_name,
              sdk_type,
              module_name + "-module-" + sdk_type + "-" + SDK_VERSION + ".zip"),
          install_path=os.path.join(
              "prebuilts/runtime/mainline",
              module_name,
              sdk_type),
          install_unzipped=True)]


def install_platform_mainline_sdk_entries(sdk_type):
  return [
      InstallEntry(
          type="module_sdk",
          source_build=SDK_SOURCE,
          source_path=os.path.join(
              "bundled-mainline-sdks",
              "platform-mainline",
              sdk_type,
              "platform-mainline-" + sdk_type + "-" + SDK_VERSION + ".zip"),
          install_path=os.path.join(
              "prebuilts/runtime/mainline/platform",
              sdk_type),
          install_unzipped=True)]


def install_impl_lib_entries(lib_name):
  return [
      InstallEntry(
          type="impl_lib",
          source_build=IMPL_LIB_SOURCE[arch],
          source_path="aosp_" + arch + "-target_files-{BUILD}.zip",
          unzip_single_file=os.path.join(
              "SYSTEM",
              "lib64" if arch.endswith("64") else "lib",
              lib_name),
          install_path=os.path.join(
              "prebuilts/runtime/mainline/platform/impl",
              arch,
              lib_name))
      for arch in ARCHES]


install_entries = (
    # Conscrypt
    install_apex_entries("conscrypt", "com.android.conscrypt") +
    install_unbundled_sdk_entries(
        "com.android.conscrypt", "conscrypt-module", "sdk",
        "prebuilts/module_sdk/conscrypt/current") +
    install_unbundled_sdk_entries(
        "com.android.conscrypt", "conscrypt-module", "test-exports",
        "prebuilts/module_sdk/conscrypt/current/test-exports") +
    install_unbundled_sdk_entries(
        "com.android.conscrypt", "conscrypt-module", "host-exports",
        "prebuilts/module_sdk/conscrypt/current/host-exports") +

    # Runtime (Bionic)
    # sdk and host-exports must always be updated together, because the linker
    # and the CRT object files gets embedded in the binaries on linux host
    # Bionic (see code and comments around host_bionic_linker_script in
    # build/soong).
    install_apex_entries("runtime", "com.android.runtime") +
    install_bundled_sdk_entries("runtime", "sdk") +
    install_bundled_sdk_entries("runtime", "host-exports") +

    # I18N
    install_apex_entries("i18n", "com.android.i18n") +
    install_bundled_sdk_entries("i18n", "sdk") +
    install_bundled_sdk_entries("i18n", "test-exports") +

    # tzdata
    install_apex_entries("tzdata", "com.android.tzdata") +
    install_bundled_sdk_entries("tzdata", "test-exports") +

    # statsd
    install_apex_entries("statsd", "com.android.os.statsd") +
    install_unbundled_sdk_entries(
        "com.android.os.statsd", "statsd-module", "sdk",
        "prebuilts/module_sdk/StatsD/current") +

    # Platform
    install_platform_mainline_sdk_entries("sdk") +
    install_platform_mainline_sdk_entries("test-exports") +
    install_impl_lib_entries("heapprofd_client_api.so") +
    install_impl_lib_entries("libartpalette-system.so") +
    install_impl_lib_entries("liblog.so") +
    install_impl_lib_entries("libbinder_ndk.so") +
    # libbinder_ndk dependencies:
    install_impl_lib_entries("libandroid_runtime_lazy.so") +
    install_impl_lib_entries("libbase.so") +
    install_impl_lib_entries("libbinder.so") +
    install_impl_lib_entries("libcutils.so") +
    install_impl_lib_entries("libutils.so") +
    install_impl_lib_entries("libvndksupport.so") +

    [])


def check_call(cmd, **kwargs):
  """Proxy for subprocess.check_call with logging."""
  msg = " ".join(cmd) if isinstance(cmd, list) else cmd
  if "cwd" in kwargs:
    msg = "In " + kwargs["cwd"] + ": " + msg
  print(msg)
  subprocess.check_call(cmd, **kwargs)


def fetch_artifact(branch, target, build, fetch_pattern, local_dir, zip_entry=None):
  """Fetches artifact from the build server."""
  fetch_artifact_path = "/google/data/ro/projects/android/fetch_artifact"
  cmd = [fetch_artifact_path, "--branch", branch, "--target", target,
         "--bid", build, fetch_pattern]
  if zip_entry:
    cmd += ["--zip_entry", zip_entry]
  check_call(cmd, cwd=local_dir)


def start_branch(git_branch_name, git_dirs):
  """Creates a new repo branch in the given projects."""
  check_call(["repo", "start", git_branch_name] + git_dirs)
  # In case the branch already exists we reset it to upstream, to get a clean
  # update CL.
  for git_dir in git_dirs:
    check_call(["git", "reset", "--hard", "@{upstream}"], cwd=git_dir)


def upload_branch(git_root, git_branch_name):
  """Uploads the CLs in the given branch in the given project."""
  # Set the branch as topic to bundle with the CLs in other git projects (if
  # any).
  check_call(["repo", "upload", "-t", "--br=" + git_branch_name, git_root])


def remove_files(git_root, subpaths, stage_removals):
  """Removes files in the work tree, optionally staging them in git."""
  if stage_removals:
    check_call(["git", "rm", "-qrf", "--ignore-unmatch"] + subpaths, cwd=git_root)
  # Need a plain rm afterwards even if git rm was executed, because git won't
  # remove directories if they have non-git files in them.
  check_call(["rm", "-rf"] + subpaths, cwd=git_root)


def commit(git_root, prebuilt_descr, installed_sources, add_paths, bug_number):
  """Commits the new prebuilts."""
  check_call(["git", "add"] +
             [path for path in add_paths
              if os.path.exists(os.path.join(git_root, path))],
             cwd=git_root)

  if installed_sources:
    message = "Update {} prebuilts.\n\n".format(prebuilt_descr)
    if len(installed_sources) == 1:
      message += "Taken from {}.".format(installed_sources[0])
    else:
      message += "Taken from:\n{}".format(
          "\n".join([s.capitalize() for s in installed_sources]))
  else:
    message = (
        "DO NOT SUBMIT: Update {prebuilt_descr} prebuilts from local build."
        .format(prebuilt_descr=prebuilt_descr))
  message += ("\n\nCL prepared by {}."
              "\n\nTest: Presubmits".format(SCRIPT_PATH))
  if bug_number:
    message += ("\nBug: {}".format(bug_number))

  msg_fd, msg_path = tempfile.mkstemp()
  try:
    with os.fdopen(msg_fd, "w") as f:
      f.write(message)
    # Do a diff first to skip the commit without error if there are no changes
    # to commit.
    check_call("git diff-index --quiet --cached HEAD -- || "
               "git commit -F " + msg_path, shell=True, cwd=git_root)
  finally:
    os.unlink(msg_path)


def install_entry(tmp_dir, local_dist, build_numbers, entry):
  """Installs one file specified by entry."""

  if not local_dist and not entry.source_build:
    print("WARNING: No CI build for {} - skipping.".format(entry.source_path))
    return None

  build_number = build_numbers[entry.source_build.branch]
  source_path = entry.source_path.replace("{BUILD}", str(build_number))

  source_dir, source_file = os.path.split(source_path)

  if local_dist:
    download_dir = os.path.join(tmp_dir, source_dir)
  else:
    download_dir = os.path.join(tmp_dir,
                                entry.source_build.branch,
                                build_number,
                                entry.source_build.target,
                                source_dir)
  os.makedirs(download_dir, exist_ok=True)
  download_file = os.path.join(download_dir, source_file)

  unzip_dir = None
  unzip_file = None
  if entry.unzip_single_file:
    unzip_dir = os.path.join(download_dir,
                             source_path.removesuffix(".zip") + "_unzip")
    os.makedirs(unzip_dir, exist_ok=True)
    unzip_file = os.path.join(unzip_dir, entry.unzip_single_file)

  if not local_dist and unzip_file:
    if not os.path.exists(unzip_file):
      # Use the --zip_entry option to fetch_artifact to avoid downloading the
      # whole zip.
      fetch_artifact(entry.source_build.branch,
                     entry.source_build.target,
                     build_number,
                     source_path,
                     unzip_dir,
                     entry.unzip_single_file)
      if not os.path.exists(unzip_file):
        sys.exit("fetch_artifact didn't create expected file {}".format(unzip_file))
  else:
    # Fetch files once by downloading them into a specific location in tmp_dir
    # only if they're not already there.
    if not os.path.exists(download_file):
      if local_dist:
        check_call(["cp", os.path.join(local_dist, source_path), download_dir])
      else:
        fetch_artifact(entry.source_build.branch,
                       entry.source_build.target,
                       build_number,
                       source_path,
                       download_dir)
      if not os.path.exists(download_file):
        sys.exit("Failed to retrieve {}".format(source_path))

  install_dir, install_file = os.path.split(entry.install_path)
  os.makedirs(install_dir, exist_ok=True)

  if entry.install_unzipped:
    check_call(["mkdir", install_file], cwd=install_dir)
    # Add -DD to not extract timestamps that may confuse the build system.
    check_call(["unzip", "-DD", download_file, "-d", install_file],
               cwd=install_dir)
  elif entry.unzip_single_file:
    if not os.path.exists(unzip_file):
      check_call(["unzip", download_file, "-d", unzip_dir, entry.unzip_single_file])
    check_call(["cp", unzip_file, install_file], cwd=install_dir)
  else:
    check_call(["cp", download_file, install_file], cwd=install_dir)

  # Return a description of the source location for inclusion in the commit
  # message.
  return (
      "branch {}, target {}, build {}".format(
          entry.source_build.branch,
          entry.source_build.target,
          build_number)
      if not local_dist else None)


def install_paths_per_git_root(roots, paths):
  """Partitions the given paths into subpaths within the given roots.

  Args:
    roots: List of root paths.
    paths: List of paths relative to the same directory as the root paths.

  Returns:
    A dict mapping each root to the subpaths under it. It's an error if some
    path doesn't go into any root.
  """
  res = collections.defaultdict(list)
  for path in paths:
    found = False
    for root in roots:
      if path.startswith(root + "/"):
        res[root].append(path[len(root) + 1:])
        found = True
        break
    if not found:
      sys.exit("Install path {} is not in any of the git roots: {}"
               .format(path, " ".join(roots)))
  return res


def get_args():
  """Parses and returns command line arguments."""
  parser = argparse.ArgumentParser(
      epilog="Either --aosp-master-build and --aosp-master-throttled-build, "
      "or --local-dist, is required.")

  parser.add_argument("--aosp-master-build", metavar="NUMBER",
                      help="Build number to fetch from aosp-master")
  parser.add_argument("--aosp-master-throttled-build", metavar="NUMBER",
                      help="Build number to fetch from aosp-master-throttled")
  parser.add_argument("--local-dist", metavar="PATH",
                      help="Take prebuilts from this local dist dir instead of "
                      "using fetch_artifact")
  parser.add_argument("--skip-apex", default=True,
                      action=argparse.BooleanOptionalAction,
                      help="Do not fetch .apex files.")
  parser.add_argument("--skip-module-sdk", action="store_true",
                      help="Do not fetch and unpack sdk and module_export zips.")
  parser.add_argument("--skip-impl-lib", action="store_true",
                      help="Do not fetch implementation libraries.")
  parser.add_argument("--skip-cls", action="store_true",
                      help="Do not create branches or git commits")
  parser.add_argument("--bug", metavar="NUMBER",
                      help="Add a 'Bug' line with this number to commit "
                      "messages.")
  parser.add_argument("--upload", action="store_true",
                      help="Upload the CLs to Gerrit")
  parser.add_argument("--tmp-dir", metavar="PATH",
                      help="Temporary directory for downloads. The default is "
                      "to create one and delete it when finished, but this one "
                      "will be kept, and any files already in it won't be "
                      "downloaded again.")

  args = parser.parse_args()
  got_build_numbers = bool(args.aosp_master_build and
                           args.aosp_master_throttled_build)
  if ((not got_build_numbers and not args.local_dist) or
      (got_build_numbers and args.local_dist)):
    sys.exit(parser.format_help())
  return args


def main():
  """Program entry point."""
  args = get_args()

  if any(path for path in GIT_PROJECT_ROOTS if not os.path.exists(path)):
    sys.exit("This script must be run in the root of the Android build tree.")

  build_numbers = None
  if args.aosp_master_build:
    build_numbers = {
        "aosp-master": args.aosp_master_build,
        "aosp-master-throttled": args.aosp_master_throttled_build,
    }

  entries = install_entries
  if args.skip_apex:
    entries = [entry for entry in entries if entry.type != "apex"]
  if args.skip_module_sdk:
    entries = [entry for entry in entries if entry.type != "module_sdk"]
  if args.skip_impl_lib:
    entries = [entry for entry in entries if entry.type != "impl_lib"]
  if not entries:
    sys.exit("All prebuilts skipped - nothing to do.")

  install_paths = [entry.install_path for entry in entries]
  install_paths_per_root = install_paths_per_git_root(
      GIT_PROJECT_ROOTS, install_paths)

  git_branch_name = PREBUILT_DESCR.lower().replace(" ", "-") + "-update"
  if args.aosp_master_build:
    git_branch_name += "-" + args.aosp_master_build

  if not args.skip_cls:
    git_paths = list(install_paths_per_root.keys())
    start_branch(git_branch_name, git_paths)

  for git_root, subpaths in install_paths_per_root.items():
    remove_files(git_root, subpaths, not args.skip_cls)

  all_installed_sources = {}

  tmp_dir_obj = None
  tmp_dir = args.tmp_dir
  if not args.tmp_dir:
    tmp_dir_obj = tempfile.TemporaryDirectory()
    tmp_dir = tmp_dir_obj.name
  tmp_dir = os.path.abspath(tmp_dir)
  try:
    for entry in entries:
      installed_source = install_entry(
          tmp_dir, args.local_dist, build_numbers, entry)
      if installed_source:
        all_installed_sources[entry.install_path] = installed_source
  finally:
    if tmp_dir_obj:
      tmp_dir_obj.cleanup()

  if not args.skip_cls:
    for git_root, subpaths in install_paths_per_root.items():
      installed_sources = set(src for path, src in all_installed_sources.items()
                              if path.startswith(git_root + "/"))
      commit(git_root, PREBUILT_DESCR, sorted(list(installed_sources)),
             subpaths, args.bug)

    if args.upload:
      # Don't upload all projects in a single repo upload call, because that
      # makes it pop up an interactive editor.
      for git_root in install_paths_per_root:
        upload_branch(git_root, git_branch_name)


if __name__ == "__main__":
  main()