blob: 700c2d525f0420dcb9b642ca902878320ec18a9a [file] [log] [blame]
#!/usr/bin/env python3
#
# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Fetch prebuilt artifacts and prepare a prebuilt commit"""
import argparse
import inspect
from functools import cache
from pathlib import Path
import os
import re
import shutil
import subprocess
import sys
from typing import KeysView, Optional, Union
from paths import (
ANDROID_BUILD_CLI_PATH,
BUILD_COMMAND_RECORD_NAME,
DOWNLOADS_PATH,
FETCH_ARTIFACT_PATH,
PROFILE_NAMES,
RUST_PREBUILT_PATH,
SOONG_PATH,
TOOLCHAIN_ARTIFACTS_PATH,
TOOLCHAIN_PATH
)
from utils import (
GitRepo,
replace_file_contents,
run_and_exit_on_failure,
run_quiet,
run_quiet_and_exit_on_failure,
VERSION_PATTERN,
version_string_type,
)
ANDROID_BP: str = "Android.bp"
BRANCH_NAME_TEMPLATE: str = "rust-update-prebuilts-%s"
BUILD_SERVER_BRANCH: str = "aosp-rust-toolchain"
BUILD_SERVER_ARCHIVE_FORMAT_PATTERN: str = "rust-%s.tar.gz"
HOST_ARCHIVE_PATTERN: str = "rust-%s-%s.tar.gz"
HOST_TARGET_DEFAULT: str = "linux-x86"
TOOLCHAIN_PATHS_SEARCH_PATTERN = 'RUST_VERSION_STAGE0:\s+str\s+=\s"[^"]+"'
TOOLCHAIN_PATHS_UPDATE_PATTERN = 'RUST_VERSION_STAGE0: str = "%s"'
RUST_PREBUILT_REPO = GitRepo(RUST_PREBUILT_PATH)
SOONG_REPO = GitRepo(SOONG_PATH)
TOOLCHAIN_REPO = GitRepo(TOOLCHAIN_PATH)
#
# String operations
#
def add_extension_prefix(filename: str, extension_prefix: str) -> str:
comps = filename.split(".", 1)
comps.insert(1, extension_prefix)
return ".".join(comps)
def artifact_ident_type(arg: str) -> Union[int, Path]:
try:
return int(arg)
except (SyntaxError, ValueError):
return Path(arg).resolve()
def make_branch_name(version: str) -> str:
return BRANCH_NAME_TEMPLATE % version
def make_commit_message(version: str, bid: int, issue: Optional[int]) -> str:
commit_message: str = f"rustc-{version} Build {bid}\n"
if issue is not None:
commit_message += f"\nBug: https://issuetracker.google.com/issues/{issue}"
commit_message += "\nTest: m rust"
return commit_message
#
# Google3 wrappers
#
@cache
def ensure_gcert_valid() -> None:
"""Ensure gcert valid for > 1 hour."""
if run_quiet("gcertstatus -quiet -check_ssh=false -check_remaining=1h") < 0:
run_and_exit_on_failure("gcert", "Failed to obtain authentication credentials")
def fetch_build_server_artifact_strict(target: str, build_id: int, build_server_pattern: str,
host_name: str) -> Path:
result = fetch_build_server_artifact(target, build_id, build_server_pattern, host_name, None)
if result is None:
sys.exit(1)
else:
return result
def fetch_build_server_artifact(target: str, build_id: int, build_server_pattern: str,
host_name: str, stderr: Optional[int] = subprocess.DEVNULL) -> Optional[Path]:
dest: Path = DOWNLOADS_PATH / host_name
if dest.exists():
print(f"Artifact {build_server_pattern} for {target} has already been downloaded as {host_name}")
else:
ensure_gcert_valid()
print(f"Downloading build server artifact {build_server_pattern} for target {target}")
build_flag = f"--bid={build_id}" if build_id else "--latest"
result = subprocess.run([
FETCH_ARTIFACT_PATH,
f"--branch={BUILD_SERVER_BRANCH}",
f"--target={target}",
build_flag,
build_server_pattern,
dest],
stderr=stderr)
if result.returncode != 0:
print(f"No file found on build server matching pattern {build_server_pattern} for target {target}")
return None
return dest
def get_lkgb() -> int:
ensure_gcert_valid()
result = subprocess.run([
ANDROID_BUILD_CLI_PATH,
"lkgb",
f"--branch={BUILD_SERVER_BRANCH}",
"--raw",
"--custom_raw_format='{o[buildId]}'"
],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL)
if result.returncode == 0:
bids = set([int(t.strip("'")) for t in result.stdout.decode().strip().split("\n")])
if len(bids) == 1:
bid = list(bids)[0]
print(f"Last Known Good Build: {bid}")
return bid
else:
sys.exit("At least one target is broken; a fully green build is required to update prebuilts")
else:
sys.exit("Unable to fetch LKGB build ID")
#
# Program logic
#
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=inspect.getdoc(sys.modules[__name__]))
parser.add_argument(
"version", metavar="VERSION", type=version_string_type,
help="Rust version string (e.g. 1.55.0)")
parser.add_argument(
"--bid", "-b", metavar="BUILD_ID", type=int,
help="Build ID to use when fetching artifacts from the build servers")
parser.add_argument(
"--download-only", "-d", action="store_true",
help="Stop after downloading the artifacts")
parser.add_argument(
"--no-pgo", action="store_true",
help="Fetch the non-PGOed version of rustc")
parser.add_argument(
"--branch", metavar="NAME", dest="branch",
help="Branch name to use for this prebuilt update")
parser.add_argument(
"--issue", "--bug", "-i", metavar="NUMBER", dest="issue", type=int,
help="Issue number to include in commit message")
parser.add_argument(
"--overwrite", "-o", dest="overwrite", action="store_true",
help="Overwrite the target branch if it exists")
return parser.parse_args()
def fetch_prebuilt_artifacts(bid: int, chained: bool) -> tuple[dict[str, Path], Path, list[Path]]:
"""
Returns a dictionary that maps target names to prebuilt artifact paths, the
manifest used by the build server, and a list of other build server
artifacts.
"""
DOWNLOADS_PATH.mkdir(exist_ok=True)
prebuilt_path_map: dict[str, Path] = {}
other_artifacts: list[Path] = []
bs_target_default = "rustc-linux_" + ("chained" if chained else "glibc")
bs_archive_name = BUILD_SERVER_ARCHIVE_FORMAT_PATTERN % bid
build_server_target_map: dict[str, str] = {
"darwin-x86": "rustc-darwin_mac",
"linux-x86": bs_target_default,
"linux-musl-x86": "rustc-linux_musl",
}
# Fetch the host-specific prebuilt archives and build commands
for host_target, bs_target in build_server_target_map.items():
host_archive_name = HOST_ARCHIVE_PATTERN % (bid, host_target if not chained else f"{host_target}-chained")
prebuilt_path_map[host_target] = fetch_build_server_artifact_strict(
bs_target, bid, bs_archive_name, host_archive_name)
host_build_command_record_name = add_extension_prefix(BUILD_COMMAND_RECORD_NAME, f"{host_target}.{bid}")
other_artifacts.append(
fetch_build_server_artifact_strict(
bs_target, bid, BUILD_COMMAND_RECORD_NAME, host_build_command_record_name))
# Fetch the manifest
manifest_name: str = f"manifest_{bid}.xml"
manifest_path: Path = fetch_build_server_artifact_strict(bs_target_default, bid, manifest_name, manifest_name)
other_artifacts.append(manifest_path)
# Fetch the profiles
if chained:
for profile_name in PROFILE_NAMES:
profile_path = fetch_build_server_artifact(
bs_target_default, bid, profile_name, add_extension_prefix(profile_name, str(bid)))
if profile_path is not None:
other_artifacts.append(profile_path)
# Print a newline to make the fetch/cache usage visually distinct
print()
return (prebuilt_path_map, manifest_path, other_artifacts)
def unpack_prebuilt_artifacts(artifact_path_map: dict[str, Path], manifest_path: Path,
version: str, overwrite: bool) -> None:
"""
Use the provided target-to-artifact path map to extract the provided
archives into the appropriate directories. If a manifest is present it
will be copied into the host target / version path.
"""
for target, artifact_path in artifact_path_map.items():
target_and_version_path: Path = RUST_PREBUILT_PATH / target / version
if target_and_version_path.exists():
if overwrite:
# Empty out the existing directory so we can overwrite the contents
RUST_PREBUILT_REPO.rm(target_and_version_path / "*")
else:
sys.exit(f"Directory {target_and_version_path} already exists and the 'overwrite' option was not set")
# Note: If the target and version path already existed and overwrite
# was specified then it will have been removed by Git and will need
# to be re-created.
target_and_version_path.mkdir(parents=True)
print(f"Extracting archive {artifact_path.name} for {target}/{version}")
run_quiet_and_exit_on_failure(
f"tar -xzf {artifact_path}",
f"Failed to extract prebuilt artifact for {target}/{version}",
cwd=target_and_version_path)
if target == HOST_TARGET_DEFAULT:
shutil.copy(manifest_path, target_and_version_path)
RUST_PREBUILT_REPO.add(target_and_version_path)
def update_symlink(targets: KeysView[str], version: str) -> None:
"""Update the symlinks in the stable directory when we update a target"""
STABLE_BINARIES = [
"rust-analyzer",
"rustfmt"
]
print("Updating stable symlinks")
for target in targets:
stable_root_path: Path = RUST_PREBUILT_PATH / target / "stable"
if not stable_root_path.exists():
stable_root_path.mkdir(parents=True)
for binary in STABLE_BINARIES:
stable_bin_path = stable_root_path / binary
if stable_bin_path.exists():
stable_bin_path.unlink()
version_bin_path = RUST_PREBUILT_PATH / target / version / "bin" / binary
# os.path.relpath() is used here because pathlib.Path.relative_to()
# requires that one path be a subcomponent of the other.
stable_bin_path.symlink_to(os.path.relpath(version_bin_path, stable_root_path))
RUST_PREBUILT_REPO.add(stable_bin_path)
def update_prebuilts(branch_name: str, overwrite: bool, version: str, bid: int,
issue: Optional[int], prebuilt_path_map: dict[str, Path],
manifest_path: Path) -> None:
RUST_PREBUILT_REPO.create_or_checkout(branch_name, overwrite)
unpack_prebuilt_artifacts(prebuilt_path_map, manifest_path, version, overwrite)
update_symlink(prebuilt_path_map.keys(), version)
commit_message = make_commit_message(version, bid, issue)
RUST_PREBUILT_REPO.amend_or_commit(commit_message)
def update_toolchain(branch_name: str, overwrite: bool, version: str, bid: int, issue: Optional[int],
other_artifacts: list[Path]) -> None:
TOOLCHAIN_REPO.create_or_checkout(branch_name, overwrite)
artifact_version_dir = TOOLCHAIN_ARTIFACTS_PATH / version
# Initialize artifact directory
if artifact_version_dir.exists():
if overwrite:
shutil.rmtree(artifact_version_dir)
else:
sys.exit(f"Toolchain artifact directory already exists: {artifact_version_dir}")
artifact_version_dir.mkdir(exist_ok=True)
# Copy over:
# * Manifest
# * Build commands
# * Profiles
for artifact in other_artifacts:
shutil.copy(artifact, artifact_version_dir)
# Update paths.py
paths_file_path = TOOLCHAIN_PATH / "paths.py"
with open(paths_file_path, "r+") as f:
replace_file_contents(f, re.sub(TOOLCHAIN_PATHS_SEARCH_PATTERN, TOOLCHAIN_PATHS_UPDATE_PATTERN % version, f.read()))
TOOLCHAIN_REPO.add(artifact_version_dir)
TOOLCHAIN_REPO.add(paths_file_path)
TOOLCHAIN_REPO.commit(make_commit_message(version, bid, issue))
def update_soong(branch_name: str, overwrite: bool, version: str, bid: int, issue: int) -> None:
"""Update the Rust version number in Soong"""
print("Updating Soong's RustDefaultVersion")
SOONG_REPO.create_or_checkout(branch_name, overwrite)
SOONG_GLOBAL_DEF_PATH: Path = SOONG_PATH / "rust" / "config" / "global.go"
with open(SOONG_GLOBAL_DEF_PATH, "r+") as f:
replace_file_contents(f, re.sub(VERSION_PATTERN, version, f.read()))
# Add the file to Git after we are sure it has been written to and closed.
SOONG_REPO.add(SOONG_GLOBAL_DEF_PATH)
SOONG_REPO.commit(make_commit_message(version, bid, issue))
def main() -> None:
args = parse_args()
branch_name: str = args.branch or make_branch_name(args.version)
bid: int = args.bid or get_lkgb()
TOOLCHAIN_ARTIFACTS_PATH.mkdir(exist_ok=True)
print()
prebuilt_path_map, manifest_path, other_artifacts = fetch_prebuilt_artifacts(bid, not args.no_pgo)
if not args.download_only:
update_prebuilts(branch_name, args.overwrite, args.version, bid, args.issue, prebuilt_path_map, manifest_path)
update_toolchain(branch_name, args.overwrite, args.version, bid, args.issue, other_artifacts)
update_soong(branch_name, args.overwrite, args.version, bid, args.issue)
print("Done")
if __name__ == "__main__":
main()