build.py - toolchain/android_rust - Git at Google

 #!/usr/bin/env python3
 #
 # Copyright (C) 2019 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

 """Creates a tarball suitable for use as a Rust prebuilt for Android."""

 import argparse
 import os
 import shutil
 import source_manager
 import subprocess
 import sys

 import boltifyer
 import build_platform
 import config
 from paths import *
 from utils import (
     ResolvedPath,
     export_profiles,
     get_prebuilt_binary_paths,
     run_and_exit_on_failure,
     run_quiet,
     run_quiet_and_exit_on_failure,
     strip_symbols)

 #
 # Constants
 #

 STDLIB_SOURCES = [
         "library/alloc",
         "library/backtrace",
         "library/core",
         "library/panic_abort",
         "library/panic_unwind",
         "library/portable-simd",
         "library/proc_macro",
         "library/profiler_builtins",
         "library/std",
         "library/stdarch",
         "library/test",
         "library/unwind",
         "vendor/backtrace",
         "vendor/cfg-if",
         "vendor/compiler_builtins",
         "vendor/getopts",
         "vendor/hashbrown",
         "vendor/libc",
         "vendor/rustc-demangle",
         "vendor/unicode-width",
 ]

 LLVM_BUILD_PATHS_OF_INTEREST: list[str] = [
     "build.ninja",
     "cmake",
     "CMakeCache.txt",
     "CMakeFiles",
     "cmake_install.cmake",
     "compile_commands.json",
     "CPackConfig.cmake",
     "CPackSourceConfig.cmake",
     "install_manifest.txt",
     "llvm.spec"
 ]

 #
 # Program logic
 #

 def parse_args(argv) -> argparse.Namespace:
     """Parses arguments and returns the parsed structure."""
     parser = argparse.ArgumentParser("Build the Rust Toolchain")
     parser.add_argument(
         "--build-name", "-b", default="dev",
         help="Release name for the dist result")
     parser.add_argument(
         "--dist", "-d", dest="dist_path", type=ResolvedPath, default=DIST_PATH_DEFAULT,
         help="Where to place distributable artifacts")
     parser.add_argument(
         "--no-patch-abort",
         help="Don't abort on patch failure. Useful for local development.")
     parser.add_argument(
         "--stage", "-s", type=int, choices=[1,2,3],
         help="Target Rust boostrap stage")

     parser.add_argument(
         "--ndk", type=ResolvedPath, default=NDK_PATH_DEFAULT,
         help="Path to location of the NDK to build against")

     pgo_group = parser.add_mutually_exclusive_group()
     pgo_group.add_argument(
         "--profile-generate", type=ResolvedPath, nargs="?", const=OUT_PATH_PROFILES,
         help="Instrument the compiler and store profiles in the specified directory")
     pgo_group.add_argument(
         "--profile-use", type=ResolvedPath, nargs="?", const=OUT_PATH_PROFILES,
         help="Use the rust.profdata and llvm.profdata files in the provided "
              "directory to optimize the compiler")

     parser.add_argument(
         "--cs-profile-generate", type=ResolvedPath, nargs="?", const=OUT_PATH_PROFILES,
         help="Instrument the LLVM libraries to generate context-sensitive profiles")
     parser.add_argument(
         "--lto", "-l", default="none", choices=["none", "thin"],
         help="Type of LTO to perform. Valid LTO types: none, thin, full")
     parser.add_argument(
         "--bolt", action="store_true",
         help="Apply BOLT optimizations that don't rely on profiling")
     parser.add_argument(
         "--emit-relocs", action="store_true",
         help="Emit relocation information")
     parser.add_argument(
         "--gc-sections", action="store_true",
         help="Garbage collect sections during linking")
     parser.add_argument(
         "--llvm-linkage", default="shared", choices=["static", "shared"],
         help="Specify if LLVM should be built as a static or shared library")
     parser.add_argument(
         "--host", default=build_platform.triple(),
         help="Override the autodetected host architecture")

     args = parser.parse_args(argv)

     if build_platform.is_darwin() and (args.profile_generate != None or args.profile_use != None):
         sys.exit("PGO is not supported on the Darwin platform")

     if args.cs_profile_generate != None and args.llvm_linkage == "static" and args.lto == None:
         sys.exit("Context-sensitive PGO with LLVM static linkage requires LTO to be enabled")

     return args


 def main(argv=None) -> None:
     """Runs the configure-build-fixup-dist pipeline."""

     args = parse_args(argv)
     args.dist_path.mkdir(exist_ok=True)
     with open(args.dist_path / BUILD_COMMAND_RECORD_NAME, "w") as f:
         f.write(" ".join(argv or sys.argv))

     # Add some output padding to make the messages easier to read
     print()

     #
     # Initialize directories
     #

     OUT_PATH.mkdir(exist_ok=True)
     OUT_PATH_WRAPPERS.mkdir(exist_ok=True)

     if OUT_PATH_PACKAGE.exists():
         shutil.rmtree(OUT_PATH_PACKAGE)

     OUT_PATH_PACKAGE.mkdir()

     args.dist_path.mkdir(exist_ok=True)

     #
     # Setup source files
     #

     source_manager.setup_files(
       RUST_SOURCE_PATH, OUT_PATH_RUST_SOURCE, PATCHES_PATH,
       no_patch_abort=args.no_patch_abort)

     #
     # Configure Rust
     #

     env = dict(os.environ)
     config.configure(args, env)

     # Flush stdout to ensure correct output ordering in the logs
     sys.stdout.flush()

     # Trigger bootstrap to trigger vendoring
     #
     # Call is not checked because this is *expected* to fail - there isn't a
     # user facing way to directly trigger the bootstrap, so we give it a
     # no-op to perform that will require it to write out the cargo config.
     run_quiet([PYTHON_PATH, OUT_PATH_RUST_SOURCE / "x.py", "--help"], cwd=OUT_PATH_RUST_SOURCE)

     # Offline fetch to regenerate lockfile
     #
     # Because some patches may have touched vendored source we will rebuild
     # Cargo.lock
     run_and_exit_on_failure(
         [CARGO_PATH, "fetch", "--offline"],
         "Failed to rebuilt Cargo.lock via cargo-fetch operation",
         cwd=OUT_PATH_RUST_SOURCE, env=env)

     #
     # Build
     #

     # We only need to perform stage 3 of the bootstrap process when we are
     # collecting profile data.
     bootstrap_stage = args.stage or ("3" if args.profile_generate or args.cs_profile_generate else "2")

     result = subprocess.run(
         [PYTHON_PATH, OUT_PATH_RUST_SOURCE / "x.py", "--stage", bootstrap_stage, "install"],
         cwd=OUT_PATH_RUST_SOURCE, env=env)

     if result.returncode != 0:
         print(f"Build stage failed with error {result.returncode}")
         tarball_path = args.dist_path / "llvm-build-config.tar.gz"
         run_quiet_and_exit_on_failure(
             ["tar", "czf", tarball_path.as_posix()] + LLVM_BUILD_PATHS_OF_INTEREST,
             "Could not generate logs/artifacts archive upon build failure",
             cwd=LLVM_BUILD_PATH)
         sys.exit(result.returncode)

     # Install sources
     if build_platform.is_linux():
         shutil.rmtree(OUT_PATH_STDLIB_SRCS, ignore_errors=True)
         for stdlib in STDLIB_SOURCES:
             shutil.copytree(OUT_PATH_RUST_SOURCE / stdlib, OUT_PATH_STDLIB_SRCS / stdlib)

     #
     # BOLT and symbol fixup
     #

     # The Rust build doesn't have an option to auto-strip binaries, so we do
     # it here, either directly or through the BOLT routine.  We only strip
     # symbols from executables and .so objects.
     if args.bolt:
         with open(BOLT_LOG_PATH, "w") as bolt_log:
             boltifyer.process_objects(OUT_PATH_PACKAGE, None, None, bolt_log)
     else:
         flag = "--strip-debug" if args.emit_relocs else "--strip-unneeded"
         for obj_path in get_prebuilt_binary_paths(OUT_PATH_PACKAGE):
             strip_symbols(obj_path, flag)

     #
     # File fixup
     #

     copy_libs = []
     # Install the libc++ library to out/package/lib64/
     if build_platform.is_darwin():
         copy_libs.append(LLVM_CXX_RUNTIME_PATH / "libc++.dylib")
     elif args.host == "x86_64-unknown-linux-musl":
         copy_libs.append(MUSL_SYSROOT64_PATH / "lib" / "libc++.so")
         copy_libs.append(MUSL_SYSROOT64_PATH / "lib" / "libc_musl.so")
     else:
         copy_libs.append(LLVM_CXX_RUNTIME_PATH / "libc++.so.1")

     lib64_path = OUT_PATH_PACKAGE / "lib64"
     lib64_path.mkdir(exist_ok=True)
     for lib in copy_libs:
         shutil.copy2(lib, lib64_path / os.path.basename(lib))

     # Some stdlib crates might include Android.mk or Android.bp files.
     # If they do, filter them out.
     if build_platform.is_linux():
         for f in OUT_PATH_STDLIB_SRCS.glob("**/Android.{mk,bp}"):
             f.unlink()

     #
     # Dist
     #

     print("Creating artifacts")

     export_profiles(args.profile_generate or args.cs_profile_generate, args.dist_path)

     if args.profile_use and args.profile_use != args.dist_path:
         for p in args.profile_use.glob("*.profdata"):
             shutil.copy(p, args.dist_path)

     shutil.make_archive((args.dist_path / f"rust-{args.build_name}").as_posix(), "gztar", OUT_PATH_PACKAGE)


 if __name__ == "__main__":
     main()
	#!/usr/bin/env python3
	#
	# Copyright (C) 2019 The Android Open Source Project
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	"""Creates a tarball suitable for use as a Rust prebuilt for Android."""

	import argparse
	import os
	import shutil
	import source_manager
	import subprocess
	import sys

	import boltifyer
	import build_platform
	import config
	from paths import *
	from utils import (
	ResolvedPath,
	export_profiles,
	get_prebuilt_binary_paths,
	run_and_exit_on_failure,
	run_quiet,
	run_quiet_and_exit_on_failure,
	strip_symbols)

	#
	# Constants
	#

	STDLIB_SOURCES = [
	"library/alloc",
	"library/backtrace",
	"library/core",
	"library/panic_abort",
	"library/panic_unwind",
	"library/portable-simd",
	"library/proc_macro",
	"library/profiler_builtins",
	"library/std",
	"library/stdarch",
	"library/test",
	"library/unwind",
	"vendor/backtrace",
	"vendor/cfg-if",
	"vendor/compiler_builtins",
	"vendor/getopts",
	"vendor/hashbrown",
	"vendor/libc",
	"vendor/rustc-demangle",
	"vendor/unicode-width",
	]

	LLVM_BUILD_PATHS_OF_INTEREST: list[str] = [
	"build.ninja",
	"cmake",
	"CMakeCache.txt",
	"CMakeFiles",
	"cmake_install.cmake",
	"compile_commands.json",
	"CPackConfig.cmake",
	"CPackSourceConfig.cmake",
	"install_manifest.txt",
	"llvm.spec"
	]

	#
	# Program logic
	#

	def parse_args(argv) -> argparse.Namespace:
	"""Parses arguments and returns the parsed structure."""
	parser = argparse.ArgumentParser("Build the Rust Toolchain")
	parser.add_argument(
	"--build-name", "-b", default="dev",
	help="Release name for the dist result")
	parser.add_argument(
	"--dist", "-d", dest="dist_path", type=ResolvedPath, default=DIST_PATH_DEFAULT,
	help="Where to place distributable artifacts")
	parser.add_argument(
	"--no-patch-abort",
	help="Don't abort on patch failure. Useful for local development.")
	parser.add_argument(
	"--stage", "-s", type=int, choices=[1,2,3],
	help="Target Rust boostrap stage")

	parser.add_argument(
	"--ndk", type=ResolvedPath, default=NDK_PATH_DEFAULT,
	help="Path to location of the NDK to build against")

	pgo_group = parser.add_mutually_exclusive_group()
	pgo_group.add_argument(
	"--profile-generate", type=ResolvedPath, nargs="?", const=OUT_PATH_PROFILES,
	help="Instrument the compiler and store profiles in the specified directory")
	pgo_group.add_argument(
	"--profile-use", type=ResolvedPath, nargs="?", const=OUT_PATH_PROFILES,
	help="Use the rust.profdata and llvm.profdata files in the provided "
	"directory to optimize the compiler")

	parser.add_argument(
	"--cs-profile-generate", type=ResolvedPath, nargs="?", const=OUT_PATH_PROFILES,
	help="Instrument the LLVM libraries to generate context-sensitive profiles")
	parser.add_argument(
	"--lto", "-l", default="none", choices=["none", "thin"],
	help="Type of LTO to perform. Valid LTO types: none, thin, full")
	parser.add_argument(
	"--bolt", action="store_true",
	help="Apply BOLT optimizations that don't rely on profiling")
	parser.add_argument(
	"--emit-relocs", action="store_true",
	help="Emit relocation information")
	parser.add_argument(
	"--gc-sections", action="store_true",
	help="Garbage collect sections during linking")
	parser.add_argument(
	"--llvm-linkage", default="shared", choices=["static", "shared"],
	help="Specify if LLVM should be built as a static or shared library")
	parser.add_argument(
	"--host", default=build_platform.triple(),
	help="Override the autodetected host architecture")

	args = parser.parse_args(argv)

	if build_platform.is_darwin() and (args.profile_generate != None or args.profile_use != None):
	sys.exit("PGO is not supported on the Darwin platform")

	if args.cs_profile_generate != None and args.llvm_linkage == "static" and args.lto == None:
	sys.exit("Context-sensitive PGO with LLVM static linkage requires LTO to be enabled")

	return args


	def main(argv=None) -> None:
	"""Runs the configure-build-fixup-dist pipeline."""

	args = parse_args(argv)
	args.dist_path.mkdir(exist_ok=True)
	with open(args.dist_path / BUILD_COMMAND_RECORD_NAME, "w") as f:
	f.write(" ".join(argv or sys.argv))

	# Add some output padding to make the messages easier to read
	print()

	#
	# Initialize directories
	#

	OUT_PATH.mkdir(exist_ok=True)
	OUT_PATH_WRAPPERS.mkdir(exist_ok=True)

	if OUT_PATH_PACKAGE.exists():
	shutil.rmtree(OUT_PATH_PACKAGE)

	OUT_PATH_PACKAGE.mkdir()

	args.dist_path.mkdir(exist_ok=True)

	#
	# Setup source files
	#

	source_manager.setup_files(
	RUST_SOURCE_PATH, OUT_PATH_RUST_SOURCE, PATCHES_PATH,
	no_patch_abort=args.no_patch_abort)

	#
	# Configure Rust
	#

	env = dict(os.environ)
	config.configure(args, env)

	# Flush stdout to ensure correct output ordering in the logs
	sys.stdout.flush()

	# Trigger bootstrap to trigger vendoring
	#
	# Call is not checked because this is expected to fail - there isn't a
	# user facing way to directly trigger the bootstrap, so we give it a
	# no-op to perform that will require it to write out the cargo config.
	run_quiet([PYTHON_PATH, OUT_PATH_RUST_SOURCE / "x.py", "--help"], cwd=OUT_PATH_RUST_SOURCE)

	# Offline fetch to regenerate lockfile
	#
	# Because some patches may have touched vendored source we will rebuild
	# Cargo.lock
	run_and_exit_on_failure(
	[CARGO_PATH, "fetch", "--offline"],
	"Failed to rebuilt Cargo.lock via cargo-fetch operation",
	cwd=OUT_PATH_RUST_SOURCE, env=env)

	#
	# Build
	#

	# We only need to perform stage 3 of the bootstrap process when we are
	# collecting profile data.
	bootstrap_stage = args.stage or ("3" if args.profile_generate or args.cs_profile_generate else "2")

	result = subprocess.run(
	[PYTHON_PATH, OUT_PATH_RUST_SOURCE / "x.py", "--stage", bootstrap_stage, "install"],
	cwd=OUT_PATH_RUST_SOURCE, env=env)

	if result.returncode != 0:
	print(f"Build stage failed with error {result.returncode}")
	tarball_path = args.dist_path / "llvm-build-config.tar.gz"
	run_quiet_and_exit_on_failure(
	["tar", "czf", tarball_path.as_posix()] + LLVM_BUILD_PATHS_OF_INTEREST,
	"Could not generate logs/artifacts archive upon build failure",
	cwd=LLVM_BUILD_PATH)
	sys.exit(result.returncode)

	# Install sources
	if build_platform.is_linux():
	shutil.rmtree(OUT_PATH_STDLIB_SRCS, ignore_errors=True)
	for stdlib in STDLIB_SOURCES:
	shutil.copytree(OUT_PATH_RUST_SOURCE / stdlib, OUT_PATH_STDLIB_SRCS / stdlib)

	#
	# BOLT and symbol fixup
	#

	# The Rust build doesn't have an option to auto-strip binaries, so we do
	# it here, either directly or through the BOLT routine. We only strip
	# symbols from executables and .so objects.
	if args.bolt:
	with open(BOLT_LOG_PATH, "w") as bolt_log:
	boltifyer.process_objects(OUT_PATH_PACKAGE, None, None, bolt_log)
	else:
	flag = "--strip-debug" if args.emit_relocs else "--strip-unneeded"
	for obj_path in get_prebuilt_binary_paths(OUT_PATH_PACKAGE):
	strip_symbols(obj_path, flag)

	#
	# File fixup
	#

	copy_libs = []
	# Install the libc++ library to out/package/lib64/
	if build_platform.is_darwin():
	copy_libs.append(LLVM_CXX_RUNTIME_PATH / "libc++.dylib")
	elif args.host == "x86_64-unknown-linux-musl":
	copy_libs.append(MUSL_SYSROOT64_PATH / "lib" / "libc++.so")
	copy_libs.append(MUSL_SYSROOT64_PATH / "lib" / "libc_musl.so")
	else:
	copy_libs.append(LLVM_CXX_RUNTIME_PATH / "libc++.so.1")

	lib64_path = OUT_PATH_PACKAGE / "lib64"
	lib64_path.mkdir(exist_ok=True)
	for lib in copy_libs:
	shutil.copy2(lib, lib64_path / os.path.basename(lib))

	# Some stdlib crates might include Android.mk or Android.bp files.
	# If they do, filter them out.
	if build_platform.is_linux():
	for f in OUT_PATH_STDLIB_SRCS.glob("**/Android.{mk,bp}"):
	f.unlink()

	#
	# Dist
	#

	print("Creating artifacts")

	export_profiles(args.profile_generate or args.cs_profile_generate, args.dist_path)

	if args.profile_use and args.profile_use != args.dist_path:
	for p in args.profile_use.glob("*.profdata"):
	shutil.copy(p, args.dist_path)

	shutil.make_archive((args.dist_path / f"rust-{args.build_name}").as_posix(), "gztar", OUT_PATH_PACKAGE)


	if __name__ == "__main__":
	main()