blob: a01e3235f6b530c487274dafe8fb03dcafe9a2c9 [file] [log] [blame]
//! Offers an easy way to build a rustc sysroot from source.
// We prefer to always borrow rather than having to figure out whether we can move or borrow (which
// depends on whether the variable is used again later).
#![allow(clippy::needless_borrows_for_generic_args)]
use std::collections::hash_map::DefaultHasher;
use std::env;
use std::ffi::{OsStr, OsString};
use std::fs;
use std::hash::{Hash, Hasher};
use std::ops::Not;
use std::path::{Path, PathBuf};
use std::process::Command;
use anyhow::{bail, Context, Result};
use tempfile::TempDir;
/// Returns where the given rustc stores its sysroot source code.
pub fn rustc_sysroot_src(mut rustc: Command) -> Result<PathBuf> {
let output = rustc
.args(["--print", "sysroot"])
.output()
.context("failed to determine sysroot")?;
if !output.status.success() {
bail!(
"failed to determine sysroot; rustc said:\n{}",
String::from_utf8_lossy(&output.stderr).trim_end()
);
}
let sysroot =
std::str::from_utf8(&output.stdout).context("sysroot folder is not valid UTF-8")?;
let sysroot = Path::new(sysroot.trim_end_matches('\n'));
let rustc_src = sysroot
.join("lib")
.join("rustlib")
.join("src")
.join("rust")
.join("library");
// There could be symlinks here, so better canonicalize to avoid busting the cache due to path
// changes.
let rustc_src = rustc_src.canonicalize().unwrap_or(rustc_src);
Ok(rustc_src)
}
/// Encode a list of rustflags for use in CARGO_ENCODED_RUSTFLAGS.
pub fn encode_rustflags(flags: &[OsString]) -> OsString {
let mut res = OsString::new();
for flag in flags {
if !res.is_empty() {
res.push(OsStr::new("\x1f"));
}
// Cargo ignores this env var if it's not UTF-8.
let flag = flag.to_str().expect("rustflags must be valid UTF-8");
if flag.contains('\x1f') {
panic!("rustflags must not contain `\\x1f` separator");
}
res.push(flag);
}
res
}
/// Make a file writeable.
#[cfg(unix)]
fn make_writeable(p: &Path) -> Result<()> {
// On Unix we avoid `set_readonly(false)`, see
// <https://rust-lang.github.io/rust-clippy/master/index.html#permissions_set_readonly_false>.
use std::fs::Permissions;
use std::os::unix::fs::PermissionsExt;
let perms = fs::metadata(p)?.permissions();
let perms = Permissions::from_mode(perms.mode() | 0o600); // read/write for owner
fs::set_permissions(p, perms).context("cannot set permissions")?;
Ok(())
}
/// Make a file writeable.
#[cfg(not(unix))]
fn make_writeable(p: &Path) -> Result<()> {
let mut perms = fs::metadata(p)?.permissions();
perms.set_readonly(false);
fs::set_permissions(p, perms).context("cannot set permissions")?;
Ok(())
}
/// The build mode to use for this sysroot.
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub enum BuildMode {
/// Do a full sysroot build. Suited for all purposes (like the regular sysroot), but only works
/// for the host or for targets that have suitable development tools installed.
Build,
/// Do a check-only sysroot build. This is only suited for check-only builds of crates, but on
/// the plus side it works for *arbitrary* targets without having any special tools installed.
Check,
}
impl BuildMode {
pub fn as_str(&self) -> &str {
use BuildMode::*;
match self {
Build => "build",
Check => "check",
}
}
}
/// Settings controlling how the sysroot will be built.
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub enum SysrootConfig {
/// Build a no-std (only core and alloc) sysroot.
NoStd,
/// Build a full sysroot with the `std` and `test` crates.
WithStd {
/// Features to enable for the `std` crate.
std_features: Vec<String>,
},
}
/// Information about a to-be-created sysroot.
pub struct SysrootBuilder {
sysroot_dir: PathBuf,
target: OsString,
config: SysrootConfig,
mode: BuildMode,
rustflags: Vec<OsString>,
cargo: Option<Command>,
rustc_version: Option<rustc_version::VersionMeta>,
}
/// Hash file name (in target/lib directory).
const HASH_FILE_NAME: &str = ".rustc-build-sysroot-hash";
impl SysrootBuilder {
/// Prepare to create a new sysroot in the given folder (that folder should later be passed to
/// rustc via `--sysroot`), for the given target.
pub fn new(sysroot_dir: &Path, target: impl Into<OsString>) -> Self {
SysrootBuilder {
sysroot_dir: sysroot_dir.to_owned(),
target: target.into(),
config: SysrootConfig::WithStd {
std_features: vec![],
},
mode: BuildMode::Build,
rustflags: vec![],
cargo: None,
rustc_version: None,
}
}
/// Sets the build mode (regular build vs check-only build).
pub fn build_mode(mut self, build_mode: BuildMode) -> Self {
self.mode = build_mode;
self
}
/// Sets the sysroot configuration (which parts of the sysroot to build and with which features).
pub fn sysroot_config(mut self, sysroot_config: SysrootConfig) -> Self {
self.config = sysroot_config;
self
}
/// Appends the given flag.
///
/// If no `--cap-lints` argument is configured, we will add `--cap-lints=warn`.
/// This emulates the usual behavior of Cargo: Lints are normally capped when building
/// dependencies, except that they are not capped when building path dependencies, except that
/// path dependencies are still capped if they are part of `-Zbuild-std`.
pub fn rustflag(mut self, rustflag: impl Into<OsString>) -> Self {
self.rustflags.push(rustflag.into());
self
}
/// Appends the given flags.
///
/// If no `--cap-lints` argument is configured, we will add `--cap-lints=warn`. See
/// [`SysrootBuilder::rustflag`] for more explanation.
pub fn rustflags(mut self, rustflags: impl IntoIterator<Item = impl Into<OsString>>) -> Self {
self.rustflags.extend(rustflags.into_iter().map(Into::into));
self
}
/// Sets the cargo command to call.
pub fn cargo(mut self, cargo: Command) -> Self {
self.cargo = Some(cargo);
self
}
/// Sets the rustc version information (in case the user has that available).
pub fn rustc_version(mut self, rustc_version: rustc_version::VersionMeta) -> Self {
self.rustc_version = Some(rustc_version);
self
}
/// Our configured target can be either a built-in target name, or a path to a target file.
/// We use the same logic as rustc to tell which is which:
/// https://github.com/rust-lang/rust/blob/8d39ec1825024f3014e1f847942ac5bbfcf055b0/compiler/rustc_session/src/config.rs#L2252-L2263
fn target_name(&self) -> &OsStr {
let path = Path::new(&self.target);
if path.extension().and_then(OsStr::to_str) == Some("json") {
// Path::file_stem and Path::extension are the last component of the path split on the
// rightmost '.' so if we have an extension we must have a file_stem.
path.file_stem().unwrap()
} else {
// The configured target doesn't end in ".json", so we assume that this is a builtin
// target.
&self.target
}
}
fn target_sysroot_dir(&self) -> PathBuf {
self.sysroot_dir
.join("lib")
.join("rustlib")
.join(self.target_name())
}
/// Computes the hash for the sysroot, so that we know whether we have to rebuild.
fn sysroot_compute_hash(
&self,
src_dir: &Path,
rustc_version: &rustc_version::VersionMeta,
) -> u64 {
let mut hasher = DefaultHasher::new();
// For now, we just hash in the information we have in `self`.
// Ideally we'd recursively hash the entire folder but that sounds slow?
src_dir.hash(&mut hasher);
self.config.hash(&mut hasher);
self.mode.hash(&mut hasher);
self.rustflags.hash(&mut hasher);
rustc_version.hash(&mut hasher);
hasher.finish()
}
fn sysroot_read_hash(&self) -> Option<u64> {
let hash_file = self.target_sysroot_dir().join("lib").join(HASH_FILE_NAME);
let hash = fs::read_to_string(&hash_file).ok()?;
hash.parse().ok()
}
/// Generate the contents of the manifest file for the sysroot build.
fn gen_manifest(&self, src_dir: &Path) -> String {
let have_sysroot_crate = src_dir.join("sysroot").exists();
let crates = match &self.config {
SysrootConfig::NoStd => format!(
r#"
[dependencies.core]
path = {src_dir_core:?}
[dependencies.alloc]
path = {src_dir_alloc:?}
[dependencies.compiler_builtins]
features = ["rustc-dep-of-std", "mem"]
version = "*"
"#,
src_dir_core = src_dir.join("core"),
src_dir_alloc = src_dir.join("alloc"),
),
SysrootConfig::WithStd { std_features } if have_sysroot_crate => format!(
r#"
[dependencies.std]
features = {std_features:?}
path = {src_dir_std:?}
[dependencies.sysroot]
path = {src_dir_sysroot:?}
"#,
std_features = std_features,
src_dir_std = src_dir.join("std"),
src_dir_sysroot = src_dir.join("sysroot"),
),
// Fallback for old rustc where the main crate was `test`, not `sysroot`
SysrootConfig::WithStd { std_features } => format!(
r#"
[dependencies.std]
features = {std_features:?}
path = {src_dir_std:?}
[dependencies.test]
path = {src_dir_test:?}
"#,
std_features = std_features,
src_dir_std = src_dir.join("std"),
src_dir_test = src_dir.join("test"),
),
};
// If we include a patch for rustc-std-workspace-std for no_std sysroot builds, we get a
// warning from Cargo that the patch is unused. If this patching ever breaks that lint will
// probably be very helpful, so it would be best to not disable it.
// Currently the only user of rustc-std-workspace-alloc is std_detect, which is only used
// by std. So we only need to patch rustc-std-workspace-core in no_std sysroot builds, or
// that patch also produces a warning.
let patches = match &self.config {
SysrootConfig::NoStd => format!(
r#"
[patch.crates-io.rustc-std-workspace-core]
path = {src_dir_workspace_core:?}
"#,
src_dir_workspace_core = src_dir.join("rustc-std-workspace-core"),
),
SysrootConfig::WithStd { .. } => format!(
r#"
[patch.crates-io.rustc-std-workspace-core]
path = {src_dir_workspace_core:?}
[patch.crates-io.rustc-std-workspace-alloc]
path = {src_dir_workspace_alloc:?}
[patch.crates-io.rustc-std-workspace-std]
path = {src_dir_workspace_std:?}
"#,
src_dir_workspace_core = src_dir.join("rustc-std-workspace-core"),
src_dir_workspace_alloc = src_dir.join("rustc-std-workspace-alloc"),
src_dir_workspace_std = src_dir.join("rustc-std-workspace-std"),
),
};
format!(
r#"
[package]
authors = ["rustc-build-sysroot"]
name = "custom-local-sysroot"
version = "0.0.0"
[lib]
# empty dummy, just so that things are being built
path = "lib.rs"
{crates}
{patches}
"#
)
}
/// Build the `self` sysroot from the given sources.
///
/// `src_dir` must be the `library` source folder, i.e., the one that contains `std/Cargo.toml`.
pub fn build_from_source(mut self, src_dir: &Path) -> Result<()> {
// A bit of preparation.
if !src_dir.join("std").join("Cargo.toml").exists() {
bail!(
"{:?} does not seem to be a rust library source folder: `src/Cargo.toml` not found",
src_dir
);
}
let sysroot_lib_dir = self.target_sysroot_dir().join("lib");
let target_name = self.target_name().to_owned();
let cargo = self.cargo.take().unwrap_or_else(|| {
Command::new(env::var_os("CARGO").unwrap_or_else(|| OsString::from("cargo")))
});
let rustc_version = match self.rustc_version.take() {
Some(v) => v,
None => rustc_version::version_meta()?,
};
// The whole point of this crate is to build the standard library in nonstandard
// configurations, which may trip lints due to untested combinations of cfgs.
// Cargo applies --cap-lints=allow or --cap-lints=warn when handling -Zbuild-std, which we
// of course are not using:
// https://github.com/rust-lang/cargo/blob/2ce45605d9db521b5fd6c1211ce8de6055fdb24e/src/cargo/core/compiler/mod.rs#L899
// https://github.com/rust-lang/cargo/blob/2ce45605d9db521b5fd6c1211ce8de6055fdb24e/src/cargo/core/compiler/unit.rs#L102-L109
// All the standard library crates are path dependencies, and they also sometimes pull in
// separately-maintained crates like backtrace by treating their crate roots as module
// roots. If we do not cap lints, we can get lint failures outside core or std.
// We cannot set --cap-lints=allow because Cargo needs to parse warnings to understand the
// output of --print=file-names for crate-types that the target does not support.
if !self.rustflags.iter().any(|flag| {
// FIXME: OsStr::as_encoded_bytes is cleaner here
flag.to_str()
.map_or(false, |f| f.starts_with("--cap-lints"))
}) {
self.rustflags.push("--cap-lints=warn".into());
}
// Check if we even need to do anything.
let cur_hash = self.sysroot_compute_hash(src_dir, &rustc_version);
if self.sysroot_read_hash() == Some(cur_hash) {
// Already done!
return Ok(());
}
// Prepare a workspace for cargo
let build_dir = TempDir::new().context("failed to create tempdir")?;
let lock_file = build_dir.path().join("Cargo.lock");
let manifest_file = build_dir.path().join("Cargo.toml");
let lib_file = build_dir.path().join("lib.rs");
fs::copy(
src_dir
.parent()
.expect("src_dir must have a parent")
.join("Cargo.lock"),
&lock_file,
)
.context("failed to copy lockfile from sysroot source")?;
make_writeable(&lock_file).context("failed to make lockfile writeable")?;
let manifest = self.gen_manifest(src_dir);
fs::write(&manifest_file, manifest.as_bytes()).context("failed to write manifest file")?;
let lib = match self.config {
SysrootConfig::NoStd => r#"#![no_std]"#,
SysrootConfig::WithStd { .. } => "",
};
fs::write(&lib_file, lib.as_bytes()).context("failed to write lib file")?;
// Run cargo.
let mut cmd = cargo;
cmd.arg(self.mode.as_str());
cmd.arg("--release");
cmd.arg("--manifest-path");
cmd.arg(&manifest_file);
cmd.arg("--target");
cmd.arg(&self.target);
// Set rustflags.
let mut flags = self.rustflags;
flags.push("-Zforce-unstable-if-unmarked".into());
cmd.env("CARGO_ENCODED_RUSTFLAGS", encode_rustflags(&flags));
// Make sure the results end up where we expect them.
let build_target_dir = build_dir.path().join("target");
cmd.env("CARGO_TARGET_DIR", &build_target_dir);
// To avoid metadata conflicts, we need to inject some custom data into the crate hash.
// bootstrap does the same at
// <https://github.com/rust-lang/rust/blob/c8e12cc8bf0de646234524924f39c85d9f3c7c37/src/bootstrap/builder.rs#L1613>.
cmd.env("__CARGO_DEFAULT_LIB_METADATA", "rustc-build-sysroot");
if cmd
.status()
.unwrap_or_else(|_| panic!("failed to execute cargo for sysroot build"))
.success()
.not()
{
bail!("sysroot build failed");
}
// Copy the output to a staging dir (so that we can do the final installation atomically.)
fs::create_dir_all(&self.sysroot_dir).context("failed to create sysroot dir")?; // TempDir expects the parent to already exist
let staging_dir =
TempDir::new_in(&self.sysroot_dir).context("failed to create staging dir")?;
let out_dir = build_target_dir
.join(&target_name)
.join("release")
.join("deps");
for entry in fs::read_dir(&out_dir).context("failed to read cargo out dir")? {
let entry = entry.context("failed to read cargo out dir entry")?;
assert!(
entry.file_type().unwrap().is_file(),
"cargo out dir must not contain directories"
);
let entry = entry.path();
fs::copy(&entry, staging_dir.path().join(entry.file_name().unwrap()))
.context("failed to copy cargo out file")?;
}
// Write the hash file (into the staging dir).
fs::write(
staging_dir.path().join(HASH_FILE_NAME),
cur_hash.to_string().as_bytes(),
)
.context("failed to write hash file")?;
// Atomic copy to final destination via rename.
if sysroot_lib_dir.exists() {
// Remove potentially outdated files.
fs::remove_dir_all(&sysroot_lib_dir).context("failed to clean sysroot target dir")?;
}
fs::create_dir_all(
sysroot_lib_dir
.parent()
.expect("target/lib dir must have a parent"),
)
.context("failed to create target directory")?;
fs::rename(staging_dir.path(), sysroot_lib_dir).context("failed installing sysroot")?;
Ok(())
}
}