Allow building and testing without rustup

build_system/abi_cafe.rs

@@ -16,10 +16,10 @@ pub(crate) fn run(
     sysroot_kind: SysrootKind,
     dirs: &Dirs,
     cg_clif_dylib: &Path,
+    rustup_toolchain_name: Option<&str>,
     bootstrap_host_compiler: &Compiler,
 ) {
     ABI_CAFE_REPO.fetch(dirs);
-    spawn_and_wait(ABI_CAFE.fetch("cargo", &bootstrap_host_compiler.rustc, dirs));
 
     eprintln!("Building sysroot for abi-cafe");
     build_sysroot::build_sysroot(
@@ -28,6 +28,7 @@ pub(crate) fn run(
         sysroot_kind,
         cg_clif_dylib,
         bootstrap_host_compiler,
+        rustup_toolchain_name,
         bootstrap_host_compiler.triple.clone(),
     );
 

build_system/bench.rs

@@ -4,7 +4,7 @@ use std::path::Path;
 use super::path::{Dirs, RelPath};
 use super::prepare::GitRepo;
 use super::rustc_info::get_file_name;
-use super::utils::{hyperfine_command, spawn_and_wait};
+use super::utils::{hyperfine_command, spawn_and_wait, Compiler};
 
 static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
     "ebobby",
@@ -13,11 +13,11 @@ static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
     "<none>",
 );
 
-pub(crate) fn benchmark(dirs: &Dirs) {
-    benchmark_simple_raytracer(dirs);
+pub(crate) fn benchmark(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
+    benchmark_simple_raytracer(dirs, bootstrap_host_compiler);
 }
 
-fn benchmark_simple_raytracer(dirs: &Dirs) {
+fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
     if std::process::Command::new("hyperfine").output().is_err() {
         eprintln!("Hyperfine not installed");
         eprintln!("Hint: Try `cargo install hyperfine` to install hyperfine");
@@ -31,8 +31,9 @@ fn benchmark_simple_raytracer(dirs: &Dirs) {
     let bench_runs = env::var("BENCH_RUNS").unwrap_or_else(|_| "10".to_string()).parse().unwrap();
 
     eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
-    let cargo_clif =
-        RelPath::DIST.to_path(dirs).join(get_file_name("cargo_clif", "bin").replace('_', "-"));
+    let cargo_clif = RelPath::DIST
+        .to_path(dirs)
+        .join(get_file_name(&bootstrap_host_compiler.rustc, "cargo_clif", "bin").replace('_', "-"));
     let manifest_path = SIMPLE_RAYTRACER_REPO.source_dir().to_path(dirs).join("Cargo.toml");
     let target_dir = RelPath::BUILD.join("simple_raytracer").to_path(dirs);
 
@@ -75,9 +76,18 @@ fn benchmark_simple_raytracer(dirs: &Dirs) {
         bench_runs,
         None,
         &[
-            Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(),
-            Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(),
-            Path::new(".").join(get_file_name("raytracer_cg_clif_opt", "bin")).to_str().unwrap(),
+            Path::new(".")
+                .join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_llvm", "bin"))
+                .to_str()
+                .unwrap(),
+            Path::new(".")
+                .join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_clif", "bin"))
+                .to_str()
+                .unwrap(),
+            Path::new(".")
+                .join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_clif_opt", "bin"))
+                .to_str()
+                .unwrap(),
         ],
     );
     bench_run.current_dir(RelPath::BUILD.to_path(dirs));

build_system/build_backend.rs

@@ -52,5 +52,5 @@ pub(crate) fn build_backend(
         .target_dir(dirs)
         .join(&bootstrap_host_compiler.triple)
         .join(channel)
-        .join(get_file_name("rustc_codegen_cranelift", "dylib"))
+        .join(get_file_name(&bootstrap_host_compiler.rustc, "rustc_codegen_cranelift", "dylib"))
 }

build_system/build_sysroot.rs

@@ -3,7 +3,7 @@ use std::path::{Path, PathBuf};
 use std::process::{self, Command};
 
 use super::path::{Dirs, RelPath};
-use super::rustc_info::{get_file_name, get_rustc_version, get_toolchain_name};
+use super::rustc_info::{get_file_name, get_rustc_version};
 use super::utils::{remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler};
 use super::SysrootKind;
 
@@ -17,6 +17,7 @@ pub(crate) fn build_sysroot(
     sysroot_kind: SysrootKind,
     cg_clif_dylib_src: &Path,
     bootstrap_host_compiler: &Compiler,
+    rustup_toolchain_name: Option<&str>,
     target_triple: String,
 ) -> Compiler {
     eprintln!("[BUILD] sysroot {:?}", sysroot_kind);
@@ -40,19 +41,30 @@ pub(crate) fn build_sysroot(
     try_hard_link(cg_clif_dylib_src, &cg_clif_dylib_path);
 
     // Build and copy rustc and cargo wrappers
-    let wrapper_base_name = get_file_name("____", "bin");
-    let toolchain_name = get_toolchain_name();
+    let wrapper_base_name = get_file_name(&bootstrap_host_compiler.rustc, "____", "bin");
     for wrapper in ["rustc-clif", "rustdoc-clif", "cargo-clif"] {
         let wrapper_name = wrapper_base_name.replace("____", wrapper);
 
         let mut build_cargo_wrapper_cmd = Command::new(&bootstrap_host_compiler.rustc);
         let wrapper_path = DIST_DIR.to_path(dirs).join(&wrapper_name);
         build_cargo_wrapper_cmd
-            .env("TOOLCHAIN_NAME", toolchain_name.clone())
             .arg(RelPath::SCRIPTS.to_path(dirs).join(&format!("{wrapper}.rs")))
             .arg("-o")
             .arg(&wrapper_path)
             .arg("-Cstrip=debuginfo");
+        if let Some(rustup_toolchain_name) = &rustup_toolchain_name {
+            build_cargo_wrapper_cmd
+                .env("TOOLCHAIN_NAME", rustup_toolchain_name)
+                .env_remove("CARGO")
+                .env_remove("RUSTC")
+                .env_remove("RUSTDOC");
+        } else {
+            build_cargo_wrapper_cmd
+                .env_remove("TOOLCHAIN_NAME")
+                .env("CARGO", &bootstrap_host_compiler.cargo)
+                .env("RUSTC", &bootstrap_host_compiler.rustc)
+                .env("RUSTDOC", &bootstrap_host_compiler.rustdoc);
+        }
         spawn_and_wait(build_cargo_wrapper_cmd);
         try_hard_link(wrapper_path, BIN_DIR.to_path(dirs).join(wrapper_name));
     }

build_system/mod.rs

@@ -78,6 +78,7 @@ pub(crate) fn main() {
     let mut channel = "release";
     let mut sysroot_kind = SysrootKind::Clif;
     let mut use_unstable_features = true;
+    let mut frozen = false;
     while let Some(arg) = args.next().as_deref() {
         match arg {
             "--out-dir" => {
@@ -96,17 +97,38 @@ pub(crate) fn main() {
                 }
             }
             "--no-unstable-features" => use_unstable_features = false,
+            "--frozen" => frozen = true,
             flag if flag.starts_with("-") => arg_error!("Unknown flag {}", flag),
             arg => arg_error!("Unexpected argument {}", arg),
         }
     }
 
-    let bootstrap_host_compiler = Compiler::bootstrap_with_triple(
-        std::env::var("HOST_TRIPLE")
+    let rustup_toolchain_name = match (env::var("CARGO"), env::var("RUSTC"), env::var("RUSTDOC")) {
+        (Ok(_), Ok(_), Ok(_)) => None,
+        (Err(_), Err(_), Err(_)) => Some(rustc_info::get_toolchain_name()),
+        _ => {
+            eprintln!("All of CARGO, RUSTC and RUSTDOC need to be set or none must be set");
+            process::exit(1);
+        }
+    };
+    let bootstrap_host_compiler = {
+        let cargo = rustc_info::get_cargo_path();
+        let rustc = rustc_info::get_rustc_path();
+        let rustdoc = rustc_info::get_rustdoc_path();
+        let triple = std::env::var("HOST_TRIPLE")
             .ok()
             .or_else(|| config::get_value("host"))
-            .unwrap_or_else(|| rustc_info::get_host_triple()),
-    );
+            .unwrap_or_else(|| rustc_info::get_host_triple(&rustc));
+        Compiler {
+            cargo,
+            rustc,
+            rustdoc,
+            rustflags: String::new(),
+            rustdocflags: String::new(),
+            triple,
+            runner: vec![],
+        }
+    };
     let target_triple = std::env::var("TARGET_TRIPLE")
         .ok()
         .or_else(|| config::get_value("target"))
@@ -120,6 +142,7 @@ pub(crate) fn main() {
         download_dir: out_dir.join("download"),
         build_dir: out_dir.join("build"),
         dist_dir: out_dir.join("dist"),
+        frozen,
     };
 
     path::RelPath::BUILD.ensure_exists(&dirs);
@@ -134,7 +157,7 @@ pub(crate) fn main() {
     }
 
     if command == Command::Prepare {
-        prepare::prepare(&dirs);
+        prepare::prepare(&dirs, &bootstrap_host_compiler.rustc);
         process::exit(0);
     }
 
@@ -158,6 +181,7 @@ pub(crate) fn main() {
                 sysroot_kind,
                 &cg_clif_dylib,
                 &bootstrap_host_compiler,
+                rustup_toolchain_name.as_deref(),
                 target_triple.clone(),
             );
         }
@@ -166,7 +190,14 @@ pub(crate) fn main() {
                 eprintln!("Abi-cafe doesn't support cross-compilation");
                 process::exit(1);
             }
-            abi_cafe::run(channel, sysroot_kind, &dirs, &cg_clif_dylib, &bootstrap_host_compiler);
+            abi_cafe::run(
+                channel,
+                sysroot_kind,
+                &dirs,
+                &cg_clif_dylib,
+                rustup_toolchain_name.as_deref(),
+                &bootstrap_host_compiler,
+            );
         }
         Command::Build => {
             build_sysroot::build_sysroot(
@@ -175,6 +206,7 @@ pub(crate) fn main() {
                 sysroot_kind,
                 &cg_clif_dylib,
                 &bootstrap_host_compiler,
+                rustup_toolchain_name.as_deref(),
                 target_triple,
             );
         }
@@ -185,9 +217,10 @@ pub(crate) fn main() {
                 sysroot_kind,
                 &cg_clif_dylib,
                 &bootstrap_host_compiler,
+                rustup_toolchain_name.as_deref(),
                 target_triple,
             );
-            bench::benchmark(&dirs);
+            bench::benchmark(&dirs, &bootstrap_host_compiler);
         }
     }
 }

build_system/path.rs

@@ -9,6 +9,7 @@ pub(crate) struct Dirs {
     pub(crate) download_dir: PathBuf,
     pub(crate) build_dir: PathBuf,
     pub(crate) dist_dir: PathBuf,
+    pub(crate) frozen: bool,
 }
 
 #[doc(hidden)]

build_system/prepare.rs

@@ -9,27 +9,19 @@ use super::rustc_info::{get_default_sysroot, get_rustc_version};
 use super::tests::LIBCORE_TESTS_SRC;
 use super::utils::{copy_dir_recursively, git_command, retry_spawn_and_wait, spawn_and_wait};
 
-pub(crate) fn prepare(dirs: &Dirs) {
+pub(crate) fn prepare(dirs: &Dirs, rustc: &Path) {
     RelPath::DOWNLOAD.ensure_fresh(dirs);
 
-    spawn_and_wait(super::build_backend::CG_CLIF.fetch("cargo", "rustc", dirs));
-
-    prepare_stdlib(dirs);
-    spawn_and_wait(super::build_sysroot::STANDARD_LIBRARY.fetch("cargo", "rustc", dirs));
-
-    prepare_coretests(dirs);
-    spawn_and_wait(super::tests::LIBCORE_TESTS.fetch("cargo", "rustc", dirs));
+    prepare_stdlib(dirs, rustc);
+    prepare_coretests(dirs, rustc);
 
     super::tests::RAND_REPO.fetch(dirs);
-    spawn_and_wait(super::tests::RAND.fetch("cargo", "rustc", dirs));
     super::tests::REGEX_REPO.fetch(dirs);
-    spawn_and_wait(super::tests::REGEX.fetch("cargo", "rustc", dirs));
     super::tests::PORTABLE_SIMD_REPO.fetch(dirs);
-    spawn_and_wait(super::tests::PORTABLE_SIMD.fetch("cargo", "rustc", dirs));
 }
 
-fn prepare_stdlib(dirs: &Dirs) {
-    let sysroot_src_orig = get_default_sysroot(Path::new("rustc")).join("lib/rustlib/src/rust");
+fn prepare_stdlib(dirs: &Dirs, rustc: &Path) {
+    let sysroot_src_orig = get_default_sysroot(rustc).join("lib/rustlib/src/rust");
     assert!(sysroot_src_orig.exists());
 
     eprintln!("[COPY] stdlib src");
@@ -44,7 +36,7 @@ fn prepare_stdlib(dirs: &Dirs) {
         &SYSROOT_SRC.to_path(dirs).join("library"),
     );
 
-    let rustc_version = get_rustc_version(Path::new("rustc"));
+    let rustc_version = get_rustc_version(rustc);
     fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap();
 
     eprintln!("[GIT] init");
@@ -53,8 +45,8 @@ fn prepare_stdlib(dirs: &Dirs) {
     apply_patches(dirs, "stdlib", &SYSROOT_SRC.to_path(dirs));
 }
 
-fn prepare_coretests(dirs: &Dirs) {
-    let sysroot_src_orig = get_default_sysroot(Path::new("rustc")).join("lib/rustlib/src/rust");
+fn prepare_coretests(dirs: &Dirs, rustc: &Path) {
+    let sysroot_src_orig = get_default_sysroot(rustc).join("lib/rustlib/src/rust");
     assert!(sysroot_src_orig.exists());
 
     eprintln!("[COPY] coretests src");

build_system/rustc_info.rs

@@ -7,9 +7,9 @@ pub(crate) fn get_rustc_version(rustc: &Path) -> String {
     String::from_utf8(version_info).unwrap()
 }
 
-pub(crate) fn get_host_triple() -> String {
+pub(crate) fn get_host_triple(rustc: &Path) -> String {
     let version_info =
-        Command::new("rustc").stderr(Stdio::inherit()).args(&["-vV"]).output().unwrap().stdout;
+        Command::new(rustc).stderr(Stdio::inherit()).args(&["-vV"]).output().unwrap().stdout;
     String::from_utf8(version_info)
         .unwrap()
         .lines()
@@ -34,6 +34,9 @@ pub(crate) fn get_toolchain_name() -> String {
 }
 
 pub(crate) fn get_cargo_path() -> PathBuf {
+    if let Ok(cargo) = std::env::var("CARGO") {
+        return PathBuf::from(cargo);
+    }
     let cargo_path = Command::new("rustup")
         .stderr(Stdio::inherit())
         .args(&["which", "cargo"])
@@ -44,6 +47,9 @@ pub(crate) fn get_cargo_path() -> PathBuf {
 }
 
 pub(crate) fn get_rustc_path() -> PathBuf {
+    if let Ok(rustc) = std::env::var("RUSTC") {
+        return PathBuf::from(rustc);
+    }
     let rustc_path = Command::new("rustup")
         .stderr(Stdio::inherit())
         .args(&["which", "rustc"])
@@ -54,6 +60,9 @@ pub(crate) fn get_rustc_path() -> PathBuf {
 }
 
 pub(crate) fn get_rustdoc_path() -> PathBuf {
+    if let Ok(rustdoc) = std::env::var("RUSTDOC") {
+        return PathBuf::from(rustdoc);
+    }
     let rustc_path = Command::new("rustup")
         .stderr(Stdio::inherit())
         .args(&["which", "rustdoc"])
@@ -73,8 +82,9 @@ pub(crate) fn get_default_sysroot(rustc: &Path) -> PathBuf {
     Path::new(String::from_utf8(default_sysroot).unwrap().trim()).to_owned()
 }
 
-pub(crate) fn get_file_name(crate_name: &str, crate_type: &str) -> String {
-    let file_name = Command::new("rustc")
+// FIXME call once for each target and pass result around in struct
+pub(crate) fn get_file_name(rustc: &Path, crate_name: &str, crate_type: &str) -> String {
+    let file_name = Command::new(rustc)
         .stderr(Stdio::inherit())
         .args(&[
             "--crate-name",

build_system/tests.rs

@@ -217,6 +217,7 @@ pub(crate) fn run_tests(
     sysroot_kind: SysrootKind,
     cg_clif_dylib: &Path,
     bootstrap_host_compiler: &Compiler,
+    rustup_toolchain_name: Option<&str>,
     target_triple: String,
 ) {
     if config::get_bool("testsuite.no_sysroot") {
@@ -226,6 +227,7 @@ pub(crate) fn run_tests(
             SysrootKind::None,
             cg_clif_dylib,
             bootstrap_host_compiler,
+            rustup_toolchain_name,
             target_triple.clone(),
         );
 
@@ -251,6 +253,7 @@ pub(crate) fn run_tests(
             sysroot_kind,
             cg_clif_dylib,
             bootstrap_host_compiler,
+            rustup_toolchain_name,
             target_triple.clone(),
         );
         // Rust's build system denies a couple of lints that trigger on several of the test