Auto merge of #54919 - alexcrichton:update-cargo, r=Mark-Simulacrum

Update Cargo, build curl/OpenSSL statically via features

In addition to to updating Cargo's submodule and Cargo's dependencies,
this also updates Cargo's build to build OpenSSL statically into Cargo
as well as libcurl unconditionally. This removes OpenSSL build logic
from the bootstrap code, and otherwise requests that even on OSX we
build curl statically.

Author: bors

config.toml.example

@@ -182,10 +182,9 @@
 # Build the profiler runtime
 #profiler = false
 
-# Indicates whether the OpenSSL linked into Cargo will be statically linked or
-# not. If static linkage is specified then the build system will download a
-# known-good version of OpenSSL, compile it, and link it to Cargo.
-#openssl-static = false
+# Indicates whether the native libraries linked into Cargo will be statically
+# linked or not.
+#cargo-native-static = false
 
 # Run the build with low priority, by setting the process group's "nice" value
 # to +10 on Unix platforms, and by using a "low priority" job object on Windows.

src/Cargo.lock

@@ -224,7 +224,8 @@ impl Step for Rustdoc {
                                            target,
                                            "check",
                                            "src/tools/rustdoc",
-                                           SourceType::InTree);
+                                           SourceType::InTree,
+                                           &[]);
 
         let _folder = builder.fold_output(|| format!("stage{}-rustdoc", compiler.stage));
         println!("Checking rustdoc artifacts ({} -> {})", &compiler.host, target);

src/bootstrap/check.rs

@@ -149,7 +149,7 @@ pub struct Config {
     pub nodejs: Option<PathBuf>,
     pub gdb: Option<PathBuf>,
     pub python: Option<PathBuf>,
-    pub openssl_static: bool,
+    pub cargo_native_static: bool,
     pub configure_args: Vec<String>,
 
     // These are either the stage0 downloaded binaries or the locally installed ones.
@@ -221,7 +221,7 @@ struct Build {
     verbose: Option<usize>,
     sanitizers: Option<bool>,
     profiler: Option<bool>,
-    openssl_static: Option<bool>,
+    cargo_native_static: Option<bool>,
     configure_args: Option<Vec<String>>,
     local_rebuild: Option<bool>,
     print_step_timings: Option<bool>,
@@ -474,7 +474,7 @@ impl Config {
         set(&mut config.verbose, build.verbose);
         set(&mut config.sanitizers, build.sanitizers);
         set(&mut config.profiler, build.profiler);
-        set(&mut config.openssl_static, build.openssl_static);
+        set(&mut config.cargo_native_static, build.cargo_native_static);
         set(&mut config.configure_args, build.configure_args);
         set(&mut config.local_rebuild, build.local_rebuild);
         set(&mut config.print_step_timings, build.print_step_timings);

src/bootstrap/config.rs

@@ -64,7 +64,7 @@ def v(*args):
 o("vendor", "build.vendor", "enable usage of vendored Rust crates")
 o("sanitizers", "build.sanitizers", "build the sanitizer runtimes (asan, lsan, msan, tsan)")
 o("dist-src", "rust.dist-src", "when building tarballs enables building a source tarball")
-o("cargo-openssl-static", "build.openssl-static", "static openssl in cargo")
+o("cargo-native-static", "build.cargo-native-static", "static native libraries in cargo")
 o("profiler", "build.profiler", "build the profiler runtime")
 o("emscripten", None, "compile the emscripten backend as well as LLVM")
 o("full-tools", None, "enable all tools")

src/bootstrap/configure.py

@@ -31,7 +31,6 @@ use channel;
 use util::{libdir, is_dylib, exe};
 use builder::{Builder, RunConfig, ShouldRun, Step};
 use compile;
-use native;
 use tool::{self, Tool};
 use cache::{INTERNER, Interned};
 use time;
@@ -990,12 +989,6 @@ impl Step for PlainSourceTarball {
                    .arg("--debug")
                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
                    .arg("cargo-vendor");
-                if let Some(dir) = builder.openssl_install_dir(builder.config.build) {
-                    builder.ensure(native::Openssl {
-                        target: builder.config.build,
-                    });
-                    cmd.env("OPENSSL_DIR", dir);
-                }
                 builder.run(&mut cmd);
             }
 

src/bootstrap/dist.rs

@@ -805,6 +805,7 @@ impl Step for Rustdoc {
             "doc",
             "src/tools/rustdoc",
             SourceType::InTree,
+            &[]
         );
 
         cargo.env("RUSTDOCFLAGS", "--document-private-items");

src/bootstrap/doc.rs

@@ -926,25 +926,6 @@ impl Build {
             (self.hosts.iter().any(|h| *h == target) || target == self.build)
     }
 
-    /// Returns the directory that OpenSSL artifacts are compiled into if
-    /// configured to do so.
-    fn openssl_dir(&self, target: Interned<String>) -> Option<PathBuf> {
-        // OpenSSL not used on Windows
-        if target.contains("windows") {
-            None
-        } else if self.config.openssl_static {
-            Some(self.out.join(&*target).join("openssl"))
-        } else {
-            None
-        }
-    }
-
-    /// Returns the directory that OpenSSL artifacts are installed into if
-    /// configured as such.
-    fn openssl_install_dir(&self, target: Interned<String>) -> Option<PathBuf> {
-        self.openssl_dir(target).map(|p| p.join("install"))
-    }
-
     /// Given `num` in the form "a.b.c" return a "release string" which
     /// describes the release version number.
     ///

src/bootstrap/lib.rs

@@ -531,189 +531,3 @@ impl Step for TestHelpers {
            .compile("rust_test_helpers");
     }
 }
-
-const OPENSSL_VERS: &'static str = "1.0.2n";
-const OPENSSL_SHA256: &'static str =
-    "370babb75f278c39e0c50e8c4e7493bc0f18db6867478341a832a982fd15a8fe";
-
-#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
-pub struct Openssl {
-    pub target: Interned<String>,
-}
-
-impl Step for Openssl {
-    type Output = ();
-
-    fn should_run(run: ShouldRun) -> ShouldRun {
-        run.never()
-    }
-
-    fn run(self, builder: &Builder) {
-        if builder.config.dry_run {
-            return;
-        }
-        let target = self.target;
-        let out = match builder.openssl_dir(target) {
-            Some(dir) => dir,
-            None => return,
-        };
-
-        let stamp = out.join(".stamp");
-        let mut contents = String::new();
-        drop(File::open(&stamp).and_then(|mut f| f.read_to_string(&mut contents)));
-        if contents == OPENSSL_VERS {
-            return
-        }
-        t!(fs::create_dir_all(&out));
-
-        let name = format!("openssl-{}.tar.gz", OPENSSL_VERS);
-        let tarball = out.join(&name);
-        if !tarball.exists() {
-            let tmp = tarball.with_extension("tmp");
-            // originally from https://www.openssl.org/source/...
-            let url = format!("https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/{}",
-                              name);
-            let mut last_error = None;
-            for _ in 0..3 {
-                let status = Command::new("curl")
-                                .arg("-o").arg(&tmp)
-                                .arg("-f")  // make curl fail if the URL does not return HTTP 200
-                                .arg(&url)
-                                .status()
-                                .expect("failed to spawn curl");
-
-                // Retry if download failed.
-                if !status.success() {
-                    last_error = Some(status.to_string());
-                    continue;
-                }
-
-                // Ensure the hash is correct.
-                let mut shasum = if target.contains("apple") ||
-                    builder.config.build.contains("netbsd") {
-                    let mut cmd = Command::new("shasum");
-                    cmd.arg("-a").arg("256");
-                    cmd
-                } else {
-                    Command::new("sha256sum")
-                };
-                let output = output(&mut shasum.arg(&tmp));
-                let found = output.split_whitespace().next().unwrap();
-
-                // If the hash is wrong, probably the download is incomplete or S3 served an error
-                // page. In any case, retry.
-                if found != OPENSSL_SHA256 {
-                    last_error = Some(format!(
-                        "downloaded openssl sha256 different\n\
-                         expected: {}\n\
-                         found:    {}\n",
-                        OPENSSL_SHA256,
-                        found
-                    ));
-                    continue;
-                }
-
-                // Everything is fine, so exit the retry loop.
-                last_error = None;
-                break;
-            }
-            if let Some(error) = last_error {
-                panic!("failed to download openssl source: {}", error);
-            }
-            t!(fs::rename(&tmp, &tarball));
-        }
-        let obj = out.join(format!("openssl-{}", OPENSSL_VERS));
-        let dst = builder.openssl_install_dir(target).unwrap();
-        drop(fs::remove_dir_all(&obj));
-        drop(fs::remove_dir_all(&dst));
-        builder.run(Command::new("tar").arg("zxf").arg(&tarball).current_dir(&out));
-
-        let mut configure = Command::new("perl");
-        configure.arg(obj.join("Configure"));
-        configure.arg(format!("--prefix={}", dst.display()));
-        configure.arg("no-dso");
-        configure.arg("no-ssl2");
-        configure.arg("no-ssl3");
-
-        let os = match &*target {
-            "aarch64-linux-android" => "linux-aarch64",
-            "aarch64-unknown-linux-gnu" => "linux-aarch64",
-            "aarch64-unknown-linux-musl" => "linux-aarch64",
-            "aarch64-unknown-netbsd" => "BSD-generic64",
-            "arm-linux-androideabi" => "android",
-            "arm-unknown-linux-gnueabi" => "linux-armv4",
-            "arm-unknown-linux-gnueabihf" => "linux-armv4",
-            "armv6-unknown-netbsd-eabihf" => "BSD-generic32",
-            "armv7-linux-androideabi" => "android-armv7",
-            "armv7-unknown-linux-gnueabihf" => "linux-armv4",
-            "armv7-unknown-netbsd-eabihf" => "BSD-generic32",
-            "i586-unknown-linux-gnu" => "linux-elf",
-            "i586-unknown-linux-musl" => "linux-elf",
-            "i686-apple-darwin" => "darwin-i386-cc",
-            "i686-linux-android" => "android-x86",
-            "i686-unknown-freebsd" => "BSD-x86-elf",
-            "i686-unknown-linux-gnu" => "linux-elf",
-            "i686-unknown-linux-musl" => "linux-elf",
-            "i686-unknown-netbsd" => "BSD-x86-elf",
-            "mips-unknown-linux-gnu" => "linux-mips32",
-            "mips64-unknown-linux-gnuabi64" => "linux64-mips64",
-            "mips64el-unknown-linux-gnuabi64" => "linux64-mips64",
-            "mipsel-unknown-linux-gnu" => "linux-mips32",
-            "powerpc-unknown-linux-gnu" => "linux-ppc",
-            "powerpc-unknown-linux-gnuspe" => "linux-ppc",
-            "powerpc-unknown-netbsd" => "BSD-generic32",
-            "powerpc64-unknown-linux-gnu" => "linux-ppc64",
-            "powerpc64le-unknown-linux-gnu" => "linux-ppc64le",
-            "powerpc64le-unknown-linux-musl" => "linux-ppc64le",
-            "s390x-unknown-linux-gnu" => "linux64-s390x",
-            "sparc-unknown-linux-gnu" => "linux-sparcv9",
-            "sparc64-unknown-linux-gnu" => "linux64-sparcv9",
-            "sparc64-unknown-netbsd" => "BSD-sparc64",
-            "x86_64-apple-darwin" => "darwin64-x86_64-cc",
-            "x86_64-linux-android" => "linux-x86_64",
-            "x86_64-unknown-freebsd" => "BSD-x86_64",
-            "x86_64-unknown-dragonfly" => "BSD-x86_64",
-            "x86_64-unknown-linux-gnu" => "linux-x86_64",
-            "x86_64-unknown-linux-gnux32" => "linux-x32",
-            "x86_64-unknown-linux-musl" => "linux-x86_64",
-            "x86_64-unknown-netbsd" => "BSD-x86_64",
-            _ => panic!("don't know how to configure OpenSSL for {}", target),
-        };
-        configure.arg(os);
-        configure.env("CC", builder.cc(target));
-        for flag in builder.cflags(target, GitRepo::Rustc) {
-            configure.arg(flag);
-        }
-        // There is no specific os target for android aarch64 or x86_64,
-        // so we need to pass some extra cflags
-        if target == "aarch64-linux-android" || target == "x86_64-linux-android" {
-            configure.arg("-mandroid");
-            configure.arg("-fomit-frame-pointer");
-        }
-        if target == "sparc64-unknown-netbsd" {
-            // Need -m64 to get assembly generated correctly for sparc64.
-            configure.arg("-m64");
-            if builder.config.build.contains("netbsd") {
-                // Disable sparc64 asm on NetBSD builders, it uses
-                // m4(1)'s -B flag, which NetBSD m4 does not support.
-                configure.arg("no-asm");
-            }
-        }
-        // Make PIE binaries
-        // Non-PIE linker support was removed in Lollipop
-        // https://source.android.com/security/enhancements/enhancements50
-        if target == "i686-linux-android" {
-            configure.arg("no-asm");
-        }
-        configure.current_dir(&obj);
-        builder.info(&format!("Configuring openssl for {}", target));
-        builder.run_quiet(&mut configure);
-        builder.info(&format!("Building openssl for {}", target));
-        builder.run_quiet(Command::new("make").arg("-j1").current_dir(&obj));
-        builder.info(&format!("Installing openssl for {}", target));
-        builder.run_quiet(Command::new("make").arg("install").arg("-j1").current_dir(&obj));
-
-        let mut f = t!(File::create(&stamp));
-        t!(f.write_all(OPENSSL_VERS.as_bytes()));
-    }
-}

src/bootstrap/native.rs

@@ -228,7 +228,8 @@ impl Step for Cargo {
                                                  self.host,
                                                  "test",
                                                  "src/tools/cargo",
-                                                 SourceType::Submodule);
+                                                 SourceType::Submodule,
+                                                 &[]);
 
         if !builder.fail_fast {
             cargo.arg("--no-fail-fast");
@@ -290,7 +291,8 @@ impl Step for Rls {
                                                  host,
                                                  "test",
                                                  "src/tools/rls",
-                                                 SourceType::Submodule);
+                                                 SourceType::Submodule,
+                                                 &[]);
 
         // Copy `src/tools/rls/test_data` to a writable drive.
         let test_workspace_path = builder.out.join("rls-test-data");
@@ -352,7 +354,8 @@ impl Step for Rustfmt {
                                                  host,
                                                  "test",
                                                  "src/tools/rustfmt",
-                                                 SourceType::Submodule);
+                                                 SourceType::Submodule,
+                                                 &[]);
 
         let dir = testdir(builder, compiler.host);
         t!(fs::create_dir_all(&dir));
@@ -407,7 +410,8 @@ impl Step for Miri {
                                                  host,
                                                  "test",
                                                  "src/tools/miri",
-                                                 SourceType::Submodule);
+                                                 SourceType::Submodule,
+                                                 &[]);
 
             // miri tests need to know about the stage sysroot
             cargo.env("MIRI_SYSROOT", builder.sysroot(compiler));
@@ -466,7 +470,8 @@ impl Step for Clippy {
                                                  host,
                                                  "test",
                                                  "src/tools/clippy",
-                                                 SourceType::Submodule);
+                                                 SourceType::Submodule,
+                                                 &[]);
 
             // clippy tests need to know about the stage sysroot
             cargo.env("SYSROOT", builder.sysroot(compiler));
@@ -1777,7 +1782,8 @@ impl Step for CrateRustdoc {
                                                  target,
                                                  test_kind.subcommand(),
                                                  "src/tools/rustdoc",
-                                                 SourceType::InTree);
+                                                 SourceType::InTree,
+                                                 &[]);
         if test_kind.subcommand() == "test" && !builder.fail_fast {
             cargo.arg("--no-fail-fast");
         }