Unrolled build for #145845

Rollup merge of #145845 - Kobzol:fix-distcheck, r=jieyouxu

Make `x test distcheck` self-contained

Before, the checked components were extracted under the checked out source root, which caused us to test some weird combination of tarball + checkout sources/aritfacts/configuration.

Now `x test distcheck` works with an external temporary directory instead, which should make it self-contained. I also moved some config from the Dockerfile to the test itself, and fixed an issue in tidy that caused `x test tidy` to fail on tarball sources.

I also removed `.args(&builder.config.configure_args)`, because it was passing all kinds of crap from the CI config to the distcheck step, which was making it less reproducible.

Fixes: #145183

r? ```@jieyouxu```

try-job: x86_64-gnu-distcheck

Author: rust-timer

src/bootstrap/src/core/build_steps/test.rs

@@ -3117,53 +3117,63 @@ impl Step for Distcheck {
     ///
     /// FIXME(#136822): dist components are under-tested.
     fn run(self, builder: &Builder<'_>) {
-        builder.info("Distcheck");
-        let dir = builder.tempdir().join("distcheck");
-        let _ = fs::remove_dir_all(&dir);
-        t!(fs::create_dir_all(&dir));
-
-        // Guarantee that these are built before we begin running.
-        builder.ensure(dist::PlainSourceTarball);
-        builder.ensure(dist::Src);
+        // Use a temporary directory completely outside the current checkout, to avoid reusing any
+        // local source code, built artifacts or configuration by accident
+        let root_dir = std::env::temp_dir().join("distcheck");
+
+        // Check that we can build some basic things from the plain source tarball
+        builder.info("Distcheck plain source tarball");
+        let plain_src_tarball = builder.ensure(dist::PlainSourceTarball);
+        let plain_src_dir = root_dir.join("distcheck-plain-src");
+        builder.clear_dir(&plain_src_dir);
+
+        let configure_args: Vec<String> = std::env::var("DISTCHECK_CONFIGURE_ARGS")
+            .map(|args| args.split(" ").map(|s| s.to_string()).collect::<Vec<String>>())
+            .unwrap_or_default();
 
         command("tar")
             .arg("-xf")
-            .arg(builder.ensure(dist::PlainSourceTarball).tarball())
+            .arg(plain_src_tarball.tarball())
             .arg("--strip-components=1")
-            .current_dir(&dir)
+            .current_dir(&plain_src_dir)
             .run(builder);
         command("./configure")
-            .args(&builder.config.configure_args)
+            .arg("--set")
+            .arg("rust.omit-git-hash=false")
+            .args(&configure_args)
             .arg("--enable-vendor")
-            .current_dir(&dir)
+            .current_dir(&plain_src_dir)
             .run(builder);
         command(helpers::make(&builder.config.host_target.triple))
             .arg("check")
-            .current_dir(&dir)
+            // Do not run the build as if we were in CI, otherwise git would be assumed to be
+            // present, but we build from a tarball here
+            .env("GITHUB_ACTIONS", "0")
+            .current_dir(&plain_src_dir)
             .run(builder);
 
         // Now make sure that rust-src has all of libstd's dependencies
         builder.info("Distcheck rust-src");
-        let dir = builder.tempdir().join("distcheck-src");
-        let _ = fs::remove_dir_all(&dir);
-        t!(fs::create_dir_all(&dir));
+        let src_tarball = builder.ensure(dist::Src);
+        let src_dir = root_dir.join("distcheck-src");
+        builder.clear_dir(&src_dir);
 
         command("tar")
             .arg("-xf")
-            .arg(builder.ensure(dist::Src).tarball())
+            .arg(src_tarball.tarball())
             .arg("--strip-components=1")
-            .current_dir(&dir)
+            .current_dir(&src_dir)
             .run(builder);
 
-        let toml = dir.join("rust-src/lib/rustlib/src/rust/library/std/Cargo.toml");
+        let toml = src_dir.join("rust-src/lib/rustlib/src/rust/library/std/Cargo.toml");
         command(&builder.initial_cargo)
             // Will read the libstd Cargo.toml
             // which uses the unstable `public-dependency` feature.
             .env("RUSTC_BOOTSTRAP", "1")
             .arg("generate-lockfile")
             .arg("--manifest-path")
             .arg(&toml)
-            .current_dir(&dir)
+            .current_dir(&src_dir)
             .run(builder);
     }
 }

src/bootstrap/src/lib.rs

@@ -1950,6 +1950,20 @@ impl Build {
         t!(fs::remove_dir_all(dir))
     }
 
+    /// Make sure that `dir` will be an empty existing directory after this function ends.
+    /// If it existed before, it will be first deleted.
+    fn clear_dir(&self, dir: &Path) {
+        if self.config.dry_run() {
+            return;
+        }
+
+        #[cfg(feature = "tracing")]
+        let _span = trace_io!("dir-clear", ?dir);
+
+        let _ = std::fs::remove_dir_all(dir);
+        self.create_dir(dir);
+    }
+
     fn read_dir(&self, dir: &Path) -> impl Iterator<Item = fs::DirEntry> {
         let iter = match fs::read_dir(dir) {
             Ok(v) => v,

src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile

@@ -33,9 +33,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
-# We are disabling CI LLVM since distcheck is an offline build.
-ENV NO_DOWNLOAD_CI_LLVM 1
+# Make distcheck builds faster
+ENV DISTCHECK_CONFIGURE_ARGS "--enable-sccache"
 
-ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --set rust.omit-git-hash=false
-ENV SCRIPT python3 ../x.py --stage 2 test distcheck
-ENV DIST_SRC 1
+ENV SCRIPT python3 ../x.py test distcheck

src/tools/tidy/src/gcc_submodule.rs

@@ -24,6 +24,12 @@ pub fn check(root_path: &Path, compiler_path: &Path, bad: &mut bool) {
         .output()
         .expect("Cannot determine git SHA of the src/gcc checkout");
 
+    // Git is not available or we are in a tarball
+    if !git_output.status.success() {
+        eprintln!("Cannot figure out the SHA of the GCC submodule");
+        return;
+    }
+
     // This can return e.g.
     // -e607be166673a8de9fc07f6f02c60426e556c5f2 src/gcc
     //  e607be166673a8de9fc07f6f02c60426e556c5f2 src/gcc (master-e607be166673a8de9fc07f6f02c60426e556c5f2.e607be)