Auto merge of rust-lang#70190 - pietroalbini:gha, r=Mark-Simulacrum

Add GitHub Actions configuration

This PR adds the GitHub Actions configuration to the rust-lang/rust repository. The configuration will be run in parallel with Azure Pipelines until the evaluation finishes: the infrastructure team will then decide whether to switch.

Since GitHub Actions doesn't currently have any way to include pieces of configuration, this also adds the `src/tools/expand-yaml-anchors` tool, which serves as a sort of templating system. Otherwise the configuration is a mostly straight port from the Azure Pipelines configuration (thanks to all the PRs opened in the past).

There are still a few small things I need to fix before we can land this, but it's mostly complete and ready for an initial review.

r? @Mark-Simulacrum

Author: bors

.github/workflows/ci.yml

@@ -443,7 +443,7 @@ dependencies = [
  "textwrap",
  "unicode-width",
  "vec_map",
- "yaml-rust",
+ "yaml-rust 0.3.5",
 ]
 
 [[package]]
@@ -1026,6 +1026,14 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "expand-yaml-anchors"
+version = "0.1.0"
+dependencies = [
+ "yaml-merge-keys",
+ "yaml-rust 0.4.3",
+]
+
 [[package]]
 name = "failure"
 version = "0.1.5"
@@ -1830,6 +1838,12 @@ dependencies = [
 name = "linkchecker"
 version = "0.1.0"
 
+[[package]]
+name = "linked-hash-map"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83"
+
 [[package]]
 name = "lock_api"
 version = "0.3.1"
@@ -4854,6 +4868,26 @@ dependencies = [
  "unicode-width",
 ]
 
+[[package]]
+name = "thiserror"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9fb62ff737e573b1e677459bea6fd023cd5d6e868c3242d3cdf3ef2f0554824"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24069c0ba08aab54289d6a25f5036d94afc61e1538bbc42ae5501df141c9027d"
+dependencies = [
+ "proc-macro2 1.0.3",
+ "quote 1.0.2",
+ "syn 1.0.11",
+]
+
 [[package]]
 name = "thread_local"
 version = "0.3.6"
@@ -5536,8 +5570,28 @@ dependencies = [
  "lzma-sys",
 ]
 
+[[package]]
+name = "yaml-merge-keys"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59893318ba3ad2b704498c7761214a10eaf42c5f838bce9fc0145bf2ba658cfa"
+dependencies = [
+ "lazy_static 1.4.0",
+ "thiserror",
+ "yaml-rust 0.4.3",
+]
+
 [[package]]
 name = "yaml-rust"
 version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e66366e18dc58b46801afbf2ca7661a9f59cc8c5962c29892b6039b4f86fa992"
+
+[[package]]
+name = "yaml-rust"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65923dd1784f44da1d2c3dbbc5e822045628c590ba72123e1c73d3c230c4434d"
+dependencies = [
+ "linked-hash-map",
+]

Cargo.lock

@@ -24,6 +24,7 @@ members = [
   "src/tools/miri",
   "src/tools/rustdoc-themes",
   "src/tools/unicode-table-generator",
+  "src/tools/expand-yaml-anchors",
 ]
 exclude = [
   "build",

Cargo.toml

@@ -21,6 +21,7 @@ use crate::doc;
 use crate::flags::Subcommand;
 use crate::install;
 use crate::native;
+use crate::run;
 use crate::test;
 use crate::tool;
 use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir};
@@ -313,6 +314,7 @@ pub enum Kind {
     Dist,
     Doc,
     Install,
+    Run,
 }
 
 impl<'a> Builder<'a> {
@@ -353,6 +355,7 @@ impl<'a> Builder<'a> {
             }
             Kind::Test => describe!(
                 crate::toolstate::ToolStateCheck,
+                test::ExpandYamlAnchors,
                 test::Tidy,
                 test::Ui,
                 test::CompileFail,
@@ -454,6 +457,7 @@ impl<'a> Builder<'a> {
                 install::Src,
                 install::Rustc
             ),
+            Kind::Run => describe!(run::ExpandYamlAnchors,),
         }
     }
 
@@ -507,6 +511,7 @@ impl<'a> Builder<'a> {
             Subcommand::Bench { ref paths, .. } => (Kind::Bench, &paths[..]),
             Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]),
             Subcommand::Install { ref paths } => (Kind::Install, &paths[..]),
+            Subcommand::Run { ref paths } => (Kind::Run, &paths[..]),
             Subcommand::Format { .. } | Subcommand::Clean { .. } => panic!(),
         };
 

src/bootstrap/builder.rs

@@ -86,6 +86,9 @@ pub enum Subcommand {
     Install {
         paths: Vec<PathBuf>,
     },
+    Run {
+        paths: Vec<PathBuf>,
+    },
 }
 
 impl Default for Subcommand {
@@ -113,6 +116,7 @@ Subcommands:
     clean       Clean out build directories
     dist        Build distribution artifacts
     install     Install distribution artifacts
+    run         Run tools contained in this repository
 
 To learn more about a subcommand, run `./x.py <subcommand> -h`",
         );
@@ -188,6 +192,7 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`",
                 || (s == "clean")
                 || (s == "dist")
                 || (s == "install")
+                || (s == "run")
         });
         let subcommand = match subcommand {
             Some(s) => s,
@@ -400,6 +405,18 @@ Arguments:
         ./x.py doc --stage 1",
                 );
             }
+            "run" => {
+                subcommand_help.push_str(
+                    "\n
+Arguments:
+    This subcommand accepts a number of paths to tools to build and run. For
+    example:
+
+        ./x.py run src/tool/expand-yaml-anchors
+
+    At least a tool needs to be called.",
+                );
+            }
             _ => {}
         };
         // Get any optional paths which occur after the subcommand
@@ -468,6 +485,13 @@ Arguments:
             "fmt" => Subcommand::Format { check: matches.opt_present("check") },
             "dist" => Subcommand::Dist { paths },
             "install" => Subcommand::Install { paths },
+            "run" => {
+                if paths.is_empty() {
+                    println!("\nrun requires at least a path!\n");
+                    usage(1, &opts, &subcommand_help, &extra_help);
+                }
+                Subcommand::Run { paths }
+            }
             _ => {
                 usage(1, &opts, &subcommand_help, &extra_help);
             }

src/bootstrap/flags.rs

@@ -140,6 +140,7 @@ mod format;
 mod install;
 mod metadata;
 mod native;
+mod run;
 mod sanity;
 mod test;
 mod tool;

src/bootstrap/lib.rs

@@ -0,0 +1,43 @@
+use crate::builder::{Builder, RunConfig, ShouldRun, Step};
+use crate::tool::Tool;
+use std::process::Command;
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct ExpandYamlAnchors;
+
+impl Step for ExpandYamlAnchors {
+    type Output = ();
+
+    /// Runs the `expand-yaml_anchors` tool.
+    ///
+    /// This tool in `src/tools` read the CI configuration files written in YAML and expands the
+    /// anchors in them, since GitHub Actions doesn't support them.
+    fn run(self, builder: &Builder<'_>) {
+        builder.info("Expanding YAML anchors in the GitHub Actions configuration");
+        try_run(
+            builder,
+            &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src),
+        );
+    }
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/expand-yaml-anchors")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(ExpandYamlAnchors);
+    }
+}
+
+fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> bool {
+    if !builder.fail_fast {
+        if !builder.try_run(cmd) {
+            let mut failures = builder.delayed_failures.borrow_mut();
+            failures.push(format!("{:?}", cmd));
+            return false;
+        }
+    } else {
+        builder.run(cmd);
+    }
+    true
+}

src/bootstrap/run.rs

@@ -750,6 +750,35 @@ impl Step for Tidy {
     }
 }
 
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct ExpandYamlAnchors;
+
+impl Step for ExpandYamlAnchors {
+    type Output = ();
+    const ONLY_HOSTS: bool = true;
+
+    /// Ensure the `generate-ci-config` tool was run locally.
+    ///
+    /// The tool in `src/tools` reads the CI definition in `src/ci/builders.yml` and generates the
+    /// appropriate configuration for all our CI providers. This step ensures the tool was called
+    /// by the user before committing CI changes.
+    fn run(self, builder: &Builder<'_>) {
+        builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded");
+        try_run(
+            builder,
+            &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src),
+        );
+    }
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/expand-yaml-anchors")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(ExpandYamlAnchors);
+    }
+}
+
 fn testdir(builder: &Builder<'_>, host: Interned<String>) -> PathBuf {
     builder.out.join(host).join("test")
 }

src/bootstrap/test.rs

@@ -378,6 +378,7 @@ bootstrap_tool!(
     RemoteTestClient, "src/tools/remote-test-client", "remote-test-client";
     RustInstaller, "src/tools/rust-installer", "fabricate", is_external_tool = true;
     RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
+    ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors";
 );
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]

src/bootstrap/tool.rs

@@ -1,3 +1,14 @@
+#####################################
+##    READ BEFORE CHANGING THIS    ##
+#####################################
+
+# We're in the process of evaluating GitHub Actions as a possible replacement
+# for Azure Pipelines, and at the moment the configuration is duplicated
+# between the two CI providers. Be sure to also change the configuration in
+# src/ci/github-actions when changing this file.
+
+#####################################
+
 #
 # Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows.
 #