Auto merge of #39284 - alexcrichton:manifesting, r=brson

rustbuild: Add manifest generation in-tree

This commit adds a new tool, `build-manifest`, which is used to generate a
distribution manifest of all produced artifacts. This tool is intended to
replace the `build-rust-manifest.py` script that's currently located on the
buildmaster. The intention is that we'll have a builder which periodically:

* Downloads all artifacts for a commit
* Runs `./x.py dist hash-and-sign`. This will generate `sha256` and `asc` files
  as well as TOML manifests.
* Upload all generated hashes and manifests to the directory the artifacts came
  from.
* Upload *all* artifacts (tarballs and hashes and manifests) to an archived
  location.
* If necessary, upload all artifacts to the main location.

This script is intended to just be the second step here where orchestrating
uploads and such will all happen externally from the build system itself.

cc #38531

Author: bors

src/Cargo.lock

@@ -10,6 +10,7 @@ members = [
   "tools/linkchecker",
   "tools/rustbook",
   "tools/tidy",
+  "tools/build-manifest",
 ]
 
 # Curiously, compiletest will segfault if compiled with opt-level=3 on 64-bit

src/Cargo.toml

@@ -78,6 +78,11 @@ pub struct Config {
     pub cargo: Option<PathBuf>,
     pub local_rebuild: bool,
 
+    // dist misc
+    pub dist_sign_folder: Option<PathBuf>,
+    pub dist_upload_addr: Option<String>,
+    pub dist_gpg_password_file: Option<PathBuf>,
+
     // libstd features
     pub debug_jemalloc: bool,
     pub use_jemalloc: bool,
@@ -123,6 +128,7 @@ struct TomlConfig {
     llvm: Option<Llvm>,
     rust: Option<Rust>,
     target: Option<HashMap<String, TomlTarget>>,
+    dist: Option<Dist>,
 }
 
 /// TOML representation of various global build decisions.
@@ -166,6 +172,13 @@ struct Llvm {
     targets: Option<String>,
 }
 
+#[derive(RustcDecodable, Default, Clone)]
+struct Dist {
+    sign_folder: Option<String>,
+    gpg_password_file: Option<String>,
+    upload_addr: Option<String>,
+}
+
 #[derive(RustcDecodable)]
 enum StringOrBool {
     String(String),
@@ -352,6 +365,12 @@ impl Config {
             }
         }
 
+        if let Some(ref t) = toml.dist {
+            config.dist_sign_folder = t.sign_folder.clone().map(PathBuf::from);
+            config.dist_gpg_password_file = t.gpg_password_file.clone().map(PathBuf::from);
+            config.dist_upload_addr = t.upload_addr.clone();
+        }
+
         return config
     }
 

src/bootstrap/config.rs

@@ -242,3 +242,33 @@
 # that this option only makes sense for MUSL targets that produce statically
 # linked binaries
 #musl-root = "..."
+
+# =============================================================================
+# Distribution options
+#
+# These options are related to distribution, mostly for the Rust project itself.
+# You probably won't need to concern yourself with any of these options
+# =============================================================================
+[dist]
+
+# This is the folder of artifacts that the build system will sign. All files in
+# this directory will be signed with the default gpg key using the system `gpg`
+# binary. The `asc` and `sha256` files will all be output into the standard dist
+# output folder (currently `build/dist`)
+#
+# This folder should be populated ahead of time before the build system is
+# invoked.
+#sign-folder = "path/to/folder/to/sign"
+
+# This is a file which contains the password of the default gpg key. This will
+# be passed to `gpg` down the road when signing all files in `sign-folder`
+# above. This should be stored in plaintext.
+#gpg-password-file = "path/to/gpg/password"
+
+# The remote address that all artifacts will eventually be uploaded to. The
+# build system generates manifests which will point to these urls, and for the
+# manifests to be correct they'll have to have the right URLs encoded.
+#
+# Note that this address should not contain a trailing slash as file names will
+# be appended to it.
+#upload-addr = "https://example.com/folder"

src/bootstrap/config.toml.example

@@ -22,7 +22,7 @@ use std::env;
 use std::fs::{self, File};
 use std::io::{Read, Write};
 use std::path::{PathBuf, Path};
-use std::process::Command;
+use std::process::{Command, Stdio};
 
 use build_helper::output;
 
@@ -876,3 +876,34 @@ fn add_env(build: &Build, cmd: &mut Command, target: &str) {
        cmd.env("CFG_PLATFORM", "x86");
     }
 }
+
+pub fn hash_and_sign(build: &Build) {
+    let compiler = Compiler::new(0, &build.config.build);
+    let mut cmd = build.tool_cmd(&compiler, "build-manifest");
+    let sign = build.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
+        panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
+    });
+    let addr = build.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
+        panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
+    });
+    let file = build.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
+        panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
+    });
+    let mut pass = String::new();
+    t!(t!(File::open(&file)).read_to_string(&mut pass));
+
+    let today = output(Command::new("date").arg("+%Y-%m-%d"));
+
+    cmd.arg(sign);
+    cmd.arg(distdir(build));
+    cmd.arg(today.trim());
+    cmd.arg(package_vers(build));
+    cmd.arg(addr);
+
+    t!(fs::create_dir_all(distdir(build)));
+
+    let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
+    t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
+    let status = t!(child.wait());
+    assert!(status.success());
+}

src/bootstrap/dist.rs

@@ -513,6 +513,9 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     rules.build("tool-compiletest", "src/tools/compiletest")
          .dep(|s| s.name("libtest"))
          .run(move |s| compile::tool(build, s.stage, s.target, "compiletest"));
+    rules.build("tool-build-manifest", "src/tools/build-manifest")
+         .dep(|s| s.name("libstd"))
+         .run(move |s| compile::tool(build, s.stage, s.target, "build-manifest"));
 
     // ========================================================================
     // Documentation targets
@@ -633,6 +636,13 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .dep(|d| d.name("dist-cargo"))
          .run(move |s| dist::extended(build, s.stage, s.target));
 
+    rules.dist("dist-sign", "hash-and-sign")
+         .host(true)
+         .only_build(true)
+         .only_host_build(true)
+         .dep(move |s| s.name("tool-build-manifest").target(&build.config.build).stage(0))
+         .run(move |_| dist::hash_and_sign(build));
+
     rules.verify();
     return rules;
 }

src/bootstrap/step.rs

@@ -0,0 +1,8 @@
+[package]
+name = "build-manifest"
+version = "0.1.0"
+authors = ["Alex Crichton <[email protected]>"]
+
+[dependencies]
+toml = "0.1"
+rustc-serialize = "0.3"