Auto merge of #42682 - alexcrichton:jobserver, r=michaelwoerister

Integrate jobserver support to parallel codegen

This commit integrates the `jobserver` crate into the compiler. The crate was
previously integrated in to Cargo as part of rust-lang/cargo#4110. The purpose
here is to two-fold:

* Primarily the compiler can cooperate with Cargo on parallelism. When you run
  `cargo build -j4` then this'll make sure that the entire build process between
  Cargo/rustc won't use more than 4 cores, whereas today you'd get 4 rustc
  instances which may all try to spawn lots of threads.

* Secondarily rustc/Cargo can now integrate with a foreign GNU `make` jobserver.
  This means that if you call cargo/rustc from `make` or another
  jobserver-compatible implementation it'll use foreign parallelism settings
  instead of creating new ones locally.

As the number of parallel codegen instances in the compiler continues to grow
over time with the advent of incremental compilation it's expected that this'll
become more of a problem, so this is intended to nip concurrent concerns in the
bud by having all the tools to cooperate!

Note that while rustc has support for itself creating a jobserver it's far more
likely that rustc will always use the jobserver configured by Cargo. Cargo today
will now set a jobserver unconditionally for rustc to use.

Author: bors

src/Cargo.lock

@@ -9,6 +9,9 @@ path = "lib.rs"
 test = false
 bench = false
 
+[dev-dependencies]
+rand = { path = "../librand" }
+
 [[test]]
 name = "coretests"
 path = "../libcore/tests/lib.rs"

src/libcore/Cargo.toml

@@ -12,6 +12,7 @@ crate-type = ["dylib"]
 arena = { path = "../libarena" }
 fmt_macros = { path = "../libfmt_macros" }
 graphviz = { path = "../libgraphviz" }
+jobserver = "0.1"
 log = "0.3"
 owning_ref = "0.3.3"
 rustc_back = { path = "../librustc_back" }

src/librustc/Cargo.toml

@@ -61,6 +61,7 @@ extern crate rustc_errors as errors;
 #[macro_use] extern crate syntax;
 extern crate syntax_pos;
 #[macro_use] #[no_link] extern crate rustc_bitflags;
+extern crate jobserver;
 
 extern crate serialize as rustc_serialize; // used by deriving
 

src/librustc/lib.rs

@@ -38,14 +38,16 @@ use syntax_pos::{Span, MultiSpan};
 use rustc_back::{LinkerFlavor, PanicStrategy};
 use rustc_back::target::Target;
 use rustc_data_structures::flock;
+use jobserver::Client;
 
-use std::path::{Path, PathBuf};
 use std::cell::{self, Cell, RefCell};
 use std::collections::HashMap;
 use std::env;
+use std::fmt;
 use std::io::Write;
+use std::path::{Path, PathBuf};
 use std::rc::Rc;
-use std::fmt;
+use std::sync::{Once, ONCE_INIT};
 use std::time::Duration;
 
 mod code_stats;
@@ -134,6 +136,10 @@ pub struct Session {
     pub print_fuel_crate: Option<String>,
     /// Always set to zero and incremented so that we can print fuel expended by a crate.
     pub print_fuel: Cell<u64>,
+
+    /// Loaded up early on in the initialization of this `Session` to avoid
+    /// false positives about a job server in our environment.
+    pub jobserver_from_env: Option<Client>,
 }
 
 pub struct PerfStats {
@@ -697,6 +703,24 @@ pub fn build_session_(sopts: config::Options,
         print_fuel_crate: print_fuel_crate,
         print_fuel: print_fuel,
         out_of_fuel: Cell::new(false),
+
+        // Note that this is unsafe because it may misinterpret file descriptors
+        // on Unix as jobserver file descriptors. We hopefully execute this near
+        // the beginning of the process though to ensure we don't get false
+        // positives, or in other words we try to execute this before we open
+        // any file descriptors ourselves.
+        //
+        // Also note that we stick this in a global because there could be
+        // multiple `Session` instances in this process, and the jobserver is
+        // per-process.
+        jobserver_from_env: unsafe {
+            static mut GLOBAL_JOBSERVER: *mut Option<Client> = 0 as *mut _;
+            static INIT: Once = ONCE_INIT;
+            INIT.call_once(|| {
+                GLOBAL_JOBSERVER = Box::into_raw(Box::new(Client::from_env()));
+            });
+            (*GLOBAL_JOBSERVER).clone()
+        },
     };
 
     sess

src/librustc/session/mod.rs

@@ -10,7 +10,9 @@ crate-type = ["dylib"]
 test = false
 
 [dependencies]
+crossbeam = "0.2"
 flate2 = "0.2"
+jobserver = "0.1.5"
 log = "0.3"
 owning_ref = "0.3.3"
 rustc = { path = "../librustc" }

src/librustc_trans/Cargo.toml

@@ -329,34 +329,38 @@ pub fn filename_for_input(sess: &Session,
 }
 
 pub fn each_linked_rlib(sess: &Session,
-                        f: &mut FnMut(CrateNum, &Path)) {
+                        f: &mut FnMut(CrateNum, &Path)) -> Result<(), String> {
     let crates = sess.cstore.used_crates(LinkagePreference::RequireStatic).into_iter();
     let fmts = sess.dependency_formats.borrow();
     let fmts = fmts.get(&config::CrateTypeExecutable)
                    .or_else(|| fmts.get(&config::CrateTypeStaticlib))
                    .or_else(|| fmts.get(&config::CrateTypeCdylib))
                    .or_else(|| fmts.get(&config::CrateTypeProcMacro));
-    let fmts = fmts.unwrap_or_else(|| {
-        bug!("could not find formats for rlibs");
-    });
+    let fmts = match fmts {
+        Some(f) => f,
+        None => return Err(format!("could not find formats for rlibs"))
+    };
     for (cnum, path) in crates {
-        match fmts[cnum.as_usize() - 1] {
-            Linkage::NotLinked | Linkage::IncludedFromDylib => continue,
-            _ => {}
+        match fmts.get(cnum.as_usize() - 1) {
+            Some(&Linkage::NotLinked) |
+            Some(&Linkage::IncludedFromDylib) => continue,
+            Some(_) => {}
+            None => return Err(format!("could not find formats for rlibs"))
         }
         let name = sess.cstore.crate_name(cnum).clone();
         let path = match path {
             LibSource::Some(p) => p,
             LibSource::MetadataOnly => {
-                sess.fatal(&format!("could not find rlib for: `{}`, found rmeta (metadata) file",
-                                    name));
+                return Err(format!("could not find rlib for: `{}`, found rmeta (metadata) file",
+                                   name))
             }
             LibSource::None => {
-                sess.fatal(&format!("could not find rlib for: `{}`", name));
+                return Err(format!("could not find rlib for: `{}`", name))
             }
         };
         f(cnum, &path);
     }
+    Ok(())
 }
 
 fn out_filename(sess: &Session,
@@ -669,7 +673,7 @@ fn link_staticlib(sess: &Session, objects: &[PathBuf], out_filename: &Path,
     let mut ab = link_rlib(sess, None, objects, out_filename, tempdir);
     let mut all_native_libs = vec![];
 
-    each_linked_rlib(sess, &mut |cnum, path| {
+    let res = each_linked_rlib(sess, &mut |cnum, path| {
         let name = sess.cstore.crate_name(cnum);
         let native_libs = sess.cstore.native_libraries(cnum);
 
@@ -694,6 +698,9 @@ fn link_staticlib(sess: &Session, objects: &[PathBuf], out_filename: &Path,
 
         all_native_libs.extend(sess.cstore.native_libraries(cnum));
     });
+    if let Err(e) = res {
+        sess.fatal(&e);
+    }
 
     ab.update_symbols();
     ab.build();

src/librustc_trans/back/link.rs

@@ -10,15 +10,16 @@
 
 use back::link;
 use back::write;
-use back::symbol_export::{self, ExportedSymbols};
-use rustc::session::{self, config};
+use back::symbol_export;
+use rustc::session::config;
+use errors::FatalError;
 use llvm;
 use llvm::archive_ro::ArchiveRO;
 use llvm::{ModuleRef, TargetMachineRef, True, False};
 use rustc::util::common::time;
 use rustc::util::common::path2cstr;
 use rustc::hir::def_id::LOCAL_CRATE;
-use back::write::{ModuleConfig, with_llvm_pmb};
+use back::write::{ModuleConfig, with_llvm_pmb, CodegenContext};
 
 use libc;
 use flate2::read::ZlibDecoder;
@@ -39,30 +40,31 @@ pub fn crate_type_allows_lto(crate_type: config::CrateType) -> bool {
     }
 }
 
-pub fn run(sess: &session::Session,
+pub fn run(cgcx: &CodegenContext,
            llmod: ModuleRef,
            tm: TargetMachineRef,
-           exported_symbols: &ExportedSymbols,
            config: &ModuleConfig,
-           temp_no_opt_bc_filename: &Path) {
-    if sess.opts.cg.prefer_dynamic {
-        sess.struct_err("cannot prefer dynamic linking when performing LTO")
+           temp_no_opt_bc_filename: &Path) -> Result<(), FatalError> {
+    let handler = cgcx.handler;
+    if cgcx.opts.cg.prefer_dynamic {
+        handler.struct_err("cannot prefer dynamic linking when performing LTO")
             .note("only 'staticlib', 'bin', and 'cdylib' outputs are \
                    supported with LTO")
             .emit();
-        sess.abort_if_errors();
+        return Err(FatalError)
     }
 
     // Make sure we actually can run LTO
-    for crate_type in sess.crate_types.borrow().iter() {
+    for crate_type in cgcx.crate_types.iter() {
         if !crate_type_allows_lto(*crate_type) {
-            sess.fatal("lto can only be run for executables, cdylibs and \
-                            static library outputs");
+            let e = handler.fatal("lto can only be run for executables, cdylibs and \
+                                   static library outputs");
+            return Err(e)
         }
     }
 
     let export_threshold =
-        symbol_export::crates_export_threshold(&sess.crate_types.borrow());
+        symbol_export::crates_export_threshold(&cgcx.crate_types);
 
     let symbol_filter = &|&(ref name, level): &(String, _)| {
         if symbol_export::is_below_threshold(level, export_threshold) {
@@ -74,7 +76,7 @@ pub fn run(sess: &session::Session,
         }
     };
 
-    let mut symbol_white_list: Vec<CString> = exported_symbols
+    let mut symbol_white_list: Vec<CString> = cgcx.exported_symbols
         .exported_symbols(LOCAL_CRATE)
         .iter()
         .filter_map(symbol_filter)
@@ -83,16 +85,11 @@ pub fn run(sess: &session::Session,
     // For each of our upstream dependencies, find the corresponding rlib and
     // load the bitcode from the archive. Then merge it into the current LLVM
     // module that we've got.
-    link::each_linked_rlib(sess, &mut |cnum, path| {
-        // `#![no_builtins]` crates don't participate in LTO.
-        if sess.cstore.is_no_builtins(cnum) {
-            return;
-        }
-
+    for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() {
         symbol_white_list.extend(
-            exported_symbols.exported_symbols(cnum)
-                            .iter()
-                            .filter_map(symbol_filter));
+            cgcx.exported_symbols.exported_symbols(cnum)
+                                 .iter()
+                                 .filter_map(symbol_filter));
 
         let archive = ArchiveRO::open(&path).expect("wanted an rlib");
         let bytecodes = archive.iter().filter_map(|child| {
@@ -102,7 +99,7 @@ pub fn run(sess: &session::Session,
             let bc_encoded = data.data();
 
             let bc_decoded = if is_versioned_bytecode_format(bc_encoded) {
-                time(sess.time_passes(), &format!("decode {}", name), || {
+                time(cgcx.time_passes, &format!("decode {}", name), || {
                     // Read the version
                     let version = extract_bytecode_format_version(bc_encoded);
 
@@ -117,44 +114,49 @@ pub fn run(sess: &session::Session,
                         let res = ZlibDecoder::new(compressed_data)
                             .read_to_end(&mut inflated);
                         if res.is_err() {
-                            sess.fatal(&format!("failed to decompress bc of `{}`",
-                                               name))
+                            let msg = format!("failed to decompress bc of `{}`",
+                                              name);
+                            Err(handler.fatal(&msg))
+                        } else {
+                            Ok(inflated)
                         }
-                        inflated
                     } else {
-                        sess.fatal(&format!("Unsupported bytecode format version {}",
-                                           version))
+                        Err(handler.fatal(&format!("Unsupported bytecode format version {}",
+                                                   version)))
                     }
-                })
+                })?
             } else {
-                time(sess.time_passes(), &format!("decode {}", name), || {
+                time(cgcx.time_passes, &format!("decode {}", name), || {
                     // the object must be in the old, pre-versioning format, so
                     // simply inflate everything and let LLVM decide if it can
                     // make sense of it
                     let mut inflated = Vec::new();
                     let res = ZlibDecoder::new(bc_encoded)
                         .read_to_end(&mut inflated);
                     if res.is_err() {
-                        sess.fatal(&format!("failed to decompress bc of `{}`",
-                                           name))
+                        let msg = format!("failed to decompress bc of `{}`",
+                                          name);
+                        Err(handler.fatal(&msg))
+                    } else {
+                        Ok(inflated)
                     }
-                    inflated
-                })
+                })?
             };
 
             let ptr = bc_decoded.as_ptr();
             debug!("linking {}", name);
-            time(sess.time_passes(), &format!("ll link {}", name), || unsafe {
-                if !llvm::LLVMRustLinkInExternalBitcode(llmod,
-                                                        ptr as *const libc::c_char,
-                                                        bc_decoded.len() as libc::size_t) {
-                    write::llvm_err(sess.diagnostic(),
-                                    format!("failed to load bc of `{}`",
-                                            name));
+            time(cgcx.time_passes, &format!("ll link {}", name), || unsafe {
+                if llvm::LLVMRustLinkInExternalBitcode(llmod,
+                                                       ptr as *const libc::c_char,
+                                                       bc_decoded.len() as libc::size_t) {
+                    Ok(())
+                } else {
+                    let msg = format!("failed to load bc of `{}`", name);
+                    Err(write::llvm_err(handler, msg))
                 }
-            });
+            })?;
         }
-    });
+    }
 
     // Internalize everything but the exported symbols of the current module
     let arr: Vec<*const libc::c_char> = symbol_white_list.iter()
@@ -167,13 +169,13 @@ pub fn run(sess: &session::Session,
                                          arr.len() as libc::size_t);
     }
 
-    if sess.no_landing_pads() {
+    if cgcx.no_landing_pads {
         unsafe {
             llvm::LLVMRustMarkAllFunctionsNounwind(llmod);
         }
     }
 
-    if sess.opts.cg.save_temps {
+    if cgcx.opts.cg.save_temps {
         let cstr = path2cstr(temp_no_opt_bc_filename);
         unsafe {
             llvm::LLVMWriteBitcodeToFile(llmod, cstr.as_ptr());
@@ -203,12 +205,13 @@ pub fn run(sess: &session::Session,
         assert!(!pass.is_null());
         llvm::LLVMRustAddPass(pm, pass);
 
-        time(sess.time_passes(), "LTO passes", ||
+        time(cgcx.time_passes, "LTO passes", ||
              llvm::LLVMRunPassManager(pm, llmod));
 
         llvm::LLVMDisposePassManager(pm);
     }
     debug!("lto done");
+    Ok(())
 }
 
 fn is_versioned_bytecode_format(bc: &[u8]) -> bool {