Auto merge of rust-lang#49219 - eddyb:proc-macro-decouple, r=alexcrichton

Decouple proc_macro from the rest of the compiler.

This PR removes all dependencies of `proc_macro` on compiler crates and allows multiple copies of `proc_macro`, built even by different compilers (but from the same source), to interoperate.

Practically, it allows:
* running proc macro tests at stage1 (I moved most from `-fulldeps` to the regular suites)
* using proc macros in the compiler itself (may require some rustbuild trickery)

On the server (i.e. compiler front-end) side:
* `server::*` traits are implemented to provide the concrete types and methods
  * the concrete types are completely separated from the `proc_macro` public API
  * the only use of the type implementing `Server` is to be passed to `Client::run`

On the client (i.e. proc macro) side (potentially using a different `proc_macro` instance!):
* `client::Client` wraps around client-side (expansion) function pointers
  * it encapsulates the `proc_macro` instance used by the client
  * its `run` method can be called by a server, to execute the client-side function
    * the client instance is bridged to the provided server, while it runs
    * ~~currently a thread is spawned, could use process isolation in the future~~
(not the case anymore, see rust-lang#56058)
* proc macro crates get a generated `static` holding a `&[ProcMacro]`
  * this describes all derives/attr/bang proc macros, replacing the "registrar" function
  * each variant of `ProcMacro` contains an appropriately typed `Client<fn(...) -> ...>`

`proc_macro` public APIs call into the server via an internal "bridge":
* only a currently running proc macro `Client` can interact with those APIs
  * server code might not be able to (if it uses a different `proc_macro` instance)
    * however, it can always create and `run` its own `Client`, but that may be inefficient
* the `bridge` uses serialization, C ABI and integer handles to avoid Rust ABI instability
* each invocation of a proc macro results in disjoint integers in its `proc_macro` handles
  * this prevents using values of those types across invocations (if they even can be kept)

r? @alexcrichton cc @jseyfried @nikomatsakis @Zoxc @thepowersgang

Author: bors

Cargo.lock

@@ -1599,12 +1599,6 @@ dependencies = [
 [[package]]
 name = "proc_macro"
 version = "0.0.0"
-dependencies = [
- "rustc_data_structures 0.0.0",
- "rustc_errors 0.0.0",
- "syntax 0.0.0",
- "syntax_pos 0.0.0",
-]
 
 [[package]]
 name = "profiler_builtins"
@@ -1933,7 +1927,6 @@ dependencies = [
  "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "polonius-engine 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "proc_macro 0.0.0",
  "rustc-rayon 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-rayon-core 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc_apfloat 0.0.0",
@@ -2351,7 +2344,6 @@ dependencies = [
  "flate2 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "proc_macro 0.0.0",
  "rustc 0.0.0",
  "rustc_data_structures 0.0.0",
  "rustc_errors 0.0.0",
@@ -2890,7 +2882,6 @@ version = "0.0.0"
 dependencies = [
  "fmt_macros 0.0.0",
  "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "proc_macro 0.0.0",
  "rustc_data_structures 0.0.0",
  "rustc_errors 0.0.0",
  "rustc_target 0.0.0",
@@ -2980,6 +2971,7 @@ name = "test"
 version = "0.0.0"
 dependencies = [
  "getopts 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
+ "proc_macro 0.0.0",
  "term 0.0.0",
 ]
 

src/bootstrap/bin/rustc.rs

@@ -129,10 +129,12 @@ fn main() {
         // Help the libc crate compile by assisting it in finding the MUSL
         // native libraries.
         if let Some(s) = env::var_os("MUSL_ROOT") {
-            let mut root = OsString::from("native=");
-            root.push(&s);
-            root.push("/lib");
-            cmd.arg("-L").arg(&root);
+            if target.contains("musl") {
+                let mut root = OsString::from("native=");
+                root.push(&s);
+                root.push("/lib");
+                cmd.arg("-L").arg(&root);
+            }
         }
 
         // Override linker if necessary.

src/bootstrap/builder.rs

@@ -390,7 +390,6 @@ impl<'a> Builder<'a> {
                 test::RunPassFullDeps,
                 test::RunFailFullDeps,
                 test::CompileFailFullDeps,
-                test::IncrementalFullDeps,
                 test::Rustdoc,
                 test::Pretty,
                 test::RunPassPretty,

src/bootstrap/test.rs

@@ -839,12 +839,6 @@ host_test!(CompileFailFullDeps {
     suite: "compile-fail-fulldeps"
 });
 
-host_test!(IncrementalFullDeps {
-    path: "src/test/incremental-fulldeps",
-    mode: "incremental",
-    suite: "incremental-fulldeps"
-});
-
 host_test!(Rustdoc {
     path: "src/test/rustdoc",
     mode: "rustdoc",
@@ -982,6 +976,11 @@ impl Step for Compiletest {
             builder.ensure(compile::Std { compiler, target: compiler.host });
         }
 
+        // HACK(eddyb) ensure that `libproc_macro` is available on the host.
+        builder.ensure(compile::Test { compiler, target: compiler.host });
+        // Also provide `rust_test_helpers` for the host.
+        builder.ensure(native::TestHelpers { target: compiler.host });
+
         builder.ensure(native::TestHelpers { target });
         builder.ensure(RemoteCopyLibs { compiler, target });
 
@@ -1049,7 +1048,11 @@ impl Step for Compiletest {
             cmd.arg("--linker").arg(linker);
         }
 
-        let hostflags = flags.clone();
+        let mut hostflags = flags.clone();
+        hostflags.push(format!(
+            "-Lnative={}",
+            builder.test_helpers_out(compiler.host).display()
+        ));
         cmd.arg("--host-rustcflags").arg(hostflags.join(" "));
 
         let mut targetflags = flags;

src/libproc_macro/Cargo.toml

@@ -5,10 +5,3 @@ version = "0.0.0"
 
 [lib]
 path = "lib.rs"
-crate-type = ["dylib"]
-
-[dependencies]
-syntax = { path = "../libsyntax" }
-syntax_pos = { path = "../libsyntax_pos" }
-rustc_errors = { path = "../librustc_errors" }
-rustc_data_structures = { path = "../librustc_data_structures" }

src/libproc_macro/bridge/buffer.rs

@@ -0,0 +1,170 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Buffer management for same-process client<->server communication.
+
+use std::io::{self, Write};
+use std::mem;
+use std::ops::{Deref, DerefMut};
+use std::slice;
+
+#[repr(C)]
+struct Slice<'a, T: 'a> {
+    data: &'a [T; 0],
+    len: usize,
+}
+
+unsafe impl<'a, T: Sync> Sync for Slice<'a, T> {}
+unsafe impl<'a, T: Sync> Send for Slice<'a, T> {}
+
+impl<T> Copy for Slice<'a, T> {}
+impl<T> Clone for Slice<'a, T> {
+    fn clone(&self) -> Self {
+        *self
+    }
+}
+
+impl<T> From<&'a [T]> for Slice<'a, T> {
+    fn from(xs: &'a [T]) -> Self {
+        Slice {
+            data: unsafe { &*(xs.as_ptr() as *const [T; 0]) },
+            len: xs.len(),
+        }
+    }
+}
+
+impl<T> Deref for Slice<'a, T> {
+    type Target = [T];
+    fn deref(&self) -> &[T] {
+        unsafe { slice::from_raw_parts(self.data.as_ptr(), self.len) }
+    }
+}
+
+#[repr(C)]
+pub struct Buffer<T: Copy> {
+    data: *mut T,
+    len: usize,
+    capacity: usize,
+    extend_from_slice: extern "C" fn(Buffer<T>, Slice<T>) -> Buffer<T>,
+    drop: extern "C" fn(Buffer<T>),
+}
+
+unsafe impl<T: Copy + Sync> Sync for Buffer<T> {}
+unsafe impl<T: Copy + Send> Send for Buffer<T> {}
+
+impl<T: Copy> Default for Buffer<T> {
+    fn default() -> Self {
+        Self::from(vec![])
+    }
+}
+
+impl<T: Copy> Deref for Buffer<T> {
+    type Target = [T];
+    fn deref(&self) -> &[T] {
+        unsafe { slice::from_raw_parts(self.data as *const T, self.len) }
+    }
+}
+
+impl<T: Copy> DerefMut for Buffer<T> {
+    fn deref_mut(&mut self) -> &mut [T] {
+        unsafe { slice::from_raw_parts_mut(self.data, self.len) }
+    }
+}
+
+impl<T: Copy> Buffer<T> {
+    pub(super) fn new() -> Self {
+        Self::default()
+    }
+
+    pub(super) fn clear(&mut self) {
+        self.len = 0;
+    }
+
+    pub(super) fn take(&mut self) -> Self {
+        mem::replace(self, Self::default())
+    }
+
+    pub(super) fn extend_from_slice(&mut self, xs: &[T]) {
+        // Fast path to avoid going through an FFI call.
+        if let Some(final_len) = self.len.checked_add(xs.len()) {
+            if final_len <= self.capacity {
+                let dst = unsafe { slice::from_raw_parts_mut(self.data, self.capacity) };
+                dst[self.len..][..xs.len()].copy_from_slice(xs);
+                self.len = final_len;
+                return;
+            }
+        }
+        let b = self.take();
+        *self = (b.extend_from_slice)(b, Slice::from(xs));
+    }
+}
+
+impl Write for Buffer<u8> {
+    fn write(&mut self, xs: &[u8]) -> io::Result<usize> {
+        self.extend_from_slice(xs);
+        Ok(xs.len())
+    }
+
+    fn write_all(&mut self, xs: &[u8]) -> io::Result<()> {
+        self.extend_from_slice(xs);
+        Ok(())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}
+
+impl<T: Copy> Drop for Buffer<T> {
+    fn drop(&mut self) {
+        let b = self.take();
+        (b.drop)(b);
+    }
+}
+
+impl<T: Copy> From<Vec<T>> for Buffer<T> {
+    fn from(mut v: Vec<T>) -> Self {
+        let (data, len, capacity) = (v.as_mut_ptr(), v.len(), v.capacity());
+        mem::forget(v);
+
+        // This utility function is nested in here because it can *only*
+        // be safely called on `Buffer`s created by *this* `proc_macro`.
+        fn to_vec<T: Copy>(b: Buffer<T>) -> Vec<T> {
+            unsafe {
+                let Buffer {
+                    data,
+                    len,
+                    capacity,
+                    ..
+                } = b;
+                mem::forget(b);
+                Vec::from_raw_parts(data, len, capacity)
+            }
+        }
+
+        extern "C" fn extend_from_slice<T: Copy>(b: Buffer<T>, xs: Slice<T>) -> Buffer<T> {
+            let mut v = to_vec(b);
+            v.extend_from_slice(&xs);
+            Buffer::from(v)
+        }
+
+        extern "C" fn drop<T: Copy>(b: Buffer<T>) {
+            mem::drop(to_vec(b));
+        }
+
+        Buffer {
+            data,
+            len,
+            capacity,
+            extend_from_slice,
+            drop,
+        }
+    }
+}