intrinsic-test: simplify linker logic

just always compile .cpp to .o to maximize thread utilization, and only invoke the (maybe custom) linker at the very end

Author: folkertdev

crates/intrinsic-test/src/arm/compile.rs

@@ -1,7 +1,7 @@
 use crate::common::cli::ProcessedCli;
 use crate::common::compile_c::{CompilationCommandBuilder, CppCompilation};
 
-pub fn build_cpp_compilation(config: &ProcessedCli) -> Option<CppCompilation> {
+pub fn configure_cpp_compiler(config: &ProcessedCli) -> Option<CppCompilation> {
     let cpp_compiler = config.cpp_compiler.as_ref()?;
 
     // -ffp-contract=off emulates Rust's approach of not fusing separate mul-add operations

crates/intrinsic-test/src/arm/mod.rs

@@ -59,28 +59,34 @@ impl SupportedArchitectureTest for ArmArchitectureTest {
         let platform_headers = &["arm_neon.h", "arm_acle.h", "arm_fp16.h"];
 
         let available_parallelism = std::thread::available_parallelism().unwrap().get();
-        let chunk_size = self.intrinsics.len().div_ceil(available_parallelism);
+        let chunk_count = Ord::min(available_parallelism, self.intrinsics.len());
+        let chunk_size = self.intrinsics.len().div_ceil(chunk_count);
 
-        let pipeline = compile::build_cpp_compilation(&self.cli_options).unwrap();
+        let cpp_compiler = compile::configure_cpp_compiler(&self.cli_options).unwrap();
 
         let notice = &build_notices("// ");
         self.intrinsics
             .par_chunks(chunk_size)
             .enumerate()
             .map(|(i, chunk)| {
                 let c_filename = format!("c_programs/mod_{i}.cpp");
+                info!("writing {c_filename}");
                 let mut file = File::create(&c_filename).unwrap();
                 write_mod_cpp(&mut file, notice, c_target, platform_headers, chunk).unwrap();
 
                 // compile this cpp file into a .o file
-                let output = pipeline.run(&[], &[format!("mod_{i}.cpp")], &format!("mod_{i}.o"))?;
+                info!("compiling {c_filename}");
+                let output = cpp_compiler
+                    .compile_object_file(&format!("mod_{i}.cpp"), &format!("mod_{i}.o"))?;
                 assert!(output.status.success());
+                info!("done compiling {c_filename}");
 
                 Ok(())
             })
             .collect::<Result<(), std::io::Error>>()
             .unwrap();
 
+        info!("writing main.cpp");
         let mut file = File::create("c_programs/main.cpp").unwrap();
         write_main_cpp(
             &mut file,
@@ -90,21 +96,35 @@ impl SupportedArchitectureTest for ArmArchitectureTest {
         )
         .unwrap();
 
-        // Files to include in the final link step.
-        let mut includes = vec![];
-        for i in 0..Ord::min(available_parallelism, self.intrinsics.len()) {
-            includes.push(format!("mod_{i}.o"));
-        }
-
-        let output = pipeline
-            .run(
-                &includes,
-                &["main.cpp".to_string()],
-                "intrinsic-test-programs",
-            )
+        // compile this cpp file into a .o file
+        info!("compiling main.cpp");
+        let output = cpp_compiler
+            .compile_object_file("main.cpp", "intrinsic-test-programs.o")
             .unwrap();
         assert!(output.status.success());
 
+        let object_files = (0..chunk_count)
+            .map(|i| format!("mod_{i}.o"))
+            .chain(["intrinsic-test-programs.o".to_owned()]);
+
+        let output = match &self.cli_options.linker {
+            Some(custom_linker) => {
+                let mut linker = std::process::Command::new(custom_linker);
+
+                linker.current_dir("c_programs");
+
+                linker.args(object_files);
+                linker.args(["-o", "intrinsic-test-programs"]);
+
+                info!("linking final C binary with {custom_linker}");
+                linker.output()
+            }
+            None => cpp_compiler.link_executable(object_files, "intrinsic-test-programs"),
+        };
+
+        let output = output.unwrap();
+        assert!(output.status.success());
+
         true
     }
 

crates/intrinsic-test/src/common/compile_c.rs

@@ -104,30 +104,19 @@ impl CompilationCommandBuilder {
             cpp_compiler.arg(format!("--target={target}"));
         }
 
-        if let (Some(linker), Some(cxx_toolchain_dir)) = (&self.linker, &self.cxx_toolchain_dir) {
+        if let Some(cxx_toolchain_dir) = &self.cxx_toolchain_dir {
             cpp_compiler.args(
                 self.include_paths
                     .iter()
-                    .map(|path| "--include-directory=".to_string() + cxx_toolchain_dir + path),
+                    .map(|path| format!("--include-directory={cxx_toolchain_dir}{path}")),
             );
-
-            CppCompilation::CustomLinker {
-                cpp_compiler,
-                linker: linker.to_owned(),
-            }
-        } else {
-            CppCompilation::Simple(cpp_compiler)
         }
+
+        CppCompilation(cpp_compiler)
     }
 }
 
-pub enum CppCompilation {
-    Simple(std::process::Command),
-    CustomLinker {
-        cpp_compiler: std::process::Command,
-        linker: String,
-    },
-}
+pub struct CppCompilation(std::process::Command);
 
 fn clone_command(command: &std::process::Command) -> std::process::Command {
     let mut cmd = std::process::Command::new(command.get_program());
@@ -144,85 +133,24 @@ fn clone_command(command: &std::process::Command) -> std::process::Command {
 }
 
 impl CppCompilation {
-    fn compile_cpp(
-        command: &std::process::Command,
-        includes: &[String],
-        inputs: &[String],
+    pub fn compile_object_file(
+        &self,
+        input: &str,
         output: &str,
     ) -> std::io::Result<std::process::Output> {
-        let mut cmd = clone_command(command);
-        cmd.args(includes);
-        cmd.args(inputs);
-        cmd.args(["-o", output]);
-
-        if output.ends_with(".o") {
-            cmd.arg("-c");
-        }
-
+        let mut cmd = clone_command(&self.0);
+        cmd.args([input, "-c", "-o", output]);
         cmd.output()
     }
 
-    pub fn run(
+    pub fn link_executable(
         &self,
-        includes: &[String],
-        inputs: &[String],
+        inputs: impl Iterator<Item = String>,
         output: &str,
     ) -> std::io::Result<std::process::Output> {
-        match self {
-            CppCompilation::Simple(command) => Self::compile_cpp(command, includes, inputs, output),
-            CppCompilation::CustomLinker { cpp_compiler, .. } if output.ends_with(".o") => {
-                // No need to invoke that custom linker if we're creating an object file.
-                Self::compile_cpp(cpp_compiler, includes, inputs, output)
-            }
-            CppCompilation::CustomLinker {
-                cpp_compiler,
-                linker,
-            } => {
-                let object_file = &format!("{output}.o");
-
-                // Build an object file using the cpp compiler.
-                let mut cmd = clone_command(cpp_compiler);
-                cmd.args(inputs);
-                cmd.args(["-c", "-o", object_file]);
-
-                let cpp_output = cmd.output()?;
-                if !cpp_output.status.success() {
-                    error!("c++ compilaton failed");
-                    return Ok(cpp_output);
-                }
-
-                trace!("using custom linker");
-
-                // Use the custom linker to turn the object file into an executable.
-                let mut cmd = std::process::Command::new(linker);
-                cmd.args(includes);
-                cmd.args([object_file, "-o", output]);
-
-                if let Some(current_dir) = cpp_compiler.get_current_dir() {
-                    cmd.current_dir(current_dir);
-                }
-
-                for (key, val) in cpp_compiler.get_envs() {
-                    cmd.env(key, val.unwrap_or_default());
-                }
-
-                let linker_output = cmd.output()?;
-                if !linker_output.status.success() {
-                    error!("custom linker failed");
-                    error!("{}", String::from_utf8_lossy(&linker_output.stderr));
-                    return Ok(linker_output);
-                }
-
-                trace!("removing {object_file}");
-                let object_file_path = match cpp_compiler.get_current_dir() {
-                    Some(current_dir) => &format!("{}/{object_file}", current_dir.display()),
-                    None => object_file,
-                };
-
-                std::fs::remove_file(object_file_path)?;
-
-                Ok(cpp_output)
-            }
-        }
+        let mut cmd = clone_command(&self.0);
+        cmd.args(inputs);
+        cmd.args(["-o", output]);
+        cmd.output()
     }
 }