intrinsic-test: combine C files for more efficient compilation

Author: folkertdev

crates/intrinsic-test/src/arm/mod.rs

@@ -4,15 +4,20 @@ mod intrinsic;
 mod json_parser;
 mod types;
 
+use std::fs::File;
+
+use rayon::prelude::*;
+
+use crate::arm::config::POLY128_OSTREAM_DEF;
 use crate::common::SupportedArchitectureTest;
 use crate::common::cli::ProcessedCli;
 use crate::common::compare::compare_outputs;
-use crate::common::gen_c::compile_c_programs;
+use crate::common::gen_c::{write_main_cpp, write_mod_cpp};
 use crate::common::gen_rust::compile_rust_programs;
 use crate::common::intrinsic::{Intrinsic, IntrinsicDefinition};
 use crate::common::intrinsic_helpers::TypeKind;
-use crate::common::write_file::{write_c_testfiles, write_rust_testfiles};
-use config::{AARCH_CONFIGURATIONS, F16_FORMATTING_DEF, POLY128_OSTREAM_DEF, build_notices};
+use crate::common::write_file::write_rust_testfiles;
+use config::{AARCH_CONFIGURATIONS, F16_FORMATTING_DEF, build_notices};
 use intrinsic::ArmIntrinsicType;
 use json_parser::get_neon_intrinsics;
 
@@ -51,24 +56,57 @@ impl SupportedArchitectureTest for ArmArchitectureTest {
     }
 
     fn build_c_file(&self) -> bool {
-        let target = &self.cli_options.target;
         let c_target = "aarch64";
+        let platform_headers = &["arm_neon.h", "arm_acle.h", "arm_fp16.h"];
 
-        let intrinsics_name_list = write_c_testfiles(
-            &self
-                .intrinsics
-                .iter()
-                .map(|i| i as &dyn IntrinsicDefinition<_>)
-                .collect::<Vec<_>>(),
-            target,
-            c_target,
-            &["arm_neon.h", "arm_acle.h", "arm_fp16.h"],
-            &build_notices("// "),
-            &[POLY128_OSTREAM_DEF],
-        );
+        let available_parallelism = std::thread::available_parallelism().unwrap().get();
+        let chunk_size = self.intrinsics.len().div_ceil(available_parallelism);
 
         let pipeline = compile::build_cpp_compilation(&self.cli_options).unwrap();
-        compile_c_programs(&pipeline, &intrinsics_name_list)
+
+        let notice = &build_notices("// ");
+        self.intrinsics
+            .par_chunks(chunk_size)
+            .enumerate()
+            .map(|(i, chunk)| {
+                let c_filename = format!("c_programs/mod_{i}.cpp");
+                let mut file = File::create(&c_filename).unwrap();
+                write_mod_cpp(&mut file, notice, c_target, platform_headers, chunk).unwrap();
+
+                // compile this cpp file into a .o file
+                let output = pipeline.run(&[], &[format!("mod_{i}.cpp")], &format!("mod_{i}.o"))?;
+                assert!(output.status.success());
+
+                Ok(())
+            })
+            .collect::<Result<(), std::io::Error>>()
+            .unwrap();
+
+        let mut file = File::create("c_programs/main.cpp").unwrap();
+        write_main_cpp(
+            &mut file,
+            c_target,
+            POLY128_OSTREAM_DEF,
+            self.intrinsics.iter().map(|i| i.name.as_str()),
+        )
+        .unwrap();
+
+        // Files to include in the final link step.
+        let mut includes = vec![];
+        for i in 0..Ord::min(available_parallelism, self.intrinsics.len()) {
+            includes.push(format!("mod_{i}.o"));
+        }
+
+        let output = pipeline
+            .run(
+                &includes,
+                &["main.cpp".to_string()],
+                "intrinsic-test-programs",
+            )
+            .unwrap();
+        assert!(output.status.success());
+
+        true
     }
 
     fn build_rust_file(&self) -> bool {

crates/intrinsic-test/src/common/argument.rs

@@ -125,19 +125,23 @@ where
     /// Creates a line for each argument that initializes an array for C from which `loads` argument
     /// values can be loaded  as a sliding window.
     /// e.g `const int32x2_t a_vals = {0x3effffff, 0x3effffff, 0x3f7fffff}`, if loads=2.
-    pub fn gen_arglists_c(&self, indentation: Indentation, loads: u32) -> String {
-        self.iter()
-            .filter(|&arg| !arg.has_constraint())
-            .map(|arg| {
-                format!(
-                    "{indentation}const {ty} {name}_vals[] = {values};",
-                    ty = arg.ty.c_scalar_type(),
-                    name = arg.name,
-                    values = arg.ty.populate_random(indentation, loads, &Language::C)
-                )
-            })
-            .collect::<Vec<_>>()
-            .join("\n")
+    pub fn gen_arglists_c(
+        &self,
+        w: &mut impl std::io::Write,
+        indentation: Indentation,
+        loads: u32,
+    ) -> std::io::Result<()> {
+        for arg in self.iter().filter(|&arg| !arg.has_constraint()) {
+            writeln!(
+                w,
+                "{indentation}const {ty} {name}_vals[] = {values};",
+                ty = arg.ty.c_scalar_type(),
+                name = arg.name,
+                values = arg.ty.populate_random(indentation, loads, &Language::C)
+            )?
+        }
+
+        Ok(())
     }
 
     /// Creates a line for each argument that initializes an array for Rust from which `loads` argument

crates/intrinsic-test/src/common/compare.rs

@@ -13,7 +13,7 @@ pub fn compare_outputs(
         .filter_map(|intrinsic_name| {
             let c = Command::new("sh")
                 .arg("-c")
-                .arg(format!("{runner} ./c_programs/{intrinsic_name}"))
+                .arg(format!("{runner} ./c_programs/intrinsic-test-programs {intrinsic_name}"))
                 .output();
 
             let rust = Command::new("sh")

crates/intrinsic-test/src/common/compile_c.rs

@@ -144,14 +144,35 @@ fn clone_command(command: &std::process::Command) -> std::process::Command {
 }
 
 impl CppCompilation {
-    pub fn run(&self, inputs: &[String], output: &str) -> std::io::Result<std::process::Output> {
-        match self {
-            CppCompilation::Simple(command) => {
-                let mut cmd = clone_command(command);
-                cmd.args(inputs);
-                cmd.args(["-o", output]);
+    fn compile_cpp(
+        command: &std::process::Command,
+        includes: &[String],
+        inputs: &[String],
+        output: &str,
+    ) -> std::io::Result<std::process::Output> {
+        let mut cmd = clone_command(command);
+        cmd.args(includes);
+        cmd.args(inputs);
+        cmd.args(["-o", output]);
+
+        if output.ends_with(".o") {
+            cmd.arg("-c");
+        }
 
-                cmd.output()
+        cmd.output()
+    }
+
+    pub fn run(
+        &self,
+        includes: &[String],
+        inputs: &[String],
+        output: &str,
+    ) -> std::io::Result<std::process::Output> {
+        match self {
+            CppCompilation::Simple(command) => Self::compile_cpp(command, includes, inputs, output),
+            CppCompilation::CustomLinker { cpp_compiler, .. } if output.ends_with(".o") => {
+                // No need to invoke that custom linker if we're creating an object file.
+                Self::compile_cpp(cpp_compiler, includes, inputs, output)
             }
             CppCompilation::CustomLinker {
                 cpp_compiler,
@@ -174,6 +195,7 @@ impl CppCompilation {
 
                 // Use the custom linker to turn the object file into an executable.
                 let mut cmd = std::process::Command::new(linker);
+                cmd.args(includes);
                 cmd.args([object_file, "-o", output]);
 
                 if let Some(current_dir) = cpp_compiler.get_current_dir() {