Combining C files is very effective

Author: folkertdev

Cargo.lock

@@ -5,7 +5,8 @@ members = [
   "examples",
 ]
 exclude = [
-  "crates/wasm-assert-instr-tests"
+  "crates/wasm-assert-instr-tests",
+  "rust_programs"
 ]
 
 [profile.release]

Cargo.toml

@@ -169,7 +169,7 @@ esac
 # Arm specific
 case "${TARGET}" in
     aarch64-unknown-linux-gnu*|armv7-unknown-linux-gnueabihf*)
-        CPPFLAGS="${TEST_CPPFLAGS}" RUSTFLAGS="${HOST_RUSTFLAGS}" RUST_LOG=warn \
+        CPPFLAGS="${TEST_CPPFLAGS}" RUSTFLAGS="${HOST_RUSTFLAGS}" RUST_LOG=trace \
             cargo run "${INTRINSIC_TEST}" "${PROFILE}" \
             --bin intrinsic-test -- intrinsics_data/arm_intrinsics.json \
             --runner "${TEST_RUNNER}" \
@@ -179,7 +179,7 @@ case "${TARGET}" in
         ;;
 
     aarch64_be-unknown-linux-gnu*)
-        CPPFLAGS="${TEST_CPPFLAGS}" RUSTFLAGS="${HOST_RUSTFLAGS}" RUST_LOG=warn \
+        CPPFLAGS="${TEST_CPPFLAGS}" RUSTFLAGS="${HOST_RUSTFLAGS}" RUST_LOG=trace \
             cargo run "${INTRINSIC_TEST}" "${PROFILE}"  \
             --bin intrinsic-test -- intrinsics_data/arm_intrinsics.json \
             --runner "${TEST_RUNNER}" \

ci/run.sh

@@ -11,7 +11,6 @@ license = "MIT OR Apache-2.0"
 edition = "2024"
 
 [dependencies]
-lazy_static = "1.4.0"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1.0"
 csv = "1.1"

crates/intrinsic-test/Cargo.toml

@@ -1,11 +1,11 @@
 use crate::common::compile_c::CompilationCommandBuilder;
-use crate::common::gen_c::compile_c_programs;
 
 pub fn compile_c_arm(
-    intrinsics_name_list: &[String],
     compiler: &str,
     target: &str,
     cxx_toolchain_dir: Option<&str>,
+    inputs: &[String],
+    output: Option<&str>,
 ) -> bool {
     // -ffp-contract=off emulates Rust's approach of not fusing separate mul-add operations
     let mut command = CompilationCommandBuilder::new()
@@ -49,16 +49,52 @@ pub fn compile_c_arm(
         command = command.add_extra_flag("-flax-vector-conversions");
     }
 
-    let compiler_commands = intrinsics_name_list
-        .iter()
-        .map(|intrinsic_name| {
-            command
-                .clone()
-                .set_input_name(intrinsic_name)
-                .set_output_name(intrinsic_name)
-                .make_string()
-        })
-        .collect::<Vec<_>>();
+    let mut command = command.into_command();
+    command.command_mut().current_dir("c_programs");
 
-    compile_c_programs(&compiler_commands)
+    for input in inputs {
+        assert!(
+            std::path::Path::new("c_programs").join(input).exists(),
+            "{}",
+            input
+        );
+    }
+    command.command_mut().args(inputs);
+
+    if let Some(output) = output {
+        trace!("running {compiler} to produce {output}");
+        if output.ends_with(".o") {
+            command.command_mut().arg("-c");
+        }
+        command.command_mut().args(["-o", output]);
+    } else {
+        trace!("running {compiler}");
+    }
+
+    trace!("running {compiler}\n{:?}", &command);
+
+    if log::log_enabled!(log::Level::Trace) {
+        command.command_mut().stdout(std::process::Stdio::inherit());
+        command.command_mut().stderr(std::process::Stdio::inherit());
+    }
+
+    let output = command.output();
+
+    trace!("{compiler} is done");
+
+    if let Ok(output) = output {
+        if output.status.success() {
+            true
+        } else {
+            error!(
+                "Failed to compile code for intrinsics: \n\nstdout:\n{}\n\nstderr:\n{}",
+                std::str::from_utf8(&output.stdout).unwrap_or(""),
+                std::str::from_utf8(&output.stderr).unwrap_or("")
+            );
+            false
+        }
+    } else {
+        error!("Command failed: {output:#?}");
+        false
+    }
 }

crates/intrinsic-test/src/arm/compile.rs

@@ -8,6 +8,7 @@ mod intrinsic;
 mod json_parser;
 mod types;
 
+use crate::arm::compile::compile_c_arm;
 use crate::common::SupportedArchitectureTest;
 use crate::common::cli::ProcessedCli;
 use crate::common::compare::compare_outputs;
@@ -54,7 +55,7 @@ impl SupportedArchitectureTest for ArmArchitectureTest {
     }
 
     fn build_c_file(&self) -> bool {
-        let compiler = self.cli_options.cpp_compiler.as_deref();
+        let compiler = self.cli_options.cpp_compiler.as_deref().unwrap();
         let target = &self.cli_options.target;
         let cxx_toolchain_dir = self.cli_options.cxx_toolchain_dir.as_deref();
         let c_target = "aarch64";
@@ -69,36 +70,24 @@ impl SupportedArchitectureTest for ArmArchitectureTest {
             .map(|(i, chunk)| {
                 let c_filename = format!("c_programs/mod_{i}.cpp");
                 let mut file = File::create(&c_filename).unwrap();
-                write_mod_cpp(&mut file, &notice, chunk).unwrap();
+                write_mod_cpp(&mut file, notice, c_target, chunk).unwrap();
 
                 // compile this cpp file into a .o file
 
-                // clang++ -march=armv8.6-a+crypto+crc+dotprod+fp16+faminmax+lut+sha3 -O2 -ffp-contract=off -Wno-narrowing --target=aarch64-unknown-linux-gnu c_file_0.c -c
-                let mut cmd = std::process::Command::new("clang++");
-                cmd.current_dir("c_programs");
-
-                cmd.arg("-march=armv8.6-a+crypto+crc+dotprod+fp16+faminmax+lut+sha3");
-                cmd.arg("-O2");
-                cmd.arg("-ffp-contract=off");
-                cmd.arg("-Wno-narrowing");
-                cmd.arg("--target=aarch64-unknown-linux-gnu");
-                cmd.arg("-c");
-                cmd.arg(format!("mod_{i}.cpp"));
-
-                let output = cmd.output();
-                eprintln!(
-                    "{}",
-                    String::from_utf8_lossy(&output.as_ref().unwrap().stderr)
+                compile_c_arm(
+                    compiler,
+                    target,
+                    cxx_toolchain_dir,
+                    &[format!("mod_{i}.cpp")],
+                    Some(&format!("mod_{i}.o")),
                 );
-                assert!(output.unwrap().status.success());
 
                 Ok(())
             })
             .collect::<Result<(), std::io::Error>>()
             .unwrap();
 
-        let c_filename = format!("c_programs/main.cpp");
-        let mut file = File::create(&c_filename).unwrap();
+        let mut file = File::create("c_programs/main.cpp").unwrap();
         write_main_cpp(
             &mut file,
             c_target,
@@ -107,38 +96,18 @@ impl SupportedArchitectureTest for ArmArchitectureTest {
         )
         .unwrap();
 
-        let mut cmd = std::process::Command::new("clang++");
-        cmd.current_dir("c_programs");
-
-        cmd.arg("-march=armv8.6-a+crypto+crc+dotprod+fp16+faminmax+lut+sha3");
-        cmd.arg("-O2");
-        cmd.arg("-ffp-contract=off");
-        cmd.arg("-Wno-narrowing");
-        cmd.arg("--target=aarch64-unknown-linux-gnu");
-        cmd.arg(format!("main.cpp"));
+        let mut inputs = vec![format!("main.cpp")];
         for i in 0..Ord::min(available_parallelism, self.intrinsics.len()) {
-            cmd.arg(format!("mod_{i}.o"));
+            inputs.push(format!("mod_{i}.o"));
         }
-        cmd.args(&["-o", "intrinsic-test-programs"]);
 
-        let output = cmd.output();
-        eprintln!(
-            "{}",
-            String::from_utf8_lossy(&output.as_ref().unwrap().stderr)
-        );
-        assert!(output.unwrap().status.success());
-
-        //        match compiler {
-        //            None => true,
-        //            Some(compiler) => compile_c_arm(
-        //                intrinsics_name_list.unwrap().as_slice(),
-        //                compiler,
-        //                target,
-        //                cxx_toolchain_dir,
-        //            ),
-        //        }
-
-        true
+        compile_c_arm(
+            compiler,
+            target,
+            cxx_toolchain_dir,
+            &inputs,
+            Some("intrinsic-test-programs"),
+        )
     }
 
     fn build_rust_file(&self) -> bool {
@@ -171,14 +140,23 @@ impl SupportedArchitectureTest for ArmArchitectureTest {
         let toolchain = self.cli_options.toolchain.as_deref();
         let linker = self.cli_options.linker.as_deref();
 
+        warn!(
+            "available parallelism: {:?} {}",
+            std::thread::available_parallelism(),
+            rayon::current_num_threads(),
+        );
+
         let notice = &build_notices("// ");
         self.intrinsics
             .par_chunks(chunk_size)
             .enumerate()
             .map(|(i, chunk)| {
                 use std::io::Write;
 
+                dbg!(chunk_size, chunk.len());
+
                 let rust_filename = format!("rust_programs/src/mod_{i}.rs");
+                trace!("generating `{rust_filename}`");
                 let mut file = File::create(rust_filename).unwrap();
 
                 write!(file, "{notice}")?;

crates/intrinsic-test/src/arm/mod.rs

@@ -90,11 +90,10 @@ impl ProcessedCli {
             (None, None)
         } else {
             (
-                Some(
-                    cli_options
-                        .toolchain
-                        .map_or_else(String::new, |t| format!("+{t}")),
-                ),
+                match cli_options.toolchain {
+                    Some(t) => Some(format!("+{t}")),
+                    None => Some(String::new()), // NOTE this is confusing
+                },
                 Some(cli_options.cppcompiler),
             )
         };

crates/intrinsic-test/src/common/cli.rs

@@ -23,7 +23,6 @@ pub fn compare_outputs(intrinsic_name_list: &Vec<String>, runner: &str, target:
             let rust = runner_command(runner)
                 .arg(format!("target/{target}/release/intrinsic-test-programs"))
                 .arg(intrinsic_name)
-                .current_dir("rust_programs")
                 .output();
 
             let (c, rust) = match (c, rust) {
@@ -33,19 +32,19 @@ pub fn compare_outputs(intrinsic_name_list: &Vec<String>, runner: &str, target:
 
             if !c.status.success() {
                 error!(
-                    "Failed to run C program for intrinsic {intrinsic_name}\nstdout: {stdout}\nstderr: {stderr}",
+                    "Failed to run C program for intrinsic `{intrinsic_name}`\nstdout: {stdout}\nstderr: {stderr}",
                     stdout = std::str::from_utf8(&c.stdout).unwrap_or(""),
                     stderr = std::str::from_utf8(&c.stderr).unwrap_or(""),
-                );
+                    );
                 return Some(FailureReason::RunC(intrinsic_name.clone()));
             }
 
             if !rust.status.success() {
                 error!(
-                    "Failed to run Rust program for intrinsic {intrinsic_name}\nstdout: {stdout}\nstderr: {stderr}",
+                    "Failed to run Rust program for intrinsic `{intrinsic_name}`\nstdout: {stdout}\nstderr: {stderr}",
                     stdout = String::from_utf8_lossy(&rust.stdout),
                     stderr = String::from_utf8_lossy(&rust.stderr),
-                );
+                    );
                 return Some(FailureReason::RunRust(intrinsic_name.clone()));
             }