Merge pull request bytecodealliance#92 from near/viktar/spectests

Viktar/spectests

Cargo.toml

@@ -255,6 +255,7 @@ cfg-if = "1.0"
 tempfile = "3.1.0"
 filecheck = "0.5.0"
 libc = "0.2.60"
+regex = "1.9.1"
 file-per-thread-logger = "0.2.0"
 tokio = { version = "1.26.0", features = [ "rt", "time" ] }
 hyper = "=1.0.0-rc.3"
@@ -416,4 +417,4 @@ overflow-checks = false
 incremental = false
 debug-assertions = false
 overflow-checks = false
-opt-level = 's'
+opt-level = 's'

ci/spectest-zkasm.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -o pipefail
+set -eux
+
+# NB: This might have false-positives locally, but it's worth it for iteration speed.
+# If you ever run into a situation when modules are out of date, run this command manually.
+if [ ! -d "tests/zkasm/node_modules" ]; then
+	npm install --prefix tests/zkasm
+fi
+TEST_PATH=../../${1:-"cranelift/zkasm_data/spectest/i64/generated"}
+npm test --prefix tests/zkasm $TEST_PATH

cranelift/filetests/Cargo.toml

@@ -39,3 +39,4 @@ cranelift.workspace = true
 env_logger = { workspace = true }
 expect-test = { workspace = true }
 smallvec = { workspace = true }
+regex = { workspace = true }

cranelift/filetests/src/test_zkasm.rs

@@ -2,6 +2,10 @@
 mod tests {
     use std::collections::HashMap;
 
+    use regex::Regex;
+    use std::fs::read_to_string;
+    use std::io::Error;
+
     use cranelift_codegen::entity::EntityRef;
     use cranelift_codegen::ir::function::FunctionParameters;
     use cranelift_codegen::ir::ExternalName;
@@ -175,6 +179,82 @@ mod tests {
         expected.assert_eq(&program);
     }
 
+    // This function asserts that none of tests generated from
+    // spectest has been changed.
+    fn check_spectests() -> Result<(), Error> {
+        let spectests_path = "../../tests/spec_testsuite/i64.wast";
+        let file_content = read_to_string(spectests_path)?;
+        let re = Regex::new(
+            r#"\(assert_return \(invoke \"(\w+)\"\s*((?:\([^\)]+\)\s*)+)\)\s*\(([^\)]+)\)\)"#,
+        )
+        .unwrap();
+        let mut test_counters = HashMap::new();
+        for cap in re.captures_iter(&file_content) {
+            let function_name = &cap[1];
+            let arguments = &cap[2];
+            let expected_result = &cap[3];
+            let assert_type = &expected_result[0..3];
+            let count = test_counters.entry(function_name.to_string()).or_insert(0);
+            *count += 1;
+            let mut testcase = String::new();
+            testcase.push_str(&format!("(module\n (import \"env\" \"assert_eq\" (func $assert_eq (param {}) (param {})))\n (func $main\n", assert_type, assert_type));
+            testcase.push_str(&format!(
+                "\t{}\n",
+                arguments
+                    .replace(") (", "\n\t")
+                    .replace("(", "")
+                    .replace(")", "")
+            ));
+            testcase.push_str(&format!("\ti64.{}\n", function_name));
+            testcase.push_str(&format!(
+                "\t{}\n\tcall $assert_eq)\n (start $main))\n",
+                expected_result.trim()
+            ));
+            let file_name = format!(
+                "../../zkasm_data/spectest/i64/{}_{}.wat",
+                function_name, count
+            );
+            let expected_test = expect_test::expect_file![file_name];
+            expected_test.assert_eq(&testcase);
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn run_spectests() {
+        check_spectests().unwrap();
+        let path = "../zkasm_data/spectest/i64/";
+        let mut failures = 0;
+        let mut count = 0;
+        for entry in std::fs::read_dir(path).expect("Directory not found") {
+            let entry = entry.expect("Failed to read entry");
+            let file_name = entry.file_name();
+            if entry.path().extension().and_then(|s| s.to_str()) != Some("wat") {
+                continue;
+            }
+            if let Some(name) = std::path::Path::new(&file_name)
+                .file_stem()
+                .and_then(|s| s.to_str())
+            {
+                let module_binary =
+                    wat::parse_file(format!("../zkasm_data/spectest/i64/{name}.wat")).unwrap();
+                let expected = expect_test::expect_file![format!(
+                    "../../zkasm_data/spectest/i64/generated/{name}.zkasm"
+                )];
+                let result = std::panic::catch_unwind(|| {
+                    let program = generate_zkasm(&module_binary);
+                    expected.assert_eq(&program);
+                });
+                count += 1;
+                if let Err(err) = result {
+                    failures += 1;
+                    println!("{} fails with {:#?}", name, err);
+                }
+            }
+        }
+        println!("Failed {} spectests out of {}", failures, count);
+    }
+
     macro_rules! testcases {
         { $($name:ident,)* } => {
           $(

cranelift/zkasm_data/spectest/i64/add_1.wat

@@ -0,0 +1,9 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 1
+	i64.const 1
+	i64.add
+	i64.const 2
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/add_2.wat

@@ -0,0 +1,9 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 1
+	i64.const 0
+	i64.add
+	i64.const 1
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/add_3.wat

@@ -0,0 +1,9 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const -1
+	i64.const -1
+	i64.add
+	i64.const -2
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/add_4.wat

@@ -0,0 +1,9 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const -1
+	i64.const 1
+	i64.add
+	i64.const 0
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/add_5.wat

@@ -0,0 +1,9 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0x7fffffffffffffff
+	i64.const 1
+	i64.add
+	i64.const 0x8000000000000000
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/add_6.wat

@@ -0,0 +1,9 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0x8000000000000000
+	i64.const -1
+	i64.add
+	i64.const 0x7fffffffffffffff
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/add_7.wat

@@ -0,0 +1,9 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0x8000000000000000
+	i64.const 0x8000000000000000
+	i64.add
+	i64.const 0
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/add_8.wat

@@ -0,0 +1,9 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0x3fffffff
+	i64.const 1
+	i64.add
+	i64.const 0x40000000
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/and_1.wat

@@ -0,0 +1,9 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 1
+	i64.const 0
+	i64.and
+	i64.const 0
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/and_2.wat

@@ -0,0 +1,9 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0
+	i64.const 1
+	i64.and
+	i64.const 0
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/and_3.wat

@@ -0,0 +1,9 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 1
+	i64.const 1
+	i64.and
+	i64.const 1
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/and_4.wat

@@ -0,0 +1,9 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0
+	i64.const 0
+	i64.and
+	i64.const 0
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/and_5.wat

@@ -0,0 +1,9 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0x7fffffffffffffff
+	i64.const 0x8000000000000000
+	i64.and
+	i64.const 0
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/and_6.wat

@@ -0,0 +1,9 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0x7fffffffffffffff
+	i64.const -1
+	i64.and
+	i64.const 0x7fffffffffffffff
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/and_7.wat

@@ -0,0 +1,9 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0xf0f0ffff
+	i64.const 0xfffff0f0
+	i64.and
+	i64.const 0xf0f0f0f0
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/and_8.wat

@@ -0,0 +1,9 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0xffffffffffffffff
+	i64.const 0xffffffffffffffff
+	i64.and
+	i64.const 0xffffffffffffffff
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/clz_1.wat

@@ -0,0 +1,8 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0xffffffffffffffff
+	i64.clz
+	i64.const 0
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/clz_2.wat

@@ -0,0 +1,8 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0
+	i64.clz
+	i64.const 64
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/clz_3.wat

@@ -0,0 +1,8 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0x00008000
+	i64.clz
+	i64.const 48
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/clz_4.wat

@@ -0,0 +1,8 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0xff
+	i64.clz
+	i64.const 56
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/clz_5.wat

@@ -0,0 +1,8 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0x8000000000000000
+	i64.clz
+	i64.const 0
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/clz_6.wat

@@ -0,0 +1,8 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 1
+	i64.clz
+	i64.const 63
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/clz_7.wat

@@ -0,0 +1,8 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 2
+	i64.clz
+	i64.const 62
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/clz_8.wat

@@ -0,0 +1,8 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0x7fffffffffffffff
+	i64.clz
+	i64.const 1
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/ctz_1.wat

@@ -0,0 +1,8 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const -1
+	i64.ctz
+	i64.const 0
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/ctz_2.wat

@@ -0,0 +1,8 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0
+	i64.ctz
+	i64.const 64
+	call $assert_eq)
+ (start $main))

cranelift/zkasm_data/spectest/i64/ctz_3.wat

@@ -0,0 +1,8 @@
+(module
+ (import "env" "assert_eq" (func $assert_eq (param i64) (param i64)))
+ (func $main
+	i64.const 0x00008000
+	i64.ctz
+	i64.const 15
+	call $assert_eq)
+ (start $main))