test(tinywasm): create wasm spec test harness

Signed-off-by: Henry Gressmann <mail@henrygressmann.de>

Author: explodingcamera

Cargo.lock

@@ -46,7 +46,7 @@ pub(crate) fn convert_module_code(
     }
 
     let body_reader = func.get_operators_reader()?;
-    let body = process_operators(body_reader.into_iter(), validator)?;
+    let body = process_operators(body_reader.original_position(), body_reader.into_iter(), validator)?;
 
     Ok(CodeSection {
         locals: locals.into_boxed_slice(),
@@ -111,12 +111,13 @@ pub(crate) fn convert_memarg(memarg: wasmparser::MemArg) -> MemArg {
 }
 
 pub fn process_operators<'a>(
+    offset: usize,
     ops: impl Iterator<Item = Result<wasmparser::Operator<'a>, wasmparser::BinaryReaderError>>,
     mut validator: FuncValidator<ValidatorResources>,
 ) -> Result<Box<[Instruction]>> {
     let mut instructions = Vec::new();
 
-    let mut offset = 0;
+    let mut offset = offset.into();
     for op in ops {
         let op = op?;
         validator.op(offset, &op)?;

crates/parser/src/conversion.rs

@@ -17,6 +17,11 @@ log={version="0.4", optional=true}
 tinywasm-parser={version="0.0.3", path="../parser", default-features=false, optional=true}
 tinywasm-types={version="0.0.3", path="../types", default-features=false}
 
+[dev-dependencies]
+wasm-testsuite={path="../wasm-testsuite"}
+wast={version="69.0"}
+owo-colors={version="3.5"}
+
 [features]
 default=["std", "parser", "logging"]
 logging=["log", "tinywasm-types/logging", "tinywasm-parser?/logging"]

crates/tinywasm/Cargo.toml

@@ -0,0 +1,126 @@
+use std::{
+    collections::HashMap,
+    fmt::{Debug, Formatter},
+};
+
+use tinywasm::{Error, Result};
+use tinywasm_types::TinyWasmModule;
+use wast::{
+    lexer::Lexer,
+    parser::{self, ParseBuffer},
+    QuoteWat, Wast,
+};
+
+fn parse_module(mut module: wast::core::Module) -> Result<TinyWasmModule, Error> {
+    let parser = tinywasm_parser::Parser::new();
+    Ok(parser.parse_module_bytes(module.encode().expect("failed to encode module"))?)
+}
+
+#[test]
+#[ignore]
+fn test_mvp() {
+    let mut test_suite = TestSuite::new();
+
+    wasm_testsuite::MVP_TESTS.iter().for_each(|name| {
+        println!("test: {}", name);
+
+        let test_group = test_suite.test_group("mvp");
+
+        let wast = wasm_testsuite::get_test_wast(name).expect("failed to get test wast");
+        let wast = std::str::from_utf8(&wast).expect("failed to convert wast to utf8");
+
+        let mut lexer = Lexer::new(&wast);
+        lexer.allow_confusing_unicode(true);
+
+        let buf = ParseBuffer::new_with_lexer(lexer).expect("failed to create parse buffer");
+        let wast_data = parser::parse::<Wast>(&buf).expect("failed to parse wat");
+
+        for directive in wast_data.directives {
+            let span = directive.span();
+
+            use wast::WastDirective::*;
+            match directive {
+                Wat(QuoteWat::Wat(wast::Wat::Module(module))) => {
+                    let module = parse_module(module).map(|_| ());
+                    test_group.module_compiles(name, span, module);
+                }
+                _ => {}
+            }
+        }
+    });
+
+    if test_suite.failed() {
+        panic!("failed one or more tests: {:#?}", test_suite);
+    }
+}
+
+struct TestSuite(HashMap<String, TestGroup>);
+
+impl TestSuite {
+    fn new() -> Self {
+        Self(HashMap::new())
+    }
+
+    fn failed(&self) -> bool {
+        self.0.values().any(|group| group.failed())
+    }
+
+    fn test_group(&mut self, name: &str) -> &mut TestGroup {
+        self.0.entry(name.to_string()).or_insert_with(TestGroup::new)
+    }
+}
+
+impl Debug for TestSuite {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        use owo_colors::OwoColorize;
+        let mut passed_count = 0;
+        let mut failed_count = 0;
+
+        for (group_name, group) in &self.0 {
+            writeln!(f, "{}", group_name.bold().underline())?;
+            for (test_name, test) in &group.tests {
+                writeln!(f, "  {}", test_name.bold())?;
+                match test.result {
+                    Ok(()) => {
+                        writeln!(f, "    Result: {}", "Passed".green())?;
+                        passed_count += 1;
+                    }
+                    Err(_) => {
+                        writeln!(f, "    Result: {}", "Failed".red())?;
+                        failed_count += 1;
+                    }
+                }
+                writeln!(f, "    Span: {:?}", test.span)?;
+            }
+        }
+
+        writeln!(f, "\n{}", "Test Summary:".bold().underline())?;
+        writeln!(f, "  Total Tests: {}", (passed_count + failed_count))?;
+        writeln!(f, "  Passed: {}", passed_count.to_string().green())?;
+        writeln!(f, "  Failed: {}", failed_count.to_string().red())?;
+        Ok(())
+    }
+}
+
+struct TestGroup {
+    tests: HashMap<String, TestCase>,
+}
+
+impl TestGroup {
+    fn new() -> Self {
+        Self { tests: HashMap::new() }
+    }
+
+    fn failed(&self) -> bool {
+        self.tests.values().any(|test| test.result.is_err())
+    }
+
+    fn module_compiles(&mut self, name: &str, span: wast::token::Span, result: Result<()>) {
+        self.tests.insert(name.to_string(), TestCase { result, span });
+    }
+}
+
+struct TestCase {
+    result: Result<()>,
+    span: wast::token::Span,
+}

crates/tinywasm/tests/mvp.rs

@@ -30,12 +30,12 @@ pub const PROPOSALS: &[&str] = &["annotations", "exception-handling", "memory64"
 
 /// List of all tests that apply to the MVP (V1) spec.
 /// Note that the tests are still for the latest spec, so the latest version of Wast is used.
-#[rustfmt::skip] 
-pub const MVP_TESTS: &[&str] = &["address.wast","address.wast","align.wast","align.wast","binary-leb128.wast","binary-leb128.wast","binary.wast","binary.wast","block.wast","block.wast","br.wast","br.wast","br_if.wast","br_if.wast","br_table.wast","br_table.wast","break-drop.wast","break-drop.wast","call.wast","call.wast","call_indirect.wast","call_indirect.wast","comments.wast","comments.wast","const.wast","const.wast","conversions.wast","conversions.wast","custom.wast","custom.wast","data.wast","data.wast","elem.wast","elem.wast","endianness.wast","endianness.wast","exports.wast","exports.wast","f32.wast","f32.wast","f32_bitwise.wast","f32_bitwise.wast","f32_cmp.wast","f32_cmp.wast","f64.wast","f64.wast","f64_bitwise.wast","f64_bitwise.wast","f64_cmp.wast","f64_cmp.wast","fac.wast","fac.wast","float_exprs.wast","float_exprs.wast","float_literals.wast","float_literals.wast","float_memory.wast","float_memory.wast","float_misc.wast","float_misc.wast","forward.wast","forward.wast","func.wast","func.wast","func_ptrs.wast","func_ptrs.wast","globals.wast","globals.wast","i32.wast","i32.wast","i64.wast","i64.wast","if.wast","if.wast","imports.wast","imports.wast","inline-module.wast","inline-module.wast","int_exprs.wast","int_exprs.wast","int_literals.wast","int_literals.wast","labels.wast","labels.wast","left-to-right.wast","left-to-right.wast","linking.wast","linking.wast","load.wast","load.wast","local_get.wast","local_get.wast","local_set.wast","local_set.wast","local_tee.wast","local_tee.wast","loop.wast","loop.wast","memory.wast","memory.wast","memory_grow.wast","memory_grow.wast","memory_redundancy.wast","memory_redundancy.wast","memory_size.wast","memory_size.wast","memory_trap.wast","memory_trap.wast","names.wast","names.wast","nop.wast","nop.wast","return.wast","return.wast","select.wast","select.wast","skip-stack-guard-page.wast","skip-stack-guard-page.wast","stack.wast","stack.wast","start.wast","start.wast","store.wast","store.wast","switch.wast","switch.wast","token.wast","token.wast","traps.wast","traps.wast","type.wast","type.wast","unreachable.wast","unreachable.wast","unreached-invalid.wast","unreached-invalid.wast","unwind.wast","unwind.wast","utf8-custom-section-id.wast","utf8-custom-section-id.wast","utf8-import-field.wast","utf8-import-field.wast","utf8-import-module.wast","utf8-import-module.wast","utf8-invalid-encoding.wast","utf8-invalid-encoding.wast"];
+#[rustfmt::skip]  // removed: "break-drop.wast",
+pub const MVP_TESTS: &[&str] = &["address.wast","align.wast","binary-leb128.wast","binary.wast","block.wast","br.wast","br_if.wast","br_table.wast","call.wast","call_indirect.wast","comments.wast","const.wast","conversions.wast","custom.wast","data.wast","elem.wast","endianness.wast","exports.wast","f32.wast","f32_bitwise.wast","f32_cmp.wast","f64.wast","f64_bitwise.wast","f64_cmp.wast","fac.wast","float_exprs.wast","float_literals.wast","float_memory.wast","float_misc.wast","forward.wast","func.wast","func_ptrs.wast","global.wast","i32.wast","i64.wast","if.wast","imports.wast","inline-module.wast","int_exprs.wast","int_literals.wast","labels.wast","left-to-right.wast","linking.wast","load.wast","local_get.wast","local_set.wast","local_tee.wast","loop.wast","memory.wast","memory_grow.wast","memory_redundancy.wast","memory_size.wast","memory_trap.wast","names.wast","nop.wast","return.wast","select.wast","skip-stack-guard-page.wast","stack.wast","start.wast","store.wast","switch.wast","token.wast","traps.wast","type.wast","unreachable.wast","unreached-invalid.wast","unwind.wast","utf8-custom-section-id.wast","utf8-import-field.wast","utf8-import-module.wast","utf8-invalid-encoding.wast"];
 
 /// List of all tests that apply to the V2 draft 1 spec.
 #[rustfmt::skip]
-pub const V2_DRAFT_1_TESTS: &[&str] = &["address.wast","address.wast","align.wast","align.wast","binary-leb128.wast","binary-leb128.wast","binary.wast","binary.wast","block.wast","block.wast","br.wast","br.wast","br_if.wast","br_if.wast","br_table.wast","br_table.wast","bulk.wast","bulk.wast","call.wast","call.wast","call_indirect.wast","call_indirect.wast","comments.wast","comments.wast","const.wast","const.wast","conversions.wast","conversions.wast","custom.wast","custom.wast","data.wast","data.wast","elem.wast","elem.wast","endianness.wast","endianness.wast","exports.wast","exports.wast","f32.wast","f32.wast","f32_bitwise.wast","f32_bitwise.wast","f32_cmp.wast","f32_cmp.wast","f64.wast","f64.wast","f64_bitwise.wast","f64_bitwise.wast","f64_cmp.wast","f64_cmp.wast","fac.wast","fac.wast","float_exprs.wast","float_exprs.wast","float_literals.wast","float_literals.wast","float_memory.wast","float_memory.wast","float_misc.wast","float_misc.wast","forward.wast","forward.wast","func.wast","func.wast","func_ptrs.wast","func_ptrs.wast","global.wast","global.wast","i32.wast","i32.wast","i64.wast","i64.wast","if.wast","if.wast","imports.wast","imports.wast","inline-module.wast","inline-module.wast","int_exprs.wast","int_exprs.wast","int_literals.wast","int_literals.wast","labels.wast","labels.wast","left-to-right.wast","left-to-right.wast","linking.wast","linking.wast","load.wast","load.wast","local_get.wast","local_get.wast","local_set.wast","local_set.wast","local_tee.wast","local_tee.wast","loop.wast","loop.wast","memory.wast","memory.wast","memory_copy.wast","memory_copy.wast","memory_fill.wast","memory_fill.wast","memory_grow.wast","memory_grow.wast","memory_init.wast","memory_init.wast","memory_redundancy.wast","memory_redundancy.wast","memory_size.wast","memory_size.wast","memory_trap.wast","memory_trap.wast","names.wast","names.wast","nop.wast","nop.wast","ref_func.wast","ref_func.wast","ref_is_null.wast","ref_is_null.wast","ref_null.wast","ref_null.wast","return.wast","return.wast","select.wast","select.wast","skip-stack-guard-page.wast","skip-stack-guard-page.wast","stack.wast","stack.wast","start.wast","start.wast","store.wast","store.wast","switch.wast","switch.wast","table-sub.wast","table-sub.wast","table.wast","table.wast","table_copy.wast","table_copy.wast","table_fill.wast","table_fill.wast","table_get.wast","table_get.wast","table_grow.wast","table_grow.wast","table_init.wast","table_init.wast","table_set.wast","table_set.wast","table_size.wast","table_size.wast","token.wast","token.wast","traps.wast","traps.wast","type.wast","type.wast","unreachable.wast","unreachable.wast","unreached-invalid.wast","unreached-invalid.wast","unreached-valid.wast","unreached-valid.wast","unwind.wast","unwind.wast","utf8-custom-section-id.wast","utf8-custom-section-id.wast","utf8-import-field.wast","utf8-import-field.wast","utf8-import-module.wast","utf8-import-module.wast","utf8-invalid-encoding.wast","utf8-invalid-encoding.wast"];
+pub const V2_DRAFT_1_TESTS: &[&str] = &["address.wast","align.wast","binary-leb128.wast","binary.wast","block.wast","br.wast","br_if.wast","br_table.wast","bulk.wast","call.wast","call_indirect.wast","comments.wast","const.wast","conversions.wast","custom.wast","data.wast","elem.wast","endianness.wast","exports.wast","f32.wast","f32_bitwise.wast","f32_cmp.wast","f64.wast","f64_bitwise.wast","f64_cmp.wast","fac.wast","float_exprs.wast","float_literals.wast","float_memory.wast","float_misc.wast","forward.wast","func.wast","func_ptrs.wast","global.wast","i32.wast","i64.wast","if.wast","imports.wast","inline-module.wast","int_exprs.wast","int_literals.wast","labels.wast","left-to-right.wast","linking.wast","load.wast","local_get.wast","local_set.wast","local_tee.wast","loop.wast","memory.wast","memory_copy.wast","memory_fill.wast","memory_grow.wast","memory_init.wast","memory_redundancy.wast","memory_size.wast","memory_trap.wast","names.wast","nop.wast","ref_func.wast","ref_is_null.wast","ref_null.wast","return.wast","select.wast","skip-stack-guard-page.wast","stack.wast","start.wast","store.wast","switch.wast","table-sub.wast","table.wast","table_copy.wast","table_fill.wast","table_get.wast","table_grow.wast","table_init.wast","table_set.wast","table_size.wast","token.wast","traps.wast","type.wast","unreachable.wast","unreached-invalid.wast","unreached-valid.wast","unwind.wast","utf8-custom-section-id.wast","utf8-import-field.wast","utf8-import-module.wast","utf8-invalid-encoding.wast"];
 
 /// Get all test file names and their contents.
 pub fn get_tests_wast(include_proposals: &[String]) -> impl Iterator<Item = (String, Cow<'static, [u8]>)> {