diff options
Diffstat (limited to 'crates')
| -rw-r--r-- | crates/tinywasm/tests/mvp.csv | 1 | ||||
| -rw-r--r-- | crates/tinywasm/tests/testsuite/mod.rs | 149 | ||||
| -rw-r--r-- | crates/tinywasm/tests/testsuite/run.rs | 128 | ||||
| -rw-r--r-- | crates/tinywasm/tests/testsuite/util.rs | 72 |
4 files changed, 350 insertions, 0 deletions
diff --git a/crates/tinywasm/tests/mvp.csv b/crates/tinywasm/tests/mvp.csv new file mode 100644 index 0000000..ed11ae2 --- /dev/null +++ b/crates/tinywasm/tests/mvp.csv @@ -0,0 +1 @@ +0.0.4,9258,7567,[{"name":"0","passed":0,"failed":54},{"name":"0","passed":0,"failed":109},{"name":"66","passed":66,"failed":25},{"name":"104","passed":104,"failed":8},{"name":"0","passed":0,"failed":171},{"name":"0","passed":0,"failed":21},{"name":"0","passed":0,"failed":30},{"name":"0","passed":0,"failed":25},{"name":"0","passed":0,"failed":22},{"name":"0","passed":0,"failed":56},{"name":"4","passed":4,"failed":4},{"name":"702","passed":702,"failed":76},{"name":"0","passed":0,"failed":93},{"name":"10","passed":10,"failed":1},{"name":"0","passed":0,"failed":61},{"name":"0","passed":0,"failed":76},{"name":"0","passed":0,"failed":1},{"name":"21","passed":21,"failed":73},{"name":"1005","passed":1005,"failed":1509},{"name":"1","passed":1,"failed":363},{"name":"2401","passed":2401,"failed":6},{"name":"1005","passed":1005,"failed":1509},{"name":"1","passed":1,"failed":363},{"name":"2401","passed":2401,"failed":6},{"name":"0","passed":0,"failed":2},{"name":"269","passed":269,"failed":591},{"name":"34","passed":34,"failed":129},{"name":"0","passed":0,"failed":6},{"name":"138","passed":138,"failed":303},{"name":"1","passed":1,"failed":4},{"name":"4","passed":4,"failed":75},{"name":"0","passed":0,"failed":16},{"name":"4","passed":4,"failed":49},{"name":"0","passed":0,"failed":96},{"name":"0","passed":0,"failed":42},{"name":"0","passed":0,"failed":118},{"name":"1","passed":1,"failed":156},{"name":"0","passed":0,"failed":1},{"name":"38","passed":38,"failed":70},{"name":"5","passed":5,"failed":46},{"name":"1","passed":1,"failed":28},{"name":"0","passed":0,"failed":1},{"name":"1","passed":1,"failed":66},{"name":"0","passed":0,"failed":60},{"name":"2","passed":2,"failed":34},{"name":"5","passed":5,"failed":48},{"name":"0","passed":0,"failed":42},{"name":"0","passed":0,"failed":43},{"name":"0","passed":0,"failed":34},{"name":"0","passed":0,"failed":19},{"name":"0","passed":0,"failed":1},{"name":"0","passed":0,"failed":6},{"name":"0","passed":0,"failed":172},{"name":"484","passed":484,"failed":1},{"name":"0","passed":0,"failed":5},{"name":"0","passed":0,"failed":21},{"name":"0","passed":0,"failed":32},{"name":"0","passed":0,"failed":11},{"name":"0","passed":0,"failed":2},{"name":"0","passed":0,"failed":10},{"name":"0","passed":0,"failed":59},{"name":"1","passed":1,"failed":27},{"name":"16","passed":16,"failed":42},{"name":"3","passed":3,"failed":33},{"name":"1","passed":1,"failed":2},{"name":"0","passed":0,"failed":59},{"name":"0","passed":0,"failed":118},{"name":"1","passed":1,"failed":49},{"name":"176","passed":176,"failed":0},{"name":"176","passed":176,"failed":0},{"name":"176","passed":176,"failed":0},{"name":"0","passed":0,"failed":176}] diff --git a/crates/tinywasm/tests/testsuite/mod.rs b/crates/tinywasm/tests/testsuite/mod.rs new file mode 100644 index 0000000..02c6d5a --- /dev/null +++ b/crates/tinywasm/tests/testsuite/mod.rs @@ -0,0 +1,149 @@ +use eyre::Result; +use std::io::{BufRead, Seek, SeekFrom}; +use std::{ + collections::BTreeMap, + fmt::{Debug, Formatter}, + io::BufReader, +}; + +mod run; +mod util; + +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +struct TestGroupResult { + name: String, + passed: usize, + failed: usize, +} + +pub struct TestSuite(BTreeMap<String, TestGroup>); + +impl TestSuite { + pub fn new() -> Self { + Self(BTreeMap::new()) + } + + pub fn failed(&self) -> bool { + self.0.values().any(|group| group.stats().1 > 0) + } + + fn test_group(&mut self, name: &str) -> &mut TestGroup { + self.0.entry(name.to_string()).or_insert_with(TestGroup::new) + } + + // create or add to a test result file + pub fn save_csv(&self, path: &str, version: &str) -> Result<()> { + use std::fs::OpenOptions; + use std::io::Write; + + let mut file = OpenOptions::new().create(true).append(true).read(true).open(path)?; + let last_line = BufReader::new(&file).lines().last().transpose()?; + + // Check if the last line starts with the current commit + if let Some(last) = last_line { + if last.starts_with(version) { + // Truncate the file size to remove the last line + let len_to_truncate = last.len() as i64; + file.set_len(file.metadata()?.len() - len_to_truncate as u64)?; + } + } + + // Seek to the end of the file for appending + file.seek(SeekFrom::End(0))?; + + let mut passed = 0; + let mut failed = 0; + + let mut groups = Vec::new(); + for group in self.0.values() { + let (group_passed, group_failed) = group.stats(); + passed += group_passed; + failed += group_failed; + + groups.push(TestGroupResult { + name: group_passed.to_string(), + passed: group_passed, + failed: group_failed, + }); + } + + let groups = serde_json::to_string(&groups)?; + let line = format!("{},{},{},{}\n", version, passed, failed, groups); + file.write_all(line.as_bytes()).expect("failed to write to csv file"); + + Ok(()) + } +} + +impl Debug for TestSuite { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + use owo_colors::OwoColorize; + let mut total_passed = 0; + let mut total_failed = 0; + + for (group_name, group) in &self.0 { + let (group_passed, group_failed) = group.stats(); + total_passed += group_passed; + total_failed += group_failed; + + writeln!(f, "{}", group_name.bold().underline())?; + writeln!(f, " Tests Passed: {}", group_passed.to_string().green())?; + writeln!(f, " Tests Failed: {}", group_failed.to_string().red())?; + + // for (test_name, test) in &group.tests { + // write!(f, " {}: ", test_name.bold())?; + // match &test.result { + // Ok(()) => { + // writeln!(f, "{}", "Passed".green())?; + // } + // Err(e) => { + // writeln!(f, "{}", "Failed".red())?; + // // writeln!(f, "Error: {:?}", e)?; + // } + // } + // writeln!(f, " Span: {:?}", test.span)?; + // } + } + + writeln!(f, "\n{}", "Total Test Summary:".bold().underline())?; + writeln!(f, " Total Tests: {}", (total_passed + total_failed))?; + writeln!(f, " Total Passed: {}", total_passed.to_string().green())?; + writeln!(f, " Total Failed: {}", total_failed.to_string().red())?; + Ok(()) + } +} + +struct TestGroup { + tests: BTreeMap<String, TestCase>, +} + +impl TestGroup { + fn new() -> Self { + Self { tests: BTreeMap::new() } + } + + fn stats(&self) -> (usize, usize) { + let mut passed_count = 0; + let mut failed_count = 0; + + for test in self.tests.values() { + match test.result { + Ok(()) => passed_count += 1, + Err(_) => failed_count += 1, + } + } + + (passed_count, failed_count) + } + + fn add_result(&mut self, name: &str, span: wast::token::Span, result: Result<()>) { + self.tests.insert(name.to_string(), TestCase { result, _span: span }); + } +} + +struct TestCase { + result: Result<()>, + _span: wast::token::Span, +} diff --git a/crates/tinywasm/tests/testsuite/run.rs b/crates/tinywasm/tests/testsuite/run.rs new file mode 100644 index 0000000..42dd825 --- /dev/null +++ b/crates/tinywasm/tests/testsuite/run.rs @@ -0,0 +1,128 @@ +use crate::testsuite::util::{parse_module, wastarg2tinywasmvalue, wastret2tinywasmvalue}; + +use super::TestSuite; +use eyre::{eyre, Result}; +use log::debug; +use tinywasm_types::TinyWasmModule; +use wast::{lexer::Lexer, parser::ParseBuffer, QuoteWat, Wast}; + +impl TestSuite { + pub fn run(&mut self, tests: &[&str]) -> Result<()> { + tests.iter().for_each(|group| { + let test_group = self.test_group(group); + + let wast = wasm_testsuite::get_test_wast(group).expect("failed to get test wast"); + let wast = std::str::from_utf8(&wast).expect("failed to convert wast to utf8"); + + let mut lexer = Lexer::new(wast); + // we need to allow confusing unicode characters since they are technically valid wasm + lexer.allow_confusing_unicode(true); + + let buf = ParseBuffer::new_with_lexer(lexer).expect("failed to create parse buffer"); + let wast_data = wast::parser::parse::<Wast>(&buf).expect("failed to parse wat"); + + let mut last_module: Option<TinyWasmModule> = None; + for (i, directive) in wast_data.directives.into_iter().enumerate() { + let span = directive.span(); + use wast::WastDirective::*; + let name = format!("{}-{}", group, i); + + match directive { + // TODO: needs to support more binary sections + Wat(QuoteWat::Wat(wast::Wat::Module(module))) => { + let result = std::panic::catch_unwind(|| parse_module(module)) + .map_err(|e| eyre!("failed to parse module: {:?}", e)) + .and_then(|res| res); + + match &result { + Err(_) => last_module = None, + Ok(m) => last_module = Some(m.clone()), + } + + test_group.add_result(&format!("{}-parse", name), span, result.map(|_| ())); + } + + // these all pass already :) + AssertMalformed { + span, + module: QuoteWat::Wat(wast::Wat::Module(module)), + message: _, + } => { + let res = std::panic::catch_unwind(|| parse_module(module).map(|_| ())); + test_group.add_result( + &format!("{}-malformed", name), + span, + match res { + Ok(Ok(_)) => Err(eyre!("expected module to be malformed")), + Err(_) | Ok(Err(_)) => Ok(()), + }, + ); + } + + AssertReturn { span, exec, results } => { + let Some(module) = last_module.as_ref() else { + // we skip tests for modules that failed to parse + println!("no module found for assert_return: {:?}", exec); + continue; + }; + + let res: Result<Result<()>, _> = std::panic::catch_unwind(|| { + let mut store = tinywasm::Store::new(); + let module = tinywasm::Module::from(module); + let instance = module.instantiate(&mut store)?; + + use wast::WastExecute::*; + let invoke = match exec { + Wat(_) => return Result::Ok(()), // not used by the testsuite + Get { module: _, global: _ } => return Result::Ok(()), + Invoke(invoke) => invoke, + }; + + let args = invoke + .args + .into_iter() + .map(wastarg2tinywasmvalue) + .collect::<Result<Vec<_>>>()?; + let res = instance.get_func(&store, invoke.name)?.call(&mut store, &args)?; + let expected = results + .into_iter() + .map(wastret2tinywasmvalue) + .collect::<Result<Vec<_>>>()?; + + if res.len() != expected.len() { + return Result::Err(eyre!("expected {} results, got {}", expected.len(), res.len())); + } + + for (i, (res, expected)) in res.iter().zip(expected).enumerate() { + if res != &expected { + return Result::Err(eyre!( + "result {} did not match: {:?} != {:?}", + i, + res, + expected + )); + } + } + + Ok(()) + }); + + let res = match res { + Err(e) => Err(eyre!("test panicked: {:?}", e)), + Ok(Err(e)) => Err(e), + Ok(Ok(())) => Ok(()), + }; + + test_group.add_result(&format!("{}-return", name), span, res); + } + Invoke(m) => { + debug!("invoke: {:?}", m); + } + _ => test_group.add_result(&format!("{}-unknown", name), span, Err(eyre!("unsupported directive"))), + } + } + }); + + Ok(()) + } +} diff --git a/crates/tinywasm/tests/testsuite/util.rs b/crates/tinywasm/tests/testsuite/util.rs new file mode 100644 index 0000000..18effa0 --- /dev/null +++ b/crates/tinywasm/tests/testsuite/util.rs @@ -0,0 +1,72 @@ +use eyre::{eyre, Result}; +use tinywasm_types::TinyWasmModule; + +pub fn parse_module(mut module: wast::core::Module) -> Result<TinyWasmModule> { + let parser = tinywasm_parser::Parser::new(); + Ok(parser.parse_module_bytes(module.encode().expect("failed to encode module"))?) +} + +pub fn wastarg2tinywasmvalue(arg: wast::WastArg) -> Result<tinywasm_types::WasmValue> { + let wast::WastArg::Core(arg) = arg else { + return Err(eyre!("unsupported arg type")); + }; + + use tinywasm_types::WasmValue; + use wast::core::WastArgCore::*; + Ok(match arg { + F32(f) => WasmValue::F32(f32::from_bits(f.bits)), + F64(f) => WasmValue::F64(f64::from_bits(f.bits)), + I32(i) => WasmValue::I32(i), + I64(i) => WasmValue::I64(i), + _ => return Err(eyre!("unsupported arg type")), + }) +} + +pub fn wastret2tinywasmvalue(arg: wast::WastRet) -> Result<tinywasm_types::WasmValue> { + let wast::WastRet::Core(arg) = arg else { + return Err(eyre!("unsupported arg type")); + }; + + use tinywasm_types::WasmValue; + use wast::core::WastRetCore::*; + Ok(match arg { + F32(f) => nanpattern2tinywasmvalue(f)?, + F64(f) => nanpattern2tinywasmvalue(f)?, + I32(i) => WasmValue::I32(i), + I64(i) => WasmValue::I64(i), + _ => return Err(eyre!("unsupported arg type")), + }) +} + +enum Bits { + U32(u32), + U64(u64), +} +trait FloatToken { + fn bits(&self) -> Bits; +} +impl FloatToken for wast::token::Float32 { + fn bits(&self) -> Bits { + Bits::U32(self.bits) + } +} +impl FloatToken for wast::token::Float64 { + fn bits(&self) -> Bits { + Bits::U64(self.bits) + } +} + +fn nanpattern2tinywasmvalue<T>(arg: wast::core::NanPattern<T>) -> Result<tinywasm_types::WasmValue> +where + T: FloatToken, +{ + use wast::core::NanPattern::*; + Ok(match arg { + CanonicalNan => tinywasm_types::WasmValue::F32(f32::NAN), + ArithmeticNan => tinywasm_types::WasmValue::F32(f32::NAN), + Value(v) => match v.bits() { + Bits::U32(v) => tinywasm_types::WasmValue::F32(f32::from_bits(v)), + Bits::U64(v) => tinywasm_types::WasmValue::F64(f64::from_bits(v)), + }, + }) +} |