diff options
Diffstat (limited to 'crates')
22 files changed, 979 insertions, 878 deletions
diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index ab0324f..3c4657d 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -19,7 +19,7 @@ argh="0.1" color-eyre={version="0.6", default-features=false} log="0.4" pretty_env_logger="0.5" -wast={version="206.0", optional=true} +wast={version="207.0", optional=true} [features] default=["wat"] diff --git a/crates/parser/src/conversion.rs b/crates/parser/src/conversion.rs index 31056a3..001d7cc 100644 --- a/crates/parser/src/conversion.rs +++ b/crates/parser/src/conversion.rs @@ -7,8 +7,7 @@ use wasmparser::{FuncValidator, OperatorsReader, ValidatorResources}; pub(crate) fn convert_module_elements<'a, T: IntoIterator<Item = wasmparser::Result<wasmparser::Element<'a>>>>( elements: T, ) -> Result<Vec<tinywasm_types::Element>> { - let elements = elements.into_iter().map(|element| convert_module_element(element?)).collect::<Result<Vec<_>>>()?; - Ok(elements) + elements.into_iter().map(|element| convert_module_element(element?)).collect::<Result<Vec<_>>>() } pub(crate) fn convert_module_element(element: wasmparser::Element<'_>) -> Result<tinywasm_types::Element> { @@ -47,8 +46,7 @@ pub(crate) fn convert_module_element(element: wasmparser::Element<'_>) -> Result pub(crate) fn convert_module_data_sections<'a, T: IntoIterator<Item = wasmparser::Result<wasmparser::Data<'a>>>>( data_sections: T, ) -> Result<Vec<tinywasm_types::Data>> { - let data_sections = data_sections.into_iter().map(|data| convert_module_data(data?)).collect::<Result<Vec<_>>>()?; - Ok(data_sections) + data_sections.into_iter().map(|data| convert_module_data(data?)).collect::<Result<Vec<_>>>() } pub(crate) fn convert_module_data(data: wasmparser::Data<'_>) -> Result<tinywasm_types::Data> { @@ -68,8 +66,7 @@ pub(crate) fn convert_module_data(data: wasmparser::Data<'_>) -> Result<tinywasm pub(crate) fn convert_module_imports<'a, T: IntoIterator<Item = wasmparser::Result<wasmparser::Import<'a>>>>( imports: T, ) -> Result<Vec<Import>> { - let imports = imports.into_iter().map(|import| convert_module_import(import?)).collect::<Result<Vec<_>>>()?; - Ok(imports) + imports.into_iter().map(|import| convert_module_import(import?)).collect::<Result<Vec<_>>>() } pub(crate) fn convert_module_import(import: wasmparser::Import<'_>) -> Result<Import> { @@ -140,8 +137,8 @@ pub(crate) fn convert_module_table(table: wasmparser::Table<'_>) -> Result<Table Ok(TableType { element_type: convert_reftype(&table.ty.element_type), size_initial, size_max }) } -pub(crate) fn convert_module_globals<'a, T: IntoIterator<Item = wasmparser::Result<wasmparser::Global<'a>>>>( - globals: T, +pub(crate) fn convert_module_globals( + globals: wasmparser::SectionLimited<'_, wasmparser::Global<'_>>, ) -> Result<Vec<Global>> { let globals = globals .into_iter() @@ -149,7 +146,6 @@ pub(crate) fn convert_module_globals<'a, T: IntoIterator<Item = wasmparser::Resu let global = global?; let ty = convert_valtype(&global.ty.content_type); let ops = global.init_expr.get_operators_reader(); - Ok(Global { init: process_const_operators(ops)?, ty: GlobalType { mutable: global.ty.mutable, ty } }) }) .collect::<Result<Vec<_>>>()?; @@ -172,7 +168,7 @@ pub(crate) fn convert_module_export(export: wasmparser::Export<'_>) -> Result<Ex pub(crate) fn convert_module_code( func: wasmparser::FunctionBody<'_>, - mut validator: FuncValidator<ValidatorResources>, + validator: &mut FuncValidator<ValidatorResources>, ) -> Result<Code> { let locals_reader = func.get_locals_reader()?; let count = locals_reader.get_count(); @@ -187,7 +183,7 @@ pub(crate) fn convert_module_code( } } - let body = process_operators(Some(&mut validator), &func)?; + let body = process_operators(Some(validator), func)?; let locals = locals.into_boxed_slice(); Ok((body, locals)) } @@ -245,18 +241,15 @@ pub(crate) fn process_const_operators(ops: OperatorsReader<'_>) -> Result<ConstI // Invalid modules will be rejected by the validator anyway (there are also tests for this in the testsuite) assert!(ops.len() >= 2); assert!(matches!(ops[ops.len() - 1], wasmparser::Operator::End)); - process_const_operator(ops[ops.len() - 2].clone()) -} -pub(crate) fn process_const_operator(op: wasmparser::Operator<'_>) -> Result<ConstInstruction> { - match op { - wasmparser::Operator::RefNull { hty } => Ok(ConstInstruction::RefNull(convert_heaptype(hty))), - wasmparser::Operator::RefFunc { function_index } => Ok(ConstInstruction::RefFunc(function_index)), - wasmparser::Operator::I32Const { value } => Ok(ConstInstruction::I32Const(value)), - wasmparser::Operator::I64Const { value } => Ok(ConstInstruction::I64Const(value)), + match &ops[ops.len() - 2] { + wasmparser::Operator::RefNull { hty } => Ok(ConstInstruction::RefNull(convert_heaptype(*hty))), + wasmparser::Operator::RefFunc { function_index } => Ok(ConstInstruction::RefFunc(*function_index)), + wasmparser::Operator::I32Const { value } => Ok(ConstInstruction::I32Const(*value)), + wasmparser::Operator::I64Const { value } => Ok(ConstInstruction::I64Const(*value)), wasmparser::Operator::F32Const { value } => Ok(ConstInstruction::F32Const(f32::from_bits(value.bits()))), wasmparser::Operator::F64Const { value } => Ok(ConstInstruction::F64Const(f64::from_bits(value.bits()))), - wasmparser::Operator::GlobalGet { global_index } => Ok(ConstInstruction::GlobalGet(global_index)), + wasmparser::Operator::GlobalGet { global_index } => Ok(ConstInstruction::GlobalGet(*global_index)), op => Err(crate::ParseError::UnsupportedOperator(format!("Unsupported const instruction: {:?}", op))), } } diff --git a/crates/parser/src/module.rs b/crates/parser/src/module.rs index 8414c17..1cd5ed5 100644 --- a/crates/parser/src/module.rs +++ b/crates/parser/src/module.rs @@ -2,12 +2,14 @@ use crate::log::debug; use crate::{conversion, ParseError, Result}; use alloc::{boxed::Box, format, vec::Vec}; use tinywasm_types::{Data, Element, Export, FuncType, Global, Import, Instruction, MemoryType, TableType, ValType}; -use wasmparser::{Payload, Validator}; +use wasmparser::{FuncValidatorAllocations, Payload, Validator}; pub(crate) type Code = (Box<[Instruction]>, Box<[ValType]>); #[derive(Default)] pub(crate) struct ModuleReader { + func_validator_allocations: Option<FuncValidatorAllocations>, + pub(crate) version: Option<u16>, pub(crate) start_func: Option<u32>, pub(crate) func_types: Vec<FuncType>, @@ -129,8 +131,9 @@ impl ModuleReader { CodeSectionEntry(function) => { debug!("Found code section entry"); let v = validator.code_section_entry(&function)?; - let func_validator = v.into_validator(Default::default()); - self.code.push(conversion::convert_module_code(function, func_validator)?); + let mut func_validator = v.into_validator(self.func_validator_allocations.take().unwrap_or_default()); + self.code.push(conversion::convert_module_code(function, &mut func_validator)?); + self.func_validator_allocations = Some(func_validator.into_allocations()); } ImportSection(reader) => { if !self.imports.is_empty() { diff --git a/crates/parser/src/visit.rs b/crates/parser/src/visit.rs index df6fc7a..332380c 100644 --- a/crates/parser/src/visit.rs +++ b/crates/parser/src/visit.rs @@ -29,12 +29,11 @@ where pub(crate) fn process_operators<R: WasmModuleResources>( validator: Option<&mut FuncValidator<R>>, - body: &FunctionBody<'_>, + body: FunctionBody<'_>, ) -> Result<Box<[Instruction]>> { let mut reader = body.get_operators_reader()?; let remaining = reader.get_binary_reader().bytes_remaining(); let mut builder = FunctionBuilder::new(remaining); - if let Some(validator) = validator { while !reader.eof() { let validate = validator.visitor(reader.original_position()); @@ -53,6 +52,7 @@ pub(crate) fn process_operators<R: WasmModuleResources>( macro_rules! define_operands { ($($name:ident, $instr:expr),*) => { $( + #[inline(always)] fn $name(&mut self) -> Self::Output { self.instructions.push($instr); Ok(()) @@ -64,15 +64,19 @@ macro_rules! define_operands { macro_rules! define_primitive_operands { ($($name:ident, $instr:expr, $ty:ty),*) => { $( + #[inline(always)] fn $name(&mut self, arg: $ty) -> Self::Output { - Ok(self.instructions.push($instr(arg))) + self.instructions.push($instr(arg)); + Ok(()) } )* }; ($($name:ident, $instr:expr, $ty:ty, $ty2:ty),*) => { $( + #[inline(always)] fn $name(&mut self, arg: $ty, arg2: $ty) -> Self::Output { - Ok(self.instructions.push($instr(arg, arg2))) + self.instructions.push($instr(arg, arg2)); + Ok(()) } )* }; @@ -81,6 +85,7 @@ macro_rules! define_primitive_operands { macro_rules! define_mem_operands { ($($name:ident, $instr:ident),*) => { $( + #[inline(always)] fn $name(&mut self, mem_arg: wasmparser::MemArg) -> Self::Output { let arg = convert_memarg(mem_arg); self.instructions.push(Instruction::$instr { @@ -100,7 +105,7 @@ pub(crate) struct FunctionBuilder { impl FunctionBuilder { pub(crate) fn new(instr_capacity: usize) -> Self { - Self { instructions: Vec::with_capacity(instr_capacity), label_ptrs: Vec::with_capacity(256) } + Self { instructions: Vec::with_capacity(instr_capacity / 4), label_ptrs: Vec::with_capacity(256) } } #[cold] @@ -108,7 +113,7 @@ impl FunctionBuilder { Err(crate::ParseError::UnsupportedOperator(format!("Unsupported instruction: {:?}", name))) } - #[inline] + #[inline(always)] fn visit(&mut self, op: Instruction) -> Result<()> { self.instructions.push(op); Ok(()) @@ -126,6 +131,7 @@ macro_rules! impl_visit_operator { (@@saturating_float_to_int $($rest:tt)* ) => {}; (@@bulk_memory $($rest:tt)* ) => {}; (@@$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident) => { + #[cold] fn $visit(&mut self $($(,$arg: $argty)*)?) -> Result<()>{ self.unsupported(stringify!($visit)) } @@ -319,6 +325,7 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder { visit_i64_trunc_sat_f64_u, Instruction::I64TruncSatF64U } + #[inline(always)] fn visit_i32_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { let arg = convert_memarg(memarg); let i32store = Instruction::I32Store { offset: arg.offset, mem_addr: arg.mem_addr }; @@ -342,6 +349,7 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder { } } + #[inline(always)] fn visit_local_get(&mut self, idx: u32) -> Self::Output { let Some(instruction) = self.instructions.last_mut() else { return self.visit(Instruction::LocalGet(idx)); @@ -357,14 +365,17 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder { Ok(()) } + #[inline(always)] fn visit_local_set(&mut self, idx: u32) -> Self::Output { self.visit(Instruction::LocalSet(idx)) } + #[inline(always)] fn visit_local_tee(&mut self, idx: u32) -> Self::Output { self.visit(Instruction::LocalTee(idx)) } + #[inline(always)] fn visit_i64_rotl(&mut self) -> Self::Output { if self.instructions.len() < 2 { return self.visit(Instruction::I64Rotl); @@ -380,6 +391,7 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder { } } + #[inline(always)] fn visit_i32_add(&mut self) -> Self::Output { if self.instructions.len() < 2 { return self.visit(Instruction::I32Add); @@ -395,35 +407,39 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder { } } + #[inline(always)] fn visit_block(&mut self, blockty: wasmparser::BlockType) -> Self::Output { self.label_ptrs.push(self.instructions.len()); self.visit(Instruction::Block(convert_blocktype(blockty), 0)) } + #[inline(always)] fn visit_loop(&mut self, ty: wasmparser::BlockType) -> Self::Output { self.label_ptrs.push(self.instructions.len()); self.visit(Instruction::Loop(convert_blocktype(ty), 0)) } + #[inline(always)] fn visit_if(&mut self, ty: wasmparser::BlockType) -> Self::Output { self.label_ptrs.push(self.instructions.len()); self.visit(Instruction::If(convert_blocktype(ty).into(), 0, 0)) } + #[inline(always)] fn visit_else(&mut self) -> Self::Output { self.label_ptrs.push(self.instructions.len()); self.visit(Instruction::Else(0)) } + #[inline(always)] fn visit_end(&mut self) -> Self::Output { let Some(label_pointer) = self.label_ptrs.pop() else { return self.visit(Instruction::Return); }; let current_instr_ptr = self.instructions.len(); - - match self.instructions[label_pointer] { - Instruction::Else(ref mut else_instr_end_offset) => { + match self.instructions.get_mut(label_pointer) { + Some(Instruction::Else(else_instr_end_offset)) => { *else_instr_end_offset = (current_instr_ptr - label_pointer) .try_into() .expect("else_instr_end_offset is too large, tinywasm does not support if blocks that large"); @@ -439,7 +455,7 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder { let if_label_pointer = self.label_ptrs.pop().ok_or_else(error)?; let if_instruction = &mut self.instructions[if_label_pointer]; - let Instruction::If(_, ref mut else_offset, ref mut end_offset) = if_instruction else { + let Instruction::If(_, else_offset, end_offset) = if_instruction else { return Err(error()); }; @@ -451,23 +467,22 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder { .try_into() .expect("else_instr_end_offset is too large, tinywasm does not support blocks that large"); } - Instruction::Block(_, ref mut end_offset) - | Instruction::Loop(_, ref mut end_offset) - | Instruction::If(_, _, ref mut end_offset) => { + Some(Instruction::Block(_, end_offset)) + | Some(Instruction::Loop(_, end_offset)) + | Some(Instruction::If(_, _, end_offset)) => { *end_offset = (current_instr_ptr - label_pointer) .try_into() .expect("else_instr_end_offset is too large, tinywasm does not support blocks that large"); } _ => { - return Err(crate::ParseError::UnsupportedOperator( - "Expected to end a block, but the last label was not a block".to_string(), - )) + unreachable!("Expected to end a block, but the last label was not a block") } }; self.visit(Instruction::EndBlockFrame) } + #[inline(always)] fn visit_br_table(&mut self, targets: wasmparser::BrTable<'_>) -> Self::Output { let def = targets.default(); let instrs = targets @@ -476,32 +491,36 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder { .collect::<Result<Vec<Instruction>, wasmparser::BinaryReaderError>>() .expect("BrTable targets are invalid, this should have been caught by the validator"); - self.instructions - .extend(IntoIterator::into_iter([Instruction::BrTable(def, instrs.len() as u32)]).chain(instrs)); - + self.instructions.extend(([Instruction::BrTable(def, instrs.len() as u32)].into_iter()).chain(instrs)); Ok(()) } + #[inline(always)] fn visit_call(&mut self, idx: u32) -> Self::Output { self.visit(Instruction::Call(idx)) } + #[inline(always)] fn visit_call_indirect(&mut self, ty: u32, table: u32, _table_byte: u8) -> Self::Output { self.visit(Instruction::CallIndirect(ty, table)) } + #[inline(always)] fn visit_memory_size(&mut self, mem: u32, mem_byte: u8) -> Self::Output { self.visit(Instruction::MemorySize(mem, mem_byte)) } + #[inline(always)] fn visit_memory_grow(&mut self, mem: u32, mem_byte: u8) -> Self::Output { self.visit(Instruction::MemoryGrow(mem, mem_byte)) } + #[inline(always)] fn visit_f32_const(&mut self, val: wasmparser::Ieee32) -> Self::Output { self.visit(Instruction::F32Const(f32::from_bits(val.bits()))) } + #[inline(always)] fn visit_f64_const(&mut self, val: wasmparser::Ieee64) -> Self::Output { self.visit(Instruction::F64Const(f64::from_bits(val.bits()))) } @@ -518,24 +537,29 @@ impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder { visit_data_drop, Instruction::DataDrop, u32 } + #[inline(always)] fn visit_elem_drop(&mut self, _elem_index: u32) -> Self::Output { self.unsupported("elem_drop") } + #[inline(always)] fn visit_table_copy(&mut self, dst_table: u32, src_table: u32) -> Self::Output { self.visit(Instruction::TableCopy { from: src_table, to: dst_table }) } // Reference Types + #[inline(always)] fn visit_ref_null(&mut self, ty: wasmparser::HeapType) -> Self::Output { self.visit(Instruction::RefNull(convert_heaptype(ty))) } + #[inline(always)] fn visit_ref_is_null(&mut self) -> Self::Output { self.visit(Instruction::RefIsNull) } + #[inline(always)] fn visit_typed_select(&mut self, ty: wasmparser::ValType) -> Self::Output { self.visit(Instruction::Select(Some(convert_valtype(&ty)))) } diff --git a/crates/tinywasm/Cargo.toml b/crates/tinywasm/Cargo.toml index 5f23389..ffd8ac9 100644 --- a/crates/tinywasm/Cargo.toml +++ b/crates/tinywasm/Cargo.toml @@ -32,8 +32,8 @@ default=["std", "parser", "logging", "archive"] logging=["_log", "tinywasm-parser?/logging", "tinywasm-types/logging"] std=["tinywasm-parser?/std", "tinywasm-types/std"] parser=["tinywasm-parser"] -unsafe=["tinywasm-types/unsafe"] archive=["tinywasm-types/archive"] +nightly=[] [[test]] name="test-mvp" diff --git a/crates/tinywasm/src/imports.rs b/crates/tinywasm/src/imports.rs index c4bae3b..f24ed73 100644 --- a/crates/tinywasm/src/imports.rs +++ b/crates/tinywasm/src/imports.rs @@ -389,7 +389,7 @@ impl Imports { match (val, &import.kind) { (ExternVal::Global(global_addr), ImportKind::Global(ty)) => { let global = store.get_global(global_addr)?; - Self::compare_types(import, &global.borrow().ty, ty)?; + Self::compare_types(import, &global.ty, ty)?; imports.globals.push(global_addr); } (ExternVal::Table(table_addr), ImportKind::Table(ty)) => { diff --git a/crates/tinywasm/src/instance.rs b/crates/tinywasm/src/instance.rs index 8a663c4..8402302 100644 --- a/crates/tinywasm/src/instance.rs +++ b/crates/tinywasm/src/instance.rs @@ -90,7 +90,7 @@ impl ModuleInstance { }; let instance = ModuleInstance::new(instance); - store.add_instance(instance.clone())?; + store.add_instance(instance.clone()); if let Some(trap) = elem_trapped { return Err(trap.into()); @@ -113,7 +113,7 @@ impl ModuleInstance { ExternalKind::Global => self.0.global_addrs.get(exports.index as usize)?, }; - Some(ExternVal::new(exports.kind.clone(), *addr)) + Some(ExternVal::new(exports.kind, *addr)) } #[inline] @@ -132,37 +132,37 @@ impl ModuleInstance { } // resolve a function address to the global store address - #[inline] + #[inline(always)] pub(crate) fn resolve_func_addr(&self, addr: FuncAddr) -> FuncAddr { - *self.0.func_addrs.get(addr as usize).expect("No func addr for func, this is a bug") + self.0.func_addrs[addr as usize] } // resolve a table address to the global store address - #[inline] + #[inline(always)] pub(crate) fn resolve_table_addr(&self, addr: TableAddr) -> TableAddr { - *self.0.table_addrs.get(addr as usize).expect("No table addr for table, this is a bug") + self.0.table_addrs[addr as usize] } // resolve a memory address to the global store address - #[inline] + #[inline(always)] pub(crate) fn resolve_mem_addr(&self, addr: MemAddr) -> MemAddr { - *self.0.mem_addrs.get(addr as usize).expect("No mem addr for mem, this is a bug") + self.0.mem_addrs[addr as usize] } // resolve a data address to the global store address - #[inline] + #[inline(always)] pub(crate) fn resolve_data_addr(&self, addr: DataAddr) -> MemAddr { - *self.0.data_addrs.get(addr as usize).expect("No data addr for data, this is a bug") + self.0.data_addrs[addr as usize] } // resolve a memory address to the global store address - #[inline] + #[inline(always)] pub(crate) fn resolve_elem_addr(&self, addr: ElemAddr) -> ElemAddr { - *self.0.elem_addrs.get(addr as usize).expect("No elem addr for elem, this is a bug") + self.0.elem_addrs[addr as usize] } // resolve a global address to the global store address - #[inline] + #[inline(always)] pub(crate) fn resolve_global_addr(&self, addr: GlobalAddr) -> GlobalAddr { self.0.global_addrs[addr as usize] } diff --git a/crates/tinywasm/src/lib.rs b/crates/tinywasm/src/lib.rs index 0bc17d6..2e9fece 100644 --- a/crates/tinywasm/src/lib.rs +++ b/crates/tinywasm/src/lib.rs @@ -6,7 +6,7 @@ #![allow(unexpected_cfgs, clippy::reserve_after_initialization)] #![warn(missing_docs, missing_debug_implementations, rust_2018_idioms, unreachable_pub)] #![cfg_attr(nightly, feature(error_in_core))] -#![cfg_attr(not(feature = "unsafe"), deny(unsafe_code))] +#![forbid(unsafe_code)] //! A tiny WebAssembly Runtime written in Rust //! @@ -23,8 +23,6 @@ //! Enables the `tinywasm-parser` crate. This is enabled by default. //!- **`archive`**\ //! Enables pre-parsing of archives. This is enabled by default. -//!- **`unsafe`**\ -//! Uses `unsafe` code to improve performance, particularly in Memory access //! //! With all these features disabled, TinyWasm only depends on `core`, `alloc` and `libm`. //! By disabling `std`, you can use TinyWasm in `no_std` environments. This requires diff --git a/crates/tinywasm/src/reference.rs b/crates/tinywasm/src/reference.rs index 6713a42..f3acc49 100644 --- a/crates/tinywasm/src/reference.rs +++ b/crates/tinywasm/src/reference.rs @@ -2,7 +2,6 @@ use core::cell::{Ref, RefCell, RefMut}; use core::ffi::CStr; use alloc::ffi::CString; -use alloc::rc::Rc; use alloc::string::{String, ToString}; use alloc::vec::Vec; @@ -142,9 +141,9 @@ impl MemoryStringExt for MemoryRef<'_> {} impl MemoryStringExt for MemoryRefMut<'_> {} /// A reference to a global instance -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct GlobalRef { - pub(crate) instance: Rc<RefCell<GlobalInstance>>, + pub(crate) instance: RefCell<GlobalInstance>, } impl GlobalRef { diff --git a/crates/tinywasm/src/runtime/interpreter/macros.rs b/crates/tinywasm/src/runtime/interpreter/macros.rs index b37dedd..8a092c0 100644 --- a/crates/tinywasm/src/runtime/interpreter/macros.rs +++ b/crates/tinywasm/src/runtime/interpreter/macros.rs @@ -10,12 +10,13 @@ // This is a bit hard to see from the spec, but it's vaild to use breaks to return // from a function, so we need to check if the label stack is empty macro_rules! break_to { - ($cf:ident, $stack:ident, $break_to_relative:ident) => {{ + ($cf:ident, $stack:ident, $module:ident, $store:ident, $break_to_relative:ident) => {{ if $cf.break_to(*$break_to_relative, &mut $stack.values, &mut $stack.blocks).is_none() { - match $stack.call_stack.is_empty() { - true => return Ok(ExecResult::Return), - false => return Ok(ExecResult::Call), + if $stack.call_stack.is_empty() { + return Ok(()); } + + call!($cf, $stack, $module, $store) } }}; } @@ -27,31 +28,35 @@ macro_rules! mem_load { }}; ($load_type:ty, $target_type:ty, $arg:expr, $stack:ident, $store:ident, $module:ident) => {{ - let (mem_addr, offset) = $arg; - - let mem_idx = $module.resolve_mem_addr(*mem_addr); - let mem = $store.get_mem(mem_idx)?; - let mem_ref = mem.borrow_mut(); - - let memory_out_of_bounds = || { - cold(); - Error::Trap(crate::Trap::MemoryOutOfBounds { - offset: *offset as usize, - len: core::mem::size_of::<$load_type>(), - max: mem_ref.max_pages(), - }) - }; + #[inline(always)] + fn mem_load_inner( + store: &Store, + module: &crate::ModuleInstance, + stack: &mut crate::runtime::Stack, + mem_addr: tinywasm_types::MemAddr, + offset: u64, + ) -> Result<()> { + let mem = store.get_mem(module.resolve_mem_addr(mem_addr))?; + let addr: usize = match offset.checked_add(stack.values.pop()?.into()).map(|a| a.try_into()) { + Some(Ok(a)) => a, + _ => { + cold(); + return Err(Error::Trap(crate::Trap::MemoryOutOfBounds { + offset: offset as usize, + len: core::mem::size_of::<$load_type>(), + max: mem.borrow().max_pages(), + })); + } + }; - let addr: usize = offset - .checked_add($stack.values.pop()?.into()) - .ok_or_else(memory_out_of_bounds)? - .try_into() - .ok() - .ok_or_else(memory_out_of_bounds)?; + const LEN: usize = core::mem::size_of::<$load_type>(); + let val = mem.borrow().load_as::<LEN, $load_type>(addr)?; + stack.values.push((val as $target_type).into()); + Ok(()) + } - const LEN: usize = core::mem::size_of::<$load_type>(); - let val = mem_ref.load_as::<LEN, $load_type>(addr)?; - $stack.values.push((val as $target_type).into()); + let (mem_addr, offset) = $arg; + mem_load_inner($store, &$module, $stack, *mem_addr, *offset)?; }}; } @@ -62,13 +67,24 @@ macro_rules! mem_store { }}; ($store_type:ty, $target_type:ty, $arg:expr, $stack:ident, $store:ident, $module:ident) => {{ - let (mem_addr, offset) = $arg; - let mem = $store.get_mem($module.resolve_mem_addr(*mem_addr))?; - let val: $store_type = $stack.values.pop()?.into(); - let val = val.to_le_bytes(); - let addr: u64 = $stack.values.pop()?.into(); + #[inline(always)] + fn mem_store_inner( + store: &Store, + module: &crate::ModuleInstance, + stack: &mut crate::runtime::Stack, + mem_addr: tinywasm_types::MemAddr, + offset: u64, + ) -> Result<()> { + let mem = store.get_mem(module.resolve_mem_addr(mem_addr))?; + let val: $store_type = stack.values.pop()?.into(); + let val = val.to_le_bytes(); + let addr: u64 = stack.values.pop()?.into(); + mem.borrow_mut().store((offset + addr) as usize, val.len(), &val)?; + Ok(()) + } - mem.borrow_mut().store((*offset + addr) as usize, val.len(), &val)?; + let (mem_addr, offset) = $arg; + mem_store_inner($store, &$module, $stack, *mem_addr, *offset)?; }}; } @@ -77,8 +93,8 @@ macro_rules! mem_store { /// for a specific conversion, which are then used in the actual conversion. /// Rust sadly doesn't have wrapping casts for floats yet, maybe never. /// Alternatively, https://crates.io/crates/az could be used for this but -/// it's not worth the dependency. -#[rustfmt::skip] +/// it's not worth the dependency. +#[rustfmt::skip] macro_rules! float_min_max { (f32, i32) => {(-2147483904.0_f32, 2147483648.0_f32)}; (f64, i32) => {(-2147483649.0_f64, 2147483648.0_f64)}; @@ -94,20 +110,17 @@ macro_rules! float_min_max { /// Convert a value on the stack macro_rules! conv { - ($from:ty, $to:ty, $stack:ident) => {{ - $stack.values.replace_top(|v| { - let a: $from = v.into(); - (a as $to).into() - }); - }}; + ($from:ty, $to:ty, $stack:ident) => { + $stack.values.replace_top(|v| (<$from>::from(v) as $to).into())? + }; } /// Convert a value on the stack with error checking macro_rules! checked_conv_float { // Direct conversion with error checking (two types) - ($from:tt, $to:tt, $stack:ident) => {{ + ($from:tt, $to:tt, $stack:ident) => { checked_conv_float!($from, $to, $to, $stack) - }}; + }; // Conversion with an intermediate unsigned type and error checking (three types) ($from:tt, $intermediate:tt, $to:tt, $stack:ident) => {{ let (min, max) = float_min_max!($from, $intermediate); @@ -127,67 +140,96 @@ macro_rules! checked_conv_float { /// Compare two values on the stack macro_rules! comp { - ($op:tt, $to:ty, $stack:ident) => {{ - let b: $to = $stack.values.pop()?.into(); - let a: $to = $stack.values.pop()?.into(); - $stack.values.push(((a $op b) as i32).into()); - }}; + ($op:tt, $to:ty, $stack:ident) => { + $stack.values.calculate(|a, b| { + ((<$to>::from(a) $op <$to>::from(b)) as i32).into() + })? + }; } /// Compare a value on the stack to zero macro_rules! comp_zero { - ($op:tt, $ty:ty, $stack:ident) => {{ - let a: $ty = $stack.values.pop()?.into(); - $stack.values.push(((a $op 0) as i32).into()); - }}; + ($op:tt, $ty:ty, $stack:ident) => { + $stack.values.replace_top(|v| { + ((<$ty>::from(v) $op 0) as i32).into() + })? + }; } /// Apply an arithmetic method to two values on the stack macro_rules! arithmetic { - ($op:ident, $to:ty, $stack:ident) => {{ - let b: $to = $stack.values.pop()?.into(); - let a: $to = $stack.values.pop()?.into(); - $stack.values.push((a.$op(b) as $to).into()); - }}; + ($op:ident, $to:ty, $stack:ident) => { + $stack.values.calculate(|a, b| { + (<$to>::from(a).$op(<$to>::from(b)) as $to).into() + })? + }; // also allow operators such as +, - - ($op:tt, $ty:ty, $stack:ident) => {{ - let b: $ty = $stack.values.pop()?.into(); - let a: $ty = $stack.values.pop()?.into(); - $stack.values.push((a $op b).into()); - }}; + ($op:tt, $ty:ty, $stack:ident) => { + $stack.values.calculate(|a, b| { + ((<$ty>::from(a) $op <$ty>::from(b)) as $ty).into() + })? + }; } /// Apply an arithmetic method to a single value on the stack macro_rules! arithmetic_single { - ($op:ident, $ty:ty, $stack:ident) => {{ + ($op:ident, $ty:ty, $stack:ident) => { arithmetic_single!($op, $ty, $ty, $stack) - }}; + }; - ($op:ident, $from:ty, $to:ty, $stack:ident) => {{ - let a: $from = $stack.values.pop()?.into(); - $stack.values.push((a.$op() as $to).into()); - }}; + ($op:ident, $from:ty, $to:ty, $stack:ident) => { + $stack.values.replace_top(|v| (<$from>::from(v).$op() as $to).into())? + }; } /// Apply an arithmetic operation to two values on the stack with error checking macro_rules! checked_int_arithmetic { - ($op:ident, $to:ty, $stack:ident) => {{ - let b: $to = $stack.values.pop()?.into(); - let a: $to = $stack.values.pop()?.into(); + ($op:ident, $to:ty, $stack:ident) => { + $stack.values.calculate_trap(|a, b| { + let a: $to = a.into(); + let b: $to = b.into(); + + if unlikely(b == 0) { + return Err(Error::Trap(crate::Trap::DivisionByZero)); + } - if unlikely(b == 0) { - return Err(Error::Trap(crate::Trap::DivisionByZero)); + let result = a.$op(b).ok_or_else(|| Error::Trap(crate::Trap::IntegerOverflow))?; + Ok((result).into()) + })? + }; +} + +macro_rules! call { + ($cf:expr, $stack:expr, $module:expr, $store:expr) => {{ + let old = $cf.block_ptr; + $cf = $stack.call_stack.pop()?; + + if old > $cf.block_ptr { + $stack.blocks.truncate(old); + } + + if $cf.module_addr != $module.id() { + $module.swap_with($cf.module_addr, $store); } - let result = a.$op(b).ok_or_else(|| Error::Trap(crate::Trap::IntegerOverflow))?; - $stack.values.push((result).into()); + continue; }}; } +macro_rules! skip { + ($code:expr) => { + match $code { + Ok(_) => continue, + Err(e) => return Err(e), + } + }; +} + pub(super) use arithmetic; pub(super) use arithmetic_single; pub(super) use break_to; +pub(super) use call; pub(super) use checked_conv_float; pub(super) use checked_int_arithmetic; pub(super) use comp; @@ -196,3 +238,4 @@ pub(super) use conv; pub(super) use float_min_max; pub(super) use mem_load; pub(super) use mem_store; +pub(super) use skip; diff --git a/crates/tinywasm/src/runtime/interpreter/mod.rs b/crates/tinywasm/src/runtime/interpreter/mod.rs index 404e4fe..d01faeb 100644 --- a/crates/tinywasm/src/runtime/interpreter/mod.rs +++ b/crates/tinywasm/src/runtime/interpreter/mod.rs @@ -1,12 +1,12 @@ use alloc::format; use alloc::string::ToString; use core::ops::{BitAnd, BitOr, BitXor, Neg}; -use tinywasm_types::{ElementKind, ValType}; +use tinywasm_types::{BlockArgs, ElementKind, ValType}; -use super::{InterpreterRuntime, Stack}; +use super::{InterpreterRuntime, RawWasmValue, Stack}; use crate::runtime::{BlockFrame, BlockType, CallFrame}; -use crate::{cold, unlikely}; -use crate::{Error, FuncContext, ModuleInstance, Result, Store, Trap}; +use crate::{cold, unlikely, ModuleInstance}; +use crate::{Error, FuncContext, Result, Store, Trap}; mod macros; mod traits; @@ -20,630 +20,756 @@ mod no_std_floats; use no_std_floats::NoStdFloatExt; impl InterpreterRuntime { - // #[inline(always)] // a small 2-3% performance improvement in some cases pub(crate) fn exec(&self, store: &mut Store, stack: &mut Stack) -> Result<()> { - let mut call_frame = stack.call_stack.pop()?; - let mut current_module = store.get_module_instance_raw(call_frame.module_addr); + let mut cf = stack.call_stack.pop()?; + let mut module = store.get_module_instance_raw(cf.module_addr); loop { - match exec_one(&mut call_frame, stack, store, ¤t_module) { - // continue to the next instruction and increment the instruction pointer - Ok(ExecResult::Ok) => call_frame.instr_ptr += 1, + use tinywasm_types::Instruction::*; + match cf.fetch_instr() { + Nop => cold(), + Unreachable => self.exec_unreachable()?, + Drop => stack.values.pop().map(|_| ())?, + Select(_valtype) => self.exec_select(stack)?, - // Continue execution at the new top of the call stack - Ok(ExecResult::Call) => { - let old = call_frame.block_ptr; - call_frame = stack.call_stack.pop()?; + Call(v) => skip!(self.exec_call(*v, store, stack, &mut cf, &mut module)), + CallIndirect(ty, table) => { + skip!(self.exec_call_indirect(*ty, *table, store, stack, &mut cf, &mut module)) + } + If(args, el, end) => skip!(self.exec_if((*args).into(), *el, *end, stack, &mut cf, &mut module)), + Loop(args, end) => self.enter_block(stack, cf.instr_ptr, *end, BlockType::Loop, args, &module), + Block(args, end) => self.enter_block(stack, cf.instr_ptr, *end, BlockType::Block, args, &module), - if old > call_frame.block_ptr { - stack.blocks.truncate(old); + Br(v) => break_to!(cf, stack, module, store, v), + BrIf(v) => { + if i32::from(stack.values.pop()?) != 0 { + break_to!(cf, stack, module, store, v); + } + } + BrTable(default, len) => { + let start = cf.instr_ptr + 1; + let end = start + *len as usize; + if end > cf.instructions().len() { + return Err(Error::Other(format!( + "br_table out of bounds: {} >= {}", + end, + cf.instructions().len() + ))); } - // keeping the pointer seperate from the call frame is about 2% faster - // than storing it in the call frame - if call_frame.module_addr != current_module.id() { - current_module.swap_with(call_frame.module_addr, store); + let idx: i32 = stack.values.pop()?.into(); + match cf.instructions()[start..end].get(idx as usize) { + None => break_to!(cf, stack, module, store, default), + Some(BrLabel(to)) => break_to!(cf, stack, module, store, to), + _ => return Err(Error::Other("br_table with invalid label".to_string())), } } - // return from the function - Ok(ExecResult::Return) => { - cold(); - return Ok(()); - } + Return => match stack.call_stack.is_empty() { + true => return Ok(()), + false => call!(cf, stack, module, store), + }, - // trap the program - Err(e) => { - cold(); - call_frame.instr_ptr += 1; - // push the call frame back onto the stack so that it can be resumed - // if the trap can be handled - stack.call_stack.push(call_frame)?; - return Err(e); - } - } - } - } -} + // We're essentially using else as a EndBlockFrame instruction for if blocks + Else(end_offset) => self.exec_else(stack, *end_offset, &mut cf)?, -enum ExecResult { - Ok, - Return, - Call, -} + // remove the label from the label stack + EndBlockFrame => self.exec_end_block(stack)?, -/// Run a single step of the interpreter -/// A seperate function is used so later, we can more easily implement -/// a step-by-step debugger (using generators once they're stable?) -// we want this be always part of the loop, rust just doesn't inline it as its too big -// this can be a 30%+ performance difference in some cases -#[inline(always)] -fn exec_one(cf: &mut CallFrame, stack: &mut Stack, store: &mut Store, module: &ModuleInstance) -> Result<ExecResult> { - let instrs = &cf.func_instance.instructions; + LocalGet(local_index) => self.exec_local_get(*local_index, stack, &cf), + LocalSet(local_index) => self.exec_local_set(*local_index, stack, &mut cf)?, + LocalTee(local_index) => self.exec_local_tee(*local_index, stack, &mut cf)?, - // A match statement is probably the fastest way to do this without - // unreasonable complexity. This *should* be optimized to a jump table. - // See https://pliniker.github.io/post/dispatchers/ - use tinywasm_types::Instruction::*; - match instrs.get(cf.instr_ptr as usize).expect("instr_ptr out of bounds, this should never happen") { - Nop => { /* do nothing */ } - Unreachable => return Err(crate::Trap::Unreachable.into()), - Drop => stack.values.pop().map(|_| ())?, - Select(_valtype) => { - // due to validation, we know that the type of the values on the stack - let cond: i32 = stack.values.pop()?.into(); - let val2 = stack.values.pop()?; + GlobalGet(global_index) => self.exec_global_get(*global_index, stack, store, &module)?, + GlobalSet(global_index) => self.exec_global_set(*global_index, stack, store, &module)?, - // if cond != 0, we already have the right value on the stack - if cond == 0 { - let _ = stack.values.pop()?; - stack.values.push(val2); - } - } + I32Const(val) => self.exec_const(*val, stack), + I64Const(val) => self.exec_const(*val, stack), + F32Const(val) => self.exec_const(*val, stack), + F64Const(val) => self.exec_const(*val, stack), - Call(v) => { - // prepare the call frame - let func_inst = store.get_func(module.resolve_func_addr(*v))?; - let wasm_func = match &func_inst.func { - crate::Function::Wasm(wasm_func) => wasm_func, - crate::Function::Host(host_func) => { - let func = &host_func.clone(); - let params = stack.values.pop_params(&host_func.ty.params)?; - let res = (func.func)(FuncContext { store, module_addr: module.id() }, ¶ms)?; - stack.values.extend_from_typed(&res); - return Ok(ExecResult::Ok); - } - }; + MemorySize(addr, byte) => self.exec_memory_size(*addr, *byte, stack, store, &module)?, + MemoryGrow(addr, byte) => self.exec_memory_grow(*addr, *byte, stack, store, &module)?, - let params = stack.values.pop_n_rev(wasm_func.ty.params.len())?; - let call_frame = CallFrame::new(wasm_func.clone(), func_inst.owner, params, stack.blocks.len() as u32); + // Bulk memory operations + MemoryCopy(from, to) => self.exec_memory_copy(*from, *to, stack, store, &module)?, + MemoryFill(addr) => self.exec_memory_fill(*addr, stack, store, &module)?, + MemoryInit(data_idx, mem_idx) => self.exec_memory_init(*data_idx, *mem_idx, stack, store, &module)?, + DataDrop(data_index) => store.get_data_mut(module.resolve_data_addr(*data_index))?.drop(), - // push the call frame - cf.instr_ptr += 1; // skip the call instruction + I32Store { mem_addr, offset } => mem_store!(i32, (mem_addr, offset), stack, store, module), + I64Store { mem_addr, offset } => mem_store!(i64, (mem_addr, offset), stack, store, module), + F32Store { mem_addr, offset } => mem_store!(f32, (mem_addr, offset), stack, store, module), + F64Store { mem_addr, offset } => mem_store!(f64, (mem_addr, offset), stack, store, module), + I32Store8 { mem_addr, offset } => mem_store!(i8, i32, (mem_addr, offset), stack, store, module), + I32Store16 { mem_addr, offset } => mem_store!(i16, i32, (mem_addr, offset), stack, store, module), + I64Store8 { mem_addr, offset } => mem_store!(i8, i64, (mem_addr, offset), stack, store, module), + I64Store16 { mem_addr, offset } => mem_store!(i16, i64, (mem_addr, offset), stack, store, module), + I64Store32 { mem_addr, offset } => mem_store!(i32, i64, (mem_addr, offset), stack, store, module), - // this is sometimes faster, and seems more efficient, but sometimes it's also a lot slower - // stack.call_stack.push(core::mem::replace(cf, call_frame))?; - // if cf.module_addr != module.id() { - // module.swap_with(cf.module_addr, store); - // } - // cf.instr_ptr -= 1; - // return Ok(ExecResult::Ok); + I32Load { mem_addr, offset } => mem_load!(i32, (mem_addr, offset), stack, store, module), + I64Load { mem_addr, offset } => mem_load!(i64, (mem_addr, offset), stack, store, module), + F32Load { mem_addr, offset } => mem_load!(f32, (mem_addr, offset), stack, store, module), + F64Load { mem_addr, offset } => mem_load!(f64, (mem_addr, offset), stack, store, module), + I32Load8S { mem_addr, offset } => mem_load!(i8, i32, (mem_addr, offset), stack, store, module), + I32Load8U { mem_addr, offset } => mem_load!(u8, i32, (mem_addr, offset), stack, store, module), + I32Load16S { mem_addr, offset } => mem_load!(i16, i32, (mem_addr, offset), stack, store, module), + I32Load16U { mem_addr, offset } => mem_load!(u16, i32, (mem_addr, offset), stack, store, module), + I64Load8S { mem_addr, offset } => mem_load!(i8, i64, (mem_addr, offset), stack, store, module), + I64Load8U { mem_addr, offset } => mem_load!(u8, i64, (mem_addr, offset), stack, store, module), + I64Load16S { mem_addr, offset } => mem_load!(i16, i64, (mem_addr, offset), stack, store, module), + I64Load16U { mem_addr, offset } => mem_load!(u16, i64, (mem_addr, offset), stack, store, module), + I64Load32S { mem_addr, offset } => mem_load!(i32, i64, (mem_addr, offset), stack, store, module), + I64Load32U { mem_addr, offset } => mem_load!(u32, i64, (mem_addr, offset), stack, store, module), - stack.call_stack.push(cf.clone())?; - stack.call_stack.push(call_frame)?; + I64Eqz => comp_zero!(==, i64, stack), + I32Eqz => comp_zero!(==, i32, stack), - // call the function - return Ok(ExecResult::Call); - } + I32Eq => comp!(==, i32, stack), + I64Eq => comp!(==, i64, stack), + F32Eq => comp!(==, f32, stack), + F64Eq => comp!(==, f64, stack), - CallIndirect(type_addr, table_addr) => { - let table = store.get_table(module.resolve_table_addr(*table_addr))?; - let table_idx: u32 = stack.values.pop()?.into(); + I32Ne => comp!(!=, i32, stack), + I64Ne => comp!(!=, i64, stack), + F32Ne => comp!(!=, f32, stack), + F64Ne => comp!(!=, f64, stack), - // verify that the table is of the right type, this should be validated by the parser already - let func_ref = { - let table = table.borrow(); - assert!(table.kind.element_type == ValType::RefFunc, "table is not of type funcref"); - table.get(table_idx)?.addr().ok_or(Trap::UninitializedElement { index: table_idx as usize })? - }; + I32LtS => comp!(<, i32, stack), + I64LtS => comp!(<, i64, stack), + I32LtU => comp!(<, u32, stack), + I64LtU => comp!(<, u64, stack), + F32Lt => comp!(<, f32, stack), + F64Lt => comp!(<, f64, stack), - let func_inst = store.get_func(func_ref)?.clone(); - let call_ty = module.func_ty(*type_addr); + I32LeS => comp!(<=, i32, stack), + I64LeS => comp!(<=, i64, stack), + I32LeU => comp!(<=, u32, stack), + I64LeU => comp!(<=, u64, stack), + F32Le => comp!(<=, f32, stack), + F64Le => comp!(<=, f64, stack), - let wasm_func = match func_inst.func { - crate::Function::Wasm(ref f) => f, - crate::Function::Host(host_func) => { - if unlikely(host_func.ty != *call_ty) { - return Err(Trap::IndirectCallTypeMismatch { - actual: host_func.ty.clone(), - expected: call_ty.clone(), - } - .into()); - } + I32GeS => comp!(>=, i32, stack), + I64GeS => comp!(>=, i64, stack), + I32GeU => comp!(>=, u32, stack), + I64GeU => comp!(>=, u64, stack), + F32Ge => comp!(>=, f32, stack), + F64Ge => comp!(>=, f64, stack), - let host_func = host_func.clone(); - let params = stack.values.pop_params(&host_func.ty.params)?; - let res = (host_func.func)(FuncContext { store, module_addr: module.id() }, ¶ms)?; - stack.values.extend_from_typed(&res); - return Ok(ExecResult::Ok); - } - }; + I32GtS => comp!(>, i32, stack), + I64GtS => comp!(>, i64, stack), + I32GtU => comp!(>, u32, stack), + I64GtU => comp!(>, u64, stack), + F32Gt => comp!(>, f32, stack), + F64Gt => comp!(>, f64, stack), - if unlikely(wasm_func.ty != *call_ty) { - return Err( - Trap::IndirectCallTypeMismatch { actual: wasm_func.ty.clone(), expected: call_ty.clone() }.into() - ); - } + I64Add => arithmetic!(wrapping_add, i64, stack), + I32Add => arithmetic!(wrapping_add, i32, stack), + F32Add => arithmetic!(+, f32, stack), + F64Add => arithmetic!(+, f64, stack), - let params = stack.values.pop_n_rev(wasm_func.ty.params.len())?; - let call_frame = CallFrame::new(wasm_func.clone(), func_inst.owner, params, stack.blocks.len() as u32); + I32Sub => arithmetic!(wrapping_sub, i32, stack), + I64Sub => arithmetic!(wrapping_sub, i64, stack), + F32Sub => arithmetic!(-, f32, stack), + F64Sub => arithmetic!(-, f64, stack), - cf.instr_ptr += 1; // skip the call instruction - stack.call_stack.push(cf.clone())?; - stack.call_stack.push(call_frame)?; + F32Div => arithmetic!(/, f32, stack), + F64Div => arithmetic!(/, f64, stack), - // call the function - return Ok(ExecResult::Call); - } + I32Mul => arithmetic!(wrapping_mul, i32, stack), + I64Mul => arithmetic!(wrapping_mul, i64, stack), + F32Mul => arithmetic!(*, f32, stack), + F64Mul => arithmetic!(*, f64, stack), - If(args, else_offset, end_offset) => { - // truthy value is on the top of the stack, so enter the then block - if i32::from(stack.values.pop()?) != 0 { - cf.enter_block( - BlockFrame::new( - cf.instr_ptr, - cf.instr_ptr + *end_offset, - stack.values.len() as u32, - BlockType::If, - &(*args).into(), - module, - ), - &mut stack.values, - &mut stack.blocks, - ); - return Ok(ExecResult::Ok); - } + // these can trap + I32DivS => checked_int_arithmetic!(checked_div, i32, stack), + I64DivS => checked_int_arithmetic!(checked_div, i64, stack), + I32DivU => checked_int_arithmetic!(checked_div, u32, stack), + I64DivU => checked_int_arithmetic!(checked_div, u64, stack), - // falsy value is on the top of the stack - if *else_offset == 0 { - cf.instr_ptr += *end_offset; - return Ok(ExecResult::Ok); - } + I32RemS => checked_int_arithmetic!(checked_wrapping_rem, i32, stack), + I64RemS => checked_int_arithmetic!(checked_wrapping_rem, i64, stack), + I32RemU => checked_int_arithmetic!(checked_wrapping_rem, u32, stack), + I64RemU => checked_int_arithmetic!(checked_wrapping_rem, u64, stack), - let label = BlockFrame::new( - cf.instr_ptr + *else_offset, - cf.instr_ptr + *end_offset, - stack.values.len() as u32, - BlockType::Else, - &(*args).into(), - module, - ); - cf.instr_ptr += *else_offset; - cf.enter_block(label, &mut stack.values, &mut stack.blocks); - } + I32And => arithmetic!(bitand, i32, stack), + I64And => arithmetic!(bitand, i64, stack), + I32Or => arithmetic!(bitor, i32, stack), + I64Or => arithmetic!(bitor, i64, stack), + I32Xor => arithmetic!(bitxor, i32, stack), + I64Xor => arithmetic!(bitxor, i64, stack), + I32Shl => arithmetic!(wasm_shl, i32, stack), + I64Shl => arithmetic!(wasm_shl, i64, stack), + I32ShrS => arithmetic!(wasm_shr, i32, stack), + I64ShrS => arithmetic!(wasm_shr, i64, stack), + I32ShrU => arithmetic!(wasm_shr, u32, stack), + I64ShrU => arithmetic!(wasm_shr, u64, stack), + I32Rotl => arithmetic!(wasm_rotl, i32, stack), + I64Rotl => arithmetic!(wasm_rotl, i64, stack), + I32Rotr => arithmetic!(wasm_rotr, i32, stack), + I64Rotr => arithmetic!(wasm_rotr, i64, stack), - Loop(args, end_offset) => { - cf.enter_block( - BlockFrame::new( - cf.instr_ptr, - cf.instr_ptr + *end_offset, - stack.values.len() as u32, - BlockType::Loop, - args, - module, - ), - &mut stack.values, - &mut stack.blocks, - ); - } + I32Clz => arithmetic_single!(leading_zeros, i32, stack), + I64Clz => arithmetic_single!(leading_zeros, i64, stack), + I32Ctz => arithmetic_single!(trailing_zeros, i32, stack), + I64Ctz => arithmetic_single!(trailing_zeros, i64, stack), + I32Popcnt => arithmetic_single!(count_ones, i32, stack), + I64Popcnt => arithmetic_single!(count_ones, i64, stack), - Block(args, end_offset) => { - cf.enter_block( - BlockFrame::new( - cf.instr_ptr, - cf.instr_ptr + *end_offset, - stack.values.len() as u32, - BlockType::Block, - args, - module, - ), - &mut stack.values, - &mut stack.blocks, - ); - } + F32ConvertI32S => conv!(i32, f32, stack), + F32ConvertI64S => conv!(i64, f32, stack), + F64ConvertI32S => conv!(i32, f64, stack), + F64ConvertI64S => conv!(i64, f64, stack), + F32ConvertI32U => conv!(u32, f32, stack), + F32ConvertI64U => conv!(u64, f32, stack), + F64ConvertI32U => conv!(u32, f64, stack), + F64ConvertI64U => conv!(u64, f64, stack), + I32Extend8S => conv!(i8, i32, stack), + I32Extend16S => conv!(i16, i32, stack), + I64Extend8S => conv!(i8, i64, stack), + I64Extend16S => conv!(i16, i64, stack), + I64Extend32S => conv!(i32, i64, stack), + I64ExtendI32U => conv!(u32, i64, stack), + I64ExtendI32S => conv!(i32, i64, stack), + I32WrapI64 => conv!(i64, i32, stack), - BrTable(default, len) => { - let start = (cf.instr_ptr + 1) as usize; - let end = start + *len as usize; - if end > cf.instructions().len() { - return Err(Error::Other(format!("br_table out of bounds: {} >= {}", end, cf.instructions().len()))); - } + F32DemoteF64 => conv!(f64, f32, stack), + F64PromoteF32 => conv!(f32, f64, stack), - let idx: i32 = stack.values.pop()?.into(); - match cf.instructions()[start..end].get(idx as usize) { - None => break_to!(cf, stack, default), - Some(BrLabel(to)) => break_to!(cf, stack, to), - _ => return Err(Error::Other("br_table with invalid label".to_string())), - } - } - - Br(v) => break_to!(cf, stack, v), - BrIf(v) => { - if i32::from(stack.values.pop()?) != 0 { - break_to!(cf, stack, v); - } - } + F32Abs => arithmetic_single!(abs, f32, stack), + F64Abs => arithmetic_single!(abs, f64, stack), + F32Neg => arithmetic_single!(neg, f32, stack), + F64Neg => arithmetic_single!(neg, f64, stack), + F32Ceil => arithmetic_single!(ceil, f32, stack), + F64Ceil => arithmetic_single!(ceil, f64, stack), + F32Floor => arithmetic_single!(floor, f32, stack), + F64Floor => arithmetic_single!(floor, f64, stack), + F32Trunc => arithmetic_single!(trunc, f32, stack), + F64Trunc => arithmetic_single!(trunc, f64, stack), + F32Nearest => arithmetic_single!(tw_nearest, f32, stack), + F64Nearest => arithmetic_single!(tw_nearest, f64, stack), + F32Sqrt => arithmetic_single!(sqrt, f32, stack), + F64Sqrt => arithmetic_single!(sqrt, f64, stack), + F32Min => arithmetic!(tw_minimum, f32, stack), + F64Min => arithmetic!(tw_minimum, f64, stack), + F32Max => arithmetic!(tw_maximum, f32, stack), + F64Max => arithmetic!(tw_maximum, f64, stack), + F32Copysign => arithmetic!(copysign, f32, stack), + F64Copysign => arithmetic!(copysign, f64, stack), - Return => match stack.call_stack.is_empty() { - true => return Ok(ExecResult::Return), - false => return Ok(ExecResult::Call), - }, + // no-op instructions since types are erased at runtime + I32ReinterpretF32 | I64ReinterpretF64 | F32ReinterpretI32 | F64ReinterpretI64 => {} - // We're essentially using else as a EndBlockFrame instruction for if blocks - Else(end_offset) => { - let block = stack.blocks.pop()?; - stack.values.truncate_keep(block.stack_ptr, block.results as u32); - cf.instr_ptr += *end_offset; - } + // unsigned versions of these are a bit broken atm + I32TruncF32S => checked_conv_float!(f32, i32, stack), + I32TruncF64S => checked_conv_float!(f64, i32, stack), + I32TruncF32U => checked_conv_float!(f32, u32, i32, stack), + I32TruncF64U => checked_conv_float!(f64, u32, i32, stack), + I64TruncF32S => checked_conv_float!(f32, i64, stack), + I64TruncF64S => checked_conv_float!(f64, i64, stack), + I64TruncF32U => checked_conv_float!(f32, u64, i64, stack), + I64TruncF64U => checked_conv_float!(f64, u64, i64, stack), - // remove the label from the label stack - EndBlockFrame => { - let block = stack.blocks.pop()?; - stack.values.truncate_keep(block.stack_ptr, block.results as u32); - } + TableGet(table_idx) => self.exec_table_get(*table_idx, stack, store, &module)?, + TableSet(table_idx) => self.exec_table_set(*table_idx, stack, store, &module)?, + TableSize(table_idx) => self.exec_table_size(*table_idx, stack, store, &module)?, + TableInit(table_idx, elem_idx) => self.exec_table_init(*elem_idx, *table_idx, store, &module)?, - LocalGet(local_index) => stack.values.push(cf.get_local(*local_index)), - LocalSet(local_index) => cf.set_local(*local_index, stack.values.pop()?), - LocalTee(local_index) => cf.set_local(*local_index, *stack.values.last()?), + I32TruncSatF32S => arithmetic_single!(trunc, f32, i32, stack), + I32TruncSatF32U => arithmetic_single!(trunc, f32, u32, stack), + I32TruncSatF64S => arithmetic_single!(trunc, f64, i32, stack), + I32TruncSatF64U => arithmetic_single!(trunc, f64, u32, stack), + I64TruncSatF32S => arithmetic_single!(trunc, f32, i64, stack), + I64TruncSatF32U => arithmetic_single!(trunc, f32, u64, stack), + I64TruncSatF64S => arithmetic_single!(trunc, f64, i64, stack), + I64TruncSatF64U => arithmetic_single!(trunc, f64, u64, stack), - GlobalGet(global_index) => { - let global = store.get_global_val(module.resolve_global_addr(*global_index))?; - stack.values.push(global); - } + // custom instructions + LocalGet2(a, b) => self.exec_local_get2(*a, *b, stack, &cf), + LocalGet3(a, b, c) => self.exec_local_get3(*a, *b, *c, stack, &cf), + LocalTeeGet(a, b) => self.exec_local_tee_get(*a, *b, stack, &mut cf), + LocalGetSet(a, b) => self.exec_local_get_set(*a, *b, &mut cf), + I64XorConstRotl(rotate_by) => self.exec_i64_xor_const_rotl(*rotate_by, stack)?, + I32LocalGetConstAdd(local, val) => self.exec_i32_local_get_const_add(*local, *val, stack, &cf), + I32StoreLocal { local, const_i32: consti32, offset, mem_addr } => { + self.exec_i32_store_local(*local, *consti32, *offset, *mem_addr, &cf, store, &module)? + } + i => { + cold(); + return Err(Error::UnsupportedFeature(format!("unimplemented instruction: {:?}", i))); + } + }; - GlobalSet(global_index) => { - let idx = module.resolve_global_addr(*global_index); - store.set_global_val(idx, stack.values.pop()?)?; + cf.instr_ptr += 1; } + } - I32Const(val) => stack.values.push((*val).into()), - I64Const(val) => stack.values.push((*val).into()), - F32Const(val) => stack.values.push((*val).into()), - F64Const(val) => stack.values.push((*val).into()), - - MemorySize(addr, byte) => { - if unlikely(*byte != 0) { - return Err(Error::UnsupportedFeature("memory.size with byte != 0".to_string())); - } + #[inline(always)] + fn exec_end_block(&self, stack: &mut Stack) -> Result<()> { + let block = stack.blocks.pop()?; + stack.values.truncate_keep(block.stack_ptr, block.results as u32); + Ok(()) + } - let mem_idx = module.resolve_mem_addr(*addr); - let mem = store.get_mem(mem_idx)?; - stack.values.push((mem.borrow().page_count() as i32).into()); - } + #[inline(always)] + fn exec_else(&self, stack: &mut Stack, end_offset: u32, cf: &mut CallFrame) -> Result<()> { + let block = stack.blocks.pop()?; + stack.values.truncate_keep(block.stack_ptr, block.results as u32); + cf.instr_ptr += end_offset as usize; + Ok(()) + } - MemoryGrow(addr, byte) => { - if unlikely(*byte != 0) { - return Err(Error::UnsupportedFeature("memory.grow with byte != 0".to_string())); - } + #[inline(always)] + #[cold] + fn exec_unreachable(&self) -> Result<()> { + Err(Error::Trap(Trap::Unreachable)) + } - let mem = store.get_mem(module.resolve_mem_addr(*addr))?; - let mut mem = mem.borrow_mut(); - let prev_size = mem.page_count() as i32; - let pages_delta: i32 = stack.values.pop()?.into(); + #[inline(always)] + fn exec_const(&self, val: impl Into<RawWasmValue>, stack: &mut Stack) { + stack.values.push(val.into()); + } - match mem.grow(pages_delta) { - Some(_) => stack.values.push(prev_size.into()), - None => stack.values.push((-1).into()), - } - } + #[inline(always)] + fn exec_i32_store_local( + &self, + local: u32, + const_i32: i32, + offset: u32, + mem_addr: u8, + cf: &CallFrame, + store: &Store, + module: &ModuleInstance, + ) -> Result<()> { + let mem_addr = module.resolve_mem_addr(mem_addr as u32); + let mem = store.get_mem(mem_addr)?; + let val = const_i32.to_le_bytes(); + let addr: u64 = cf.get_local(local).into(); + mem.borrow_mut().store((offset as u64 + addr) as usize, val.len(), &val)?; + Ok(()) + } - // Bulk memory operations - MemoryCopy(from, to) => { - let size: i32 = stack.values.pop()?.into(); - let src: i32 = stack.values.pop()?.into(); - let dst: i32 = stack.values.pop()?.into(); + #[inline(always)] + fn exec_i32_local_get_const_add(&self, local: u32, val: i32, stack: &mut Stack, cf: &CallFrame) { + let local: i32 = cf.get_local(local).into(); + stack.values.push((local + val).into()); + } - let mem = store.get_mem(module.resolve_mem_addr(*from))?; - let mut mem = mem.borrow_mut(); + #[inline(always)] + fn exec_i64_xor_const_rotl(&self, rotate_by: i64, stack: &mut Stack) -> Result<()> { + let val: i64 = stack.values.pop()?.into(); + let res = stack.values.last_mut()?; + let mask: i64 = (*res).into(); + *res = (val ^ mask).rotate_left(rotate_by as u32).into(); + Ok(()) + } - if from == to { - // copy within the same memory - mem.copy_within(dst as usize, src as usize, size as usize)?; - } else { - // copy between two memories - let mem2 = store.get_mem(module.resolve_mem_addr(*to))?; - let mut mem2 = mem2.borrow_mut(); - mem2.copy_from_slice(dst as usize, mem.load(src as usize, size as usize)?)?; - } - } + #[inline(always)] + fn exec_local_get(&self, local_index: u32, stack: &mut Stack, cf: &CallFrame) { + stack.values.push(cf.get_local(local_index)); + } - MemoryFill(addr) => { - let size: i32 = stack.values.pop()?.into(); - let val: i32 = stack.values.pop()?.into(); - let dst: i32 = stack.values.pop()?.into(); + #[inline(always)] + fn exec_local_get2(&self, a: u32, b: u32, stack: &mut Stack, cf: &CallFrame) { + stack.values.push(cf.get_local(a)); + stack.values.push(cf.get_local(b)); + } - let mem = store.get_mem(module.resolve_mem_addr(*addr))?; - mem.borrow_mut().fill(dst as usize, size as usize, val as u8)?; - } + #[inline(always)] + fn exec_local_get3(&self, a: u32, b: u32, c: u32, stack: &mut Stack, cf: &CallFrame) { + stack.values.push(cf.get_local(a)); + stack.values.push(cf.get_local(b)); + stack.values.push(cf.get_local(c)); + } - MemoryInit(data_index, mem_index) => { - let size = i32::from(stack.values.pop()?) as usize; - let offset = i32::from(stack.values.pop()?) as usize; - let dst = i32::from(stack.values.pop()?) as usize; + #[inline(always)] + fn exec_local_get_set(&self, a: u32, b: u32, cf: &mut CallFrame) { + cf.set_local(b, cf.get_local(a)) + } - let data = match &store.get_data(module.resolve_data_addr(*data_index))?.data { - Some(data) => data, - None => return Err(Trap::MemoryOutOfBounds { offset: 0, len: 0, max: 0 }.into()), - }; + #[inline(always)] + fn exec_local_set(&self, local_index: u32, stack: &mut Stack, cf: &mut CallFrame) -> Result<()> { + cf.set_local(local_index, stack.values.pop()?); + Ok(()) + } - if unlikely(offset + size > data.len()) { - return Err(Trap::MemoryOutOfBounds { offset, len: size, max: data.len() }.into()); - } + #[inline(always)] + fn exec_local_tee(&self, local_index: u32, stack: &mut Stack, cf: &mut CallFrame) -> Result<()> { + cf.set_local(local_index, *stack.values.last()?); + Ok(()) + } - let mem = store.get_mem(module.resolve_mem_addr(*mem_index))?; - mem.borrow_mut().store(dst, size, &data[offset..(offset + size)])?; // mem.store checks bounds - } + #[inline(always)] + fn exec_local_tee_get(&self, a: u32, b: u32, stack: &mut Stack, cf: &mut CallFrame) { + let last = + stack.values.last().expect("localtee: stack is empty. this should have been validated by the parser"); + cf.set_local(a, *last); + stack.values.push(match a == b { + true => *last, + false => cf.get_local(b), + }); + } - DataDrop(data_index) => store.get_data_mut(module.resolve_data_addr(*data_index))?.drop(), + #[inline(always)] + fn exec_global_get( + &self, + global_index: u32, + stack: &mut Stack, + store: &Store, + module: &ModuleInstance, + ) -> Result<()> { + let global = store.get_global_val(module.resolve_global_addr(global_index))?; + stack.values.push(global); + Ok(()) + } - I32Store { mem_addr, offset } => mem_store!(i32, (mem_addr, offset), stack, store, module), - I64Store { mem_addr, offset } => mem_store!(i64, (mem_addr, offset), stack, store, module), - F32Store { mem_addr, offset } => mem_store!(f32, (mem_addr, offset), stack, store, module), - F64Store { mem_addr, offset } => mem_store!(f64, (mem_addr, offset), stack, store, module), - I32Store8 { mem_addr, offset } => mem_store!(i8, i32, (mem_addr, offset), stack, store, module), - I32Store16 { mem_addr, offset } => mem_store!(i16, i32, (mem_addr, offset), stack, store, module), - I64Store8 { mem_addr, offset } => mem_store!(i8, i64, (mem_addr, offset), stack, store, module), - I64Store16 { mem_addr, offset } => mem_store!(i16, i64, (mem_addr, offset), stack, store, module), - I64Store32 { mem_addr, offset } => mem_store!(i32, i64, (mem_addr, offset), stack, store, module), + #[inline(always)] + fn exec_global_set( + &self, + global_index: u32, + stack: &mut Stack, + store: &mut Store, + module: &ModuleInstance, + ) -> Result<()> { + let idx = module.resolve_global_addr(global_index); + store.set_global_val(idx, stack.values.pop()?)?; + Ok(()) + } - I32Load { mem_addr, offset } => mem_load!(i32, (mem_addr, offset), stack, store, module), - I64Load { mem_addr, offset } => mem_load!(i64, (mem_addr, offset), stack, store, module), - F32Load { mem_addr, offset } => mem_load!(f32, (mem_addr, offset), stack, store, module), - F64Load { mem_addr, offset } => mem_load!(f64, (mem_addr, offset), stack, store, module), - I32Load8S { mem_addr, offset } => mem_load!(i8, i32, (mem_addr, offset), stack, store, module), - I32Load8U { mem_addr, offset } => mem_load!(u8, i32, (mem_addr, offset), stack, store, module), - I32Load16S { mem_addr, offset } => mem_load!(i16, i32, (mem_addr, offset), stack, store, module), - I32Load16U { mem_addr, offset } => mem_load!(u16, i32, (mem_addr, offset), stack, store, module), - I64Load8S { mem_addr, offset } => mem_load!(i8, i64, (mem_addr, offset), stack, store, module), - I64Load8U { mem_addr, offset } => mem_load!(u8, i64, (mem_addr, offset), stack, store, module), - I64Load16S { mem_addr, offset } => mem_load!(i16, i64, (mem_addr, offset), stack, store, module), - I64Load16U { mem_addr, offset } => mem_load!(u16, i64, (mem_addr, offset), stack, store, module), - I64Load32S { mem_addr, offset } => mem_load!(i32, i64, (mem_addr, offset), stack, store, module), - I64Load32U { mem_addr, offset } => mem_load!(u32, i64, (mem_addr, offset), stack, store, module), + #[inline(always)] + fn exec_table_get( + &self, + table_index: u32, + stack: &mut Stack, + store: &Store, + module: &ModuleInstance, + ) -> Result<()> { + let table_idx = module.resolve_table_addr(table_index); + let table = store.get_table(table_idx)?; + let idx: u32 = stack.values.pop()?.into(); + let v = table.borrow().get_wasm_val(idx)?; + stack.values.push(v.into()); + Ok(()) + } - I64Eqz => comp_zero!(==, i64, stack), - I32Eqz => comp_zero!(==, i32, stack), + #[inline(always)] + fn exec_table_set( + &self, + table_index: u32, + stack: &mut Stack, + store: &Store, + module: &ModuleInstance, + ) -> Result<()> { + let table_idx = module.resolve_table_addr(table_index); + let table = store.get_table(table_idx)?; + let val = stack.values.pop()?.into(); + let idx = stack.values.pop()?.into(); + table.borrow_mut().set(idx, val)?; + Ok(()) + } - I32Eq => comp!(==, i32, stack), - I64Eq => comp!(==, i64, stack), - F32Eq => comp!(==, f32, stack), - F64Eq => comp!(==, f64, stack), + #[inline(always)] + fn exec_table_size( + &self, + table_index: u32, + stack: &mut Stack, + store: &Store, + module: &ModuleInstance, + ) -> Result<()> { + let table_idx = module.resolve_table_addr(table_index); + let table = store.get_table(table_idx)?; + stack.values.push(table.borrow().size().into()); + Ok(()) + } - I32Ne => comp!(!=, i32, stack), - I64Ne => comp!(!=, i64, stack), - F32Ne => comp!(!=, f32, stack), - F64Ne => comp!(!=, f64, stack), + #[inline(always)] + fn exec_table_init(&self, elem_index: u32, table_index: u32, store: &Store, module: &ModuleInstance) -> Result<()> { + let table_idx = module.resolve_table_addr(table_index); + let table = store.get_table(table_idx)?; + let elem = store.get_elem(module.resolve_elem_addr(elem_index))?; - I32LtS => comp!(<, i32, stack), - I64LtS => comp!(<, i64, stack), - I32LtU => comp!(<, u32, stack), - I64LtU => comp!(<, u64, stack), - F32Lt => comp!(<, f32, stack), - F64Lt => comp!(<, f64, stack), + if let ElementKind::Passive = elem.kind { + return Err(Trap::TableOutOfBounds { offset: 0, len: 0, max: 0 }.into()); + } - I32LeS => comp!(<=, i32, stack), - I64LeS => comp!(<=, i64, stack), - I32LeU => comp!(<=, u32, stack), - I64LeU => comp!(<=, u64, stack), - F32Le => comp!(<=, f32, stack), - F64Le => comp!(<=, f64, stack), + let Some(items) = elem.items.as_ref() else { + return Err(Trap::TableOutOfBounds { offset: 0, len: 0, max: 0 }.into()); + }; - I32GeS => comp!(>=, i32, stack), - I64GeS => comp!(>=, i64, stack), - I32GeU => comp!(>=, u32, stack), - I64GeU => comp!(>=, u64, stack), - F32Ge => comp!(>=, f32, stack), - F64Ge => comp!(>=, f64, stack), + table.borrow_mut().init(module.func_addrs(), 0, items)?; + Ok(()) + } - I32GtS => comp!(>, i32, stack), - I64GtS => comp!(>, i64, stack), - I32GtU => comp!(>, u32, stack), - I64GtU => comp!(>, u64, stack), - F32Gt => comp!(>, f32, stack), - F64Gt => comp!(>, f64, stack), + #[inline(always)] + fn exec_select(&self, stack: &mut Stack) -> Result<()> { + let cond: i32 = stack.values.pop()?.into(); + let val2 = stack.values.pop()?; + // if cond != 0, we already have the right value on the stack + if cond == 0 { + *stack.values.last_mut()? = val2; + } + Ok(()) + } - I64Add => arithmetic!(wrapping_add, i64, stack), - I32Add => arithmetic!(wrapping_add, i32, stack), - F32Add => arithmetic!(+, f32, stack), - F64Add => arithmetic!(+, f64, stack), + #[inline(always)] + fn exec_memory_size( + &self, + addr: u32, + byte: u8, + stack: &mut Stack, + store: &Store, + module: &ModuleInstance, + ) -> Result<()> { + if unlikely(byte != 0) { + return Err(Error::UnsupportedFeature("memory.size with byte != 0".to_string())); + } - I32Sub => arithmetic!(wrapping_sub, i32, stack), - I64Sub => arithmetic!(wrapping_sub, i64, stack), - F32Sub => arithmetic!(-, f32, stack), - F64Sub => arithmetic!(-, f64, stack), + let mem_idx = module.resolve_mem_addr(addr); + let mem = store.get_mem(mem_idx)?; + stack.values.push((mem.borrow().page_count() as i32).into()); + Ok(()) + } - F32Div => arithmetic!(/, f32, stack), - F64Div => arithmetic!(/, f64, stack), + #[inline(always)] + fn exec_memory_grow( + &self, + addr: u32, + byte: u8, + stack: &mut Stack, + store: &Store, + module: &ModuleInstance, + ) -> Result<()> { + if unlikely(byte != 0) { + return Err(Error::UnsupportedFeature("memory.grow with byte != 0".to_string())); + } - I32Mul => arithmetic!(wrapping_mul, i32, stack), - I64Mul => arithmetic!(wrapping_mul, i64, stack), - F32Mul => arithmetic!(*, f32, stack), - F64Mul => arithmetic!(*, f64, stack), + let mut mem = store.get_mem(module.resolve_mem_addr(addr))?.borrow_mut(); + let prev_size = mem.page_count() as i32; + let pages_delta = stack.values.last_mut()?; + *pages_delta = match mem.grow(i32::from(*pages_delta)) { + Some(_) => prev_size.into(), + None => (-1).into(), + }; - // these can trap - I32DivS => checked_int_arithmetic!(checked_div, i32, stack), - I64DivS => checked_int_arithmetic!(checked_div, i64, stack), - I32DivU => checked_int_arithmetic!(checked_div, u32, stack), - I64DivU => checked_int_arithmetic!(checked_div, u64, stack), + Ok(()) + } - I32RemS => checked_int_arithmetic!(checked_wrapping_rem, i32, stack), - I64RemS => checked_int_arithmetic!(checked_wrapping_rem, i64, stack), - I32RemU => checked_int_arithmetic!(checked_wrapping_rem, u32, stack), - I64RemU => checked_int_arithmetic!(checked_wrapping_rem, u64, stack), + #[inline(always)] + fn exec_memory_copy( + &self, + from: u32, + to: u32, + stack: &mut Stack, + store: &Store, + module: &ModuleInstance, + ) -> Result<()> { + let size: i32 = stack.values.pop()?.into(); + let src: i32 = stack.values.pop()?.into(); + let dst: i32 = stack.values.pop()?.into(); - I32And => arithmetic!(bitand, i32, stack), - I64And => arithmetic!(bitand, i64, stack), - I32Or => arithmetic!(bitor, i32, stack), - I64Or => arithmetic!(bitor, i64, stack), - I32Xor => arithmetic!(bitxor, i32, stack), - I64Xor => arithmetic!(bitxor, i64, stack), - I32Shl => arithmetic!(wasm_shl, i32, stack), - I64Shl => arithmetic!(wasm_shl, i64, stack), - I32ShrS => arithmetic!(wasm_shr, i32, stack), - I64ShrS => arithmetic!(wasm_shr, i64, stack), - I32ShrU => arithmetic!(wasm_shr, u32, stack), - I64ShrU => arithmetic!(wasm_shr, u64, stack), - I32Rotl => arithmetic!(wasm_rotl, i32, stack), - I64Rotl => arithmetic!(wasm_rotl, i64, stack), - I32Rotr => arithmetic!(wasm_rotr, i32, stack), - I64Rotr => arithmetic!(wasm_rotr, i64, stack), + if from == to { + let mut mem_from = store.get_mem(module.resolve_mem_addr(from))?.borrow_mut(); + // copy within the same memory + mem_from.copy_within(dst as usize, src as usize, size as usize)?; + } else { + // copy between two memories + let mem_from = store.get_mem(module.resolve_mem_addr(from))?.borrow(); + let mut mem_to = store.get_mem(module.resolve_mem_addr(to))?.borrow_mut(); + mem_to.copy_from_slice(dst as usize, mem_from.load(src as usize, size as usize)?)?; + } + Ok(()) + } - I32Clz => arithmetic_single!(leading_zeros, i32, stack), - I64Clz => arithmetic_single!(leading_zeros, i64, stack), - I32Ctz => arithmetic_single!(trailing_zeros, i32, stack), - I64Ctz => arithmetic_single!(trailing_zeros, i64, stack), - I32Popcnt => arithmetic_single!(count_ones, i32, stack), - I64Popcnt => arithmetic_single!(count_ones, i64, stack), + #[inline(always)] + fn exec_memory_fill(&self, addr: u32, stack: &mut Stack, store: &Store, module: &ModuleInstance) -> Result<()> { + let size: i32 = stack.values.pop()?.into(); + let val: i32 = stack.values.pop()?.into(); + let dst: i32 = stack.values.pop()?.into(); - F32ConvertI32S => conv!(i32, f32, stack), - F32ConvertI64S => conv!(i64, f32, stack), - F64ConvertI32S => conv!(i32, f64, stack), - F64ConvertI64S => conv!(i64, f64, stack), - F32ConvertI32U => conv!(u32, f32, stack), - F32ConvertI64U => conv!(u64, f32, stack), - F64ConvertI32U => conv!(u32, f64, stack), - F64ConvertI64U => conv!(u64, f64, stack), - I32Extend8S => conv!(i8, i32, stack), - I32Extend16S => conv!(i16, i32, stack), - I64Extend8S => conv!(i8, i64, stack), - I64Extend16S => conv!(i16, i64, stack), - I64Extend32S => conv!(i32, i64, stack), - I64ExtendI32U => conv!(u32, i64, stack), - I64ExtendI32S => conv!(i32, i64, stack), - I32WrapI64 => conv!(i64, i32, stack), + let mem = store.get_mem(module.resolve_mem_addr(addr))?; + mem.borrow_mut().fill(dst as usize, size as usize, val as u8)?; + Ok(()) + } - F32DemoteF64 => conv!(f64, f32, stack), - F64PromoteF32 => conv!(f32, f64, stack), + #[inline(always)] + fn exec_memory_init( + &self, + data_index: u32, + mem_index: u32, + stack: &mut Stack, + store: &Store, + module: &ModuleInstance, + ) -> Result<()> { + let size = i32::from(stack.values.pop()?) as usize; + let offset = i32::from(stack.values.pop()?) as usize; + let dst = i32::from(stack.values.pop()?) as usize; - F32Abs => arithmetic_single!(abs, f32, stack), - F64Abs => arithmetic_single!(abs, f64, stack), - F32Neg => arithmetic_single!(neg, f32, stack), - F64Neg => arithmetic_single!(neg, f64, stack), - F32Ceil => arithmetic_single!(ceil, f32, stack), - F64Ceil => arithmetic_single!(ceil, f64, stack), - F32Floor => arithmetic_single!(floor, f32, stack), - F64Floor => arithmetic_single!(floor, f64, stack), - F32Trunc => arithmetic_single!(trunc, f32, stack), - F64Trunc => arithmetic_single!(trunc, f64, stack), - F32Nearest => arithmetic_single!(tw_nearest, f32, stack), - F64Nearest => arithmetic_single!(tw_nearest, f64, stack), - F32Sqrt => arithmetic_single!(sqrt, f32, stack), - F64Sqrt => arithmetic_single!(sqrt, f64, stack), - F32Min => arithmetic!(tw_minimum, f32, stack), - F64Min => arithmetic!(tw_minimum, f64, stack), - F32Max => arithmetic!(tw_maximum, f32, stack), - F64Max => arithmetic!(tw_maximum, f64, stack), - F32Copysign => arithmetic!(copysign, f32, stack), - F64Copysign => arithmetic!(copysign, f64, stack), + let data = match &store.get_data(module.resolve_data_addr(data_index))?.data { + Some(data) => data, + None => return Err(Trap::MemoryOutOfBounds { offset: 0, len: 0, max: 0 }.into()), + }; - // no-op instructions since types are erased at runtime - I32ReinterpretF32 | I64ReinterpretF64 | F32ReinterpretI32 | F64ReinterpretI64 => {} + if unlikely(offset + size > data.len()) { + return Err(Trap::MemoryOutOfBounds { offset, len: size, max: data.len() }.into()); + } - // unsigned versions of these are a bit broken atm - I32TruncF32S => checked_conv_float!(f32, i32, stack), - I32TruncF64S => checked_conv_float!(f64, i32, stack), - I32TruncF32U => checked_conv_float!(f32, u32, i32, stack), - I32TruncF64U => checked_conv_float!(f64, u32, i32, stack), - I64TruncF32S => checked_conv_float!(f32, i64, stack), - I64TruncF64S => checked_conv_float!(f64, i64, stack), - I64TruncF32U => checked_conv_float!(f32, u64, i64, stack), - I64TruncF64U => checked_conv_float!(f64, u64, i64, stack), + let mem = store.get_mem(module.resolve_mem_addr(mem_index))?; + mem.borrow_mut().store(dst, size, &data[offset..(offset + size)])?; + Ok(()) + } - TableGet(table_index) => { - let table_idx = module.resolve_table_addr(*table_index); - let table = store.get_table(table_idx)?; - let idx: u32 = stack.values.pop()?.into(); - let v = table.borrow().get_wasm_val(idx)?; - stack.values.push(v.into()); - } + #[inline(always)] + fn exec_call( + &self, + v: u32, + store: &mut Store, + stack: &mut Stack, + cf: &mut CallFrame, + module: &mut ModuleInstance, + ) -> Result<()> { + let func_inst = store.get_func(module.resolve_func_addr(v))?; + let wasm_func = match &func_inst.func { + crate::Function::Wasm(wasm_func) => wasm_func, + crate::Function::Host(host_func) => { + let func = &host_func.clone(); + let params = stack.values.pop_params(&host_func.ty.params)?; + let res = (func.func)(FuncContext { store, module_addr: module.id() }, ¶ms)?; + stack.values.extend_from_typed(&res); + cf.instr_ptr += 1; + return Ok(()); + } + }; - TableSet(table_index) => { - let table_idx = module.resolve_table_addr(*table_index); - let table = store.get_table(table_idx)?; - let val = stack.values.pop()?.into(); - let idx = stack.values.pop()?.into(); - table.borrow_mut().set(idx, val)?; - } + let params = stack.values.pop_n_rev(wasm_func.ty.params.len())?; + let new_call_frame = CallFrame::new(wasm_func.clone(), func_inst.owner, params, stack.blocks.len() as u32); - TableSize(table_index) => { - let table = store.get_table(module.resolve_table_addr(*table_index))?; - stack.values.push(table.borrow().size().into()); + cf.instr_ptr += 1; // skip the call instruction + stack.call_stack.push(core::mem::replace(cf, new_call_frame))?; + if cf.module_addr != module.id() { + module.swap_with(cf.module_addr, store); } + Ok(()) + } - TableInit(table_index, elem_index) => { - let table = store.get_table(module.resolve_table_addr(*table_index))?; - let elem = store.get_elem(module.resolve_elem_addr(*elem_index))?; + #[inline(always)] + fn exec_call_indirect( + &self, + type_addr: u32, + table_addr: u32, + store: &mut Store, + stack: &mut Stack, + cf: &mut CallFrame, + module: &mut ModuleInstance, + ) -> Result<()> { + let table = store.get_table(module.resolve_table_addr(table_addr))?; + let table_idx: u32 = stack.values.pop()?.into(); - if let ElementKind::Passive = elem.kind { - return Err(Trap::TableOutOfBounds { offset: 0, len: 0, max: 0 }.into()); - } + // verify that the table is of the right type, this should be validated by the parser already + let func_ref = { + let table = table.borrow(); + assert!(table.kind.element_type == ValType::RefFunc, "table is not of type funcref"); + table.get(table_idx)?.addr().ok_or(Trap::UninitializedElement { index: table_idx as usize })? + }; - let Some(items) = elem.items.as_ref() else { - return Err(Trap::TableOutOfBounds { offset: 0, len: 0, max: 0 }.into()); - }; + let func_inst = store.get_func(func_ref)?.clone(); + let call_ty = module.func_ty(type_addr); - table.borrow_mut().init(module.func_addrs(), 0, items)?; - } - - I32TruncSatF32S => arithmetic_single!(trunc, f32, i32, stack), - I32TruncSatF32U => arithmetic_single!(trunc, f32, u32, stack), - I32TruncSatF64S => arithmetic_single!(trunc, f64, i32, stack), - I32TruncSatF64U => arithmetic_single!(trunc, f64, u32, stack), - I64TruncSatF32S => arithmetic_single!(trunc, f32, i64, stack), - I64TruncSatF32U => arithmetic_single!(trunc, f32, u64, stack), - I64TruncSatF64S => arithmetic_single!(trunc, f64, i64, stack), - I64TruncSatF64U => arithmetic_single!(trunc, f64, u64, stack), + let wasm_func = match func_inst.func { + crate::Function::Wasm(ref f) => f, + crate::Function::Host(host_func) => { + if unlikely(host_func.ty != *call_ty) { + return Err(Trap::IndirectCallTypeMismatch { + actual: host_func.ty.clone(), + expected: call_ty.clone(), + } + .into()); + } - // custom instructions - LocalGet2(a, b) => stack.values.extend_from_slice(&[cf.get_local(*a), cf.get_local(*b)]), - LocalGet3(a, b, c) => stack.values.extend_from_slice(&[cf.get_local(*a), cf.get_local(*b), cf.get_local(*c)]), - LocalTeeGet(a, b) => { - #[inline(always)] - fn local_tee_get(cf: &mut CallFrame, stack: &mut Stack, a: u32, b: u32) { - let last = match stack.values.last() { - Ok(v) => v, - Err(_) => unreachable!("localtee: stack is empty. this should have been validated by the parser"), - }; + let host_func = host_func.clone(); + let params = stack.values.pop_params(&host_func.ty.params)?; + let res = (host_func.func)(FuncContext { store, module_addr: module.id() }, ¶ms)?; + stack.values.extend_from_typed(&res); - cf.set_local(a, *last); - stack.values.push(cf.get_local(b)); + cf.instr_ptr += 1; + return Ok(()); } + }; - local_tee_get(cf, stack, *a, *b); - } - LocalGetSet(a, b) => cf.set_local(*b, cf.get_local(*a)), - I64XorConstRotl(rotate_by) => { - let val: i64 = stack.values.pop()?.into(); - let mask: i64 = stack.values.pop()?.into(); - let res = val ^ mask; - stack.values.push(res.rotate_left(*rotate_by as u32).into()); + if unlikely(wasm_func.ty != *call_ty) { + return Err( + Trap::IndirectCallTypeMismatch { actual: wasm_func.ty.clone(), expected: call_ty.clone() }.into() + ); } - I32LocalGetConstAdd(local, val) => { - let local: i32 = cf.get_local(*local).into(); - stack.values.push((local + *val).into()); + + let params = stack.values.pop_n_rev(wasm_func.ty.params.len())?; + let new_call_frame = CallFrame::new(wasm_func.clone(), func_inst.owner, params, stack.blocks.len() as u32); + + cf.instr_ptr += 1; // skip the call instruction + stack.call_stack.push(core::mem::replace(cf, new_call_frame))?; + if cf.module_addr != module.id() { + module.swap_with(cf.module_addr, store); } - I32StoreLocal { local, const_i32: consti32, offset, mem_addr } => { - let (mem_addr, offset) = (*mem_addr as u32, *offset); - let mem = store.get_mem(module.resolve_mem_addr(mem_addr))?; - let val = consti32.to_le_bytes(); - let addr: u64 = cf.get_local(*local).into(); - mem.borrow_mut().store((offset as u64 + addr) as usize, val.len(), &val)?; + Ok(()) + } + + #[inline(always)] + fn exec_if( + &self, + args: BlockArgs, + else_offset: u32, + end_offset: u32, + stack: &mut Stack, + cf: &mut CallFrame, + module: &mut ModuleInstance, + ) -> Result<()> { + // truthy value is on the top of the stack, so enter the then block + if i32::from(stack.values.pop()?) != 0 { + self.enter_block(stack, cf.instr_ptr, end_offset, BlockType::If, &args, module); + cf.instr_ptr += 1; + return Ok(()); } - i => { - cold(); - return Err(Error::UnsupportedFeature(format!("unimplemented instruction: {:?}", i))); + + // falsy value is on the top of the stack + if else_offset == 0 { + cf.instr_ptr += end_offset as usize + 1; + return Ok(()); } - }; - Ok(ExecResult::Ok) + let old = cf.instr_ptr; + cf.instr_ptr += else_offset as usize; + + self.enter_block(stack, old + else_offset as usize, end_offset - else_offset, BlockType::Else, &args, module); + + cf.instr_ptr += 1; + Ok(()) + } + + #[inline(always)] + fn enter_block( + &self, + stack: &mut super::Stack, + instr_ptr: usize, + end_instr_offset: u32, + ty: BlockType, + args: &BlockArgs, + module: &ModuleInstance, + ) { + let (params, results) = match args { + BlockArgs::Empty => (0, 0), + BlockArgs::Type(_) => (0, 1), + BlockArgs::FuncType(t) => { + let ty = module.func_ty(*t); + (ty.params.len() as u8, ty.results.len() as u8) + } + }; + + stack.blocks.push(BlockFrame { + instr_ptr, + end_instr_offset, + stack_ptr: stack.values.len() as u32 - params as u32, + results, + params, + ty, + }); + } } diff --git a/crates/tinywasm/src/runtime/stack/block_stack.rs b/crates/tinywasm/src/runtime/stack/block_stack.rs index a7b8ece..9a823fd 100644 --- a/crates/tinywasm/src/runtime/stack/block_stack.rs +++ b/crates/tinywasm/src/runtime/stack/block_stack.rs @@ -1,18 +1,15 @@ -use crate::{unlikely, Error, ModuleInstance, Result}; +use crate::{cold, unlikely, Error, Result}; use alloc::vec::Vec; -use tinywasm_types::BlockArgs; #[derive(Debug, Clone)] pub(crate) struct BlockStack(Vec<BlockFrame>); impl BlockStack { pub(crate) fn new() -> Self { - let mut vec = Vec::new(); - vec.reserve(128); - Self(vec) + Self(Vec::with_capacity(128)) } - #[inline] + #[inline(always)] pub(crate) fn len(&self) -> usize { self.0.len() } @@ -35,55 +32,36 @@ impl BlockStack { Some(&self.0[self.0.len() - index as usize - 1]) } - #[inline] + #[inline(always)] pub(crate) fn pop(&mut self) -> Result<BlockFrame> { - self.0.pop().ok_or(Error::BlockStackUnderflow) + match self.0.pop() { + Some(frame) => Ok(frame), + None => { + cold(); + Err(Error::BlockStackUnderflow) + } + } } /// keep the top `len` blocks and discard the rest - #[inline] + #[inline(always)] pub(crate) fn truncate(&mut self, len: u32) { self.0.truncate(len as usize); } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub(crate) struct BlockFrame { - // position of the instruction pointer when the block was entered - pub(crate) instr_ptr: u32, - // position of the end instruction of the block - pub(crate) end_instr_ptr: u32, - - // position of the stack pointer when the block was entered - pub(crate) stack_ptr: u32, + pub(crate) instr_ptr: usize, // position of the instruction pointer when the block was entered + pub(crate) end_instr_offset: u32, // position of the end instruction of the block + pub(crate) stack_ptr: u32, // position of the stack pointer when the block was entered pub(crate) results: u8, pub(crate) params: u8, pub(crate) ty: BlockType, } -impl BlockFrame { - #[inline(always)] - pub(crate) fn new( - instr_ptr: u32, - end_instr_ptr: u32, - stack_ptr: u32, - ty: BlockType, - args: &BlockArgs, - module: &ModuleInstance, - ) -> Self { - let (params, results) = match args { - BlockArgs::Empty => (0, 0), - BlockArgs::Type(_) => (0, 1), - BlockArgs::FuncType(t) => { - let ty = module.func_ty(*t); - (ty.params.len() as u8, ty.results.len() as u8) - } - }; - - Self { instr_ptr, end_instr_ptr, stack_ptr, results, params, ty } - } -} +impl BlockFrame {} #[derive(Debug, Copy, Clone)] #[allow(dead_code)] diff --git a/crates/tinywasm/src/runtime/stack/call_stack.rs b/crates/tinywasm/src/runtime/stack/call_stack.rs index 8002385..5dc754f 100644 --- a/crates/tinywasm/src/runtime/stack/call_stack.rs +++ b/crates/tinywasm/src/runtime/stack/call_stack.rs @@ -1,11 +1,8 @@ -use alloc::{boxed::Box, rc::Rc, vec::Vec}; -use tinywasm_types::{Instruction, LocalAddr, ModuleInstanceAddr, WasmFunction}; - use crate::runtime::{BlockType, RawWasmValue}; -use crate::unlikely; +use crate::{cold, unlikely}; use crate::{Error, Result, Trap}; - -use super::BlockFrame; +use alloc::{boxed::Box, rc::Rc, vec::Vec}; +use tinywasm_types::{Instruction, LocalAddr, ModuleInstanceAddr, WasmFunction}; const CALL_STACK_SIZE: usize = 1024; @@ -19,10 +16,8 @@ impl CallStack { pub(crate) fn new(initial_frame: CallFrame) -> Self { let mut stack = Vec::new(); stack.reserve_exact(CALL_STACK_SIZE); - - let mut stack = Self { stack }; - stack.push(initial_frame).unwrap(); - stack + stack.push(initial_frame); + Self { stack } } #[inline] @@ -30,17 +25,20 @@ impl CallStack { self.stack.is_empty() } - #[inline] + #[inline(always)] pub(crate) fn pop(&mut self) -> Result<CallFrame> { match self.stack.pop() { Some(frame) => Ok(frame), - None => Err(Error::CallStackUnderflow), + None => { + cold(); + Err(Error::CallStackUnderflow) + } } } - #[inline] + #[inline(always)] pub(crate) fn push(&mut self, call_frame: CallFrame) -> Result<()> { - if unlikely(self.stack.len() >= CALL_STACK_SIZE) { + if unlikely(self.stack.len() >= self.stack.capacity()) { return Err(Trap::CallStackOverflow.into()); } self.stack.push(call_frame); @@ -50,7 +48,7 @@ impl CallStack { #[derive(Debug, Clone)] pub(crate) struct CallFrame { - pub(crate) instr_ptr: u32, + pub(crate) instr_ptr: usize, pub(crate) block_ptr: u32, pub(crate) func_instance: Rc<WasmFunction>, pub(crate) module_addr: ModuleInstanceAddr, @@ -58,20 +56,15 @@ pub(crate) struct CallFrame { } impl CallFrame { - /// Push a new label to the label stack and ensure the stack has the correct values - pub(crate) fn enter_block( - &mut self, - block_frame: BlockFrame, - values: &mut super::ValueStack, - blocks: &mut super::BlockStack, - ) { - if block_frame.params > 0 { - let start = (block_frame.stack_ptr - block_frame.params as u32) as usize; - let end = block_frame.stack_ptr as usize; - values.extend_from_within(start..end); + #[inline(always)] + pub(crate) fn fetch_instr(&self) -> &Instruction { + match self.func_instance.instructions.get(self.instr_ptr) { + Some(instr) => instr, + None => { + cold(); + panic!("Instruction pointer out of bounds"); + } } - - blocks.push(block_frame); } /// Break to a block at the given index (relative to the current frame) @@ -108,7 +101,7 @@ impl CallFrame { values.break_to(break_to.stack_ptr, break_to.results); // (the inst_ptr will be incremented by 1 before the next instruction is executed) - self.instr_ptr = break_to.end_instr_ptr; + self.instr_ptr = break_to.instr_ptr + break_to.end_instr_offset as usize; // we also want to trim the label stack, including the block blocks.truncate(blocks.len() as u32 - (break_to_relative + 1)); @@ -118,8 +111,7 @@ impl CallFrame { Some(()) } - // TODO: perf: a lot of time is spent here - #[inline(always)] // about 10% faster with this + #[inline(always)] pub(crate) fn new( wasm_func_inst: Rc<WasmFunction>, owner: ModuleInstanceAddr, @@ -138,12 +130,12 @@ impl CallFrame { Self { instr_ptr: 0, func_instance: wasm_func_inst, module_addr: owner, locals, block_ptr } } - #[inline] + #[inline(always)] pub(crate) fn set_local(&mut self, local_index: LocalAddr, value: RawWasmValue) { self.locals[local_index as usize] = value; } - #[inline] + #[inline(always)] pub(crate) fn get_local(&self, local_index: LocalAddr) -> RawWasmValue { self.locals[local_index as usize] } diff --git a/crates/tinywasm/src/runtime/stack/value_stack.rs b/crates/tinywasm/src/runtime/stack/value_stack.rs index 8a3f3fa..1573a5d 100644 --- a/crates/tinywasm/src/runtime/stack/value_stack.rs +++ b/crates/tinywasm/src/runtime/stack/value_stack.rs @@ -1,5 +1,3 @@ -use core::ops::Range; - use crate::{cold, runtime::RawWasmValue, unlikely, Error, Result}; use alloc::vec::Vec; use tinywasm_types::{ValType, WasmValue}; @@ -13,43 +11,48 @@ pub(crate) struct ValueStack { impl Default for ValueStack { fn default() -> Self { - let mut vec = Vec::new(); - vec.reserve(MIN_VALUE_STACK_SIZE); // gives a slight performance over with_capacity - Self { stack: vec } + Self { stack: Vec::with_capacity(MIN_VALUE_STACK_SIZE) } } } impl ValueStack { #[inline] - pub(crate) fn extend_from_within(&mut self, range: Range<usize>) { - self.stack.extend_from_within(range); - } - - #[inline] pub(crate) fn extend_from_typed(&mut self, values: &[WasmValue]) { self.stack.extend(values.iter().map(|v| RawWasmValue::from(*v))); } - #[inline] - pub(crate) fn extend_from_slice(&mut self, values: &[RawWasmValue]) { - self.stack.extend_from_slice(values); + #[inline(always)] + pub(crate) fn replace_top(&mut self, func: fn(RawWasmValue) -> RawWasmValue) -> Result<()> { + let v = self.last_mut()?; + *v = func(*v); + Ok(()) } - #[inline] - pub(crate) fn replace_top(&mut self, func: impl FnOnce(RawWasmValue) -> RawWasmValue) { - let len = self.stack.len(); - if unlikely(len == 0) { - return; - } - let top = self.stack[len - 1]; - self.stack[len - 1] = func(top); + #[inline(always)] + pub(crate) fn calculate(&mut self, func: fn(RawWasmValue, RawWasmValue) -> RawWasmValue) -> Result<()> { + let v2 = self.pop()?; + let v1 = self.last_mut()?; + *v1 = func(*v1, v2); + Ok(()) } - #[inline] + #[inline(always)] + pub(crate) fn calculate_trap( + &mut self, + func: fn(RawWasmValue, RawWasmValue) -> Result<RawWasmValue>, + ) -> Result<()> { + let v2 = self.pop()?; + let v1 = self.last_mut()?; + *v1 = func(*v1, v2)?; + Ok(()) + } + + #[inline(always)] pub(crate) fn len(&self) -> usize { self.stack.len() } + #[inline] pub(crate) fn truncate_keep(&mut self, n: u32, end_keep: u32) { let total_to_keep = n + end_keep; let len = self.stack.len() as u32; @@ -71,6 +74,17 @@ impl ValueStack { } #[inline] + pub(crate) fn last_mut(&mut self) -> Result<&mut RawWasmValue> { + match self.stack.last_mut() { + Some(v) => Ok(v), + None => { + cold(); + Err(Error::ValueStackUnderflow) + } + } + } + + #[inline] pub(crate) fn last(&self) -> Result<&RawWasmValue> { match self.stack.last() { Some(v) => Ok(v), @@ -81,7 +95,7 @@ impl ValueStack { } } - #[inline] + #[inline(always)] pub(crate) fn pop(&mut self) -> Result<RawWasmValue> { match self.stack.pop() { Some(v) => Ok(v), diff --git a/crates/tinywasm/src/runtime/value.rs b/crates/tinywasm/src/runtime/value.rs index 2865308..e5769bf 100644 --- a/crates/tinywasm/src/runtime/value.rs +++ b/crates/tinywasm/src/runtime/value.rs @@ -86,9 +86,9 @@ impl_from_raw_wasm_value!(u32, |x| x as u64, |x: [u8; 8]| u32::from_ne_bytes(x[0 impl_from_raw_wasm_value!(u64, |x| x, |x: [u8; 8]| u64::from_ne_bytes(x[0..8].try_into().unwrap())); impl_from_raw_wasm_value!(i8, |x| x as u64, |x: [u8; 8]| i8::from_ne_bytes(x[0..1].try_into().unwrap())); impl_from_raw_wasm_value!(i16, |x| x as u64, |x: [u8; 8]| i16::from_ne_bytes(x[0..2].try_into().unwrap())); -impl_from_raw_wasm_value!(f32, |x| f32::to_bits(x) as u64, |x: [u8; 8]| f32::from_bits(u32::from_ne_bytes( +impl_from_raw_wasm_value!(f32, |x| f32::to_bits(x) as u64, |x: [u8; 8]| f32::from_ne_bytes( x[0..4].try_into().unwrap() -))); +)); impl_from_raw_wasm_value!(f64, f64::to_bits, |x: [u8; 8]| f64::from_bits(u64::from_ne_bytes( x[0..8].try_into().unwrap() ))); diff --git a/crates/tinywasm/src/store/global.rs b/crates/tinywasm/src/store/global.rs index cbba1a2..6cc778c 100644 --- a/crates/tinywasm/src/store/global.rs +++ b/crates/tinywasm/src/store/global.rs @@ -1,3 +1,5 @@ +use core::cell::Cell; + use alloc::{format, string::ToString}; use tinywasm_types::*; @@ -8,19 +10,19 @@ use crate::{runtime::RawWasmValue, unlikely, Error, Result}; /// See <https://webassembly.github.io/spec/core/exec/runtime.html#global-instances> #[derive(Debug)] pub(crate) struct GlobalInstance { - pub(crate) value: RawWasmValue, + pub(crate) value: Cell<RawWasmValue>, pub(crate) ty: GlobalType, pub(crate) _owner: ModuleInstanceAddr, // index into store.module_instances } impl GlobalInstance { pub(crate) fn new(ty: GlobalType, value: RawWasmValue, owner: ModuleInstanceAddr) -> Self { - Self { ty, value, _owner: owner } + Self { ty, value: value.into(), _owner: owner } } #[inline] pub(crate) fn get(&self) -> WasmValue { - self.value.attach_type(self.ty.ty) + self.value.get().attach_type(self.ty.ty) } pub(crate) fn set(&mut self, val: WasmValue) -> Result<()> { @@ -36,7 +38,7 @@ impl GlobalInstance { return Err(Error::Other("global is immutable".to_string())); } - self.value = val.into(); + self.value.set(val.into()); Ok(()) } } diff --git a/crates/tinywasm/src/store/memory.rs b/crates/tinywasm/src/store/memory.rs index 00bcc97..5ef4366 100644 --- a/crates/tinywasm/src/store/memory.rs +++ b/crates/tinywasm/src/store/memory.rs @@ -32,6 +32,7 @@ impl MemoryInstance { } } + #[inline(never)] #[cold] fn trap_oob(&self, addr: usize, len: usize) -> Error { Error::Trap(crate::Trap::MemoryOutOfBounds { offset: addr, len, max: self.data.len() }) @@ -46,20 +47,7 @@ impl MemoryInstance { return Err(self.trap_oob(addr, data.len())); } - // WebAssembly doesn't require alignment for stores - #[cfg(not(feature = "unsafe"))] self.data[addr..end].copy_from_slice(data); - - #[cfg(feature = "unsafe")] - // SAFETY: we checked that `end` is in bounds above, this is the same as `copy_from_slice` - // src must is for reads of count * size_of::<T>() bytes. - // dst must is for writes of count * size_of::<T>() bytes. - // Both src and dst are properly aligned. - // The region of memory beginning at src does not overlap with the region of memory beginning at dst with the same size. - unsafe { - core::ptr::copy_nonoverlapping(data.as_ptr(), self.data[addr..end].as_mut_ptr(), len); - } - Ok(()) } @@ -88,14 +76,10 @@ impl MemoryInstance { if end > self.data.len() { return Err(self.trap_oob(addr, SIZE)); } - - #[cfg(not(feature = "unsafe"))] - let val = T::from_le_bytes(self.data[addr..end].try_into().expect("slice size mismatch")); - - #[cfg(feature = "unsafe")] - // SAFETY: we checked that `end` is in bounds above. All types that implement `Into<RawWasmValue>` are valid - // to load from unaligned addresses. - let val = unsafe { core::ptr::read_unaligned(self.data[addr..end].as_ptr() as *const T) }; + let val = T::from_le_bytes(match self.data[addr..end].try_into() { + Ok(bytes) => bytes, + Err(_) => unreachable!("checked bounds above"), + }); Ok(val) } @@ -168,37 +152,20 @@ impl MemoryInstance { } } -#[allow(unsafe_code)] /// A trait for types that can be loaded from memory -/// -/// # Safety -/// Only implemented for primitive types, unsafe to not allow it for other types. -/// Only actually unsafe to implement if the `unsafe` feature is enabled since there might be -/// UB for loading things things like packed structs -pub(crate) unsafe trait MemLoadable<const T: usize>: Sized + Copy { +pub(crate) trait MemLoadable<const T: usize>: Sized + Copy { /// Load a value from memory - #[allow(unused)] fn from_le_bytes(bytes: [u8; T]) -> Self; - /// Load a value from memory - #[allow(unused)] - fn from_be_bytes(bytes: [u8; T]) -> Self; } macro_rules! impl_mem_loadable_for_primitive { ($($type:ty, $size:expr),*) => { $( - #[allow(unused)] - #[allow(unsafe_code)] - unsafe impl MemLoadable<$size> for $type { - #[inline] + impl MemLoadable<$size> for $type { + #[inline(always)] fn from_le_bytes(bytes: [u8; $size]) -> Self { <$type>::from_le_bytes(bytes) } - - #[inline] - fn from_be_bytes(bytes: [u8; $size]) -> Self { - <$type>::from_be_bytes(bytes) - } } )* } diff --git a/crates/tinywasm/src/store/mod.rs b/crates/tinywasm/src/store/mod.rs index fddf3f4..8c6e698 100644 --- a/crates/tinywasm/src/store/mod.rs +++ b/crates/tinywasm/src/store/mod.rs @@ -1,4 +1,4 @@ -use alloc::{boxed::Box, format, rc::Rc, string::ToString, vec::Vec}; +use alloc::{boxed::Box, format, string::ToString, vec::Vec}; use core::cell::RefCell; use core::sync::atomic::{AtomicUsize, Ordering}; use tinywasm_types::*; @@ -31,7 +31,6 @@ static STORE_ID: AtomicUsize = AtomicUsize::new(0); pub struct Store { id: usize, module_instances: Vec<ModuleInstance>, - module_instance_count: usize, pub(crate) data: StoreData, pub(crate) runtime: Runtime, @@ -74,14 +73,7 @@ impl PartialEq for Store { impl Default for Store { fn default() -> Self { let id = STORE_ID.fetch_add(1, Ordering::Relaxed); - - Self { - id, - module_instances: Vec::new(), - module_instance_count: 0, - data: StoreData::default(), - runtime: Runtime::Default, - } + Self { id, module_instances: Vec::new(), data: StoreData::default(), runtime: Runtime::Default } } } @@ -92,9 +84,9 @@ impl Default for Store { /// See <https://webassembly.github.io/spec/core/exec/runtime.html#store> pub(crate) struct StoreData { pub(crate) funcs: Vec<FunctionInstance>, - pub(crate) tables: Vec<Rc<RefCell<TableInstance>>>, - pub(crate) memories: Vec<Rc<RefCell<MemoryInstance>>>, - pub(crate) globals: Vec<Rc<RefCell<GlobalInstance>>>, + pub(crate) tables: Vec<RefCell<TableInstance>>, + pub(crate) memories: Vec<RefCell<MemoryInstance>>, + pub(crate) globals: Vec<GlobalInstance>, pub(crate) elements: Vec<ElementInstance>, pub(crate) datas: Vec<DataInstance>, } @@ -106,14 +98,12 @@ impl Store { } pub(crate) fn next_module_instance_idx(&self) -> ModuleInstanceAddr { - self.module_instance_count as ModuleInstanceAddr + self.module_instances.len() as ModuleInstanceAddr } - pub(crate) fn add_instance(&mut self, instance: ModuleInstance) -> Result<()> { - assert!(instance.id() == self.module_instance_count as ModuleInstanceAddr); + pub(crate) fn add_instance(&mut self, instance: ModuleInstance) { + assert!(instance.id() == self.module_instances.len() as ModuleInstanceAddr); self.module_instances.push(instance); - self.module_instance_count += 1; - Ok(()) } #[cold] @@ -129,13 +119,13 @@ impl Store { /// Get the memory at the actual index in the store #[inline] - pub(crate) fn get_mem(&self, addr: MemAddr) -> Result<&Rc<RefCell<MemoryInstance>>> { + pub(crate) fn get_mem(&self, addr: MemAddr) -> Result<&RefCell<MemoryInstance>> { self.data.memories.get(addr as usize).ok_or_else(|| Self::not_found_error("memory")) } /// Get the table at the actual index in the store #[inline] - pub(crate) fn get_table(&self, addr: TableAddr) -> Result<&Rc<RefCell<TableInstance>>> { + pub(crate) fn get_table(&self, addr: TableAddr) -> Result<&RefCell<TableInstance>> { self.data.tables.get(addr as usize).ok_or_else(|| Self::not_found_error("table")) } @@ -159,7 +149,7 @@ impl Store { /// Get the global at the actual index in the store #[inline] - pub(crate) fn get_global(&self, addr: GlobalAddr) -> Result<&Rc<RefCell<GlobalInstance>>> { + pub(crate) fn get_global(&self, addr: GlobalAddr) -> Result<&GlobalInstance> { self.data.globals.get(addr as usize).ok_or_else(|| Self::not_found_error("global")) } @@ -170,14 +160,14 @@ impl Store { .globals .get(addr as usize) .ok_or_else(|| Self::not_found_error("global")) - .map(|global| global.borrow().value) + .map(|global| global.value.get()) } /// Set the global at the actual index in the store #[inline] pub(crate) fn set_global_val(&mut self, addr: MemAddr, value: RawWasmValue) -> Result<()> { let global = self.data.globals.get(addr as usize).ok_or_else(|| Self::not_found_error("global")); - global.map(|global| global.borrow_mut().value = value) + global.map(|global| global.value.set(value)) } } @@ -199,7 +189,7 @@ impl Store { let table_count = self.data.tables.len(); let mut table_addrs = Vec::with_capacity(table_count); for (i, table) in tables.into_iter().enumerate() { - self.data.tables.push(Rc::new(RefCell::new(TableInstance::new(table, idx)))); + self.data.tables.push(RefCell::new(TableInstance::new(table, idx))); table_addrs.push((i + table_count) as TableAddr); } Ok(table_addrs) @@ -213,7 +203,7 @@ impl Store { if let MemoryArch::I64 = mem.arch { return Err(Error::UnsupportedFeature("64-bit memories".to_string())); } - self.data.memories.push(Rc::new(RefCell::new(MemoryInstance::new(mem, idx)))); + self.data.memories.push(RefCell::new(MemoryInstance::new(mem, idx))); mem_addrs.push((i + mem_count) as MemAddr); } Ok(mem_addrs) @@ -232,11 +222,11 @@ impl Store { let mut global_addrs = imported_globals; for (i, global) in new_globals.iter().enumerate() { - self.data.globals.push(Rc::new(RefCell::new(GlobalInstance::new( + self.data.globals.push(GlobalInstance::new( global.ty, self.eval_const(&global.init, &global_addrs, func_addrs)?, idx, - )))); + )); global_addrs.push((i + global_count) as Addr); } @@ -255,8 +245,7 @@ impl Store { let addr = globals.get(*addr as usize).copied().ok_or_else(|| { Error::Other(format!("global {} not found. This should have been caught by the validator", addr)) })?; - let global = self.data.globals[addr as usize].clone(); - let val = i64::from(global.borrow().value); + let val: i64 = self.data.globals[addr as usize].value.get().into(); // check if the global is actually a null reference match val < 0 { @@ -374,12 +363,12 @@ impl Store { } pub(crate) fn add_global(&mut self, ty: GlobalType, value: RawWasmValue, idx: ModuleInstanceAddr) -> Result<Addr> { - self.data.globals.push(Rc::new(RefCell::new(GlobalInstance::new(ty, value, idx)))); + self.data.globals.push(GlobalInstance::new(ty, value, idx)); Ok(self.data.globals.len() as Addr - 1) } pub(crate) fn add_table(&mut self, table: TableType, idx: ModuleInstanceAddr) -> Result<TableAddr> { - self.data.tables.push(Rc::new(RefCell::new(TableInstance::new(table, idx)))); + self.data.tables.push(RefCell::new(TableInstance::new(table, idx))); Ok(self.data.tables.len() as TableAddr - 1) } @@ -387,7 +376,7 @@ impl Store { if let MemoryArch::I64 = mem.arch { return Err(Error::UnsupportedFeature("64-bit memories".to_string())); } - self.data.memories.push(Rc::new(RefCell::new(MemoryInstance::new(mem, idx)))); + self.data.memories.push(RefCell::new(MemoryInstance::new(mem, idx))); Ok(self.data.memories.len() as MemAddr - 1) } @@ -401,10 +390,7 @@ impl Store { use tinywasm_types::ConstInstruction::*; let val = match const_instr { I32Const(i) => *i, - GlobalGet(addr) => { - let global = self.data.globals[*addr as usize].borrow(); - i32::from(global.value) - } + GlobalGet(addr) => i32::from(self.data.globals[*addr as usize].value.get()), _ => return Err(Error::Other("expected i32".to_string())), }; Ok(val) @@ -430,8 +416,7 @@ impl Store { let global = self.data.globals.get(*addr as usize).expect("global not found. This should be unreachable"); - - global.borrow().value + global.value.get() } RefNull(t) => RawWasmValue::from(t.default_value()), RefFunc(idx) => RawWasmValue::from(*module_func_addrs.get(*idx as usize).ok_or_else(|| { diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index 76e5679..33271d5 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -13,8 +13,7 @@ rkyv={version="0.7", optional=true, default-features=false, features=["size_32", bytecheck={version="0.7", optional=true} [features] -default=["std", "logging", "archive", "unsafe"] +default=["std", "logging", "archive"] std=["rkyv?/std"] archive=["dep:rkyv", "dep:bytecheck"] logging=["dep:log"] -unsafe=[] diff --git a/crates/types/src/archive.rs b/crates/types/src/archive.rs index 9bf095b..52146e7 100644 --- a/crates/types/src/archive.rs +++ b/crates/types/src/archive.rs @@ -65,18 +65,6 @@ impl TinyWasmModule { Ok(root.deserialize(&mut rkyv::Infallible).unwrap()) } - #[cfg(feature = "unsafe")] - #[allow(unsafe_code)] - /// Creates a TinyWasmModule from a slice of bytes. - /// - /// # Safety - /// This function is only safe to call if the bytes have been created by - /// a trusted source. Otherwise, it may cause undefined behavior. - pub unsafe fn from_twasm_unchecked(wasm: &[u8]) -> Self { - let len = validate_magic(wasm).unwrap(); - rkyv::archived_root::<TinyWasmModule>(&wasm[len..]).deserialize(&mut rkyv::Infallible).unwrap() - } - /// Serializes the TinyWasmModule into a vector of bytes. /// AlignedVec can be deferenced as a slice of bytes and /// implements io::Write when the `std` feature is enabled. @@ -101,14 +89,4 @@ mod tests { let wasm2 = TinyWasmModule::from_twasm(&twasm).unwrap(); assert_eq!(wasm, wasm2); } - - #[cfg(feature = "unsafe")] - #[test] - fn test_serialize_unchecked() { - let wasm = TinyWasmModule::default(); - let twasm = wasm.serialize_twasm(); - #[allow(unsafe_code)] - let wasm2 = unsafe { TinyWasmModule::from_twasm_unchecked(&twasm) }; - assert_eq!(wasm, wasm2); - } } diff --git a/crates/types/src/instructions.rs b/crates/types/src/instructions.rs index ee624bd..6290bb4 100644 --- a/crates/types/src/instructions.rs +++ b/crates/types/src/instructions.rs @@ -85,6 +85,7 @@ pub enum ConstInstruction { #[cfg_attr(feature = "archive", derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize), archive(check_bytes))] // should be kept as small as possible (16 bytes max) #[rustfmt::skip] +#[non_exhaustive] pub enum Instruction { // > Custom Instructions BrLabel(LabelAddr), diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index c5c8071..5e38bfc 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -4,8 +4,7 @@ ))] #![warn(missing_debug_implementations, rust_2018_idioms, unreachable_pub)] #![no_std] -#![cfg_attr(not(feature = "unsafe"), forbid(unsafe_code))] -#![cfg_attr(feature = "unsafe", deny(unused_unsafe))] +#![forbid(unsafe_code)] //! Types used by [`tinywasm`](https://docs.rs/tinywasm) and [`tinywasm_parser`](https://docs.rs/tinywasm_parser). @@ -95,7 +94,7 @@ pub struct TinyWasmModule { /// A WebAssembly External Kind. /// /// See <https://webassembly.github.io/spec/core/syntax/types.html#external-types> -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, Copy, PartialEq)] #[cfg_attr(feature = "archive", derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize), archive(check_bytes))] pub enum ExternalKind { /// A WebAssembly Function. |