diff options
| -rw-r--r-- | crates/parser/src/conversion.rs | 4 | ||||
| -rw-r--r-- | crates/tinywasm/src/func.rs | 6 | ||||
| -rw-r--r-- | crates/tinywasm/src/imports.rs | 80 | ||||
| -rw-r--r-- | crates/tinywasm/src/instance.rs | 272 | ||||
| -rw-r--r-- | crates/tinywasm/src/interpreter/executor.rs | 154 | ||||
| -rw-r--r-- | crates/tinywasm/src/interpreter/stack/call_stack.rs | 31 | ||||
| -rw-r--r-- | crates/tinywasm/src/interpreter/stack/value_stack.rs | 36 | ||||
| -rw-r--r-- | crates/tinywasm/src/store/mod.rs | 252 | ||||
| -rw-r--r-- | crates/types/src/lib.rs | 25 |
9 files changed, 411 insertions, 449 deletions
diff --git a/crates/parser/src/conversion.rs b/crates/parser/src/conversion.rs index 08f3c83..f7fcb34 100644 --- a/crates/parser/src/conversion.rs +++ b/crates/parser/src/conversion.rs @@ -241,8 +241,8 @@ pub(crate) fn process_const_operators(ops: OperatorsReader<'_>) -> Result<Box<[C // In practice, the len can never be something other than 2, // but we'll keep this here since it's part of the spec // Invalid modules will be rejected by the validator anyway (there are also tests for this in the testsuite) - assert!(ops.len() >= 2); - assert!(matches!(ops[ops.len() - 1], wasmparser::Operator::End)); + debug_assert!(ops.len() >= 2); + debug_assert!(matches!(ops[ops.len() - 1], wasmparser::Operator::End)); let mut out = Vec::with_capacity(ops.len().saturating_sub(1)); for op in ops.iter().take(ops.len() - 1) { diff --git a/crates/tinywasm/src/func.rs b/crates/tinywasm/src/func.rs index d79951a..e0929e4 100644 --- a/crates/tinywasm/src/func.rs +++ b/crates/tinywasm/src/func.rs @@ -25,8 +25,7 @@ impl Function { store.value_stack.clear(); store.value_stack.extend_from_wasmvalues(params)?; let locals_base = store.value_stack.enter_locals(&wasm_func.func.params, &wasm_func.func.locals)?; - let stack_offset = wasm_func.func.locals; - let callframe = CallFrame::new(self.addr, wasm_func.owner, locals_base, stack_offset); + let callframe = CallFrame::new(self.addr, locals_base, wasm_func.func.locals); // Execute until completion and then collect result values from the stack. InterpreterRuntime::exec(store, callframe)?; @@ -56,8 +55,7 @@ impl Function { store.value_stack.clear(); store.value_stack.extend_from_wasmvalues(params)?; let locals_base = store.value_stack.enter_locals(&wasm_func.func.params, &wasm_func.func.locals)?; - let stack_offset = wasm_func.func.locals; - let callframe = CallFrame::new(self.addr, wasm_func.owner, locals_base, stack_offset); + let callframe = CallFrame::new(self.addr, locals_base, wasm_func.func.locals); Ok(FuncExecution { store, diff --git a/crates/tinywasm/src/imports.rs b/crates/tinywasm/src/imports.rs index fff2a8f..ba6076e 100644 --- a/crates/tinywasm/src/imports.rs +++ b/crates/tinywasm/src/imports.rs @@ -98,10 +98,7 @@ impl From<&Import> for ExternName { #[derive(Default, Clone)] #[cfg_attr(feature = "debug", derive(Debug))] pub struct Imports { - globals: BTreeMap<ExternName, Global>, - tables: BTreeMap<ExternName, Table>, - memories: BTreeMap<ExternName, Memory>, - function_handles: BTreeMap<ExternName, Function>, + externs: BTreeMap<ExternName, Extern>, modules: BTreeMap<String, crate::ModuleInstance>, } @@ -112,30 +109,15 @@ pub(crate) struct ResolvedImports { pub(crate) funcs: Vec<FuncAddr>, } -impl ResolvedImports { - pub(crate) const fn new() -> Self { - Self { globals: Vec::new(), tables: Vec::new(), memories: Vec::new(), funcs: Vec::new() } - } -} - impl Imports { /// Create a new empty import set pub const fn new() -> Self { - Self { - globals: BTreeMap::new(), - tables: BTreeMap::new(), - memories: BTreeMap::new(), - function_handles: BTreeMap::new(), - modules: BTreeMap::new(), - } + Self { externs: BTreeMap::new(), modules: BTreeMap::new() } } /// Merge two import sets pub fn merge(mut self, other: Self) -> Self { - self.globals.extend(other.globals); - self.tables.extend(other.tables); - self.memories.extend(other.memories); - self.function_handles.extend(other.function_handles); + self.externs.extend(other.externs); self.modules.extend(other.modules); self } @@ -151,38 +133,13 @@ impl Imports { /// Define an import value. pub fn define(&mut self, module: &str, name: &str, value: impl Into<Extern>) -> &mut Self { let name = ExternName { module: module.to_string(), name: name.to_string() }; - match value.into() { - Extern::Global(v) => { - self.globals.insert(name, v); - } - Extern::Table(v) => { - self.tables.insert(name, v); - } - Extern::Memory(v) => { - self.memories.insert(name, v); - } - Extern::Function(v) => { - self.function_handles.insert(name, v); - } - } + self.externs.insert(name, value.into()); self } pub(crate) fn take_defined(&self, import: &Import) -> Option<Extern> { let name = ExternName::from(import); - if let Some(v) = self.globals.get(&name) { - return Some(Extern::Global(*v)); - } - if let Some(v) = self.tables.get(&name) { - return Some(Extern::Table(*v)); - } - if let Some(v) = self.memories.get(&name) { - return Some(Extern::Memory(*v)); - } - if let Some(v) = self.function_handles.get(&name) { - return Some(Extern::Function(v.clone())); - } - None + self.externs.get(&name).cloned() } #[cfg(not(feature = "debug"))] @@ -242,13 +199,21 @@ impl Imports { Ok(()) } - pub(crate) fn link( - self, - store: &mut crate::Store, - module: &Module, - _idx: ModuleInstanceAddr, - ) -> Result<ResolvedImports> { - let mut imports = ResolvedImports::new(); + pub(crate) fn link(&self, store: &mut crate::Store, module: &Module) -> Result<ResolvedImports> { + let (global_count, table_count, mem_count, func_count) = + module.imports.iter().fold((0, 0, 0, 0), |(g, t, m, f), import| match import.kind { + ImportKind::Global(_) => (g + 1, t, m, f), + ImportKind::Table(_) => (g, t + 1, m, f), + ImportKind::Memory(_) => (g, t, m + 1, f), + ImportKind::Function(_) => (g, t, m, f + 1), + }); + + let mut imports = ResolvedImports { + globals: Vec::with_capacity(global_count), + tables: Vec::with_capacity(table_count), + memories: Vec::with_capacity(mem_count), + funcs: Vec::with_capacity(func_count), + }; for import in &*module.imports { if let Some(defined) = self.take_defined(import) { @@ -299,9 +264,8 @@ impl Imports { let Some(instance) = self.modules.get(&name.module) else { return Err(LinkingError::unknown_import(import).into()); }; - if instance.0.store_id != store.id() { - return Err(crate::Error::InvalidStore); - } + instance.validate_store(store)?; + let val = instance.export_addr(&import.name).ok_or_else(|| LinkingError::unknown_import(import))?; { diff --git a/crates/tinywasm/src/instance.rs b/crates/tinywasm/src/instance.rs index 627604e..f5125ba 100644 --- a/crates/tinywasm/src/instance.rs +++ b/crates/tinywasm/src/instance.rs @@ -1,3 +1,5 @@ +use core::hint::cold_path; + use alloc::boxed::Box; use alloc::sync::Arc; use alloc::{format, rc::Rc}; @@ -25,132 +27,121 @@ pub enum ExternItem { /// See <https://webassembly.github.io/spec/core/exec/runtime.html#module-instances> #[derive(Clone)] #[cfg_attr(feature = "debug", derive(Debug))] -pub struct ModuleInstance(pub(crate) Rc<ModuleInstanceInner>); +pub struct ModuleInstance(Rc<ModuleInstanceInner>); #[cfg_attr(feature = "debug", derive(Debug))] -pub(crate) struct ModuleInstanceInner { - pub(crate) store_id: usize, - pub(crate) idx: ModuleInstanceAddr, - pub(crate) types: Arc<[Arc<FuncType>]>, - pub(crate) func_addrs: Box<[FuncAddr]>, - pub(crate) table_addrs: Box<[TableAddr]>, - pub(crate) mem_addrs: Box<[MemAddr]>, - pub(crate) global_addrs: Box<[GlobalAddr]>, - pub(crate) elem_addrs: Box<[ElemAddr]>, - pub(crate) data_addrs: Box<[DataAddr]>, - pub(crate) func_start: Option<FuncAddr>, - pub(crate) exports: Arc<[Export]>, +struct ModuleInstanceInner { + store_id: usize, + idx: ModuleInstanceAddr, + types: Arc<[Arc<FuncType>]>, + func_addrs: Box<[FuncAddr]>, + table_addrs: Box<[TableAddr]>, + mem_addrs: Box<[MemAddr]>, + global_addrs: Box<[GlobalAddr]>, + elem_addrs: Box<[ElemAddr]>, + data_addrs: Box<[DataAddr]>, + func_start: Option<FuncAddr>, + exports: Arc<[Export]>, } -// impl ModuleInstance { -// #[cfg(feature = "parser")] -// /// Parse a module from bytes. Requires `parser` feature. -// pub fn from_wasm_bytes(wasm: &[u8]) -> Result<Self> { -// let data = tinywasm_parser::Parser::new().parse_module_bytes(wasm)?; -// Ok(data.into()) -// } - -// #[cfg(all(feature = "parser", feature = "std"))] -// /// Parse a module from a file. Requires `parser` and `std` features. -// pub fn from_wasm_file(path: impl AsRef<crate::std::path::Path> + Clone) -> Result<Self> { -// let data = tinywasm_parser::Parser::new().parse_module_file(path)?; -// Ok(data.into()) -// } - -// #[cfg(all(feature = "parser", feature = "std"))] -// /// Parse a module from a stream. Requires `parser` and `std` features. -// pub fn from_wasm_stream(stream: impl crate::std::io::Read) -> Result<Self> { -// let data = tinywasm_parser::Parser::new().parse_module_stream(stream)?; -// Ok(data.into()) -// } - -// /// Instantiate the module in the given store -// /// -// /// Runs the start function if it exists -// /// -// /// If you want to run the start function yourself, use `ModuleInstance::instantiate` -// /// -// /// See <https://webassembly.github.io/spec/core/exec/modules.html#exec-instantiation> -// pub fn instantiate(self, store: &mut Store, imports: Option<Imports>) -> Result<ModuleInstance> { -// let instance = ModuleInstance::instantiate(store, self, imports)?; -// let _ = instance.start(store)?; -// Ok(instance) -// } -// } +impl ModuleInstance { + #[inline] + pub(crate) fn idx(&self) -> ModuleInstanceAddr { + self.0.idx + } -impl ModuleInstanceInner { #[inline] pub(crate) fn func_ty(&self, addr: FuncAddr) -> &Arc<FuncType> { - match self.types.get(addr as usize) { + match self.0.types.get(addr as usize) { Some(ty) => ty, - None => unreachable!("invalid function address: {addr}"), + None => { + cold_path(); + unreachable!("invalid function address: {addr}") + } } } #[inline] pub(crate) fn func_addrs(&self) -> &[FuncAddr] { - &self.func_addrs + &self.0.func_addrs } // resolve a function address to the global store address #[inline] pub(crate) fn resolve_func_addr(&self, addr: FuncAddr) -> FuncAddr { - match self.func_addrs.get(addr as usize) { + match self.0.func_addrs.get(addr as usize) { Some(addr) => *addr, - None => unreachable!("invalid function address: {addr}"), + None => { + cold_path(); + unreachable!("invalid function address: {addr}") + } } } // resolve a table address to the global store address #[inline] pub(crate) fn resolve_table_addr(&self, addr: TableAddr) -> TableAddr { - match self.table_addrs.get(addr as usize) { + match self.0.table_addrs.get(addr as usize) { Some(addr) => *addr, - None => unreachable!("invalid table address: {addr}"), + None => { + cold_path(); + unreachable!("invalid table address: {addr}") + } } } // resolve a memory address to the global store address #[inline] pub(crate) fn resolve_mem_addr(&self, addr: MemAddr) -> MemAddr { - match self.mem_addrs.get(addr as usize) { + match self.0.mem_addrs.get(addr as usize) { Some(addr) => *addr, - None => unreachable!("invalid memory address: {addr}"), + None => { + cold_path(); + unreachable!("invalid memory address: {addr}") + } } } // resolve a data address to the global store address #[inline] pub(crate) fn resolve_data_addr(&self, addr: DataAddr) -> DataAddr { - match self.data_addrs.get(addr as usize) { + match self.0.data_addrs.get(addr as usize) { Some(addr) => *addr, - None => unreachable!("invalid data address: {addr}"), + None => { + cold_path(); + unreachable!("invalid data address: {addr}") + } } } - // resolve a memory address to the global store address + // resolve an element address to the global store address #[inline] pub(crate) fn resolve_elem_addr(&self, addr: ElemAddr) -> ElemAddr { - match self.elem_addrs.get(addr as usize) { + match self.0.elem_addrs.get(addr as usize) { Some(addr) => *addr, - None => unreachable!("invalid element address: {addr}"), + None => { + cold_path(); + unreachable!("invalid element address: {addr}") + } } } // resolve a global address to the global store address #[inline] pub(crate) fn resolve_global_addr(&self, addr: GlobalAddr) -> GlobalAddr { - match self.global_addrs.get(addr as usize) { + match self.0.global_addrs.get(addr as usize) { Some(addr) => *addr, - None => unreachable!("invalid global address: {addr}"), + None => { + cold_path(); + unreachable!("invalid global address: {addr}") + } } } -} -impl ModuleInstance { #[inline] - fn validate_store(&self, store: &Store) -> Result<()> { + pub(crate) fn validate_store(&self, store: &Store) -> Result<()> { if self.0.store_id != store.id() { + cold_path(); return Err(Error::InvalidStore); } Ok(()) @@ -175,20 +166,21 @@ impl ModuleInstance { /// See <https://webassembly.github.io/spec/core/exec/modules.html#exec-instantiation> pub fn instantiate_no_start(store: &mut Store, module: &Module, imports: Option<Imports>) -> Result<Self> { let idx = store.next_module_instance_idx(); - let mut addrs = imports.unwrap_or_default().link(store, module, idx)?; + let mut addrs = imports.unwrap_or_default().link(store, module)?; addrs.funcs.extend(store.init_funcs(&module.funcs, idx)); - addrs.tables.extend(store.init_tables(&module.table_types, idx)); + addrs.tables.extend(store.init_tables(&module.table_types)); match module.local_memory_allocation { LocalMemoryAllocation::Skip => {} - LocalMemoryAllocation::Lazy => addrs.memories.extend(store.init_lazy_memories(&module.memory_types, idx)?), - LocalMemoryAllocation::Eager => addrs.memories.extend(store.init_memories(&module.memory_types, idx)?), + LocalMemoryAllocation::Lazy => addrs.memories.extend(store.init_lazy_memories(&module.memory_types)?), + LocalMemoryAllocation::Eager => addrs.memories.extend(store.init_memories(&module.memory_types)?), } - let global_addrs = store.init_globals(addrs.globals, &module.globals, &addrs.funcs, idx)?; + + store.init_globals(&mut addrs.globals, &module.globals, &addrs.funcs)?; let (elem_addrs, elem_trapped) = - store.init_elements(&addrs.tables, &addrs.funcs, &global_addrs, &module.elements, idx)?; + store.init_elements(&addrs.tables, &addrs.funcs, &addrs.globals, &module.elements)?; let (data_addrs, data_trapped) = - store.init_data(&addrs.memories, &global_addrs, &addrs.funcs, &module.data, idx)?; + store.init_data(&addrs.memories, &addrs.globals, &addrs.funcs, &module.data)?; let instance = ModuleInstanceInner { store_id: store.id(), @@ -197,25 +189,25 @@ impl ModuleInstance { func_addrs: addrs.funcs.into_boxed_slice(), table_addrs: addrs.tables.into_boxed_slice(), mem_addrs: addrs.memories.into_boxed_slice(), - global_addrs: global_addrs.into_boxed_slice(), + global_addrs: addrs.globals.into_boxed_slice(), elem_addrs, data_addrs, func_start: module.start_func, exports: module.exports.clone(), }; - let instance = Rc::new(instance); + let instance = ModuleInstance(Rc::new(instance)); store.add_instance(instance.clone()); match (elem_trapped, data_trapped) { (Some(trap), _) | (_, Some(trap)) => Err(trap.into()), - _ => Ok(ModuleInstance(instance)), + _ => Ok(instance), } } /// Get a export by name pub fn export_addr(&self, name: &str) -> Option<ExternVal> { - let exports = self.0.exports.iter().find(|e| e.name == name.into())?; + let exports = self.0.exports.iter().find(|e| *e.name == *name)?; let addr = match exports.kind { ExternalKind::Func => self.0.func_addrs.get(exports.index as usize)?, ExternalKind::Table => self.0.table_addrs.get(exports.index as usize)?, @@ -228,53 +220,28 @@ impl ModuleInstance { /// Returns an iterator over all exported extern values for this instance. pub fn exports(&self) -> impl Iterator<Item = (&str, ExternItem)> + '_ { self.0.exports.iter().map(move |export| { - let name = export.name.as_ref(); let item = match export.kind { ExternalKind::Func => { - let idx = export.index as usize; - let func_addr = *self - .0 - .func_addrs - .get(idx) - .unwrap_or_else(|| unreachable!("invalid function export index: {}", export.index)); - let ty = self.0.func_ty(export.index).clone(); + let func_addr = self.resolve_func_addr(export.index); ExternItem::Func(Function { item: crate::StoreItem::new(self.0.store_id, func_addr), module_addr: self.id(), addr: func_addr, - ty, + ty: self.func_ty(export.index).clone(), }) } ExternalKind::Table => { - let idx = export.index as usize; - let table_addr = *self - .0 - .table_addrs - .get(idx) - .unwrap_or_else(|| unreachable!("invalid table export index: {}", export.index)); - ExternItem::Table(Table::from_store_addr(self.0.store_id, table_addr)) + ExternItem::Table(Table::from_store_addr(self.0.store_id, self.resolve_table_addr(export.index))) } ExternalKind::Memory => { - let idx = export.index as usize; - let mem_addr = *self - .0 - .mem_addrs - .get(idx) - .unwrap_or_else(|| unreachable!("invalid memory export index: {}", export.index)); - ExternItem::Memory(Memory::from_store_addr(self.0.store_id, mem_addr)) + ExternItem::Memory(Memory::from_store_addr(self.0.store_id, self.resolve_mem_addr(export.index))) } ExternalKind::Global => { - let idx = export.index as usize; - let global_addr = *self - .0 - .global_addrs - .get(idx) - .unwrap_or_else(|| unreachable!("invalid global export index: {}", export.index)); - ExternItem::Global(Global::from_store_addr(self.0.store_id, global_addr)) + ExternItem::Global(Global::from_store_addr(self.0.store_id, self.resolve_global_addr(export.index))) } }; - (name, item) + (export.name.as_ref(), item) }) } @@ -292,25 +259,19 @@ impl ModuleInstance { /// Get any exported extern value by name. pub fn extern_item(&self, name: &str) -> Result<ExternItem> { match self.require_export(name)? { - ExternVal::Func(func_addr) => { - let export = self - .0 - .exports - .iter() - .find(|e| e.name == name.into()) - .ok_or_else(|| Error::Other(format!("Export not found: {name}")))?; + ExternVal::Func(addr) => { + let export = self.0.exports.iter().find(|e| e.name == name.into()); + let export = export.ok_or_else(|| Error::Other(format!("Export not found: {name}")))?; Ok(ExternItem::Func(Function { - item: crate::StoreItem::new(self.0.store_id, func_addr), + item: crate::StoreItem::new(self.0.store_id, addr), module_addr: self.id(), - addr: func_addr, - ty: self.0.func_ty(export.index).clone(), + addr, + ty: self.func_ty(export.index).clone(), })) } - ExternVal::Memory(mem_addr) => Ok(ExternItem::Memory(Memory::from_store_addr(self.0.store_id, mem_addr))), - ExternVal::Table(table_addr) => Ok(ExternItem::Table(Table::from_store_addr(self.0.store_id, table_addr))), - ExternVal::Global(global_addr) => { - Ok(ExternItem::Global(Global::from_store_addr(self.0.store_id, global_addr))) - } + ExternVal::Memory(addr) => Ok(ExternItem::Memory(Memory::from_store_addr(self.0.store_id, addr))), + ExternVal::Table(addr) => Ok(ExternItem::Table(Table::from_store_addr(self.0.store_id, addr))), + ExternVal::Global(addr) => Ok(ExternItem::Global(Global::from_store_addr(self.0.store_id, addr))), } } @@ -318,17 +279,19 @@ impl ModuleInstance { pub fn func_untyped(&self, store: &Store, name: &str) -> Result<Function> { self.validate_store(store)?; - let export = self.require_export(name)?; - let ExternVal::Func(func_addr) = export else { - return Err(Error::Other(format!("Export is not a function: {name}"))); + let func_addr = match self.require_export(name)? { + ExternVal::Func(func_addr) => func_addr, + _ => { + cold_path(); + return Err(Error::Other(format!("Export is not a function: {name}"))); + } }; - let ty = store.state.get_func(func_addr).ty(); Ok(Function { item: crate::StoreItem::new(self.0.store_id, func_addr), addr: func_addr, module_addr: self.id(), - ty: ty.clone(), + ty: store.state.get_func(func_addr).ty().clone(), }) } @@ -384,13 +347,16 @@ impl ModuleInstance { func: &Function, func_name: &str, ) -> Result<()> { - let expected = FuncType::new(&P::wasm_types(), &R::wasm_types()); - if *func.ty != expected { + if *func.ty.params() != *P::wasm_types() || *func.ty.results() != *R::wasm_types() { + cold_path(); + #[cfg(feature = "debug")] return Err(Error::Other(format!( - "function type mismatch for {func_name}: expected {expected:?}, actual {:?}", + "function type mismatch for {func_name}: expected {:?}, actual {:?}", + FuncType::new(&P::wasm_types(), &R::wasm_types()), func.ty ))); + #[cfg(not(feature = "debug"))] return Err(Error::Other(format!("function type mismatch for {func_name}"))); } @@ -400,10 +366,10 @@ impl ModuleInstance { /// Get a memory export by name. pub fn memory(&self, name: &str) -> Result<Memory> { - let ExternVal::Memory(mem_addr) = self.require_export(name)? else { - return Err(Error::Other(format!("Export is not a memory: {name}"))); - }; - Ok(Memory::from_store_addr(self.0.store_id, mem_addr)) + match self.require_export(name)? { + ExternVal::Memory(mem_addr) => Ok(Memory::from_store_addr(self.0.store_id, mem_addr)), + _ => Err(Error::Other(format!("Export is not a memory: {name}"))), + } } /// Get a memory by its module-local index. @@ -420,11 +386,10 @@ impl ModuleInstance { /// Get a table export by name. pub fn table(&self, name: &str) -> Result<Table> { - let export = self.require_export(name)?; - let ExternVal::Table(table_addr) = export else { - return Err(Error::Other(format!("Export is not a table: {name}"))); - }; - Ok(Table::from_store_addr(self.0.store_id, table_addr)) + match self.require_export(name)? { + ExternVal::Table(table_addr) => Ok(Table::from_store_addr(self.0.store_id, table_addr)), + _ => Err(Error::Other(format!("Export is not a table: {name}"))), + } } /// Get a table by its module-local index. @@ -446,12 +411,10 @@ impl ModuleInstance { /// Get a global export by name. pub fn global(&self, name: &str) -> Result<Global> { - let export = self.require_export(name)?; - let ExternVal::Global(global_addr) = export else { - return Err(Error::Other(format!("Export is not a global: {name}"))); - }; - - Ok(Global::from_store_addr(self.0.store_id, global_addr)) + match self.require_export(name)? { + ExternVal::Global(global_addr) => Ok(Global::from_store_addr(self.0.store_id, global_addr)), + _ => Err(Error::Other(format!("Export is not a global: {name}"))), + } } /// Set the value of a mutable global export by name. @@ -492,13 +455,12 @@ impl ModuleInstance { } }; - let func_addr = self.0.resolve_func_addr(func_index); - let ty = store.state.get_func(func_addr).ty(); + let func_addr = self.resolve_func_addr(func_index); Ok(Some(Function { item: crate::StoreItem::new(self.0.store_id, func_addr), module_addr: self.id(), addr: func_addr, - ty: ty.clone(), + ty: store.state.get_func(func_addr).ty().clone(), })) } @@ -508,9 +470,9 @@ impl ModuleInstance { /// /// See <https://webassembly.github.io/spec/core/syntax/modules.html#syntax-start> pub fn start(&self, store: &mut Store) -> Result<Option<()>> { - let Some(func) = self.start_func(store)? else { - return Ok(None); - }; - func.call(store, &[]).map(|_| Some(())) + match self.start_func(store)? { + Some(func) => func.call(store, &[]).map(|_| Some(())), + None => Ok(None), + } } } diff --git a/crates/tinywasm/src/interpreter/executor.rs b/crates/tinywasm/src/interpreter/executor.rs index 204618a..4cacea6 100644 --- a/crates/tinywasm/src/interpreter/executor.rs +++ b/crates/tinywasm/src/interpreter/executor.rs @@ -16,7 +16,6 @@ use super::num_helpers::*; use super::values::*; use crate::engine::FuelPolicy; use crate::func::{FuncContext, HostFunction}; -use crate::instance::ModuleInstanceInner; use crate::interpreter::Value128; use crate::*; @@ -25,15 +24,15 @@ const FUEL_COST_CALL_TOTAL: u32 = 5; pub(crate) struct Executor<'store, const BUDGETED: bool> { cf: CallFrame, func: Arc<WasmFunction>, - module: Rc<ModuleInstanceInner>, + module: ModuleInstance, store: &'store mut Store, } impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { pub(crate) fn new(store: &'store mut Store, cf: CallFrame) -> Self { - let module = store.get_module_instance_raw(cf.module_addr).clone(); - let func = store.state.get_wasm_func(cf.func_addr).clone(); - Self { module, store, cf, func } + let wasm_func = store.state.get_wasm_func(cf.func_addr); + let module = store.get_module_instance_internal(wasm_func.owner); + Self { module, cf, func: wasm_func.func.clone(), store } } #[inline(always)] @@ -138,11 +137,11 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { Select64 => self.store.value_stack.select::<Value64>()?, Select128 => self.store.value_stack.select::<Value128>()?, SelectMulti(counts) => self.store.value_stack.select_multi(*counts), - Call(v) => { self.exec_call_direct::<false>(*v)?; return Ok(None); } - CallSelf => { self.exec_call_self::<false>()?; return Ok(None); } + Call(v) => { self.exec_call_direct(*v)?; return Ok(None); } + CallSelf => { self.exec_call_self()?; return Ok(None); } CallIndirect(ty, table) => { self.exec_call_indirect::<false>(*ty, *table)?; return Ok(None); } - ReturnCall(v) => { self.exec_call_direct::<true>(*v)?; return Ok(None); } - ReturnCallSelf => { self.exec_call_self::<true>()?; return Ok(None); } + ReturnCall(v) => { self.exec_return_call_direct(*v)?; return Ok(None); } + ReturnCallSelf => { self.exec_return_call_self()?; return Ok(None); } ReturnCallIndirect(ty, table) => { self.exec_call_indirect::<true>(*ty, *table)?; return Ok(None); } Jump(ip) => { self.exec_jump(*ip); return Ok(None); } JumpIfZero(ip) => if self.exec_jump_if_zero(*ip) { return Ok(None) }, @@ -769,7 +768,6 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { self.jump_if(cmp_i64(lhs, rhs, op), target_ip) } - #[inline(always)] fn exec_branch_table(&mut self, default_ip: u32, start: u32, len: u32) { let idx = self.store.value_stack.pop::<i32>(); let target_ip = if idx >= 0 && (idx as u32) < len { @@ -781,121 +779,119 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { self.cf.instr_ptr = target_ip; } - fn exec_call<const IS_RETURN_CALL: bool>( - &mut self, - wasm_func: WasmFunctionInstance, - func_addr: FuncAddr, - ) -> Result<(), Trap> { + fn exec_call(&mut self, wasm_func: WasmFunctionInstance, func_addr: FuncAddr) -> Result<(), Trap> { if !Arc::ptr_eq(&self.func, &wasm_func.func) { self.func = wasm_func.func.clone(); } - if IS_RETURN_CALL { - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, wasm_func.func.params); - } else if self.store.call_stack.is_at_limit() { + let Ok(locals_base) = self.store.value_stack.enter_locals(&wasm_func.func.params, &wasm_func.func.locals) + else { cold_path(); return Err(Trap::CallStackOverflow); - } - - let locals_base = match self.store.value_stack.enter_locals(&wasm_func.func.params, &wasm_func.func.locals) { - Ok(base) => base, - Err(err) => { - cold_path(); - if IS_RETURN_CALL { - return Err(err); - } - return Err(Trap::CallStackOverflow); - } }; - let new_call_frame = CallFrame::new(func_addr, wasm_func.owner, locals_base, wasm_func.func.locals); + self.store.call_stack.push(self.cf)?; + self.cf = CallFrame::new(func_addr, locals_base, wasm_func.func.locals); + if wasm_func.owner != self.module.idx() { + self.module = self.store.get_module_instance_internal(wasm_func.owner); + } + + Ok(()) + } - if !IS_RETURN_CALL { - self.cf.incr_instr_ptr(); // skip the call instruction - self.store.call_stack.push(self.cf)?; + fn exec_return_call(&mut self, wasm_func: WasmFunctionInstance, func_addr: FuncAddr) -> Result<(), Trap> { + if !Arc::ptr_eq(&self.func, &wasm_func.func) { + self.func = wasm_func.func.clone(); } - self.cf = new_call_frame; - if self.cf.module_addr != self.module.idx { - self.module = self.store.get_module_instance_raw(self.cf.module_addr).clone(); + self.store.value_stack.truncate_keep_counts(self.cf.locals_base, wasm_func.func.params); + let locals_base = self.store.value_stack.enter_locals(&wasm_func.func.params, &wasm_func.func.locals)?; + self.cf = CallFrame::new(func_addr, locals_base, wasm_func.func.locals); + if wasm_func.owner != self.module.idx() { + self.module = self.store.get_module_instance_internal(wasm_func.owner); } Ok(()) } + fn exec_call_host(&mut self, host_func: Rc<HostFunction>) -> Result<(), Trap> { let params = self.store.value_stack.pop_types(host_func.ty.params()).collect::<Box<_>>(); - let res = match host_func.call(FuncContext { store: self.store, module_addr: self.module.idx }, ¶ms) { + let res = match host_func.call(FuncContext { store: self.store, module_addr: self.module.idx() }, ¶ms) { Ok(res) => res, Err(err) => { cold_path(); return Err(Trap::HostFunction(Box::new(err))); } }; + self.store.value_stack.extend_from_wasmvalues(&res)?; self.cf.incr_instr_ptr(); Ok(()) } - fn exec_call_direct<const IS_RETURN_CALL: bool>(&mut self, v: u32) -> Result<(), Trap> { + + fn exec_call_direct(&mut self, v: u32) -> Result<(), Trap> { + self.charge_call_fuel(FUEL_COST_CALL_TOTAL); + let addr = self.module.resolve_func_addr(v); + match self.store.state.get_func(addr) { + crate::FunctionInstance::Wasm(wasm_func) => self.exec_call(wasm_func.clone(), addr), + crate::FunctionInstance::Host(host_func) => self.exec_call_host(host_func.clone()), + } + } + + fn exec_return_call_direct(&mut self, v: u32) -> Result<(), Trap> { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); let addr = self.module.resolve_func_addr(v); match self.store.state.get_func(addr) { - crate::FunctionInstance::Wasm(wasm_func) => self.exec_call::<IS_RETURN_CALL>(wasm_func.clone(), addr), + crate::FunctionInstance::Wasm(wasm_func) => self.exec_return_call(wasm_func.clone(), addr), crate::FunctionInstance::Host(host_func) => self.exec_call_host(host_func.clone()), } } - fn exec_call_self<const IS_RETURN_CALL: bool>(&mut self) -> Result<(), Trap> { + fn exec_call_self(&mut self) -> Result<(), Trap> { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - let params = self.func.params; - let locals = self.func.locals; - if IS_RETURN_CALL { - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, params); - } else if self.store.call_stack.is_at_limit() { + self.store.call_stack.push(self.cf)?; + let Ok(locals_base) = self.store.value_stack.enter_locals(&self.func.params, &self.func.locals) else { cold_path(); return Err(Trap::CallStackOverflow); - } + }; + self.cf = CallFrame::new(self.cf.func_addr, locals_base, self.func.locals); + + Ok(()) + } + + fn exec_return_call_self(&mut self) -> Result<(), Trap> { + self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - let locals_base = match self.store.value_stack.enter_locals(¶ms, &locals) { + self.store.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.params); + let locals_base = match self.store.value_stack.enter_locals(&self.func.params, &self.func.locals) { Ok(base) => base, Err(err) => { cold_path(); - if IS_RETURN_CALL { - return Err(err); - } - return Err(Trap::CallStackOverflow); + return Err(err); } }; - let new_call_frame = CallFrame::new(self.cf.func_addr, self.cf.module_addr, locals_base, locals); - if !IS_RETURN_CALL { - self.cf.incr_instr_ptr(); - self.store.call_stack.push(self.cf)?; - } - self.cf = new_call_frame; + self.cf = CallFrame::new(self.cf.func_addr, locals_base, self.func.locals); Ok(()) } fn exec_call_indirect<const IS_RETURN_CALL: bool>(&mut self, type_addr: u32, table_addr: u32) -> Result<(), Trap> { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); + // verify that the table is of the right type, this should be validated by the parser already - let func_ref = { - let table_idx: u32 = self.store.value_stack.pop::<i32>() as u32; - let table = self.store.state.get_table(self.module.resolve_table_addr(table_addr)); - assert!(table.kind.element_type == WasmType::RefFunc, "table is not of type funcref"); + let table_idx: u32 = self.store.value_stack.pop::<i32>() as u32; + let table = self.store.state.get_table(self.module.resolve_table_addr(table_addr)); + debug_assert!(table.kind.element_type == WasmType::RefFunc, "table is not of type funcref"); - let Ok(table) = table.get(table_idx) else { - cold_path(); - return Err(Trap::UndefinedElement { index: table_idx as usize }); - }; + let Ok(table) = table.get(table_idx) else { + cold_path(); + return Err(Trap::UndefinedElement { index: table_idx as usize }); + }; - match table.addr() { - Some(addr) => addr, - None => { - cold_path(); - return Err(Trap::UninitializedElement { index: table_idx as usize }); - } - } + let Some(func_ref) = table.addr() else { + cold_path(); + return Err(Trap::UninitializedElement { index: table_idx as usize }); }; let call_ty = self.module.func_ty(type_addr); @@ -909,7 +905,10 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { }); } - self.exec_call::<IS_RETURN_CALL>(wasm_func.clone(), func_ref) + match IS_RETURN_CALL { + true => self.exec_return_call(wasm_func.clone(), func_ref), + false => self.exec_call(wasm_func.clone(), func_ref), + } } crate::FunctionInstance::Host(host_func) => { if host_func.ty != *call_ty { @@ -931,12 +930,13 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { let Some(cf) = self.store.call_stack.pop() else { return true }; if cf.func_addr != self.cf.func_addr { - self.func = self.store.state.get_wasm_func(cf.func_addr).clone(); - - if cf.module_addr != self.module.idx { - self.module = self.store.get_module_instance_raw(cf.module_addr).clone(); + let wasm_func = self.store.state.get_wasm_func(cf.func_addr); + self.func = wasm_func.func.clone(); + if wasm_func.owner != self.module.idx() { + self.module = self.store.get_module_instance_internal(wasm_func.owner); } } + self.cf = cf; false } diff --git a/crates/tinywasm/src/interpreter/stack/call_stack.rs b/crates/tinywasm/src/interpreter/stack/call_stack.rs index 0592e31..03cf5fe 100644 --- a/crates/tinywasm/src/interpreter/stack/call_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/call_stack.rs @@ -2,7 +2,7 @@ use crate::{Result, Trap}; use core::hint::cold_path; use alloc::vec::Vec; -use tinywasm_types::{FuncAddr, ModuleInstanceAddr, ValueCounts}; +use tinywasm_types::{FuncAddr, ValueCounts}; #[cfg_attr(feature = "debug", derive(Debug))] pub(crate) struct CallStack { @@ -27,13 +27,9 @@ impl CallStack { } #[inline(always)] - pub(crate) fn is_at_limit(&self) -> bool { - self.stack.len() == self.max_size - } - - #[inline(always)] - pub(crate) fn push(&mut self, call_frame: CallFrame) -> Result<(), Trap> { + pub(crate) fn push(&mut self, mut call_frame: CallFrame) -> Result<(), Trap> { self.ensure_capacity_for(self.stack.len() + 1)?; + call_frame.incr_instr_ptr(); self.stack.push(call_frame); Ok(()) } @@ -50,13 +46,10 @@ impl CallStack { } let target_capacity = required_len.max(self.stack.capacity().max(1).saturating_mul(2)).min(self.max_size); - match self.stack.try_reserve(target_capacity.saturating_sub(self.stack.len())) { - Ok(()) => {} - Err(_) => { - cold_path(); - return Err(Trap::CallStackOverflow); - } - } + let Ok(()) = self.stack.try_reserve(target_capacity.saturating_sub(self.stack.len())) else { + cold_path(); + return Err(Trap::CallStackOverflow); + }; Ok(()) } } @@ -65,7 +58,6 @@ impl CallStack { #[cfg_attr(feature = "debug", derive(Debug))] pub(crate) struct CallFrame { pub(crate) instr_ptr: u32, - pub(crate) module_addr: ModuleInstanceAddr, pub(crate) func_addr: FuncAddr, pub(crate) locals_base: StackBase, pub(crate) stack_offset: ValueCounts, @@ -80,13 +72,8 @@ pub(crate) struct StackBase { } impl CallFrame { - pub(crate) fn new( - func_addr: FuncAddr, - module_addr: ModuleInstanceAddr, - locals_base: StackBase, - stack_offset: ValueCounts, - ) -> Self { - Self { instr_ptr: 0, func_addr, module_addr, locals_base, stack_offset } + pub(crate) fn new(func_addr: FuncAddr, locals_base: StackBase, stack_offset: ValueCounts) -> Self { + Self { instr_ptr: 0, func_addr, locals_base, stack_offset } } #[inline] diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index 5ef1e10..80ea044 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -38,11 +38,6 @@ impl<T: Copy + Default> Stack<T> { } #[inline(always)] - pub(crate) fn truncate(&mut self, len: usize) { - self.data.truncate(len); - } - - #[inline(always)] pub(crate) fn push(&mut self, value: T) -> Result<(), Trap> { self.ensure_capacity_for(self.data.len() + 1)?; self.data.push(value); @@ -97,10 +92,13 @@ impl<T: Copy + Default> Stack<T> { return; } - let keep = (len - n).min(end_keep); - if keep > 0 { - self.data.copy_within(len - keep..len, n); + if end_keep == 0 { + self.data.truncate(n); + return; } + + let keep = (len - n).min(end_keep); + self.data.copy_within(len - keep..len, n); self.data.truncate(n + keep); } @@ -111,8 +109,8 @@ impl<T: Copy + Default> Stack<T> { let start = self.data.len() - param_count; let end = start + local_count; self.ensure_capacity_for(end)?; - self.data.resize(end, T::default()); + Ok(start as u32) } @@ -231,25 +229,9 @@ impl ValueStack { } pub(crate) fn enter_locals(&mut self, params: &ValueCounts, locals: &ValueCounts) -> Result<StackBase, Trap> { - let len32 = self.stack_32.len(); - let len64 = self.stack_64.len(); - let locals_base32 = self.stack_32.enter_locals(params.c32 as usize, locals.c32 as usize)?; - let locals_base64 = match self.stack_64.enter_locals(params.c64 as usize, locals.c64 as usize) { - Ok(base) => base, - Err(err) => { - self.stack_32.truncate(len32); - return Err(err); - } - }; - let locals_base128 = match self.stack_128.enter_locals(params.c128 as usize, locals.c128 as usize) { - Ok(base) => base, - Err(err) => { - self.stack_32.truncate(len32); - self.stack_64.truncate(len64); - return Err(err); - } - }; + let locals_base64 = self.stack_64.enter_locals(params.c64 as usize, locals.c64 as usize)?; + let locals_base128 = self.stack_128.enter_locals(params.c128 as usize, locals.c128 as usize)?; Ok(StackBase { s32: locals_base32, s64: locals_base64, s128: locals_base128 }) } diff --git a/crates/tinywasm/src/store/mod.rs b/crates/tinywasm/src/store/mod.rs index 773f4cb..7e70062 100644 --- a/crates/tinywasm/src/store/mod.rs +++ b/crates/tinywasm/src/store/mod.rs @@ -1,10 +1,9 @@ -use alloc::rc::Rc; use alloc::sync::Arc; use alloc::{boxed::Box, format, string::ToString, vec::Vec}; +use core::hint::cold_path; use core::sync::atomic::{AtomicUsize, Ordering}; use tinywasm_types::*; -use crate::instance::ModuleInstanceInner; use crate::interpreter::stack::{CallStack, ValueStack}; use crate::interpreter::{TinyWasmValue, ValueRef}; use crate::{Engine, Error, ModuleInstance, Result, Trap}; @@ -34,7 +33,7 @@ static STORE_ID: AtomicUsize = AtomicUsize::new(0); /// See <https://webassembly.github.io/spec/core/exec/runtime.html#store> pub struct Store { id: usize, - module_instances: Vec<Rc<ModuleInstanceInner>>, + module_instances: Vec<ModuleInstance>, pub(crate) engine: Engine, pub(crate) execution_fuel: u32, @@ -71,14 +70,17 @@ impl Store { /// Get a module instance by the internal id pub fn get_module_instance(&self, addr: ModuleInstanceAddr) -> Option<ModuleInstance> { - Some(ModuleInstance(self.module_instances.get(addr as usize)?.clone())) + self.module_instances.get(addr as usize).cloned() } #[inline] - pub(crate) fn get_module_instance_raw(&self, addr: ModuleInstanceAddr) -> &Rc<ModuleInstanceInner> { + pub(crate) fn get_module_instance_internal(&self, addr: ModuleInstanceAddr) -> ModuleInstance { match self.module_instances.get(addr as usize) { - Some(instance) => instance, - None => unreachable!("module instance {addr} not found. This should be unreachable"), + Some(instance) => instance.clone(), + None => { + cold_path(); + unreachable!("module instance {addr} not found. This should be unreachable") + } } } } @@ -114,20 +116,21 @@ impl State { pub(crate) fn get_func(&self, addr: FuncAddr) -> &FunctionInstance { match self.funcs.get(addr as usize) { Some(func) => func, - None => unreachable!("function {addr} not found. This should be unreachable"), + None => { + cold_path(); + unreachable!("function {addr} not found. This should be unreachable") + } } } /// Get a wasm function at the actual index in the store, panicking if it's a host function (which should be guaranteed by the validator) - pub(crate) fn get_wasm_func(&self, addr: FuncAddr) -> &Arc<WasmFunction> { + pub(crate) fn get_wasm_func(&self, addr: FuncAddr) -> &WasmFunctionInstance { match self.funcs.get(addr as usize) { - Some(func) => match func { - FunctionInstance::Wasm(wasm_func) => &wasm_func.func, - FunctionInstance::Host(_) => unreachable!( - "expected a wasm function at address {addr}, but found a host function. This should be unreachable" - ), - }, - None => unreachable!("function {addr} not found. This should be unreachable"), + Some(FunctionInstance::Wasm(wasm_func)) => wasm_func, + _ => { + cold_path(); + unreachable!("function {addr} not found. This should be unreachable") + } } } @@ -135,7 +138,10 @@ impl State { pub(crate) fn get_mem(&self, addr: MemAddr) -> &MemoryInstance { match self.memories.get(addr as usize) { Some(mem) => mem, - None => unreachable!("memory {addr} not found. This should be unreachable"), + None => { + cold_path(); + unreachable!("memory {addr} not found. This should be unreachable") + } } } @@ -143,7 +149,10 @@ impl State { pub(crate) fn get_mem_mut(&mut self, addr: MemAddr) -> &mut MemoryInstance { match self.memories.get_mut(addr as usize) { Some(mem) => mem, - None => unreachable!("memory {addr} not found. This should be unreachable"), + None => { + cold_path(); + unreachable!("memory {addr} not found. This should be unreachable") + } } } @@ -151,7 +160,10 @@ impl State { pub(crate) fn get_mems_mut(&mut self, addr: MemAddr, addr2: MemAddr) -> (&mut MemoryInstance, &mut MemoryInstance) { match self.memories.get_disjoint_mut([addr as usize, addr2 as usize]) { Ok([mem_a, mem_b]) => (mem_a, mem_b), - Err(_) => unreachable!("memory {addr} or {addr2} not found. This should be unreachable"), + Err(_) => { + cold_path(); + unreachable!("memory {addr} or {addr2} not found. This should be unreachable") + } } } @@ -159,7 +171,10 @@ impl State { pub(crate) fn get_table(&self, addr: TableAddr) -> &TableInstance { match self.tables.get(addr as usize) { Some(table) => table, - None => unreachable!("table {addr} not found. This should be unreachable"), + None => { + cold_path(); + unreachable!("table {addr} not found. This should be unreachable") + } } } @@ -167,7 +182,10 @@ impl State { pub(crate) fn get_table_mut(&mut self, addr: TableAddr) -> &mut TableInstance { match self.tables.get_mut(addr as usize) { Some(table) => table, - None => unreachable!("table {addr} not found. This should be unreachable"), + None => { + cold_path(); + unreachable!("table {addr} not found. This should be unreachable") + } } } @@ -179,7 +197,10 @@ impl State { ) -> (&mut TableInstance, &mut TableInstance) { match self.tables.get_disjoint_mut([addr as usize, addr2 as usize]) { Ok([table_a, table_b]) => (table_a, table_b), - Err(_) => unreachable!("table {addr} or {addr2} not found. This should be unreachable"), + Err(_) => { + cold_path(); + unreachable!("table {addr} or {addr2} not found. This should be unreachable") + } } } @@ -187,7 +208,10 @@ impl State { pub(crate) fn get_data_mut(&mut self, addr: DataAddr) -> &mut DataInstance { match self.data.get_mut(addr as usize) { Some(data) => data, - None => unreachable!("data {addr} not found. This should be unreachable"), + None => { + cold_path(); + unreachable!("data {addr} not found. This should be unreachable") + } } } @@ -195,7 +219,10 @@ impl State { pub(crate) fn get_elem_mut(&mut self, addr: ElemAddr) -> &mut ElementInstance { match self.elements.get_mut(addr as usize) { Some(elem) => elem, - None => unreachable!("element {addr} not found. This should be unreachable"), + None => { + cold_path(); + unreachable!("element {addr} not found. This should be unreachable") + } } } @@ -203,7 +230,10 @@ impl State { pub(crate) fn get_global(&self, addr: GlobalAddr) -> &GlobalInstance { match self.globals.get(addr as usize) { Some(global) => global, - None => unreachable!("global {addr} not found. This should be unreachable"), + None => { + cold_path(); + unreachable!("global {addr} not found. This should be unreachable") + } } } @@ -211,7 +241,10 @@ impl State { pub(crate) fn get_global_mut(&mut self, addr: GlobalAddr) -> &mut GlobalInstance { match self.globals.get_mut(addr as usize) { Some(global) => global, - None => unreachable!("global {addr} not found. This should be unreachable"), + None => { + cold_path(); + unreachable!("global {addr} not found. This should be unreachable") + } } } @@ -219,7 +252,10 @@ impl State { pub(crate) fn get_global_val(&self, addr: MemAddr) -> TinyWasmValue { match self.globals.get(addr as usize) { Some(global) => global.value.get(), - None => unreachable!("global {addr} not found. This should be unreachable"), + None => { + cold_path(); + unreachable!("global {addr} not found. This should be unreachable") + } } } @@ -227,7 +263,10 @@ impl State { pub(crate) fn set_global_val(&mut self, addr: MemAddr, value: TinyWasmValue) { match self.globals.get_mut(addr as usize) { Some(global) => global.value.set(value), - None => unreachable!("global {addr} not found. This should be unreachable"), + None => { + cold_path(); + unreachable!("global {addr} not found. This should be unreachable") + } } } } @@ -242,8 +281,8 @@ impl Store { self.module_instances.len() as ModuleInstanceAddr } - pub(crate) fn add_instance(&mut self, instance: Rc<ModuleInstanceInner>) { - assert!(instance.idx == self.module_instances.len() as ModuleInstanceAddr); + pub(crate) fn add_instance(&mut self, instance: ModuleInstance) { + debug_assert!(instance.idx() == self.module_instances.len() as ModuleInstanceAddr); self.module_instances.push(instance); } @@ -263,78 +302,83 @@ impl Store { // Linking related functions impl Store { /// Add functions to the store, returning their addresses in the store - pub(crate) fn init_funcs(&mut self, funcs: &[Arc<WasmFunction>], idx: ModuleInstanceAddr) -> Vec<FuncAddr> { - let func_count = self.state.funcs.len(); - let mut func_addrs = Vec::with_capacity(func_count); - for (i, func) in funcs.iter().enumerate() { - self.state.funcs.push(FunctionInstance::new_wasm(func.clone(), idx)); - func_addrs.push((i + func_count) as FuncAddr); - } - func_addrs + pub(crate) fn init_funcs( + &mut self, + funcs: &[Arc<WasmFunction>], + idx: ModuleInstanceAddr, + ) -> impl ExactSizeIterator<Item = FuncAddr> { + let start = self.state.funcs.len() as FuncAddr; + self.state.funcs.extend(funcs.iter().map(|func| FunctionInstance::new_wasm(func.clone(), idx))); + start..start + funcs.len() as FuncAddr } /// Add tables to the store, returning their addresses in the store - pub(crate) fn init_tables(&mut self, tables: &[TableType], _idx: ModuleInstanceAddr) -> Vec<TableAddr> { - let table_count = self.state.tables.len(); - let mut table_addrs = Vec::with_capacity(table_count); - for (i, table) in tables.iter().enumerate() { - self.state.tables.push(TableInstance::new(table.clone())); - table_addrs.push((i + table_count) as TableAddr); - } - table_addrs + pub(crate) fn init_tables(&mut self, tables: &[TableType]) -> impl ExactSizeIterator<Item = TableAddr> { + let start = self.state.tables.len() as TableAddr; + self.state.tables.extend(tables.iter().map(|table| TableInstance::new(table.clone()))); + start..start + tables.len() as TableAddr } /// Add memories to the store, returning their addresses in the store - pub(crate) fn init_memories(&mut self, memories: &[MemoryType], _idx: ModuleInstanceAddr) -> Result<Vec<MemAddr>> { - let mem_count = self.state.memories.len(); - let mut mem_addrs = Vec::with_capacity(mem_count); - for (i, mem) in memories.iter().enumerate() { - self.state.memories.push(MemoryInstance::new(*mem, &self.engine.config().memory_backend)?); - mem_addrs.push((i + mem_count) as MemAddr); + pub(crate) fn init_memories(&mut self, memories: &[MemoryType]) -> Result<impl ExactSizeIterator<Item = MemAddr>> { + let start = self.state.memories.len() as MemAddr; + self.state.memories.reserve_exact(memories.len()); + for &mem in memories { + self.state.memories.push(MemoryInstance::new(mem, &self.engine.config().memory_backend)?); } - Ok(mem_addrs) + Ok(start..start + memories.len() as MemAddr) } pub(crate) fn init_lazy_memories( &mut self, memories: &[MemoryType], - _idx: ModuleInstanceAddr, - ) -> Result<Vec<MemAddr>> { - let mem_count = self.state.memories.len(); - let mut mem_addrs = Vec::with_capacity(mem_count); - for (i, mem) in memories.iter().enumerate() { - self.state.memories.push(MemoryInstance::new_lazy(*mem, &self.engine.config().memory_backend)?); - mem_addrs.push((i + mem_count) as MemAddr); + ) -> Result<impl ExactSizeIterator<Item = MemAddr>> { + let start = self.state.memories.len() as MemAddr; + self.state.memories.reserve_exact(memories.len()); + for &mem in memories { + self.state.memories.push(MemoryInstance::new_lazy(mem, &self.engine.config().memory_backend)?); } - Ok(mem_addrs) + Ok(start..start + memories.len() as MemAddr) } /// Add globals to the store, returning their addresses in the store pub(crate) fn init_globals( &mut self, - mut imported_globals: Vec<GlobalAddr>, + out: &mut Vec<Addr>, new_globals: &[Global], func_addrs: &[FuncAddr], - _idx: ModuleInstanceAddr, - ) -> Result<Vec<Addr>> { - let global_count = self.state.globals.len(); - imported_globals.reserve_exact(new_globals.len()); - let mut global_addrs = imported_globals; + ) -> Result<()> { + let start = self.state.globals.len() as Addr; + out.reserve_exact(new_globals.len()); + self.state.globals.reserve_exact(new_globals.len()); for (i, global) in new_globals.iter().enumerate() { - let value = self.eval_const(&global.init, &global_addrs, func_addrs)?; + let value = match self.eval_const(&global.init, out, func_addrs) { + Ok(val) => val, + Err(e) => { + cold_path(); + return Err(e); + } + }; + self.state.globals.push(GlobalInstance::new(global.ty, value)); - global_addrs.push((i + global_count) as Addr); + out.push(start + i as Addr); } - Ok(global_addrs) + Ok(()) } fn elem_addr(&self, item: &ElementItem, globals: &[Addr], funcs: &[FuncAddr]) -> Result<Option<u32>> { let res = match item { - ElementItem::Func(addr) => Some(funcs.get(*addr as usize).copied().ok_or_else(|| { - Error::Other(format!("function {addr} not found. This should have been caught by the validator")) - })?), + ElementItem::Func(addr) => match funcs.get(*addr as usize) { + Some(func_addr) => Some(*func_addr), + None => { + cold_path(); + return Err(Error::Other(format!( + "function {addr} not found. This should have been caught by the validator" + ))); + } + }, ElementItem::Expr(expr) => self.eval_ref_const(expr, globals, funcs)?, }; @@ -349,7 +393,6 @@ impl Store { func_addrs: &[FuncAddr], global_addrs: &[Addr], elements: &[Element], - _idx: ModuleInstanceAddr, ) -> Result<(Box<[Addr]>, Option<Trap>)> { let elem_count = self.state.elements.len(); let mut elem_addrs = Vec::with_capacity(elem_count); @@ -408,7 +451,6 @@ impl Store { global_addrs: &[Addr], func_addrs: &[FuncAddr], data: &[Data], - _idx: ModuleInstanceAddr, ) -> Result<(Box<[Addr]>, Option<Trap>)> { let data_count = self.state.data.len(); let mut data_addrs = Vec::with_capacity(data_count); @@ -479,21 +521,31 @@ impl Store { use tinywasm_types::ConstInstruction::*; let resolve_global = |idx: u32| -> Result<TinyWasmValue> { - let addr = module_global_addrs.get(idx as usize).ok_or_else(|| { - Error::Other(format!("global {idx} not found. This should have been caught by the validator")) - })?; - let global = self - .state - .globals - .get(*addr as usize) - .ok_or_else(|| Error::Other(format!("global {addr} not found")))?; + let Some(addr) = module_global_addrs.get(idx as usize) else { + cold_path(); + return Err(Error::Other(format!( + "global {idx} not found. This should have been caught by the validator" + ))); + }; + + let Some(global) = self.state.globals.get(*addr as usize) else { + cold_path(); + return Err(Error::Other(format!("global {addr} not found"))); + }; + Ok(global.value.get()) }; let resolve_func = |idx: u32| -> Result<u32> { - module_func_addrs.get(idx as usize).copied().ok_or_else(|| { - Error::Other(format!("function {idx} not found. This should have been caught by the validator")) - }) + match module_func_addrs.get(idx as usize).copied() { + Some(func_addr) => Ok(func_addr), + None => { + cold_path(); + Err(Error::Other(format!( + "function {idx} not found. This should have been caught by the validator" + ))) + } + } }; if const_instrs.len() == 1 { @@ -507,8 +559,12 @@ impl Store { RefFunc(None) => TinyWasmValue::ValueRef(ValueRef::NULL), RefExtern(None) => TinyWasmValue::ValueRef(ValueRef::NULL), RefFunc(Some(idx)) => TinyWasmValue::ValueRef(ValueRef::from_addr(Some(resolve_func(*idx)?))), - _ => return Err(Error::Other("unsupported const instruction".to_string())), + _ => { + cold_path(); + return Err(Error::Other("unsupported const instruction".to_string())); + } }; + return Ok(val); } @@ -526,12 +582,14 @@ impl Store { stack.push(TinyWasmValue::ValueRef(ValueRef::from_addr(Some(resolve_func(*idx)?)))) } RefExtern(Some(_)) => { + cold_path(); return Err(Error::Other("ref.extern constants are not supported in init expressions".to_string())); } I32Add | I32Sub | I32Mul => { let rhs = stack.pop().ok_or_else(|| Error::Other("const stack underflow".to_string()))?; let lhs = stack.pop().ok_or_else(|| Error::Other("const stack underflow".to_string()))?; let (TinyWasmValue::Value32(lhs), TinyWasmValue::Value32(rhs)) = (lhs, rhs) else { + cold_path(); return Err(Error::Other("type mismatch in const i32 op".to_string())); }; let lhs = lhs as i32; @@ -545,26 +603,36 @@ impl Store { stack.push(TinyWasmValue::Value32(out as u32)); } I64Add | I64Sub | I64Mul => { - let rhs = stack.pop().ok_or_else(|| Error::Other("const stack underflow".to_string()))?; - let lhs = stack.pop().ok_or_else(|| Error::Other("const stack underflow".to_string()))?; - let (TinyWasmValue::Value64(lhs), TinyWasmValue::Value64(rhs)) = (lhs, rhs) else { + let rhs = stack.pop(); + let lhs = stack.pop(); + let (Some(TinyWasmValue::Value64(lhs)), Some(TinyWasmValue::Value64(rhs))) = (lhs, rhs) else { + cold_path(); return Err(Error::Other("type mismatch in const i64 op".to_string())); }; + let lhs = lhs as i64; let rhs = rhs as i64; let out = match instr { I64Add => lhs.wrapping_add(rhs), I64Sub => lhs.wrapping_sub(rhs), I64Mul => lhs.wrapping_mul(rhs), - _ => unreachable!(), + _ => { + cold_path(); + unreachable!("invalid const instruction in i64 op") + } }; stack.push(TinyWasmValue::Value64(out as u64)); } } } - let value = stack.pop().ok_or_else(|| Error::Other("empty const expression".to_string()))?; + let Some(value) = stack.pop() else { + cold_path(); + return Err(Error::Other("empty const expression".to_string())); + }; + if !stack.is_empty() { + cold_path(); return Err(Error::Other("const expression did not reduce to single value".to_string())); } Ok(value) diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index 6196beb..3eb14ba 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -93,7 +93,7 @@ pub struct ModuleInner { /// Optimized and validated WebAssembly functions /// /// Contains data from to the `code`, `func`, and `type` sections of the original WebAssembly module. - pub funcs: Arc<[Arc<WasmFunction>]>, + pub funcs: Box<[Arc<WasmFunction>]>, /// A vector of type definitions, indexed by `TypeAddr` /// @@ -108,32 +108,32 @@ pub struct ModuleInner { /// Global components of the WebAssembly module. /// /// Corresponds to the `global` section of the original WebAssembly module. - pub globals: Arc<[Global]>, + pub globals: Box<[Global]>, /// Table components of the WebAssembly module used to initialize tables. /// /// Corresponds to the `table` section of the original WebAssembly module. - pub table_types: Arc<[TableType]>, + pub table_types: Box<[TableType]>, /// Memory components of the WebAssembly module used to initialize memories. /// /// Corresponds to the `memory` section of the original WebAssembly module. - pub memory_types: Arc<[MemoryType]>, + pub memory_types: Box<[MemoryType]>, /// Imports of the WebAssembly module. /// /// Corresponds to the `import` section of the original WebAssembly module. - pub imports: Arc<[Import]>, + pub imports: Box<[Import]>, /// Data segments of the WebAssembly module. /// /// Corresponds to the `data` section of the original WebAssembly module. - pub data: Arc<[Data]>, + pub data: Box<[Data]>, /// Element segments of the WebAssembly module. /// /// Corresponds to the `elem` section of the original WebAssembly module. - pub elements: Arc<[Element]>, + pub elements: Box<[Element]>, /// How instantiation should prepare the module's local memories. pub local_memory_allocation: LocalMemoryAllocation, @@ -397,15 +397,16 @@ impl ValueCounts { impl<'a> FromIterator<&'a WasmType> for ValueCounts { #[inline] fn from_iter<I: IntoIterator<Item = &'a WasmType>>(iter: I) -> Self { - iter.into_iter().fold(Self::default(), |mut counts, ty| { + let mut counts = Self::default(); + + for ty in iter { match ty { - WasmType::I32 | WasmType::F32 => counts.c32 += 1, + WasmType::I32 | WasmType::F32 | WasmType::RefExtern | WasmType::RefFunc => counts.c32 += 1, WasmType::I64 | WasmType::F64 => counts.c64 += 1, WasmType::V128 => counts.c128 += 1, - WasmType::RefExtern | WasmType::RefFunc => counts.c32 += 1, } - counts - }) + } + counts } } |