diff options
Diffstat (limited to 'crates')
| -rw-r--r-- | crates/tinywasm/src/interpreter/executor.rs | 94 | ||||
| -rw-r--r-- | crates/tinywasm/src/interpreter/simd/macros.rs | 44 | ||||
| -rw-r--r-- | crates/tinywasm/src/interpreter/stack/value_stack.rs | 111 | ||||
| -rw-r--r-- | crates/tinywasm/src/interpreter/values.rs | 59 | ||||
| -rw-r--r-- | crates/tinywasm/src/lib.rs | 5 | ||||
| -rw-r--r-- | crates/tinywasm/src/reference.rs | 6 | ||||
| -rw-r--r-- | crates/tinywasm/src/store/memory.rs | 9 | ||||
| -rw-r--r-- | crates/tinywasm/tests/testsuite/run.rs | 2 |
8 files changed, 179 insertions, 151 deletions
diff --git a/crates/tinywasm/src/interpreter/executor.rs b/crates/tinywasm/src/interpreter/executor.rs index 63ddee7..fd7d49e 100644 --- a/crates/tinywasm/src/interpreter/executor.rs +++ b/crates/tinywasm/src/interpreter/executor.rs @@ -1,3 +1,5 @@ +use core::hint::cold_path; + #[cfg(not(feature = "std"))] #[allow(unused_imports)] use super::no_std_floats::NoStdFloatExt; @@ -17,9 +19,6 @@ use crate::instance::ModuleInstanceInner; use crate::interpreter::Value128; use crate::*; -#[cfg(feature = "std")] -const TIME_BUDGET_CHECK_INTERVAL: usize = 2048; -const FUEL_ACCOUNTING_INTERVAL: usize = 1024; const FUEL_COST_CALL_TOTAL: u32 = 5; pub(crate) struct Executor<'store, const BUDGETED: bool> { @@ -119,11 +118,14 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { let next = match self.func.instructions.0.get(self.cf.instr_ptr as usize) { Some(instr) => instr, - None => unreachable!( - "Instruction pointer out of bounds: {} ({} instructions)", - self.cf.instr_ptr, - self.func.instructions.0.len() - ), + None => { + cold_path(); + unreachable!( + "Instruction pointer out of bounds: {} ({} instructions)", + self.cf.instr_ptr, + self.func.instructions.0.len() + ) + } }; #[rustfmt::skip] @@ -722,7 +724,11 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { } let res = self.store.stack.values.enter_locals(&wasm_func.func.params, &wasm_func.func.locals); - let locals_base = res.map_err(|err| if IS_RETURN_CALL { err } else { Error::Trap(Trap::CallStackOverflow) })?; + let locals_base = res.map_err(|err| { + cold_path(); + if IS_RETURN_CALL { err } else { Error::Trap(Trap::CallStackOverflow) } + })?; + let new_call_frame = CallFrame::new(func_addr, wasm_func.owner, locals_base, wasm_func.func.locals); if !IS_RETURN_CALL { @@ -763,7 +769,10 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { } let res = self.store.stack.values.enter_locals(¶ms, &locals); - let locals_base = res.map_err(|err| if IS_RETURN_CALL { err } else { Error::Trap(Trap::CallStackOverflow) })?; + let locals_base = res.map_err(|err| { + cold_path(); + if IS_RETURN_CALL { err } else { Error::Trap(Trap::CallStackOverflow) } + })?; let new_call_frame = CallFrame::new(self.cf.func_addr, self.cf.module_addr, locals_base, locals); if !IS_RETURN_CALL { @@ -781,15 +790,23 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { let table_idx: u32 = self.store.stack.values.pop::<i32>() as u32; let table = self.store.state.get_table(self.module.resolve_table_addr(table_addr)); assert!(table.kind.element_type == WasmType::RefFunc, "table is not of type funcref"); - let table = - table.get(table_idx).map_err(|_| Error::from(Trap::UndefinedElement { index: table_idx as usize }))?; - table.addr().ok_or_else(|| Error::from(Trap::UninitializedElement { index: table_idx as usize }))? + + let table = table.get(table_idx).map_err(|_| { + cold_path(); + Error::from(Trap::UndefinedElement { index: table_idx as usize }) + })?; + + table.addr().ok_or_else(|| { + cold_path(); + Error::from(Trap::UninitializedElement { index: table_idx as usize }) + })? }; let call_ty = self.module.func_ty(type_addr); match self.store.state.get_func(func_ref) { crate::FunctionInstance::Wasm(wasm_func) => { if wasm_func.ty() != call_ty { + cold_path(); return Err(Trap::IndirectCallTypeMismatch { actual: wasm_func.ty().clone(), expected: call_ty.clone(), @@ -801,6 +818,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { } crate::FunctionInstance::Host(host_func) => { if host_func.ty != *call_ty { + cold_path(); return Err(Trap::IndirectCallTypeMismatch { actual: host_func.ty.clone(), expected: call_ty.clone(), @@ -833,6 +851,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { fn local_mem_addr<const N: usize>(&self, memarg: MemoryArg, addr_local: u8) -> Result<usize> { let addr = u64::from(self.store.stack.values.local_get::<u32>(&self.cf, u16::from(addr_local))); let Some(Ok(addr)) = memarg.offset().checked_add(addr).map(|a| a.try_into()) else { + cold_path(); return Err(Error::Trap(Trap::MemoryOutOfBounds { offset: addr as usize, len: N, max: 0 })); }; Ok(addr) @@ -850,9 +869,11 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { let addr = u64::from(self.store.stack.values.local_get::<u32>(&self.cf, u16::from(addr_local))); let value = cast(self.store.stack.values.local_get::<T>(&self.cf, u16::from(value_local))).to_mem_bytes(); let Some(effective_addr) = memarg.offset().checked_add(addr) else { + cold_path(); return Err(Error::Trap(Trap::MemoryOutOfBounds { offset: addr as usize, len: N, max: 0 })); }; let Ok(effective_addr) = usize::try_from(effective_addr) else { + cold_path(); return Err(Error::Trap(Trap::MemoryOutOfBounds { offset: addr as usize, len: N, max: 0 })); }; mem.store(effective_addr, value.len(), &value)?; @@ -970,6 +991,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { let data_len = data.data.as_ref().map_or(0, |d| d.len()); if ((size + offset) as usize > data_len) || ((dst + size) as usize > mem.len()) { + cold_path(); return Err(Trap::MemoryOutOfBounds { offset: offset as usize, len: size as usize, max: data_len }.into()); } @@ -977,7 +999,11 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { return Ok(()); } - let Some(data) = &data.data else { return Err(Trap::MemoryOutOfBounds { offset: 0, len: 0, max: 0 }.into()) }; + let Some(data) = &data.data else { + cold_path(); + return Err(Trap::MemoryOutOfBounds { offset: 0, len: 0, max: 0 }.into()); + }; + mem.store(dst as usize, size as usize, &data[offset as usize..((offset + size) as usize)]) } fn exec_table_copy(&mut self, dst_table: u32, src_table: u32) -> Result<()> { @@ -1012,6 +1038,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { let val = self.store.stack.values.pop::<i32>() as u64; let mem = self.store.state.get_mem(self.module.resolve_mem_addr(mem_addr)); let Some(Ok(addr)) = offset.checked_add(val).map(TryInto::try_into) else { + cold_path(); return Err(Error::Trap(Trap::MemoryOutOfBounds { offset: val as usize, len: LOAD_SIZE, max: 0 })); }; let val = mem.load_as::<LOAD_SIZE, LOAD>(addr)?.to_mem_bytes(); @@ -1038,11 +1065,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { self.store.stack.values.pop::<i32>() as u32 as u64 }; - let Some(addr) = base.checked_add(offset) else { - return Err(Error::Trap(Trap::MemoryOutOfBounds { offset: base as usize, len: LOAD_SIZE, max: 0 })); - }; - - let Ok(addr) = usize::try_from(addr) else { + let Some(Ok(addr)) = base.checked_add(offset).map(usize::try_from) else { + cold_path(); return Err(Error::Trap(Trap::MemoryOutOfBounds { offset: base as usize, len: LOAD_SIZE, max: 0 })); }; @@ -1068,10 +1092,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { false => self.store.stack.values.pop::<i32>() as u32 as u64, }; - let Some(effective_addr) = offset.checked_add(addr) else { - return Err(Error::Trap(Trap::MemoryOutOfBounds { offset: addr as usize, len: N, max: 0 })); - }; - let Ok(effective_addr) = usize::try_from(effective_addr) else { + let Some(Ok(effective_addr)) = offset.checked_add(addr).map(usize::try_from) else { + cold_path(); return Err(Error::Trap(Trap::MemoryOutOfBounds { offset: addr as usize, len: N, max: 0 })); }; @@ -1095,15 +1117,12 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { false => u64::from(self.store.stack.values.pop::<i32>() as u32), }; - let Some(effective_addr) = offset.checked_add(addr) else { - return Err(Error::Trap(Trap::MemoryOutOfBounds { offset: addr as usize, len: N, max: 0 })); - }; - let Ok(effective_addr) = usize::try_from(effective_addr) else { + let Some(Ok(effective_addr)) = offset.checked_add(addr).map(usize::try_from) else { + cold_path(); return Err(Error::Trap(Trap::MemoryOutOfBounds { offset: addr as usize, len: N, max: 0 })); }; mem.store(effective_addr, val.len(), &val)?; - Ok(()) } @@ -1148,6 +1167,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { let table_len = table.size(); if size < 0 || ((size + offset) as usize > elem_len) || ((dst + size) > table_len) { + cold_path(); return Err(Trap::TableOutOfBounds { offset: offset as usize, len: size as usize, max: elem_len }.into()); } @@ -1156,10 +1176,12 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { } if let ElementKind::Active { .. } = elem.kind { + cold_path(); return Err(Error::Other("table.init with active element".to_string())); } let Some(items) = elem.items.as_ref() else { + cold_path(); return Err(Trap::TableOutOfBounds { offset: 0, len: 0, max: 0 }.into()); }; @@ -1187,6 +1209,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { let i = self.store.stack.values.pop::<i32>(); if i + n > table.size() { + cold_path(); return Err(Error::Trap(Trap::TableOutOfBounds { offset: i as usize, len: n as usize, @@ -1205,10 +1228,13 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { impl<'store> Executor<'store, false> { #[inline(always)] pub(crate) fn run_to_completion(&mut self) -> Result<()> { - if self.exec::<{ usize::MAX }>()?.is_some() { - return Ok(()); + // direct loop+match has worse codegen than a fixed for loop here for some reason (like ~5% worse) + // ideally we use `loop_match` / `become` once thats stabilized + loop { + if self.exec::<1024>()?.is_some() { + return Ok(()); + } } - unreachable!(); } #[cfg(feature = "std")] @@ -1221,7 +1247,7 @@ impl<'store> Executor<'store, false> { } loop { - if self.exec::<TIME_BUDGET_CHECK_INTERVAL>()?.is_some() { + if self.exec::<1024>()?.is_some() { return Ok(ExecState::Completed); } @@ -1241,11 +1267,11 @@ impl<'store> Executor<'store, true> { } loop { - if self.exec::<FUEL_ACCOUNTING_INTERVAL>()?.is_some() { + if self.exec::<1024>()?.is_some() { return Ok(ExecState::Completed); } - self.store.execution_fuel = self.store.execution_fuel.saturating_sub(FUEL_ACCOUNTING_INTERVAL as u32); + self.store.execution_fuel = self.store.execution_fuel.saturating_sub(1024_u32); if self.store.execution_fuel == 0 { return Ok(ExecState::Suspended(self.cf)); } diff --git a/crates/tinywasm/src/interpreter/simd/macros.rs b/crates/tinywasm/src/interpreter/simd/macros.rs index 8cb5a3a..9509594 100644 --- a/crates/tinywasm/src/interpreter/simd/macros.rs +++ b/crates/tinywasm/src/interpreter/simd/macros.rs @@ -1,33 +1,12 @@ #![allow(unused_macros)] macro_rules! simd_impl { - ($(wasm => $wasm:block)? $(x86 => $x86:block)? generic => $generic:block) => {{ - #[cfg(any(target_arch = "wasm32", target_arch = "wasm64"))] - { - simd_impl!(@pick_wasm $( $wasm )? ; $generic) - } - - #[cfg(all( - not(any(target_arch = "wasm32", target_arch = "wasm64")), - feature = "simd-x86", - target_arch = "x86_64", - target_feature = "sse4.2", - target_feature = "avx", - target_feature = "avx2", - target_feature = "bmi1", - target_feature = "bmi2", - target_feature = "fma", - target_feature = "lzcnt", - target_feature = "movbe", - target_feature = "popcnt" - ))] - { - simd_impl!(@pick_x86 $( $x86 )? ; $generic) - } + ($(wasm => $wasm:block)? $(x86 => $x86:block)? generic => $generic:block) => { + cfg_select! { + any(target_arch = "wasm32", target_arch = "wasm64") => { + simd_impl!(@pick_wasm $( $wasm )? ; $generic) + }, - #[allow(unreachable_code)] - #[cfg(not(any( - any(target_arch = "wasm32", target_arch = "wasm64"), all( feature = "simd-x86", target_arch = "x86_64", @@ -40,12 +19,15 @@ macro_rules! simd_impl { target_feature = "lzcnt", target_feature = "movbe", target_feature = "popcnt" - ) - )))] - { - $generic + ) => { + simd_impl!(@pick_x86 $( $x86 )? ; $generic) + }, + + _ => { + $generic + } } - }}; + }; (@pick_wasm $wasm:block ; $generic:block) => { $wasm diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index 8db3c85..189282f 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -1,8 +1,10 @@ +use core::hint::cold_path; + use alloc::boxed::Box; use alloc::vec::Vec; use tinywasm_types::{ExternRef, FuncRef, LocalAddr, ValueCounts, WasmType, WasmValue}; -use crate::{Result, Trap, engine::Config, interpreter::*, unlikely}; +use crate::{Result, Trap, engine::Config, interpreter::*}; use super::{CallFrame, StackBase}; @@ -21,7 +23,7 @@ pub(crate) struct Stack<T: Copy + Default> { } impl<T: Copy + Default> Stack<T> { - pub(crate) fn with_size(size: usize) -> Self { + pub(crate) fn new(size: usize) -> Self { let mut data = Vec::with_capacity(size); data.resize_with(size, T::default); Self { data: data.into_boxed_slice(), len: 0 } @@ -33,113 +35,132 @@ impl<T: Copy + Default> Stack<T> { #[inline(always)] pub(crate) fn push(&mut self, value: T) -> Result<()> { - if unlikely(self.len >= self.data.len()) { + if let Some(slot) = self.data.get_mut(self.len) { + *slot = value; + self.len += 1; + } else { + cold_path(); return Err(Trap::ValueStackOverflow.into()); } - self.data[self.len] = value; - self.len += 1; Ok(()) } #[inline(always)] pub(crate) fn pop(&mut self) -> T { - if self.len == 0 { - unreachable!("ValueStack underflow, this is a bug"); - } self.len -= 1; - self.data[self.len] + *self.data.get(self.len).unwrap_or_else(|| { + cold_path(); + unreachable!("ValueStack underflow, this is a bug"); + }) } #[inline(always)] pub(crate) fn last(&self) -> &T { - if self.len == 0 { + self.data.get(self.len - 1).unwrap_or_else(|| { + cold_path(); unreachable!("ValueStack underflow, this is a bug"); - } - &self.data[self.len - 1] + }) } #[inline(always)] - pub(crate) fn get(&self, index: usize) -> T { - match self.data.get(index) { - Some(v) => *v, - None => unreachable!("Stack index out of bounds, this is a bug"), - } + pub(crate) fn get(&self, index: usize) -> &T { + self.data.get(index).unwrap_or_else(|| { + cold_path(); + unreachable!("Stack index out of bounds, this is a bug"); + }) } #[inline(always)] pub(crate) fn set(&mut self, index: usize, value: T) { - match self.data.get_mut(index) { - Some(v) => *v = value, - None => unreachable!("Stack index out of bounds, this is a bug"), - } + *self.data.get_mut(index).unwrap_or_else(|| { + cold_path(); + unreachable!("Stack index out of bounds, this is a bug"); + }) = value; } #[inline(always)] pub(crate) fn get_mut(&mut self, index: usize) -> &mut T { - match self.data.get_mut(index) { - Some(v) => v, - None => unreachable!("Stack index out of bounds, this is a bug"), - } + self.data.get_mut(index).unwrap_or_else(|| { + cold_path(); + unreachable!("Stack index out of bounds, this is a bug"); + }) } + #[inline(always)] pub(crate) fn truncate_keep(&mut self, n: usize, end_keep: usize) { - debug_assert!(n <= self.len); let len = self.len; + debug_assert!(n <= len); + if n >= len { return; } - if end_keep == 0 { - self.len = n; - return; + let dropped = len - n; + let keep = dropped.min(end_keep); + + if keep > 0 { + self.data.copy_within(len - keep..len, n); } - let keep = (len - n).min(end_keep); - self.data.copy_within((len - keep)..len, n); self.len = n + keep; } + #[inline(always)] pub(crate) fn enter_locals(&mut self, param_count: usize, local_count: usize) -> Result<u32> { + let len = self.len; debug_assert!(param_count <= local_count); - let start = self.len - param_count; + debug_assert!(param_count <= len); + + let start = len - param_count; let end = start + local_count; - if unlikely(end > self.data.len()) { + if end > self.data.len() { + cold_path(); return Err(Trap::ValueStackOverflow.into()); } - let init_start = start + param_count; - if init_start != end { - self.data[init_start..end].fill(T::default()); + if len != end { + self.data[len..end].fill(T::default()); } + self.len = end; Ok(start as u32) } + #[inline(always)] pub(crate) fn select_many(&mut self, count: usize, condition: bool) { if count == 0 { return; } - if self.len < count * 2 { + + let len = self.len; + let needed = count.checked_mul(2).unwrap_or_else(|| { + cold_path(); + unreachable!("Stack underflow, this is a bug"); + }); + + if len < needed { + cold_path(); unreachable!("Stack underflow, this is a bug"); } if !condition { - let start = self.len - (count * 2); - let second_start = self.len - count; - self.data.copy_within(second_start..self.len, start); + let dst = len - needed; + let src = len - count; + self.data.copy_within(src..len, dst); } - self.len -= count; + + self.len = len - count; } } impl ValueStack { pub(crate) fn new(config: &Config) -> Self { Self { - stack_32: Stack::with_size(config.stack_32_size), - stack_64: Stack::with_size(config.stack_64_size), - stack_128: Stack::with_size(config.stack_128_size), - stack_ref: Stack::with_size(config.stack_ref_size), + stack_32: Stack::new(config.stack_32_size), + stack_64: Stack::new(config.stack_64_size), + stack_128: Stack::new(config.stack_128_size), + stack_ref: Stack::new(config.stack_ref_size), } } diff --git a/crates/tinywasm/src/interpreter/values.rs b/crates/tinywasm/src/interpreter/values.rs index b8e15f4..7726e57 100644 --- a/crates/tinywasm/src/interpreter/values.rs +++ b/crates/tinywasm/src/interpreter/values.rs @@ -22,53 +22,52 @@ pub enum TinyWasmValue { } impl TinyWasmValue { - /// Asserts that the value is a 32-bit value and returns it (panics if the value is the wrong size) - pub fn unwrap_32(&self) -> Value32 { + /// Converts the value to a 32-bit value (returns None if the value is not a 32-bit value) + pub fn as_32(self) -> Option<Value32> { match self { - Self::Value32(v) => *v, - _ => panic!("Expected Value32"), + Self::Value32(v) => Some(v), + _ => None, } } - /// Asserts that the value is a 64-bit value and returns it (panics if the value is the wrong size) - pub fn unwrap_64(&self) -> Value64 { + /// Converts the value to a 64-bit value (returns None if the value is not a 64-bit value) + pub fn as_64(self) -> Option<Value64> { match self { - Self::Value64(v) => *v, - _ => panic!("Expected Value64"), + Self::Value64(v) => Some(v), + _ => None, } } - /// Asserts that the value is a 128-bit value and returns it (panics if the value is the wrong size) - pub fn unwrap_128(&self) -> Value128 { + /// Converts the value to a 128-bit value (returns None if the value is not a 128-bit value) + pub fn as_128(self) -> Option<Value128> { match self { - Self::Value128(v) => *v, - _ => panic!("Expected Value128"), + Self::Value128(v) => Some(v), + _ => None, } } - /// Asserts that the value is a reference value and returns it (panics if the value is the wrong size) - pub fn unwrap_ref(&self) -> ValueRef { + /// Converts the value to a reference value (returns None if the value is not a reference value) + pub fn as_ref(self) -> Option<ValueRef> { match self { - Self::ValueRef(v) => *v, - _ => panic!("Expected ValueRef"), + Self::ValueRef(v) => Some(v), + _ => None, } } /// Attaches a type to the value (panics if the size of the value is not the same as the type) - pub fn attach_type(&self, ty: WasmType) -> WasmValue { + pub fn attach_type(self, ty: WasmType) -> Option<WasmValue> { match (self, ty) { - (Self::Value32(v), WasmType::I32) => WasmValue::I32(*v as i32), - (Self::Value64(v), WasmType::I64) => WasmValue::I64(*v as i64), - (Self::Value32(v), WasmType::F32) => WasmValue::F32(f32::from_bits(*v)), - (Self::Value64(v), WasmType::F64) => WasmValue::F64(f64::from_bits(*v)), - (Self::ValueRef(v), WasmType::RefExtern) => WasmValue::RefExtern(ExternRef::new(*v)), - (Self::ValueRef(v), WasmType::RefFunc) => WasmValue::RefFunc(FuncRef::new(*v)), - (Self::Value128(v), WasmType::V128) => WasmValue::V128((*v).into()), - - (_, WasmType::I32 | WasmType::F32) => panic!("Expected Value32"), - (_, WasmType::I64 | WasmType::F64) => panic!("Expected Value64"), - (_, WasmType::RefExtern | WasmType::RefFunc) => panic!("Expected ValueRef"), - (_, WasmType::V128) => panic!("Expected Value128"), + (Self::Value32(v), WasmType::I32) => Some(WasmValue::I32(v as i32)), + (Self::Value64(v), WasmType::I64) => Some(WasmValue::I64(v as i64)), + (Self::Value32(v), WasmType::F32) => Some(WasmValue::F32(f32::from_bits(v))), + (Self::Value64(v), WasmType::F64) => Some(WasmValue::F64(f64::from_bits(v))), + (Self::ValueRef(v), WasmType::RefExtern) => Some(WasmValue::RefExtern(ExternRef::new(v))), + (Self::ValueRef(v), WasmType::RefFunc) => Some(WasmValue::RefFunc(FuncRef::new(v))), + (Self::Value128(v), WasmType::V128) => Some(WasmValue::V128((v).into())), + (_, WasmType::I32 | WasmType::F32) => None, + (_, WasmType::I64 | WasmType::F64) => None, + (_, WasmType::RefExtern | WasmType::RefFunc) => None, + (_, WasmType::V128) => None, } } } @@ -132,7 +131,7 @@ macro_rules! impl_internalvalue { #[inline(always)] fn local_get(stack: &ValueStack, frame: &CallFrame, index: LocalAddr) -> Self { - $to_outer(stack.$stack.get(frame.locals_base.$stack_base as usize + index as usize)) + $to_outer(*stack.$stack.get(frame.locals_base.$stack_base as usize + index as usize)) } #[inline(always)] diff --git a/crates/tinywasm/src/lib.rs b/crates/tinywasm/src/lib.rs index 37f0013..6180de3 100644 --- a/crates/tinywasm/src/lib.rs +++ b/crates/tinywasm/src/lib.rs @@ -128,12 +128,9 @@ pub mod types { pub use tinywasm_types::*; } -#[cold] -pub(crate) fn cold() {} - pub(crate) fn unlikely(b: bool) -> bool { if b { - cold(); + core::hint::cold_path(); }; b } diff --git a/crates/tinywasm/src/reference.rs b/crates/tinywasm/src/reference.rs index 3808293..e5719c6 100644 --- a/crates/tinywasm/src/reference.rs +++ b/crates/tinywasm/src/reference.rs @@ -1,4 +1,5 @@ use core::ffi::CStr; +use core::hint::cold_path; use alloc::string::{String, ToString}; use alloc::{ffi::CString, format}; @@ -291,16 +292,19 @@ impl Global { /// Get the current value of the global. pub fn get(&self, store: &Store) -> Result<WasmValue> { let global = self.instance(store)?; - Ok(global.value.get().attach_type(global.ty.ty)) + let value = global.value.get().attach_type(global.ty.ty); + Ok(value.unwrap_or_else(|| unreachable!("Global value type does not match global type, this is a bug"))) } /// Set the current value of the global. pub fn set(&self, store: &mut Store, value: WasmValue) -> Result<()> { let global = self.instance_mut(store)?; if !global.ty.mutable { + cold_path(); return Err(Error::Other("global is immutable".to_string())); } if WasmType::from(value) != global.ty.ty { + cold_path(); return Err(Error::Other("invalid global value type".to_string())); } global.value.set(value.into()); diff --git a/crates/tinywasm/src/store/memory.rs b/crates/tinywasm/src/store/memory.rs index d20fac0..a5b18d4 100644 --- a/crates/tinywasm/src/store/memory.rs +++ b/crates/tinywasm/src/store/memory.rs @@ -1,8 +1,10 @@ +use core::hint::cold_path; + use alloc::vec; use alloc::vec::Vec; use tinywasm_types::{MemoryArch, MemoryType}; -use crate::{Error, Result, cold, interpreter::Value128, log}; +use crate::{Error, Result, interpreter::Value128, log}; /// A WebAssembly Memory Instance /// @@ -30,17 +32,16 @@ impl MemoryInstance { } const fn trap_oob(&self, addr: usize, len: usize) -> Error { + cold_path(); Error::Trap(crate::Trap::MemoryOutOfBounds { offset: addr, len, max: self.data.len() }) } pub(crate) fn store(&mut self, addr: usize, len: usize, data: &[u8]) -> Result<()> { let Some(end) = addr.checked_add(len) else { - cold(); return Err(self.trap_oob(addr, data.len())); }; if end > self.data.len() || end < addr { - cold(); return Err(self.trap_oob(addr, data.len())); } self.data[addr..end].copy_from_slice(data); @@ -49,12 +50,10 @@ impl MemoryInstance { pub(crate) fn load(&self, addr: usize, len: usize) -> Result<&[u8]> { let Some(end) = addr.checked_add(len) else { - cold(); return Err(self.trap_oob(addr, len)); }; if end > self.data.len() || end < addr { - cold(); return Err(self.trap_oob(addr, len)); } diff --git a/crates/tinywasm/tests/testsuite/run.rs b/crates/tinywasm/tests/testsuite/run.rs index a39ffa7..5b1d11c 100644 --- a/crates/tinywasm/tests/testsuite/run.rs +++ b/crates/tinywasm/tests/testsuite/run.rs @@ -470,7 +470,7 @@ impl TestSuite { let expected = expected_alternatives .iter() .filter_map(|alts| alts.first()) - .find(|exp| module_global.attach_type(WasmType::from(*exp)).eq_loose(exp)); + .find(|exp| module_global.attach_type(WasmType::from(*exp)).unwrap().eq_loose(exp)); if expected.is_none() { test_group.add_result( |