diff options
Diffstat (limited to 'crates')
| -rw-r--r-- | crates/parser/src/lib.rs | 46 | ||||
| -rw-r--r-- | crates/parser/src/macros.rs | 117 | ||||
| -rw-r--r-- | crates/parser/src/module.rs | 15 | ||||
| -rw-r--r-- | crates/parser/src/optimize.rs | 794 | ||||
| -rw-r--r-- | crates/parser/src/visit.rs | 8 | ||||
| -rw-r--r-- | crates/tinywasm/src/engine.rs | 14 | ||||
| -rw-r--r-- | crates/tinywasm/src/imports.rs | 24 | ||||
| -rw-r--r-- | crates/tinywasm/src/instance.rs | 26 | ||||
| -rw-r--r-- | crates/tinywasm/src/interpreter/executor.rs | 357 | ||||
| -rw-r--r-- | crates/tinywasm/src/interpreter/num_helpers.rs | 2 | ||||
| -rw-r--r-- | crates/tinywasm/src/interpreter/simd/mod.rs | 5 | ||||
| -rw-r--r-- | crates/tinywasm/src/interpreter/stack/call_stack.rs | 2 | ||||
| -rw-r--r-- | crates/tinywasm/src/interpreter/stack/value_stack.rs | 113 | ||||
| -rw-r--r-- | crates/tinywasm/src/interpreter/values.rs | 72 | ||||
| -rw-r--r-- | crates/tinywasm/src/store/memory/mod.rs | 52 | ||||
| -rw-r--r-- | crates/tinywasm/src/store/memory/vec.rs | 32 | ||||
| -rw-r--r-- | crates/tinywasm/tests/testsuite/util.rs | 31 | ||||
| -rw-r--r-- | crates/types/src/instructions.rs | 14 | ||||
| -rw-r--r-- | crates/types/src/lib.rs | 1 |
19 files changed, 1254 insertions, 471 deletions
diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index 7b3dfa2..949afa2 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -47,11 +47,22 @@ pub use tinywasm_types::Module; pub struct ParserOptions { /// Whether to optimize local memory allocation by skipping allocation of unused local memories. pub optimize_local_memory_allocation: bool, + /// Whether to run the peephole rewrite optimizer. + pub optimize_rewrite: bool, + /// Whether to remove `Nop` and `MergeBarrier` instructions after rewriting. + pub optimize_remove_nop: bool, + /// Whether to invert conditional branches over an unconditional jump. + pub optimize_branch_inversion: bool, } impl Default for ParserOptions { fn default() -> Self { - Self { optimize_local_memory_allocation: true } + Self { + optimize_local_memory_allocation: true, + optimize_rewrite: true, + optimize_remove_nop: true, + optimize_branch_inversion: false, + } } } @@ -66,6 +77,39 @@ impl ParserOptions { pub const fn optimize_local_memory_allocation(&self) -> bool { self.optimize_local_memory_allocation } + + /// Enable or disable the peephole rewrite optimizer. + pub const fn with_rewrite_optimization(mut self, enabled: bool) -> Self { + self.optimize_rewrite = enabled; + self + } + + /// Returns whether the peephole rewrite optimizer is enabled. + pub const fn optimize_rewrite(&self) -> bool { + self.optimize_rewrite + } + + /// Enable or disable `Nop`/`MergeBarrier` removal after rewriting. + pub const fn with_nop_removal_optimization(mut self, enabled: bool) -> Self { + self.optimize_remove_nop = enabled; + self + } + + /// Returns whether `Nop`/`MergeBarrier` removal is enabled. + pub const fn optimize_remove_nop(&self) -> bool { + self.optimize_remove_nop + } + + /// Enable or disable the optimization that inverts conditional branches over an unconditional jump. + pub const fn with_branch_inversion_optimization(mut self, enabled: bool) -> Self { + self.optimize_branch_inversion = enabled; + self + } + + /// Returns whether conditional branch inversion optimization is enabled. + pub const fn optimize_branch_inversion(&self) -> bool { + self.optimize_branch_inversion + } } /// A WebAssembly parser diff --git a/crates/parser/src/macros.rs b/crates/parser/src/macros.rs index 97f0080..a6eab61 100644 --- a/crates/parser/src/macros.rs +++ b/crates/parser/src/macros.rs @@ -216,5 +216,120 @@ pub(crate) mod optimize { }; } - pub(crate) use {replace, rewrite}; + macro_rules! define_local_source_resolver { + ( + $name:ident, + get = $get:ident, + tee = $tee:ident, + set = $set:ident, + copy = $copy:ident, + binop_local_local_tee = $lltee:ident, + binop_local_local_set = $llset:ident, + binop_local_const_tee = $lctee:ident, + binop_local_const_set = $lcset:ident + $(, load_local_tee = $loadtee:ident, load_local_set = $loadset:ident)? + ) => { + fn $name(instrs: &mut [Instruction], read: usize, instr: Instruction) -> Option<(Instruction, u16)> { + Some(match instr { + Instruction::$get(local) => (Instruction::Nop, local), + Instruction::$tee(local) => { + let replacement = if let Some([(prev_idx, Instruction::$get(src))]) = previous_non_nop::<1>(instrs, read) { + instrs[prev_idx] = Instruction::Nop; + if src == local { Instruction::Nop } else { Instruction::$copy(src, local) } + } else { + Instruction::$set(local) + }; + (replacement, local) + } + Instruction::$lltee(op, a, b, local) => (Instruction::$llset(op, a, b, local), local), + Instruction::$lctee(op, src, c, local) => (Instruction::$lcset(op, src, c, local), local), + $(Instruction::$loadtee(memarg, addr, local) => (Instruction::$loadset(memarg, addr, local), local.into()),)? + _ => return None, + }) + } + }; + } + + macro_rules! fold_local_binop { + ( + $instrs:ident, $read:expr, $dst:expr, + source = $source:ident, + op = $op:ident, + const = $const:ident, + local_local = $local_local:ident, + local_const = $local_const:expr + ) => {{ + if let Some([(lhs_idx, lhs_src), (rhs_idx, rhs_src), (op_idx, raw_op)]) = + previous_non_nop::<3>($instrs, $read) + && let Some((lhs_instr, lhs)) = $source($instrs, lhs_idx, lhs_src) + && let Some(op) = $op(raw_op) + { + if let Some((rhs_instr, rhs)) = $source($instrs, rhs_idx, rhs_src) { + $instrs[lhs_idx] = lhs_instr; + $instrs[rhs_idx] = rhs_instr; + $instrs[op_idx] = Instruction::Nop; + $instrs[$read] = Instruction::$local_local(op, lhs, rhs, $dst); + } else if let Some(imm) = $const(rhs_src, raw_op) { + $instrs[lhs_idx] = lhs_instr; + $instrs[rhs_idx] = Instruction::Nop; + $instrs[op_idx] = Instruction::Nop; + $instrs[$read] = $local_const($dst, lhs, op, imm); + } + } + }}; + } + + macro_rules! rewrite_local_set_direct { + ( + $instrs:ident, $read:ident, $dst:expr, + get = $get:ident, + copy = $copy:ident, + binop_local_local = $ll:ident, + binop_local_local_set = $llset:ident, + binop_local_const = $lc:ident, + binop_local_const_set = $lcset:expr + $(, const_instr = $const_instr:ident, set_local_const = $set_local_const:ident)? + ) => {{ + rewrite!($instrs, $read, [$get(src)] => if src == $dst { Instruction::Nop } else { Instruction::$copy(src, $dst) }); + $(rewrite!($instrs, $read, [$const_instr(c)] => Instruction::$set_local_const($dst, c));)? + rewrite!($instrs, $read, [$ll(op, a, b)] => Instruction::$llset(op, a, b, $dst)); + rewrite!($instrs, $read, [$lc(op, src, c)] => { replace!($instrs, $read, 1 => $lcset($dst, src, op, c)); }); + }}; + } + + macro_rules! rewrite_local_tee_direct { + ( + $instrs:ident, $read:ident, $dst:expr, + get = $get:ident, + binop_local_local = $ll:ident, + binop_local_local_tee = $lltee:ident, + binop_local_const = $lc:ident, + binop_local_const_tee = $lctee:ident + ) => {{ + rewrite!($instrs, $read, [$get(src)] if (src == $dst) => [Instruction::$get(src), Instruction::Nop]); + rewrite!($instrs, $read, [$ll(op, a, b)] => Instruction::$lltee(op, a, b, $dst)); + rewrite!($instrs, $read, [$lc(op, src, c)] => Instruction::$lctee(op, src, c, $dst)); + }}; + } + + macro_rules! rewrite_drop_tee_direct { + ( + $instrs:ident, $read:ident, + tee = $tee:ident, + set = $set:ident, + binop_local_local_tee = $lltee:ident, + binop_local_local_set = $llset:ident, + binop_local_const_tee = $lctee:ident, + binop_local_const_set = $lcset:ident + ) => {{ + rewrite!($instrs, $read, [$tee(local)] => [Instruction::$set(local), Instruction::Nop]); + rewrite!($instrs, $read, [$lltee(op, a, b, dst)] => Instruction::$llset(op, a, b, dst)); + rewrite!($instrs, $read, [$lctee(op, src, c, dst)] => Instruction::$lcset(op, src, c, dst)); + }}; + } + + pub(crate) use { + define_local_source_resolver, fold_local_binop, replace, rewrite, rewrite_drop_tee_direct, + rewrite_local_set_direct, rewrite_local_tee_direct, + }; } diff --git a/crates/parser/src/module.rs b/crates/parser/src/module.rs index 164127f..548280e 100644 --- a/crates/parser/src/module.rs +++ b/crates/parser/src/module.rs @@ -200,17 +200,26 @@ impl ModuleReader { { let ty = self.func_types.get(ty_idx as usize).expect("No func type for func, this is a bug").clone(); let params = ValueCounts::from_iter(ty.params()); + let results = ValueCounts::from_iter(ty.results()); let self_func = (imported_func_count + func_idx) as u32; let local_mem_alloc = optimize_local_memory_allocation && local_memory_allocation != LocalMemoryAllocation::Eager; - let optimized = - optimize::optimize_instructions(instructions, &mut data, self_func, import_mem_count, local_mem_alloc); + let optimized = optimize::optimize_instructions( + instructions, + &mut data, + options, + self_func, + import_mem_count, + local_mem_alloc, + ); if optimized.uses_local_memory { local_memory_allocation = LocalMemoryAllocation::Eager; } - funcs.push(WasmFunction { instructions: optimized.instructions.into(), data, locals, params, ty }.into()); + funcs.push( + WasmFunction { instructions: optimized.instructions.into(), data, locals, params, results, ty }.into(), + ); } Ok(ModuleInner { diff --git a/crates/parser/src/optimize.rs b/crates/parser/src/optimize.rs index 51d0944..baaaf66 100644 --- a/crates/parser/src/optimize.rs +++ b/crates/parser/src/optimize.rs @@ -1,6 +1,7 @@ +use crate::ParserOptions; use crate::macros::optimize::*; use alloc::vec::Vec; -use tinywasm_types::{BinOp, BinOp128, CmpOp, Instruction, WasmFunctionData}; +use tinywasm_types::{BinOp, BinOp128, CmpOp, ConstIdx, Instruction, WasmFunctionData}; pub(crate) struct OptimizeResult { pub(crate) instructions: Vec<Instruction>, @@ -10,12 +11,27 @@ pub(crate) struct OptimizeResult { pub(crate) fn optimize_instructions( mut instructions: Vec<Instruction>, function_data: &mut WasmFunctionData, + options: &ParserOptions, self_func_addr: u32, imported_memory_count: u32, track_local_memory_usage: bool, ) -> OptimizeResult { - let uses_local_memory = rewrite(&mut instructions, self_func_addr, imported_memory_count, track_local_memory_usage); - remove_nop(&mut instructions, function_data); + let uses_local_memory = if options.optimize_rewrite() { + rewrite( + &mut instructions, + self_func_addr, + imported_memory_count, + track_local_memory_usage, + options.optimize_branch_inversion(), + ) + } else { + track_local_memory_usage + && instructions.iter().any(|instr| instr.memory_addr().is_some_and(|mem| mem >= imported_memory_count)) + }; + + if options.optimize_remove_nop() { + remove_nop(&mut instructions, function_data); + } OptimizeResult { instructions, uses_local_memory } } @@ -24,6 +40,7 @@ fn rewrite( self_func_addr: u32, imported_memory_count: u32, track_local_memory_usage: bool, + optimize_branch_inversion: bool, ) -> bool { use Instruction::*; let mut uses_local_memory = false; @@ -38,71 +55,71 @@ fn rewrite( instr @ (I32Add | I32Mul | I32And | I32Or | I32Xor) => { let Some(op) = int_bin_op_32(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); - rewrite!(instrs, i, [LocalGet32(local), I32Const(c)] => BinOpLocalConst32(op, local, c)); - rewrite!(instrs, i, [I32Const(c), LocalGet32(local)] => BinOpLocalConst32(op, local, c)); + rewrite!(instrs, i, [LocalGet32(local), Const32(c)] => BinOpLocalConst32(op, local, c)); + rewrite!(instrs, i, [Const32(c), LocalGet32(local)] => BinOpLocalConst32(op, local, c)); if matches!(op, BinOp::IAdd) { - rewrite!(instrs, i, [I32Const(c)] => AddConst32(c)); + rewrite!(instrs, i, [Const32(c)] => AddConst32(c)); } } instr @ (I32Sub | I32Shl | I32ShrS | I32ShrU | I32Rotl | I32Rotr) => { let Some(op) = int_bin_op_32(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); - rewrite!(instrs, i, [LocalGet32(local), I32Const(c)] => BinOpLocalConst32(op, local, c)); + rewrite!(instrs, i, [LocalGet32(local), Const32(c)] => BinOpLocalConst32(op, local, c)); } instr @ (I64Add | I64Mul | I64And | I64Or | I64Xor) => { let Some(op) = int_bin_op_64(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); - rewrite!(instrs, i, [LocalGet64(local), I64Const(c)] => BinOpLocalConst64(op, local, c)); - rewrite!(instrs, i, [I64Const(c), LocalGet64(local)] => BinOpLocalConst64(op, local, c)); + rewrite!(instrs, i, [LocalGet64(local), Const64(c)] => BinOpLocalConst64(op, local, c)); + rewrite!(instrs, i, [Const64(c), LocalGet64(local)] => BinOpLocalConst64(op, local, c)); if matches!(op, BinOp::IAdd) { - rewrite!(instrs, i, [I64Const(c)] => AddConst64(c)); + rewrite!(instrs, i, [Const64(c)] => AddConst64(c)); } } instr @ (I64Sub | I64Shl | I64ShrS | I64ShrU | I64Rotl | I64Rotr) => { let Some(op) = int_bin_op_64(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); - rewrite!(instrs, i, [LocalGet64(local), I64Const(c)] => BinOpLocalConst64(op, local, c)); + rewrite!(instrs, i, [LocalGet64(local), Const64(c)] => BinOpLocalConst64(op, local, c)); } instr @ (F32Add | F32Mul | F32Min | F32Max) => { let Some(op) = float_bin_op_32(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); - rewrite!(instrs, i, [LocalGet32(local), F32Const(c)] => BinOpLocalConst32(op, local, f32_const_bits(c))); - rewrite!(instrs, i, [F32Const(c), LocalGet32(local)] => BinOpLocalConst32(op, local, f32_const_bits(c))); + rewrite!(instrs, i, [LocalGet32(local), Const32(c)] => BinOpLocalConst32(op, local, c)); + rewrite!(instrs, i, [Const32(c), LocalGet32(local)] => BinOpLocalConst32(op, local, c)); } instr @ (F32Sub | F32Div | F32Copysign) => { let Some(op) = float_bin_op_32(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); - rewrite!(instrs, i, [LocalGet32(local), F32Const(c)] => BinOpLocalConst32(op, local, f32_const_bits(c))); + rewrite!(instrs, i, [LocalGet32(local), Const32(c)] => BinOpLocalConst32(op, local, c)); } instr @ (F64Add | F64Mul | F64Min | F64Max) => { let Some(op) = float_bin_op_64(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); - rewrite!(instrs, i, [LocalGet64(local), F64Const(c)] => BinOpLocalConst64(op, local, f64_const_bits(c))); - rewrite!(instrs, i, [F64Const(c), LocalGet64(local)] => BinOpLocalConst64(op, local, f64_const_bits(c))); + rewrite!(instrs, i, [LocalGet64(local), Const64(c)] => BinOpLocalConst64(op, local, c)); + rewrite!(instrs, i, [Const64(c), LocalGet64(local)] => BinOpLocalConst64(op, local, c)); } instr @ (F64Sub | F64Div | F64Copysign) => { let Some(op) = float_bin_op_64(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); - rewrite!(instrs, i, [LocalGet64(local), F64Const(c)] => BinOpLocalConst64(op, local, f64_const_bits(c))); + rewrite!(instrs, i, [LocalGet64(local), Const64(c)] => BinOpLocalConst64(op, local, c)); } instr @ (V128And | V128Or | V128Xor | I64x2Add | I64x2Mul) => { let Some(op) = bin_op_128(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet128(a), LocalGet128(b)] => BinOpLocalLocal128(op, a, b)); - rewrite!(instrs, i, [LocalGet128(local), V128Const(c)] => BinOpLocalConst128(op, local, c)); - rewrite!(instrs, i, [V128Const(c), LocalGet128(local)] => BinOpLocalConst128(op, local, c)); + rewrite!(instrs, i, [LocalGet128(local), Const128(c)] => BinOpLocalConst128(op, local, c)); + rewrite!(instrs, i, [Const128(c), LocalGet128(local)] => BinOpLocalConst128(op, local, c)); } V128AndNot => { rewrite!(instrs, i, [LocalGet128(a), LocalGet128(b)] => BinOpLocalLocal128(BinOp128::AndNot, a, b)); - rewrite!(instrs, i, [LocalGet128(local), V128Const(c)] => BinOpLocalConst128(BinOp128::AndNot, local, c)); + rewrite!(instrs, i, [LocalGet128(local), Const128(c)] => BinOpLocalConst128(BinOp128::AndNot, local, c)); } - I32Store(memarg) => { + I32Store(memarg) | F32Store(memarg) => { rewrite!(instrs, i, [LocalGet32(addr_local), LocalGet32(value_local)] if (let (Ok(addr_local), Ok(value_local)) = (u8::try_from(addr_local), u8::try_from(value_local))) => StoreLocalLocal32(memarg, addr_local, value_local) ); } - I64Store(memarg) => { + I64Store(memarg) | F64Store(memarg) => { rewrite!(instrs, i, [LocalGet32(addr_local), LocalGet64(value_local)] if (let (Ok(addr_local), Ok(value_local)) = (u8::try_from(addr_local), u8::try_from(value_local))) => @@ -116,48 +133,48 @@ fn rewrite( StoreLocalLocal128(memarg, addr_local, value_local) ); } - I32Load(memarg) => { + I32Load(memarg) | F32Load(memarg) => { rewrite!(instrs, i, [LocalGet32(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) => LoadLocal32(memarg, addr_local) ); } MemoryFill(mem) => { - rewrite!(instrs, i, [I32Const(val), I32Const(size)] => MemoryFillImm(mem, val as u8, size)) + rewrite!(instrs, i, [Const32(val), Const32(size)] => MemoryFillImm(mem, val as u8, size)) } LocalGet32(dst) => rewrite!(instrs, i, [LocalSet32(src)] if (src == dst) => [LocalTee32(src), Nop]), LocalGet64(dst) => rewrite!(instrs, i, [LocalSet64(src)] if (src == dst) => [LocalTee64(src), Nop]), LocalGet128(dst) => rewrite!(instrs, i, [LocalSet128(src)] if (src == dst) => [LocalTee128(src), Nop]), LocalSet32(dst) => { - if let Some([(lhs_idx, lhs_src), (rhs_idx, rhs_src), (op_idx, raw_op)]) = previous_non_nop_3(instrs, i) - && let Some((lhs_instr, lhs)) = stack_source_local_32(lhs_src) - && let Some(op) = scalar_bin_op_32(raw_op) - { - if let Some((rhs_instr, rhs)) = stack_source_local_32(rhs_src) { - instrs[lhs_idx] = lhs_instr; - instrs[rhs_idx] = rhs_instr; - instrs[op_idx] = Nop; - instrs[i] = BinOpLocalLocalSet32(op, lhs, rhs, dst); - } else if let Some(imm) = scalar_const_32(rhs_src, raw_op) { - instrs[lhs_idx] = lhs_instr; - instrs[rhs_idx] = Nop; - instrs[op_idx] = Nop; - instrs[i] = match (dst == lhs, op) { - (true, BinOp::IAdd) => IncLocal32(dst, imm), - (true, BinOp::ISub) => IncLocal32(dst, imm.wrapping_neg()), - _ => BinOpLocalConstSet32(op, lhs, imm, dst), - }; + fold_local_binop!( + instrs, i, dst, + source = resolve_local_source_32, + op = scalar_bin_op_32, + const = scalar_const_32, + local_local = BinOpLocalLocalSet32, + local_const = |dst, lhs, op, imm| match (dst == lhs, op) { + (true, BinOp::IAdd) => Instruction::IncLocal32(dst, imm), + (true, BinOp::ISub) => Instruction::IncLocal32(dst, imm.wrapping_neg()), + _ => Instruction::BinOpLocalConstSet32(op, lhs, imm, dst), } - } - rewrite!(instrs, i, [LocalGet32(src)] => if src == dst { Nop } else { LocalCopy32(src, dst) }); - rewrite!(instrs, i, [I32Const(c)] => SetLocalConst32(dst, c)); - rewrite!(instrs, i, [F32Const(c)] => SetLocalConst32(dst, i32::from_ne_bytes(c.to_bits().to_ne_bytes()))); - rewrite!(instrs, i, [BinOpLocalLocal32(op, a, b)] => BinOpLocalLocalSet32(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConst32(op, src, c)] => match (dst == src, op) { - (true, BinOp::IAdd) => IncLocal32(dst, c), - (true, BinOp::ISub) => IncLocal32(dst, c.wrapping_neg()), - _ => BinOpLocalConstSet32(op, src, c, dst), - }); + ); + rewrite_local_set_direct!( + instrs, + i, + dst, + get = LocalGet32, + copy = LocalCopy32, + binop_local_local = BinOpLocalLocal32, + binop_local_local_set = BinOpLocalLocalSet32, + binop_local_const = BinOpLocalConst32, + binop_local_const_set = |dst, src, op, c| match (dst == src, op) { + (true, BinOp::IAdd) => IncLocal32(dst, c), + (true, BinOp::ISub) => IncLocal32(dst, c.wrapping_neg()), + _ => BinOpLocalConstSet32(op, src, c, dst), + }, + const_instr = Const32, + set_local_const = SetLocalConst32 + ); rewrite!(instrs, i, [LoadLocal32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => LoadLocalSet32(memarg, addr, dst)); rewrite!(instrs, i, [LocalGet32(addr), I32Load(memarg)] if @@ -166,32 +183,58 @@ fn rewrite( ); } LocalSet64(dst) => { - rewrite!(instrs, i, [LocalGet64(src)] => if src == dst { Nop } else { LocalCopy64(src, dst) }); - rewrite!(instrs, i, - [LocalTee64(src), I64Const(c), instr] if (let Some(op) = int_bin_op_64(instr)) => - [LocalSet64(src), Nop, Nop, match (dst == src, op) { + fold_local_binop!( + instrs, i, dst, + source = resolve_local_source_64, + op = scalar_bin_op_64, + const = scalar_const_64, + local_local = BinOpLocalLocalSet64, + local_const = |dst, lhs, op, imm| match (dst == lhs, op) { + (true, BinOp::IAdd) => Instruction::IncLocal64(dst, imm), + (true, BinOp::ISub) => Instruction::IncLocal64(dst, imm.wrapping_neg()), + _ => Instruction::BinOpLocalConstSet64(op, lhs, imm, dst), + } + ); + rewrite_local_set_direct!( + instrs, + i, + dst, + get = LocalGet64, + copy = LocalCopy64, + binop_local_local = BinOpLocalLocal64, + binop_local_local_set = BinOpLocalLocalSet64, + binop_local_const = BinOpLocalConst64, + binop_local_const_set = |dst, src, op, c| match (dst == src, op) { (true, BinOp::IAdd) => IncLocal64(dst, c), (true, BinOp::ISub) => IncLocal64(dst, c.wrapping_neg()), _ => BinOpLocalConstSet64(op, src, c, dst), - }] + }, + const_instr = Const64, + set_local_const = SetLocalConst64 ); - rewrite!(instrs, i, - [LocalTee64(src), F64Const(c), instr] if (let Some(op) = float_bin_op_64(instr)) => - [LocalSet64(src), Nop, Nop, BinOpLocalConstSet64(op, src, f64_const_bits(c), dst)] - ); - rewrite!(instrs, i, [I64Const(c)] => SetLocalConst64(dst, c)); - rewrite!(instrs, i, [F64Const(c)] => SetLocalConst64(dst, i64::from_ne_bytes(c.to_bits().to_ne_bytes()))); - rewrite!(instrs, i, [BinOpLocalLocal64(op, a, b)] => BinOpLocalLocalSet64(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConst64(op, src, c)] => match (dst == src, op) { - (true, BinOp::IAdd) => IncLocal64(dst, c), - (true, BinOp::ISub) => IncLocal64(dst, c.wrapping_neg()), - _ => BinOpLocalConstSet64(op, src, c, dst), - }); } LocalSet128(dst) => { - rewrite!(instrs, i, [LocalGet128(src)] => if src == dst { Nop } else { LocalCopy128(src, dst) }); - rewrite!(instrs, i, [BinOpLocalLocal128(op, a, b)] => BinOpLocalLocalSet128(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConst128(op, src, c)] => BinOpLocalConstSet128(op, src, c, dst)); + fold_local_binop!( + instrs, i, dst, + source = resolve_local_source_128, + op = bin_op_128, + const = const_128, + local_local = BinOpLocalLocalSet128, + local_const = |dst, lhs, op, imm| Instruction::BinOpLocalConstSet128(op, lhs, imm, dst) + ); + rewrite_local_set_direct!( + instrs, + i, + dst, + get = LocalGet128, + copy = LocalCopy128, + binop_local_local = BinOpLocalLocal128, + binop_local_local_set = BinOpLocalLocalSet128, + binop_local_const = BinOpLocalConst128, + binop_local_const_set = |dst, src, op, c| BinOpLocalConstSet128(op, src, c, dst), + const_instr = Const128, + set_local_const = SetLocalConst128 + ); rewrite!(instrs, i, [LocalGet32(addr), V128Load(memarg)] if (let (Ok(addr), Ok(dst)) = (u8::try_from(addr), u8::try_from(dst))) => @@ -199,153 +242,353 @@ fn rewrite( ); } LocalTee32(dst) => { - if let Some([(lhs_idx, lhs_src), (rhs_idx, rhs_src), (op_idx, raw_op)]) = previous_non_nop_3(instrs, i) - && let Some((lhs_instr, lhs)) = stack_source_local_32(lhs_src) - && let Some(op) = scalar_bin_op_32(raw_op) - { - if let Some((rhs_instr, rhs)) = stack_source_local_32(rhs_src) { - instrs[lhs_idx] = lhs_instr; - instrs[rhs_idx] = rhs_instr; - instrs[op_idx] = Nop; - instrs[i] = BinOpLocalLocalTee32(op, lhs, rhs, dst); - } else if let Some(imm) = scalar_const_32(rhs_src, raw_op) { - instrs[lhs_idx] = lhs_instr; - instrs[rhs_idx] = Nop; - instrs[op_idx] = Nop; - instrs[i] = BinOpLocalConstTee32(op, lhs, imm, dst); - } - } - rewrite!(instrs, i, [LocalGet32(src)] if (src == dst) => [LocalGet32(src), Nop]); - rewrite!(instrs, i, [BinOpLocalLocal32(op, a, b)] => BinOpLocalLocalTee32(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConst32(op, src, c)] => BinOpLocalConstTee32(op, src, c, dst)); - rewrite!(instrs, i, [I32Const(c), I32And] => AndConstTee32(c, dst)); - rewrite!(instrs, i, [I32Const(c), I32Sub] => SubConstTee32(c, dst)); + fold_local_binop!( + instrs, i, dst, + source = resolve_local_source_32, + op = scalar_bin_op_32, + const = scalar_const_32, + local_local = BinOpLocalLocalTee32, + local_const = |dst, lhs, op, imm| Instruction::BinOpLocalConstTee32(op, lhs, imm, dst) + ); + rewrite_local_tee_direct!( + instrs, + i, + dst, + get = LocalGet32, + binop_local_local = BinOpLocalLocal32, + binop_local_local_tee = BinOpLocalLocalTee32, + binop_local_const = BinOpLocalConst32, + binop_local_const_tee = BinOpLocalConstTee32 + ); + rewrite!(instrs, i, [Const32(c), I32And] => AndConstTee32(c, dst)); + rewrite!(instrs, i, [Const32(c), I32Sub] => SubConstTee32(c, dst)); rewrite!(instrs, i, [LocalGet32(addr), I32Load(memarg)] if (let (Ok(addr), Ok(dst)) = (u8::try_from(addr), u8::try_from(dst))) => LoadLocalTee32(memarg, addr, dst) ); rewrite!(instrs, i, + [LocalGet32(addr), F32Load(memarg)] if + (let (Ok(addr), Ok(dst)) = (u8::try_from(addr), u8::try_from(dst))) => + LoadLocalTee32(memarg, addr, dst) + ); + rewrite!(instrs, i, [LoadLocal32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => LoadLocalTee32(memarg, addr, dst) ); } LocalTee64(dst) => { - rewrite!(instrs, i, [LocalGet64(src)] if (src == dst) => [LocalGet64(src), Nop]); - rewrite!(instrs, i, [BinOpLocalLocal64(op, a, b)] => BinOpLocalLocalTee64(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConst64(op, src, c)] => BinOpLocalConstTee64(op, src, c, dst)); - rewrite!(instrs, i, [I64Const(c), I64And] => AndConstTee64(c, dst)); - rewrite!(instrs, i, [I64Const(c), I64Sub] => SubConstTee64(c, dst)); + fold_local_binop!( + instrs, i, dst, + source = resolve_local_source_64, + op = scalar_bin_op_64, + const = scalar_const_64, + local_local = BinOpLocalLocalTee64, + local_const = |dst, lhs, op, imm| Instruction::BinOpLocalConstTee64(op, lhs, imm, dst) + ); + rewrite_local_tee_direct!( + instrs, + i, + dst, + get = LocalGet64, + binop_local_local = BinOpLocalLocal64, + binop_local_local_tee = BinOpLocalLocalTee64, + binop_local_const = BinOpLocalConst64, + binop_local_const_tee = BinOpLocalConstTee64 + ); + rewrite!(instrs, i, [Const64(c), I64And] => AndConstTee64(c, dst)); + rewrite!(instrs, i, [Const64(c), I64Sub] => SubConstTee64(c, dst)); } LocalTee128(dst) => { - rewrite!(instrs, i, [LocalGet128(src)] if (src == dst) => [LocalGet128(src), Nop]); - rewrite!(instrs, i, [BinOpLocalLocal128(op, a, b)] => BinOpLocalLocalTee128(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConst128(op, src, c)] => BinOpLocalConstTee128(op, src, c, dst)); + fold_local_binop!( + instrs, i, dst, + source = resolve_local_source_128, + op = bin_op_128, + const = const_128, + local_local = BinOpLocalLocalTee128, + local_const = |dst, lhs, op, imm| Instruction::BinOpLocalConstTee128(op, lhs, imm, dst) + ); + rewrite_local_tee_direct!( + instrs, + i, + dst, + get = LocalGet128, + binop_local_local = BinOpLocalLocal128, + binop_local_local_tee = BinOpLocalLocalTee128, + binop_local_const = BinOpLocalConst128, + binop_local_const_tee = BinOpLocalConstTee128 + ); rewrite!(instrs, i, [LocalGet32(addr), V128Load(memarg)] if (let (Ok(addr), Ok(dst)) = (u8::try_from(addr), u8::try_from(dst))) => LoadLocalTee128(memarg, addr, dst) ); } - Drop32 => { - rewrite!(instrs, i, [LocalTee32(local)] => [LocalSet32(local), Nop]); - rewrite!(instrs, i, [BinOpLocalLocalTee32(op, a, b, dst)] => BinOpLocalLocalSet32(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConstTee32(op, src, c, dst)] => BinOpLocalConstSet32(op, src, c, dst)); - } - Drop64 => { - rewrite!(instrs, i, [LocalTee64(local)] => [LocalSet64(local), Nop]); - rewrite!(instrs, i, [BinOpLocalLocalTee64(op, a, b, dst)] => BinOpLocalLocalSet64(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConstTee64(op, src, c, dst)] => BinOpLocalConstSet64(op, src, c, dst)); - } - Drop128 => { - rewrite!(instrs, i, [LocalTee128(local)] => [LocalSet128(local), Nop]); - rewrite!(instrs, i, [BinOpLocalLocalTee128(op, a, b, dst)] => BinOpLocalLocalSet128(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConstTee128(op, src, c, dst)] => BinOpLocalConstSet128(op, src, c, dst)); + Drop32 => rewrite_drop_tee_direct!( + instrs, + i, + tee = LocalTee32, + set = LocalSet32, + binop_local_local_tee = BinOpLocalLocalTee32, + binop_local_local_set = BinOpLocalLocalSet32, + binop_local_const_tee = BinOpLocalConstTee32, + binop_local_const_set = BinOpLocalConstSet32 + ), + Drop64 => rewrite_drop_tee_direct!( + instrs, + i, + tee = LocalTee64, + set = LocalSet64, + binop_local_local_tee = BinOpLocalLocalTee64, + binop_local_local_set = BinOpLocalLocalSet64, + binop_local_const_tee = BinOpLocalConstTee64, + binop_local_const_set = BinOpLocalConstSet64 + ), + Drop128 => rewrite_drop_tee_direct!( + instrs, + i, + tee = LocalTee128, + set = LocalSet128, + binop_local_local_tee = BinOpLocalLocalTee128, + binop_local_local_set = BinOpLocalLocalSet128, + binop_local_const_tee = BinOpLocalConstTee128, + binop_local_const_set = BinOpLocalConstSet128 + ), + Jump(ip) => { + let target = resolve_jump_target(instrs, ip); + if target == next_non_nop(instrs, i + 1) as u32 { + instrs[i] = Nop; + } else if target != ip { + instrs[i] = Jump(target); + } } JumpIfZero(ip) => { + let target = resolve_jump_target(instrs, ip); + rewrite!(instrs, i, [LocalGet32(local), I32Eqz] => { + replace!(instrs, i, 2 => [Nop, Nop, JumpIfLocalNonZero32 { target_ip: target, local }]); + continue; + }); rewrite!(instrs, i, [I32Eqz] => { - replace!(instrs, i, 1 => [Nop, JumpIfNonZero32(ip)]); + replace!(instrs, i, 1 => [Nop, JumpIfNonZero32(target)]); + continue; + }); + rewrite!(instrs, i, [LocalGet32(local)] => JumpIfLocalZero32 { target_ip: target, local }); + rewrite!(instrs, i, [LocalGet64(local), I64Eqz] => { + replace!(instrs, i, 2 => [Nop, Nop, JumpIfLocalNonZero64 { target_ip: target, local }]); continue; }); rewrite!(instrs, i, [I64Eqz] => { - replace!(instrs, i, 1 => [Nop, JumpIfNonZero64(ip)]); + replace!(instrs, i, 1 => [Nop, JumpIfNonZero64(target)]); continue; }); rewrite!(instrs, i, - [LocalGet32(local), I32Const(imm), cmp] if (let Some(op) = cmp_op(cmp)) => - JumpCmpLocalConst32 { target_ip: ip, local, imm, op: inverse_cmp_op(op) } + [LocalGet32(local), Const32(imm), cmp] if (let Some(op) = cmp_op(cmp)) => + match (imm, inverse_cmp_op(op)) { + (0, CmpOp::Eq) => JumpIfLocalZero32 { target_ip: target, local }, + (0, CmpOp::Ne) => JumpIfLocalNonZero32 { target_ip: target, local }, + (imm, op) => JumpCmpLocalConst32 { target_ip: target, local, imm, op }, + } ); rewrite!(instrs, i, - [LocalGet64(local), I64Const(imm), cmp] if + [LocalGet64(local), Const64(imm), cmp] if (let Some(op) = cmp_op_64(cmp) && let Ok(imm) = i32::try_from(imm)) => - JumpCmpLocalConst64 { target_ip: ip, local, imm, op: inverse_cmp_op(op) } + match (imm, inverse_cmp_op(op)) { + (0, CmpOp::Eq) => JumpIfLocalZero64 { target_ip: target, local }, + (0, CmpOp::Ne) => JumpIfLocalNonZero64 { target_ip: target, local }, + (imm, op) => JumpCmpLocalConst64 { target_ip: target, local, imm, op }, + } ); rewrite!(instrs, i, [LocalGet32(left), LocalGet32(right), cmp] if (let Some(op) = cmp_op(cmp)) => - JumpCmpLocalLocal32 { target_ip: ip, left, right, op: inverse_cmp_op(op) } + JumpCmpLocalLocal32 { target_ip: target, left, right, op: inverse_cmp_op(op) } ); rewrite!(instrs, i, [LocalGet64(left), LocalGet64(right), cmp] if (let Some(op) = cmp_op_64(cmp)) => - JumpCmpLocalLocal64 { target_ip: ip, left, right, op: inverse_cmp_op(op) } + JumpCmpLocalLocal64 { target_ip: target, left, right, op: inverse_cmp_op(op) } ); - rewrite!(instrs, i, [I32Const(imm), cmp] if (let Some(op) = cmp_op(cmp)) => match (imm, inverse_cmp_op(op)) { - (0, CmpOp::Eq) => JumpIfZero32(ip), - (0, CmpOp::Ne) => JumpIfNonZero32(ip), - (imm, op) => JumpCmpStackConst32 { target_ip: ip, imm, op }, + rewrite!(instrs, i, [Const32(imm), cmp] if (let Some(op) = cmp_op(cmp)) => match (imm, inverse_cmp_op(op)) { + (0, CmpOp::Eq) => JumpIfZero32(target), + (0, CmpOp::Ne) => JumpIfNonZero32(target), + (imm, op) => JumpCmpStackConst32 { target_ip: target, imm, op }, }); - rewrite!(instrs, i, [I64Const(imm), cmp] if (let Some(op) = cmp_op_64(cmp)) => match (imm, inverse_cmp_op(op)) { - (0, CmpOp::Eq) => JumpIfZero64(ip), - (0, CmpOp::Ne) => JumpIfNonZero64(ip), - (imm, op) => JumpCmpStackConst64 { target_ip: ip, imm, op }, + rewrite!(instrs, i, [Const64(imm), cmp] if (let Some(op) = cmp_op_64(cmp)) => match (imm, inverse_cmp_op(op)) { + (0, CmpOp::Eq) => JumpIfZero64(target), + (0, CmpOp::Ne) => JumpIfNonZero64(target), + (imm, op) => JumpCmpStackConst64 { target_ip: target, imm, op }, }); + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + if let JumpIfZero(current) = &mut instrs[i] { + *current = target; + } } JumpIfNonZero(ip) => { + let target = resolve_jump_target(instrs, ip); + rewrite!(instrs, i, [LocalGet32(local), I32Eqz] => { + replace!(instrs, i, 2 => [Nop, Nop, JumpIfLocalZero32 { target_ip: target, local }]); + continue; + }); rewrite!(instrs, i, [I32Eqz] => { - replace!(instrs, i, 1 => [Nop, JumpIfZero32(ip)]); + replace!(instrs, i, 1 => [Nop, JumpIfZero32(target)]); + continue; + }); + rewrite!(instrs, i, [LocalGet32(local)] => JumpIfLocalNonZero32 { target_ip: target, local }); + rewrite!(instrs, i, [LocalGet64(local), I64Eqz] => { + replace!(instrs, i, 2 => [Nop, Nop, JumpIfLocalZero64 { target_ip: target, local }]); continue; }); rewrite!(instrs, i, [I64Eqz] => { - replace!(instrs, i, 1 => [Nop, JumpIfZero64(ip)]); + replace!(instrs, i, 1 => [Nop, JumpIfZero64(target)]); continue; }); rewrite!(instrs, i, - [LocalGet32(local), I32Const(imm), cmp] if (let Some(op) = cmp_op(cmp)) => - JumpCmpLocalConst32 { target_ip: ip, local, imm, op } + [LocalGet32(local), Const32(imm), cmp] if (let Some(op) = cmp_op(cmp)) => + match (imm, op) { + (0, CmpOp::Eq) => JumpIfLocalZero32 { target_ip: target, local }, + (0, CmpOp::Ne) => JumpIfLocalNonZero32 { target_ip: target, local }, + (imm, op) => JumpCmpLocalConst32 { target_ip: target, local, imm, op }, + } ); rewrite!(instrs, i, - [LocalGet64(local), I64Const(imm), cmp] if + [LocalGet64(local), Const64(imm), cmp] if (let Some(op) = cmp_op_64(cmp) && let Ok(imm) = i32::try_from(imm)) => - JumpCmpLocalConst64 { target_ip: ip, local, imm, op } + match (imm, op) { + (0, CmpOp::Eq) => JumpIfLocalZero64 { target_ip: target, local }, + (0, CmpOp::Ne) => JumpIfLocalNonZero64 { target_ip: target, local }, + (imm, op) => JumpCmpLocalConst64 { target_ip: target, local, imm, op }, + } ); rewrite!(instrs, i, [LocalGet32(left), LocalGet32(right), cmp] if (let Some(op) = cmp_op(cmp)) => - JumpCmpLocalLocal32 { target_ip: ip, left, right, op } + JumpCmpLocalLocal32 { target_ip: target, left, right, op } ); rewrite!(instrs, i, [LocalGet64(left), LocalGet64(right), cmp] if (let Some(op) = cmp_op_64(cmp)) => - JumpCmpLocalLocal64 { target_ip: ip, left, right, op } + JumpCmpLocalLocal64 { target_ip: target, left, right, op } ); - rewrite!(instrs, i, [I32Const(imm), cmp] if (let Some(op) = cmp_op(cmp)) => match (imm, op) { - (0, CmpOp::Eq) => JumpIfZero32(ip), - (0, CmpOp::Ne) => JumpIfNonZero32(ip), - (imm, op) => JumpCmpStackConst32 { target_ip: ip, imm, op }, + rewrite!(instrs, i, [Const32(imm), cmp] if (let Some(op) = cmp_op(cmp)) => match (imm, op) { + (0, CmpOp::Eq) => JumpIfZero32(target), + (0, CmpOp::Ne) => JumpIfNonZero32(target), + (imm, op) => JumpCmpStackConst32 { target_ip: target, imm, op }, }); - rewrite!(instrs, i, [I64Const(imm), cmp] if (let Some(op) = cmp_op_64(cmp)) => match (imm, op) { - (0, CmpOp::Eq) => JumpIfZero64(ip), - (0, CmpOp::Ne) => JumpIfNonZero64(ip), - (imm, op) => JumpCmpStackConst64 { target_ip: ip, imm, op }, + rewrite!(instrs, i, [Const64(imm), cmp] if (let Some(op) = cmp_op_64(cmp)) => match (imm, op) { + (0, CmpOp::Eq) => JumpIfZero64(target), + (0, CmpOp::Ne) => JumpIfNonZero64(target), + (imm, op) => JumpCmpStackConst64 { target_ip: target, imm, op }, }); + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + if let JumpIfNonZero(current) = &mut instrs[i] { + *current = target; + } + } + JumpIfZero32(ip) => { + let target = resolve_jump_target(instrs, ip); + rewrite!(instrs, i, [LocalGet32(local)] => JumpIfLocalZero32 { target_ip: target, local }); + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + if let JumpIfZero32(current) = &mut instrs[i] { + *current = target; + } + } + JumpIfNonZero32(ip) => { + let target = resolve_jump_target(instrs, ip); + rewrite!(instrs, i, [LocalGet32(local)] => JumpIfLocalNonZero32 { target_ip: target, local }); + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + if let JumpIfNonZero32(current) = &mut instrs[i] { + *current = target; + } + } + JumpIfZero64(ip) => { + let target = resolve_jump_target(instrs, ip); + rewrite!(instrs, i, [LocalGet64(local)] => JumpIfLocalZero64 { target_ip: target, local }); + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + if let JumpIfZero64(current) = &mut instrs[i] { + *current = target; + } + } + JumpIfNonZero64(ip) => { + let target = resolve_jump_target(instrs, ip); + rewrite!(instrs, i, [LocalGet64(local)] => JumpIfLocalNonZero64 { target_ip: target, local }); + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + if let JumpIfNonZero64(current) = &mut instrs[i] { + *current = target; + } + } + JumpCmpStackConst32 { target_ip, imm: 0, op } => { + match op { + CmpOp::Eq => instrs[i] = JumpIfZero32(target_ip), + CmpOp::Ne => instrs[i] = JumpIfNonZero32(target_ip), + _ => {} + } + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + } + JumpCmpStackConst64 { target_ip, imm: 0, op } => { + match op { + CmpOp::Eq => instrs[i] = JumpIfZero64(target_ip), + CmpOp::Ne => instrs[i] = JumpIfNonZero64(target_ip), + _ => {} + } + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + } + JumpCmpLocalConst32 { target_ip, local, imm: 0, op } => { + match op { + CmpOp::Eq => instrs[i] = JumpIfLocalZero32 { target_ip, local }, + CmpOp::Ne => instrs[i] = JumpIfLocalNonZero32 { target_ip, local }, + _ => {} + } + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + } + JumpCmpLocalConst64 { target_ip, local, imm: 0, op } => { + match op { + CmpOp::Eq => instrs[i] = JumpIfLocalZero64 { target_ip, local }, + CmpOp::Ne => instrs[i] = JumpIfLocalNonZero64 { target_ip, local }, + _ => {} + } + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + } + JumpCmpStackConst32 { .. } + | JumpCmpStackConst64 { .. } + | JumpCmpLocalConst32 { .. } + | JumpCmpLocalConst64 { .. } + | JumpCmpLocalLocal32 { .. } + | JumpCmpLocalLocal64 { .. } + | JumpIfLocalZero32 { .. } + | JumpIfLocalNonZero32 { .. } + | JumpIfLocalZero64 { .. } + | JumpIfLocalNonZero64 { .. } => { + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); } - JumpCmpStackConst32 { target_ip, imm: 0, op } => match op { - CmpOp::Eq => instrs[i] = JumpIfZero32(target_ip), - CmpOp::Ne => instrs[i] = JumpIfNonZero32(target_ip), - _ => {} - }, - JumpCmpStackConst64 { target_ip, imm: 0, op } => match op { - CmpOp::Eq => instrs[i] = JumpIfZero64(target_ip), - CmpOp::Ne => instrs[i] = JumpIfNonZero64(target_ip), - _ => {} - }, _ => {} } @@ -437,31 +680,71 @@ fn scalar_bin_op_32(instr: Instruction) -> Option<BinOp> { int_bin_op_32(instr).or_else(|| float_bin_op_32(instr)) } -fn stack_source_local_32(instr: Instruction) -> Option<(Instruction, u16)> { - Some(match instr { - Instruction::LocalGet32(local) => (Instruction::Nop, local), - Instruction::LocalTee32(local) => (Instruction::LocalSet32(local), local), - Instruction::BinOpLocalLocalTee32(op, a, b, local) => { - (Instruction::BinOpLocalLocalSet32(op, a, b, local), local) - } - Instruction::BinOpLocalConstTee32(op, src, c, local) => { - (Instruction::BinOpLocalConstSet32(op, src, c, local), local) - } - Instruction::LoadLocalTee32(memarg, addr, local) => { - (Instruction::LoadLocalSet32(memarg, addr, local), local.into()) - } - _ => return None, - }) +fn scalar_bin_op_64(instr: Instruction) -> Option<BinOp> { + int_bin_op_64(instr).or_else(|| float_bin_op_64(instr)) } fn scalar_const_32(instr: Instruction, op_instr: Instruction) -> Option<i32> { match instr { - Instruction::I32Const(c) if int_bin_op_32(op_instr).is_some() => Some(c), - Instruction::F32Const(c) if float_bin_op_32(op_instr).is_some() => Some(f32_const_bits(c)), + Instruction::Const32(c) if int_bin_op_32(op_instr).is_some() || float_bin_op_32(op_instr).is_some() => Some(c), + _ => None, + } +} + +fn scalar_const_64(instr: Instruction, op_instr: Instruction) -> Option<i64> { + match instr { + Instruction::Const64(c) if int_bin_op_64(op_instr).is_some() || float_bin_op_64(op_instr).is_some() => Some(c), _ => None, } } +fn const_128(instr: Instruction, op_instr: Instruction) -> Option<ConstIdx> { + match instr { + Instruction::Const128(c) if bin_op_128(op_instr).is_some() => Some(c), + _ => None, + } +} + +define_local_source_resolver!( + resolve_local_source_32, + get = LocalGet32, + tee = LocalTee32, + set = LocalSet32, + copy = LocalCopy32, + binop_local_local_tee = BinOpLocalLocalTee32, + binop_local_local_set = BinOpLocalLocalSet32, + binop_local_const_tee = BinOpLocalConstTee32, + binop_local_const_set = BinOpLocalConstSet32, + load_local_tee = LoadLocalTee32, + load_local_set = LoadLocalSet32 +); + +define_local_source_resolver!( + resolve_local_source_64, + get = LocalGet64, + tee = LocalTee64, + set = LocalSet64, + copy = LocalCopy64, + binop_local_local_tee = BinOpLocalLocalTee64, + binop_local_local_set = BinOpLocalLocalSet64, + binop_local_const_tee = BinOpLocalConstTee64, + binop_local_const_set = BinOpLocalConstSet64 +); + +define_local_source_resolver!( + resolve_local_source_128, + get = LocalGet128, + tee = LocalTee128, + set = LocalSet128, + copy = LocalCopy128, + binop_local_local_tee = BinOpLocalLocalTee128, + binop_local_local_set = BinOpLocalLocalSet128, + binop_local_const_tee = BinOpLocalConstTee128, + binop_local_const_set = BinOpLocalConstSet128, + load_local_tee = LoadLocalTee128, + load_local_set = LoadLocalSet128 +); + fn bin_op_128(instr: Instruction) -> Option<BinOp128> { Some(match instr { Instruction::V128And => BinOp128::And, @@ -474,14 +757,6 @@ fn bin_op_128(instr: Instruction) -> Option<BinOp128> { }) } -fn f32_const_bits(value: f32) -> i32 { - i32::from_ne_bytes(value.to_bits().to_ne_bytes()) -} - -fn f64_const_bits(value: f64) -> i64 { - i64::from_ne_bytes(value.to_bits().to_ne_bytes()) -} - fn cmp_op_64(instr: Instruction) -> Option<CmpOp> { Some(match instr { Instruction::I64Eq => CmpOp::Eq, @@ -513,8 +788,8 @@ fn inverse_cmp_op(op: CmpOp) -> CmpOp { } } -fn previous_non_nop_3(instrs: &[Instruction], read: usize) -> Option<[(usize, Instruction); 3]> { - let mut out = [(0usize, Instruction::Nop); 3]; +fn previous_non_nop<const N: usize>(instrs: &[Instruction], read: usize) -> Option<[(usize, Instruction); N]> { + let mut out = [(0usize, Instruction::Nop); N]; let mut filled = 0usize; for idx in (0..read).rev() { @@ -526,9 +801,9 @@ fn previous_non_nop_3(instrs: &[Instruction], read: usize) -> Option<[(usize, In continue; } - out[2 - filled] = (idx, instr); + out[N - 1 - filled] = (idx, instr); filled += 1; - if filled == 3 { + if filled == N { return Some(out); } } @@ -536,6 +811,149 @@ fn previous_non_nop_3(instrs: &[Instruction], read: usize) -> Option<[(usize, In None } +fn next_non_nop(instrs: &[Instruction], mut idx: usize) -> usize { + while idx < instrs.len() && matches!(instrs[idx], Instruction::Nop | Instruction::MergeBarrier) { + idx += 1; + } + idx +} + +fn resolve_jump_target(instrs: &[Instruction], target: u32) -> u32 { + let mut idx = next_non_nop(instrs, target as usize); + let mut steps = 0usize; + + while idx < instrs.len() && steps < instrs.len() { + match instrs[idx] { + Instruction::Jump(next) => { + idx = next_non_nop(instrs, next as usize); + steps += 1; + } + _ => break, + } + } + + idx as u32 +} + +fn jump_target(instr: Instruction) -> Option<u32> { + Some(match instr { + Instruction::Jump(ip) + | Instruction::JumpIfZero(ip) + | Instruction::JumpIfNonZero(ip) + | Instruction::JumpIfZero32(ip) + | Instruction::JumpIfNonZero32(ip) + | Instruction::JumpIfZero64(ip) + | Instruction::JumpIfNonZero64(ip) => ip, + Instruction::JumpCmpStackConst32 { target_ip, .. } + | Instruction::JumpCmpStackConst64 { target_ip, .. } + | Instruction::JumpIfLocalZero32 { target_ip, .. } + | Instruction::JumpIfLocalNonZero32 { target_ip, .. } + | Instruction::JumpIfLocalZero64 { target_ip, .. } + | Instruction::JumpIfLocalNonZero64 { target_ip, .. } + | Instruction::JumpCmpLocalConst32 { target_ip, .. } + | Instruction::JumpCmpLocalConst64 { target_ip, .. } + | Instruction::JumpCmpLocalLocal32 { target_ip, .. } + | Instruction::JumpCmpLocalLocal64 { target_ip, .. } => target_ip, + _ => return None, + }) +} + +fn set_jump_target(instr: &mut Instruction, target: u32) { + match instr { + Instruction::Jump(ip) + | Instruction::JumpIfZero(ip) + | Instruction::JumpIfNonZero(ip) + | Instruction::JumpIfZero32(ip) + | Instruction::JumpIfNonZero32(ip) + | Instruction::JumpIfZero64(ip) + | Instruction::JumpIfNonZero64(ip) + | Instruction::JumpCmpStackConst32 { target_ip: ip, .. } + | Instruction::JumpCmpStackConst64 { target_ip: ip, .. } + | Instruction::JumpIfLocalZero32 { target_ip: ip, .. } + | Instruction::JumpIfLocalNonZero32 { target_ip: ip, .. } + | Instruction::JumpIfLocalZero64 { target_ip: ip, .. } + | Instruction::JumpIfLocalNonZero64 { target_ip: ip, .. } + | Instruction::JumpCmpLocalConst32 { target_ip: ip, .. } + | Instruction::JumpCmpLocalConst64 { target_ip: ip, .. } + | Instruction::JumpCmpLocalLocal32 { target_ip: ip, .. } + | Instruction::JumpCmpLocalLocal64 { target_ip: ip, .. } => *ip = target, + _ => {} + } +} + +fn invert_jump(instr: Instruction, target: u32) -> Option<Instruction> { + Some(match instr { + Instruction::JumpCmpStackConst32 { imm, op, .. } => { + Instruction::JumpCmpStackConst32 { target_ip: target, imm, op: inverse_cmp_op(op) } + } + Instruction::JumpCmpStackConst64 { imm, op, .. } => { + Instruction::JumpCmpStackConst64 { target_ip: target, imm, op: inverse_cmp_op(op) } + } + Instruction::JumpCmpLocalConst32 { local, imm, op, .. } => { + Instruction::JumpCmpLocalConst32 { target_ip: target, local, imm, op: inverse_cmp_op(op) } + } + Instruction::JumpCmpLocalConst64 { local, imm, op, .. } => { + Instruction::JumpCmpLocalConst64 { target_ip: target, local, imm, op: inverse_cmp_op(op) } + } + Instruction::JumpCmpLocalLocal32 { left, right, op, .. } => { + Instruction::JumpCmpLocalLocal32 { target_ip: target, left, right, op: inverse_cmp_op(op) } + } + Instruction::JumpCmpLocalLocal64 { left, right, op, .. } => { + Instruction::JumpCmpLocalLocal64 { target_ip: target, left, right, op: inverse_cmp_op(op) } + } + _ => return None, + }) +} + +fn canonicalize_jump_like(instrs: &mut [Instruction], idx: usize) { + let Some(target) = jump_target(instrs[idx]) else { + return; + }; + + let target = resolve_jump_target(instrs, target); + if matches!(instrs[idx], Instruction::Jump(_)) && target == next_non_nop(instrs, idx + 1) as u32 { + instrs[idx] = Instruction::Nop; + } else { + set_jump_target(&mut instrs[idx], target); + } +} + +fn invert_conditional_over_jump(instrs: &mut [Instruction], idx: usize) { + let Some(target) = jump_target(instrs[idx]) else { + return; + }; + if matches!(instrs[idx], Instruction::Jump(_)) { + return; + } + + let target_idx = next_non_nop(instrs, target as usize); + if target_idx >= instrs.len() || target_idx <= idx + 1 { + return; + } + + let Some(jump_idx) = ((idx + 1)..target_idx) + .rev() + .find(|&candidate| !matches!(instrs[candidate], Instruction::Nop | Instruction::MergeBarrier)) + else { + return; + }; + + let Instruction::Jump(exit_target) = instrs[jump_idx] else { + return; + }; + if next_non_nop(instrs, jump_idx + 1) != target_idx { + return; + } + + let exit_target = resolve_jump_target(instrs, exit_target); + let Some(inverted) = invert_jump(instrs[idx], exit_target) else { + return; + }; + + instrs[idx] = inverted; + instrs[jump_idx] = Instruction::Nop; +} + fn remove_nop(instructions: &mut Vec<Instruction>, function_data: &mut WasmFunctionData) { let old_len = instructions.len(); if old_len == 0 { @@ -574,6 +992,10 @@ fn remove_nop(instructions: &mut Vec<Instruction>, function_data: &mut WasmFunct | Instruction::JumpIfNonZero32(ip) | Instruction::JumpIfZero64(ip) | Instruction::JumpIfNonZero64(ip) + | Instruction::JumpIfLocalZero32 { target_ip: ip, .. } + | Instruction::JumpIfLocalNonZero32 { target_ip: ip, .. } + | Instruction::JumpIfLocalZero64 { target_ip: ip, .. } + | Instruction::JumpIfLocalNonZero64 { target_ip: ip, .. } | Instruction::JumpCmpStackConst32 { target_ip: ip, .. } | Instruction::JumpCmpStackConst64 { target_ip: ip, .. } | Instruction::JumpCmpLocalConst32 { target_ip: ip, .. } diff --git a/crates/parser/src/visit.rs b/crates/parser/src/visit.rs index 5e1ca15..9a094ed 100644 --- a/crates/parser/src/visit.rs +++ b/crates/parser/src/visit.rs @@ -108,7 +108,7 @@ impl<'a, R: WasmModuleResources> wasmparser::VisitOperator<'a> for FunctionBuild define_operands! { // basic instructions - visit_global_get(GlobalGet, u32), visit_i32_const(I32Const, i32), visit_i64_const(I64Const, i64), visit_return(Return), + visit_global_get(GlobalGet, u32), visit_i32_const(Const32, i32), visit_i64_const(Const64, i64), visit_return(Return), visit_call(Call, u32), visit_call_indirect(CallIndirect, u32, u32), visit_return_call_indirect(ReturnCallIndirect, u32, u32), visit_return_call(ReturnCall, u32), visit_memory_size(MemorySize, u32), visit_memory_grow(MemoryGrow, u32), visit_unreachable(Unreachable), visit_nop(Nop), visit_i32_eqz(I32Eqz), visit_i32_eq(I32Eq), visit_i32_ne(I32Ne), visit_i32_lt_s(I32LtS), visit_i32_lt_u(I32LtU), @@ -360,11 +360,11 @@ impl<'a, R: WasmModuleResources> wasmparser::VisitOperator<'a> for FunctionBuild } fn visit_f32_const(&mut self, val: wasmparser::Ieee32) -> Self::Output { - self.instructions.push(Instruction::F32Const(f32::from_bits(val.bits()))); + self.instructions.push(Instruction::Const32(i32::from_ne_bytes(val.bits().to_ne_bytes()))); } fn visit_f64_const(&mut self, val: wasmparser::Ieee64) -> Self::Output { - self.instructions.push(Instruction::F64Const(f64::from_bits(val.bits()))); + self.instructions.push(Instruction::Const64(i64::from_ne_bytes(val.bits().to_ne_bytes()))); } fn visit_table_copy(&mut self, dst_table: u32, src_table: u32) -> Self::Output { @@ -485,7 +485,7 @@ impl<R: WasmModuleResources> wasmparser::VisitSimdOperator<'_> for FunctionBuild } fn visit_v128_const(&mut self, value: wasmparser::V128) -> Self::Output { - self.instructions.push(Instruction::V128Const(self.data.v128_constants.len() as u32)); + self.instructions.push(Instruction::Const128(self.data.v128_constants.len() as u32)); self.data.v128_constants.push(*value.bytes()); } } diff --git a/crates/tinywasm/src/engine.rs b/crates/tinywasm/src/engine.rs index 3b7b8b8..652a07c 100644 --- a/crates/tinywasm/src/engine.rs +++ b/crates/tinywasm/src/engine.rs @@ -1,5 +1,3 @@ -use alloc::sync::Arc; - /// Memory backend types and traits. pub use crate::store::{LazyLinearMemory, LinearMemory, MemoryBackend, PagedMemory, VecMemory}; @@ -9,27 +7,21 @@ pub use crate::store::{LazyLinearMemory, LinearMemory, MemoryBackend, PagedMemor #[derive(Clone, Default)] #[cfg_attr(feature = "debug", derive(Debug))] pub struct Engine { - pub(crate) inner: Arc<EngineInner>, + pub(crate) config: Config, } impl Engine { /// Create a new engine with the given configuration pub fn new(config: Config) -> Self { - Self { inner: Arc::new(EngineInner { config }) } + Self { config } } /// Get a reference to the engine's configuration pub fn config(&self) -> &Config { - &self.inner.config + &self.config } } -#[derive(Default)] -#[cfg_attr(feature = "debug", derive(Debug))] -pub(crate) struct EngineInner { - pub(crate) config: Config, -} - /// Fuel accounting policy for budgeted execution. #[non_exhaustive] #[derive(Default, Clone, Copy)] diff --git a/crates/tinywasm/src/imports.rs b/crates/tinywasm/src/imports.rs index ba6076e..8198b55 100644 --- a/crates/tinywasm/src/imports.rs +++ b/crates/tinywasm/src/imports.rs @@ -2,6 +2,7 @@ use alloc::collections::BTreeMap; use alloc::string::{String, ToString}; use alloc::vec::Vec; use core::fmt::Debug; +use core::hint::cold_path; use crate::{Function, Global, LinkingError, Memory, Result, Table, log}; use tinywasm_types::*; @@ -145,6 +146,7 @@ impl Imports { #[cfg(not(feature = "debug"))] fn compare_types<T: PartialEq>(import: &Import, actual: &T, expected: &T) -> Result<()> { if expected != actual { + cold_path(); log::error!("failed to link import {}", import.name); return Err(LinkingError::incompatible_import_type(import).into()); } @@ -154,6 +156,7 @@ impl Imports { #[cfg(feature = "debug")] fn compare_types<T: PartialEq + Debug>(import: &Import, actual: &T, expected: &T) -> Result<()> { if expected != actual { + cold_path(); log::error!("failed to link import {}: expected {:?}, got {:?}", import.name, expected, actual); return Err(LinkingError::incompatible_import_type(import).into()); } @@ -163,12 +166,17 @@ impl Imports { fn compare_table_types(import: &Import, expected: &TableType, actual: &TableType) -> Result<()> { Self::compare_types(import, &actual.element_type, &expected.element_type)?; if actual.size_initial > expected.size_initial { + cold_path(); return Err(LinkingError::incompatible_import_type(import).into()); } match (expected.size_max, actual.size_max) { - (None, Some(_)) => Err(LinkingError::incompatible_import_type(import).into()), + (None, Some(_)) => { + cold_path(); + Err(LinkingError::incompatible_import_type(import).into()) + } (Some(expected_max), Some(actual_max)) if actual_max < expected_max => { + cold_path(); Err(LinkingError::incompatible_import_type(import).into()) } _ => Ok(()), @@ -209,10 +217,10 @@ impl Imports { }); let mut imports = ResolvedImports { - globals: Vec::with_capacity(global_count), - tables: Vec::with_capacity(table_count), - memories: Vec::with_capacity(mem_count), - funcs: Vec::with_capacity(func_count), + globals: Vec::with_capacity(global_count + module.globals.len()), + tables: Vec::with_capacity(table_count + module.table_types.len()), + memories: Vec::with_capacity(mem_count + module.memory_types.len()), + funcs: Vec::with_capacity(func_count + module.funcs.len()), }; for import in &*module.imports { @@ -220,6 +228,7 @@ impl Imports { match defined { Extern::Global(global) => { let ImportKind::Global(import_ty) = &import.kind else { + cold_path(); return Err(LinkingError::incompatible_import_type(import).into()); }; let global_instance = store.state.get_global(global.0.addr); @@ -228,6 +237,7 @@ impl Imports { } Extern::Table(table) => { let ImportKind::Table(import_ty) = &import.kind else { + cold_path(); return Err(LinkingError::incompatible_import_type(import).into()); }; let table_instance = store.state.get_table(table.0.addr); @@ -238,6 +248,7 @@ impl Imports { } Extern::Memory(memory) => { let ImportKind::Memory(import_ty) = &import.kind else { + cold_path(); return Err(LinkingError::incompatible_import_type(import).into()); }; let mem = store.state.get_mem(memory.0.addr); @@ -246,6 +257,7 @@ impl Imports { } Extern::Function(func_handle) => { let ImportKind::Function(ty) = &import.kind else { + cold_path(); return Err(LinkingError::incompatible_import_type(import).into()); }; let import_func_type = module @@ -262,6 +274,7 @@ impl Imports { let name = ExternName::from(import); let Some(instance) = self.modules.get(&name.module) else { + cold_path(); return Err(LinkingError::unknown_import(import).into()); }; instance.validate_store(store)?; @@ -271,6 +284,7 @@ impl Imports { { // check if the kind matches if val.kind() != (&import.kind).into() { + cold_path(); return Err(LinkingError::incompatible_import_type(import).into()); } diff --git a/crates/tinywasm/src/instance.rs b/crates/tinywasm/src/instance.rs index f5125ba..2ec7916 100644 --- a/crates/tinywasm/src/instance.rs +++ b/crates/tinywasm/src/instance.rs @@ -200,7 +200,10 @@ impl ModuleInstance { store.add_instance(instance.clone()); match (elem_trapped, data_trapped) { - (Some(trap), _) | (_, Some(trap)) => Err(trap.into()), + (Some(trap), _) | (_, Some(trap)) => { + cold_path(); + Err(trap.into()) + } _ => Ok(instance), } } @@ -247,13 +250,25 @@ impl ModuleInstance { #[inline] fn require_export(&self, name: &str) -> Result<ExternVal> { - self.export_addr(name).ok_or_else(|| Error::Other(format!("Export not found: {name}"))) + match self.export_addr(name) { + Some(addr) => Ok(addr), + None => { + cold_path(); + Err(Error::Other(format!("Export not found: {name}"))) + } + } } #[inline] #[cfg(feature = "guest_debug")] fn index_addr<T: Copy>(slice: &[T], idx: u32, kind: &str) -> Result<T> { - slice.get(idx as usize).copied().ok_or_else(|| Error::Other(format!("{kind} index out of bounds: {idx}"))) + match slice.get(idx as usize) { + Some(addr) => Ok(*addr), + None => { + cold_path(); + Err(Error::Other(format!("{kind} index out of bounds: {idx}"))) + } + } } /// Get any exported extern value by name. @@ -368,7 +383,10 @@ impl ModuleInstance { pub fn memory(&self, name: &str) -> Result<Memory> { match self.require_export(name)? { ExternVal::Memory(mem_addr) => Ok(Memory::from_store_addr(self.0.store_id, mem_addr)), - _ => Err(Error::Other(format!("Export is not a memory: {name}"))), + _ => { + cold_path(); + Err(Error::Other(format!("Export is not a memory: {name}"))) + } } } diff --git a/crates/tinywasm/src/interpreter/executor.rs b/crates/tinywasm/src/interpreter/executor.rs index a800433..856f1f0 100644 --- a/crates/tinywasm/src/interpreter/executor.rs +++ b/crates/tinywasm/src/interpreter/executor.rs @@ -120,65 +120,65 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { fn exec(&mut self) -> Result<Option<()>, Trap> { macro_rules! stack_op { (unary $ty:ty, |$v:ident| $expr:expr) => {{ - let $v = self.store.value_stack.pop::<$ty>(); - self.store.value_stack.push::<$ty>($expr)?; + let $v = <$ty>::stack_pop(&mut self.store.value_stack); + <$ty>::stack_push(&mut self.store.value_stack, $expr)?; }}; (binary $ty:ty, |$lhs:ident, $rhs:ident| $expr:expr) => {{ - let $rhs = self.store.value_stack.pop::<$ty>(); - let $lhs = self.store.value_stack.pop::<$ty>(); - self.store.value_stack.push::<$ty>($expr)?; + let $rhs = <$ty>::stack_pop(&mut self.store.value_stack); + let $lhs = <$ty>::stack_pop(&mut self.store.value_stack); + <$ty>::stack_push(&mut self.store.value_stack, $expr)?; }}; (binary try $ty:ty, |$lhs:ident, $rhs:ident| $expr:expr) => {{ - let $rhs = self.store.value_stack.pop::<$ty>(); - let $lhs = self.store.value_stack.pop::<$ty>(); - self.store.value_stack.push::<$ty>($expr?)?; + let $rhs = <$ty>::stack_pop(&mut self.store.value_stack); + let $lhs = <$ty>::stack_pop(&mut self.store.value_stack); + <$ty>::stack_push(&mut self.store.value_stack, $expr?)?; }}; (unary $from:ty => $to:ty, |$v:ident| $expr:expr) => {{ - let $v = self.store.value_stack.pop::<$from>(); - self.store.value_stack.push::<$to>($expr)?; + let $v = <$from>::stack_pop(&mut self.store.value_stack); + <$to>::stack_push(&mut self.store.value_stack, $expr)?; }}; (binary $from:ty => $to:ty, |$lhs:ident, $rhs:ident| $expr:expr) => {{ - let $rhs = self.store.value_stack.pop::<$from>(); - let $lhs = self.store.value_stack.pop::<$from>(); - self.store.value_stack.push::<$to>($expr)?; + let $rhs = <$from>::stack_pop(&mut self.store.value_stack); + let $lhs = <$from>::stack_pop(&mut self.store.value_stack); + <$to>::stack_push(&mut self.store.value_stack, $expr)?; }}; (binary_into2 $from:ty => $to:ty, |$lhs:ident, $rhs:ident| $expr:expr) => {{ - let $rhs = self.store.value_stack.pop::<$from>(); - let $lhs = self.store.value_stack.pop::<$from>(); + let $rhs = <$from>::stack_pop(&mut self.store.value_stack); + let $lhs = <$from>::stack_pop(&mut self.store.value_stack); let out = $expr; - self.store.value_stack.push::<$to>(out.0)?; - self.store.value_stack.push::<$to>(out.1)?; + <$to>::stack_push(&mut self.store.value_stack, out.0)?; + <$to>::stack_push(&mut self.store.value_stack, out.1)?; }}; (binary $lhs_ty:ty, $rhs_ty:ty, |$lhs:ident, $rhs:ident| $expr:expr) => { stack_op!(binary $lhs_ty, $rhs_ty => $rhs_ty, |$lhs, $rhs| $expr) }; (binary $lhs_ty:ty, $rhs_ty:ty => $res:ty, |$lhs:ident, $rhs:ident| $expr:expr) => {{ - let $rhs = self.store.value_stack.pop::<$rhs_ty>(); - let $lhs = self.store.value_stack.pop::<$lhs_ty>(); - self.store.value_stack.push::<$res>($expr)?; + let $rhs = <$rhs_ty>::stack_pop(&mut self.store.value_stack); + let $lhs = <$lhs_ty>::stack_pop(&mut self.store.value_stack); + <$res>::stack_push(&mut self.store.value_stack, $expr)?; }}; (ternary $ty:ty, |$a:ident, $b:ident, $c:ident| $expr:expr) => {{ - let $c = self.store.value_stack.pop::<$ty>(); - let $b = self.store.value_stack.pop::<$ty>(); - let $a = self.store.value_stack.pop::<$ty>(); - self.store.value_stack.push::<$ty>($expr)?; + let $c = <$ty>::stack_pop(&mut self.store.value_stack); + let $b = <$ty>::stack_pop(&mut self.store.value_stack); + let $a = <$ty>::stack_pop(&mut self.store.value_stack); + <$ty>::stack_push(&mut self.store.value_stack, $expr)?; }}; (quaternary_into2 $from:ty => $to:ty, |$a:ident, $b:ident, $c:ident, $d:ident| $expr:expr) => {{ - let $d = self.store.value_stack.pop::<$from>(); - let $c = self.store.value_stack.pop::<$from>(); - let $b = self.store.value_stack.pop::<$from>(); - let $a = self.store.value_stack.pop::<$from>(); + let $d = <$from>::stack_pop(&mut self.store.value_stack); + let $c = <$from>::stack_pop(&mut self.store.value_stack); + let $b = <$from>::stack_pop(&mut self.store.value_stack); + let $a = <$from>::stack_pop(&mut self.store.value_stack); let out = $expr; - self.store.value_stack.push::<$to>(out.0)?; - self.store.value_stack.push::<$to>(out.1)?; + <$to>::stack_push(&mut self.store.value_stack, out.0)?; + <$to>::stack_push(&mut self.store.value_stack, out.1)?; }}; (local_set_pop $ty:ty, $local_index:expr) => {{ - let val = self.store.value_stack.pop::<$ty>(); - self.store.value_stack.local_set(&self.cf, *$local_index, val); + let val = <$ty>::stack_pop(&mut self.store.value_stack); + <$ty>::local_set(&mut self.store.value_stack, &self.cf, *$local_index, val); }}; (local_tee $ty:ty, $local_index:expr) => {{ - let val = self.store.value_stack.peek::<$ty>(); - self.store.value_stack.local_set(&self.cf, *$local_index, val); + let val = <$ty>::stack_peek(&self.store.value_stack); + <$ty>::local_set(&mut self.store.value_stack, &self.cf, *$local_index, val); }}; } @@ -199,12 +199,12 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { match next { Nop | MergeBarrier => {} Unreachable => return Err(Trap::Unreachable), - Drop32 => self.store.value_stack.drop::<Value32>(), - Drop64 => self.store.value_stack.drop::<Value64>(), - Drop128 => self.store.value_stack.drop::<Value128>(), - Select32 => self.store.value_stack.select::<Value32>()?, - Select64 => self.store.value_stack.select::<Value64>()?, - Select128 => self.store.value_stack.select::<Value128>()?, + Drop32 => { _ = Value32::stack_pop(&mut self.store.value_stack)}, + Drop64 => { _ = Value64::stack_pop(&mut self.store.value_stack)}, + Drop128 => { _ = Value128::stack_pop(&mut self.store.value_stack)}, + Select32 => Value32::stack_select(&mut self.store.value_stack)?, + Select64 => Value64::stack_select(&mut self.store.value_stack)?, + Select128 => Value128::stack_select(&mut self.store.value_stack)?, SelectMulti(counts) => self.store.value_stack.select_multi(*counts), Call(v) => { self.exec_call_direct(*v)?; return Ok(None); } CallSelf => { self.exec_call_self()?; return Ok(None); } @@ -213,12 +213,16 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { ReturnCallSelf => { self.exec_return_call_self()?; return Ok(None); } ReturnCallIndirect(ty, table) => { self.exec_call_indirect::<true>(*ty, *table)?; return Ok(None); } Jump(ip) => { self.exec_jump(*ip); return Ok(None); } - JumpIfZero(ip) => { let condition = self.store.value_stack.pop::<i32>() == 0; if self.jump_if(condition, *ip) { return Ok(None) }} - JumpIfNonZero(ip) => { let condition = self.store.value_stack.pop::<i32>() != 0; if self.jump_if(condition, *ip) { return Ok(None) }} - JumpIfZero32(ip) => { let condition = self.store.value_stack.pop::<Value32>() == 0; if self.jump_if(condition, *ip) { return Ok(None) }} - JumpIfNonZero32(ip) => { let condition = self.store.value_stack.pop::<Value32>() != 0; if self.jump_if(condition, *ip) { return Ok(None) }} - JumpIfZero64(ip) => { let condition = self.store.value_stack.pop::<Value64>() == 0; if self.jump_if(condition, *ip) { return Ok(None) }} - JumpIfNonZero64(ip) => { let condition = self.store.value_stack.pop::<Value64>() != 0; if self.jump_if(condition, *ip) { return Ok(None) }} + JumpIfZero(ip) => { let condition = <i32>::stack_pop(&mut self.store.value_stack) == 0; if self.jump_if(condition, *ip) { return Ok(None) }} + JumpIfNonZero(ip) => { let condition = <i32>::stack_pop(&mut self.store.value_stack) != 0; if self.jump_if(condition, *ip) { return Ok(None) }} + JumpIfZero32(ip) => { let condition = <Value32>::stack_pop(&mut self.store.value_stack) == 0; if self.jump_if(condition, *ip) { return Ok(None) }} + JumpIfNonZero32(ip) => { let condition = <Value32>::stack_pop(&mut self.store.value_stack) != 0; if self.jump_if(condition, *ip) { return Ok(None) }} + JumpIfZero64(ip) => { let condition = <Value64>::stack_pop(&mut self.store.value_stack) == 0; if self.jump_if(condition, *ip) { return Ok(None) }} + JumpIfNonZero64(ip) => { let condition = <Value64>::stack_pop(&mut self.store.value_stack) != 0; if self.jump_if(condition, *ip) { return Ok(None) }} + JumpIfLocalZero32 { target_ip, local } => if self.exec_jump_local_zero_32(*target_ip, *local) { return Ok(None) }, + JumpIfLocalNonZero32 { target_ip, local } => if self.exec_jump_local_non_zero_32(*target_ip, *local) { return Ok(None) }, + JumpIfLocalZero64 { target_ip, local } => if self.exec_jump_local_zero_64(*target_ip, *local) { return Ok(None) }, + JumpIfLocalNonZero64 { target_ip, local } => if self.exec_jump_local_non_zero_64(*target_ip, *local) { return Ok(None) }, JumpCmpStackConst32 { target_ip, imm, op } => if self.exec_jump_cmp_stack_const_32(*target_ip, *imm, *op) { return Ok(None) }, JumpCmpStackConst64 { target_ip, imm, op } => if self.exec_jump_cmp_stack_const_64(*target_ip, *imm, *op) { return Ok(None) }, JumpCmpLocalConst32 { target_ip, local, imm, op } => if self.exec_jump_cmp_local_const_32(*target_ip, *local, *imm, *op) { return Ok(None) }, @@ -234,69 +238,91 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { DropKeep128(base, keep) => self.store.value_stack.stack_128.truncate_keep((self.cf.stack_base().s128 + *base as u32) as usize, *keep as usize), BranchTable(default_ip, start, len) => { self.exec_branch_table(*default_ip, *start, *len); return Ok(None); } Return => { if self.exec_return() { return Ok(Some(())); } return Ok(None); } - LocalGet32(local_index) => self.store.value_stack.push(self.store.value_stack.local_get::<Value32>(&self.cf, *local_index))?, - LocalGet64(local_index) => self.store.value_stack.push(self.store.value_stack.local_get::<Value64>(&self.cf, *local_index))?, - LocalGet128(local_index) => self.store.value_stack.push(self.store.value_stack.local_get::<Value128>(&self.cf, *local_index))?, + LocalGet32(local_index) => self.store.value_stack.push(Value32::local_get(&self.store.value_stack, &self.cf, *local_index))?, + LocalGet64(local_index) => self.store.value_stack.push(Value64::local_get(&self.store.value_stack, &self.cf, *local_index))?, + LocalGet128(local_index) => self.store.value_stack.push(Value128::local_get(&self.store.value_stack, &self.cf, *local_index))?, LocalSet32(local_index) => stack_op!(local_set_pop Value32, local_index), LocalSet64(local_index) => stack_op!(local_set_pop Value64, local_index), LocalSet128(local_index) => stack_op!(local_set_pop Value128, local_index), - LocalCopy32(from, to) => self.store.value_stack.local_set(&self.cf, *to, self.store.value_stack.local_get::<Value32>(&self.cf, *from)), - LocalCopy64(from, to) => self.store.value_stack.local_set(&self.cf, *to, self.store.value_stack.local_get::<Value64>(&self.cf, *from)), - LocalCopy128(from, to) => self.store.value_stack.local_set(&self.cf, *to, self.store.value_stack.local_get::<Value128>(&self.cf, *from)), + LocalCopy32(from, to) => Value32::local_copy(&mut self.store.value_stack, &self.cf, *from, *to), + LocalCopy64(from, to) => Value64::local_copy(&mut self.store.value_stack, &self.cf, *from, *to), + LocalCopy128(from, to) => Value128::local_copy(&mut self.store.value_stack, &self.cf, *from, *to), AddConst32(c) => stack_op!(unary i32, |v| v.wrapping_add(*c)), AddConst64(c) => stack_op!(unary i64, |v| v.wrapping_add(*c)), IncLocal32(local_index, delta) => { - let value = self.store.value_stack.local_get::<Value32>(&self.cf, *local_index).wrapping_add(*delta as u32); - self.store.value_stack.local_set::<Value32>(&self.cf, *local_index, value); + let value = Value32::local_get(&self.store.value_stack, &self.cf, *local_index).wrapping_add(*delta as u32); + Value32::local_set(&mut self.store.value_stack, &self.cf, *local_index, value); } IncLocal64(local_index, delta) => { - let value = self.store.value_stack.local_get::<Value64>(&self.cf, *local_index).wrapping_add(*delta as u64); - self.store.value_stack.local_set::<Value64>(&self.cf, *local_index, value); + let value = Value64::local_get(&self.store.value_stack, &self.cf, *local_index).wrapping_add(*delta as u64); + Value64::local_set(&mut self.store.value_stack, &self.cf, *local_index, value); + } + BinOpLocalLocal32(op, a, b) => self.store.value_stack.push(self.exec_binop_32(*op, Value32::local_get(&self.store.value_stack, &self.cf, *a), Value32::local_get(&self.store.value_stack, &self.cf, *b)))?, + BinOpLocalLocal64(op, a, b) => self.store.value_stack.push(self.exec_binop_64(*op, Value64::local_get(&self.store.value_stack, &self.cf, *a), Value64::local_get(&self.store.value_stack, &self.cf, *b)))?, + BinOpLocalLocal128(op, a, b) => self.store.value_stack.push(self.exec_binop_128(*op, Value128::local_get(&self.store.value_stack, &self.cf, *a), Value128::local_get(&self.store.value_stack, &self.cf, *b)))?, + BinOpLocalLocalSet32(op, a, b, dst) => { + let value = self.exec_binop_32(*op, Value32::local_get(&self.store.value_stack, &self.cf, *a), Value32::local_get(&self.store.value_stack, &self.cf, *b)); + Value32::local_set(&mut self.store.value_stack, &self.cf, *dst, value); + } + BinOpLocalLocalSet64(op, a, b, dst) => { + let value = self.exec_binop_64(*op, Value64::local_get(&self.store.value_stack, &self.cf, *a), Value64::local_get(&self.store.value_stack, &self.cf, *b)); + Value64::local_set(&mut self.store.value_stack, &self.cf, *dst, value); + } + BinOpLocalLocalSet128(op, a, b, dst) => { + let value = self.exec_binop_128(*op, Value128::local_get(&self.store.value_stack, &self.cf, *a), Value128::local_get(&self.store.value_stack, &self.cf, *b)); + Value128::local_set(&mut self.store.value_stack, &self.cf, *dst, value); } - BinOpLocalLocal32(op, a, b) => self.store.value_stack.push(self.exec_binop_32(*op, self.store.value_stack.local_get::<Value32>(&self.cf, *a), self.store.value_stack.local_get::<Value32>(&self.cf, *b)))?, - BinOpLocalLocal64(op, a, b) => self.store.value_stack.push(self.exec_binop_64(*op, self.store.value_stack.local_get::<Value64>(&self.cf, *a), self.store.value_stack.local_get::<Value64>(&self.cf, *b)))?, - BinOpLocalLocal128(op, a, b) => self.store.value_stack.push(self.exec_binop_128(*op, self.store.value_stack.local_get::<Value128>(&self.cf, *a), self.store.value_stack.local_get::<Value128>(&self.cf, *b)))?, - BinOpLocalLocalSet32(op, a, b, dst) => self.store.value_stack.local_set::<Value32>(&self.cf, *dst, self.exec_binop_32(*op, self.store.value_stack.local_get::<Value32>(&self.cf, *a), self.store.value_stack.local_get::<Value32>(&self.cf, *b))), - BinOpLocalLocalSet64(op, a, b, dst) => self.store.value_stack.local_set::<Value64>(&self.cf, *dst, self.exec_binop_64(*op, self.store.value_stack.local_get::<Value64>(&self.cf, *a), self.store.value_stack.local_get::<Value64>(&self.cf, *b))), - BinOpLocalLocalSet128(op, a, b, dst) => self.store.value_stack.local_set::<Value128>(&self.cf, *dst, self.exec_binop_128(*op, self.store.value_stack.local_get::<Value128>(&self.cf, *a), self.store.value_stack.local_get::<Value128>(&self.cf, *b))), BinOpLocalLocalTee32(op, a, b, dst) => { - let value = self.exec_binop_32(*op, self.store.value_stack.local_get::<Value32>(&self.cf, *a), self.store.value_stack.local_get::<Value32>(&self.cf, *b)); - self.store.value_stack.local_set::<Value32>(&self.cf, *dst, value); + let value = self.exec_binop_32(*op, Value32::local_get(&self.store.value_stack, &self.cf, *a), Value32::local_get(&self.store.value_stack, &self.cf, *b)); + Value32::local_set(&mut self.store.value_stack, &self.cf, *dst, value); self.store.value_stack.push(value)?; } BinOpLocalLocalTee64(op, a, b, dst) => { - let value = self.exec_binop_64(*op, self.store.value_stack.local_get::<Value64>(&self.cf, *a), self.store.value_stack.local_get::<Value64>(&self.cf, *b)); - self.store.value_stack.local_set::<Value64>(&self.cf, *dst, value); + let value = self.exec_binop_64(*op, Value64::local_get(&self.store.value_stack, &self.cf, *a), Value64::local_get(&self.store.value_stack, &self.cf, *b)); + Value64::local_set(&mut self.store.value_stack, &self.cf, *dst, value); self.store.value_stack.push(value)?; } BinOpLocalLocalTee128(op, a, b, dst) => { - let value = self.exec_binop_128(*op, self.store.value_stack.local_get::<Value128>(&self.cf, *a), self.store.value_stack.local_get::<Value128>(&self.cf, *b)); - self.store.value_stack.local_set::<Value128>(&self.cf, *dst, value); + let value = self.exec_binop_128(*op, Value128::local_get(&self.store.value_stack, &self.cf, *a), Value128::local_get(&self.store.value_stack, &self.cf, *b)); + Value128::local_set(&mut self.store.value_stack, &self.cf, *dst, value); self.store.value_stack.push(value)?; } - BinOpLocalConst32(op, local_index, c) => self.store.value_stack.push(self.exec_binop_32(*op, self.store.value_stack.local_get::<Value32>(&self.cf, *local_index), *c as u32))?, - BinOpLocalConst64(op, local_index, c) => self.store.value_stack.push(self.exec_binop_64(*op, self.store.value_stack.local_get::<Value64>(&self.cf, *local_index), *c as u64))?, - BinOpLocalConst128(op, local_index, c) => self.store.value_stack.push(self.exec_binop_128(*op, self.store.value_stack.local_get::<Value128>(&self.cf, *local_index), self.get_v128_const(*c)))?, - BinOpLocalConstSet32(op, local_index, c, dst) => self.store.value_stack.local_set::<Value32>(&self.cf, *dst, self.exec_binop_32(*op, self.store.value_stack.local_get::<Value32>(&self.cf, *local_index), *c as u32)), - BinOpLocalConstSet64(op, local_index, c, dst) => self.store.value_stack.local_set::<Value64>(&self.cf, *dst, self.exec_binop_64(*op, self.store.value_stack.local_get::<Value64>(&self.cf, *local_index), *c as u64)), - BinOpLocalConstSet128(op, local_index, c, dst) => self.store.value_stack.local_set::<Value128>(&self.cf, *dst, self.exec_binop_128(*op, self.store.value_stack.local_get::<Value128>(&self.cf, *local_index), self.get_v128_const(*c))), + BinOpLocalConst32(op, local_index, c) => self.store.value_stack.push(self.exec_binop_32(*op, Value32::local_get(&self.store.value_stack, &self.cf, *local_index), *c as u32))?, + BinOpLocalConst64(op, local_index, c) => self.store.value_stack.push(self.exec_binop_64(*op, Value64::local_get(&self.store.value_stack, &self.cf, *local_index), *c as u64))?, + BinOpLocalConst128(op, local_index, c) => self.store.value_stack.push(self.exec_binop_128(*op, Value128::local_get(&self.store.value_stack, &self.cf, *local_index), self.get_v128_const(*c)))?, + BinOpLocalConstSet32(op, local_index, c, dst) => { + let value = self.exec_binop_32(*op, Value32::local_get(&self.store.value_stack, &self.cf, *local_index), *c as u32); + Value32::local_set(&mut self.store.value_stack, &self.cf, *dst, value); + }, + BinOpLocalConstSet64(op, local_index, c, dst) => { + let value = self.exec_binop_64(*op, Value64::local_get(&self.store.value_stack, &self.cf, *local_index), *c as u64); + Value64::local_set(&mut self.store.value_stack, &self.cf, *dst, value); + }, + BinOpLocalConstSet128(op, local_index, c, dst) => { + let value = self.exec_binop_128(*op, Value128::local_get(&self.store.value_stack, &self.cf, *local_index), self.get_v128_const(*c)); + Value128::local_set(&mut self.store.value_stack, &self.cf, *dst, value); + }, BinOpLocalConstTee32(op, local_index, c, dst) => { - let value = self.exec_binop_32(*op, self.store.value_stack.local_get::<Value32>(&self.cf, *local_index), *c as u32); - self.store.value_stack.local_set::<Value32>(&self.cf, *dst, value); + let value = self.exec_binop_32(*op, Value32::local_get(&self.store.value_stack, &self.cf, *local_index), *c as u32); + Value32::local_set(&mut self.store.value_stack, &self.cf, *dst, value); self.store.value_stack.push(value)?; } BinOpLocalConstTee64(op, local_index, c, dst) => { - let value = self.exec_binop_64(*op, self.store.value_stack.local_get::<Value64>(&self.cf, *local_index), *c as u64); - self.store.value_stack.local_set::<Value64>(&self.cf, *dst, value); + let value = self.exec_binop_64(*op, Value64::local_get(&self.store.value_stack, &self.cf, *local_index), *c as u64); + Value64::local_set(&mut self.store.value_stack, &self.cf, *dst, value); self.store.value_stack.push(value)?; } BinOpLocalConstTee128(op, local_index, c, dst) => { - let value = self.exec_binop_128(*op, self.store.value_stack.local_get::<Value128>(&self.cf, *local_index), self.get_v128_const(*c)); - self.store.value_stack.local_set::<Value128>(&self.cf, *dst, value); + let value = self.exec_binop_128(*op, Value128::local_get(&self.store.value_stack, &self.cf, *local_index), self.get_v128_const(*c)); + Value128::local_set(&mut self.store.value_stack, &self.cf, *dst, value); self.store.value_stack.push(value)?; } - SetLocalConst32(local_index, c) => self.store.value_stack.local_set::<i32>(&self.cf, *local_index, *c), - SetLocalConst64(local_index, c) => self.store.value_stack.local_set::<i64>(&self.cf, *local_index, *c), + SetLocalConst32(local_index, c) => i32::local_set(&mut self.store.value_stack, &self.cf, *local_index, *c), + SetLocalConst64(local_index, c) => i64::local_set(&mut self.store.value_stack, &self.cf, *local_index, *c), + SetLocalConst128(local_index, c) => { + let value = self.get_v128_const(*c); + Value128::local_set(&mut self.store.value_stack, &self.cf, *local_index, value); + } StoreLocalLocal32(m, addr_local, value_local) => self.exec_store_local_local::<u32, 4>(*m, *addr_local, *value_local)?, StoreLocalLocal64(m, addr_local, value_local) => self.exec_store_local_local::<i64, 8>(*m, *addr_local, *value_local)?, StoreLocalLocal128(m, addr_local, value_local) => self.exec_store_local_local::<Value128, 16>(*m, *addr_local, *value_local)?, @@ -316,10 +342,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { GlobalSet32(global_index) => self.exec_global_set_32(*global_index), GlobalSet64(global_index) => self.exec_global_set::<Value64>(*global_index), GlobalSet128(global_index) => self.exec_global_set::<Value128>(*global_index), - I32Const(val) => self.exec_const(*val)?, - I64Const(val) => self.exec_const(*val)?, - F32Const(val) => self.exec_const(*val)?, - F64Const(val) => self.exec_const(*val)?, + Const32(val) => self.exec_const(*val)?, + Const64(val) => self.exec_const(*val)?, I64Eqz => stack_op!(unary i64 => i32, |v| i32::from(v == 0)), I32Eqz => stack_op!(unary i32, |v| i32::from(v == 0)), I32Eq => stack_op!(binary i32, |a, b| i32::from(a == b)), @@ -556,7 +580,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { V128Store64Lane(arg, lane) => self.exec_mem_store_lane::<i64, 8>(arg.mem_addr(), arg.offset(), *lane)?, V128Load32Zero(arg) => self.exec_mem_load::<i32, 4, Value128>(arg.mem_addr(), arg.offset(), |v| Value128::from_i32x4([v, 0, 0, 0]))?, V128Load64Zero(arg) => self.exec_mem_load::<i64, 8, Value128>(arg.mem_addr(), arg.offset(), |v| Value128::from_i64x2([v, 0]))?, - V128Const(arg) => self.exec_const(self.get_v128_const(*arg))?, + Const128(arg) => self.exec_const(self.get_v128_const(*arg))?, I8x16ExtractLaneS(lane) => stack_op!(unary Value128 => i32, |v| v.extract_lane_i8(*lane) as i32), I8x16ExtractLaneU(lane) => stack_op!(unary Value128 => i32, |v| v.extract_lane_u8(*lane) as i32), I16x8ExtractLaneS(lane) => stack_op!(unary Value128 => i32, |v| v.extract_lane_i16(*lane) as i32), @@ -812,43 +836,63 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { } #[inline(always)] + fn exec_jump_local_zero_32(&mut self, target_ip: u32, local: LocalAddr) -> bool { + self.jump_if(Value32::local_get(&self.store.value_stack, &self.cf, local) == 0, target_ip) + } + + #[inline(always)] + fn exec_jump_local_non_zero_32(&mut self, target_ip: u32, local: LocalAddr) -> bool { + self.jump_if(Value32::local_get(&self.store.value_stack, &self.cf, local) != 0, target_ip) + } + + #[inline(always)] + fn exec_jump_local_zero_64(&mut self, target_ip: u32, local: LocalAddr) -> bool { + self.jump_if(Value64::local_get(&self.store.value_stack, &self.cf, local) == 0, target_ip) + } + + #[inline(always)] + fn exec_jump_local_non_zero_64(&mut self, target_ip: u32, local: LocalAddr) -> bool { + self.jump_if(Value64::local_get(&self.store.value_stack, &self.cf, local) != 0, target_ip) + } + + #[inline(always)] fn exec_jump_cmp_stack_const_32(&mut self, target_ip: u32, imm: i32, op: CmpOp) -> bool { - let condition = cmp_i32(self.store.value_stack.pop::<i32>(), imm, op); + let condition = cmp_i32(<i32>::stack_pop(&mut self.store.value_stack), imm, op); self.jump_if(condition, target_ip) } #[inline(always)] fn exec_jump_cmp_stack_const_64(&mut self, target_ip: u32, imm: i64, op: CmpOp) -> bool { - let condition = cmp_i64(self.store.value_stack.pop::<i64>(), imm, op); + let condition = cmp_i64(<i64>::stack_pop(&mut self.store.value_stack), imm, op); self.jump_if(condition, target_ip) } #[inline(always)] fn exec_jump_cmp_local_const_32(&mut self, target_ip: u32, local: LocalAddr, imm: i32, op: CmpOp) -> bool { - self.jump_if(cmp_i32(self.store.value_stack.local_get::<i32>(&self.cf, local), imm, op), target_ip) + self.jump_if(cmp_i32(i32::local_get(&self.store.value_stack, &self.cf, local), imm, op), target_ip) } #[inline(always)] fn exec_jump_cmp_local_const_64(&mut self, target_ip: u32, local: LocalAddr, imm: i32, op: CmpOp) -> bool { - self.jump_if(cmp_i64(self.store.value_stack.local_get::<i64>(&self.cf, local), i64::from(imm), op), target_ip) + self.jump_if(cmp_i64(i64::local_get(&self.store.value_stack, &self.cf, local), i64::from(imm), op), target_ip) } #[inline(always)] fn exec_jump_cmp_local_local_32(&mut self, target_ip: u32, left: LocalAddr, right: LocalAddr, op: CmpOp) -> bool { - let lhs = self.store.value_stack.local_get::<i32>(&self.cf, left); - let rhs = self.store.value_stack.local_get::<i32>(&self.cf, right); + let lhs = i32::local_get(&self.store.value_stack, &self.cf, left); + let rhs = i32::local_get(&self.store.value_stack, &self.cf, right); self.jump_if(cmp_i32(lhs, rhs, op), target_ip) } #[inline(always)] fn exec_jump_cmp_local_local_64(&mut self, target_ip: u32, left: LocalAddr, right: LocalAddr, op: CmpOp) -> bool { - let lhs = self.store.value_stack.local_get::<i64>(&self.cf, left); - let rhs = self.store.value_stack.local_get::<i64>(&self.cf, right); + let lhs = i64::local_get(&self.store.value_stack, &self.cf, left); + let rhs = i64::local_get(&self.store.value_stack, &self.cf, right); self.jump_if(cmp_i64(lhs, rhs, op), target_ip) } fn exec_branch_table(&mut self, default_ip: u32, start: u32, len: u32) { - let idx = self.store.value_stack.pop::<i32>(); + let idx = <i32>::stack_pop(&mut self.store.value_stack); let target_ip = if idx >= 0 && (idx as u32) < len { self.func.data.branch_table_targets.get((start + idx as u32) as usize).copied().unwrap_or(default_ip) } else { @@ -884,7 +928,11 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { } self.store.value_stack.truncate_keep_counts(self.cf.locals_base, wasm_func.func.params); - let locals_base = self.store.value_stack.enter_locals(&wasm_func.func.params, &wasm_func.func.locals)?; + let Ok(locals_base) = self.store.value_stack.enter_locals(&wasm_func.func.params, &wasm_func.func.locals) + else { + cold_path(); + return Err(Trap::CallStackOverflow); + }; self.cf = CallFrame::new(func_addr, locals_base, wasm_func.func.locals); if wasm_func.owner != self.module.idx() { self.module = self.store.get_module_instance_internal(wasm_func.owner); @@ -943,14 +991,10 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); self.store.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.params); - let locals_base = match self.store.value_stack.enter_locals(&self.func.params, &self.func.locals) { - Ok(base) => base, - Err(err) => { - cold_path(); - return Err(err); - } + let Ok(locals_base) = self.store.value_stack.enter_locals(&self.func.params, &self.func.locals) else { + cold_path(); + return Err(Trap::CallStackOverflow); }; - self.cf = CallFrame::new(self.cf.func_addr, locals_base, self.func.locals); Ok(()) } @@ -959,7 +1003,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); // verify that the table is of the right type, this should be validated by the parser already - let table_idx: u32 = self.store.value_stack.pop::<i32>() as u32; + let table_idx: u32 = <i32>::stack_pop(&mut self.store.value_stack) as u32; let table = self.store.state.get_table(self.module.resolve_table_addr(table_addr)); debug_assert!(table.kind.element_type == WasmType::RefFunc, "table is not of type funcref"); @@ -1004,9 +1048,11 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { } fn exec_return(&mut self) -> bool { - let results = ValueCounts::from_iter(self.func.ty.results()); - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, results); - let Some(cf) = self.store.call_stack.pop() else { return true }; + self.store.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.results); + let Some(cf) = self.store.call_stack.pop() else { + cold_path(); + return true; + }; if cf.func_addr != self.cf.func_addr { let wasm_func = self.store.state.get_wasm_func(cf.func_addr); @@ -1026,8 +1072,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { addr_local: u8, value_local: u8, ) -> Result<(), Trap> { - let addr = u64::from(self.store.value_stack.local_get::<u32>(&self.cf, u16::from(addr_local))); - let value = self.store.value_stack.local_get::<T>(&self.cf, u16::from(value_local)).to_mem_bytes(); + let addr = u64::from(u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local))); + let value = T::local_get(&self.store.value_stack, &self.cf, u16::from(value_local)).to_mem_bytes(); let mem = self.store.state.get_mem_mut(self.module.resolve_mem_addr(memarg.mem_addr())); mem.store(addr, memarg.offset(), value)?; Ok(()) @@ -1039,7 +1085,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { addr_local: u8, ) -> Result<T, Trap> { let mem = self.store.state.get_mem(self.module.resolve_mem_addr(memarg.mem_addr())); - let addr = u64::from(self.store.value_stack.local_get::<u32>(&self.cf, u16::from(addr_local))); + let addr = u64::from(u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local))); let bytes = mem.load(addr, memarg.offset())?; Ok(T::from_mem_bytes(bytes)) } @@ -1051,7 +1097,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { dst_local: u8, ) -> Result<(), Trap> { let value = self.exec_load_local_value::<T, N>(memarg, addr_local)?; - self.store.value_stack.local_set(&self.cf, u16::from(dst_local), value); + T::local_set(&mut self.store.value_stack, &self.cf, u16::from(dst_local), value); self.store.value_stack.push(value)?; Ok(()) } @@ -1063,7 +1109,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { dst_local: u8, ) -> Result<(), Trap> { let value = self.exec_load_local_value::<T, N>(memarg, addr_local)?; - self.store.value_stack.local_set(&self.cf, u16::from(dst_local), value); + T::local_set(&mut self.store.value_stack, &self.cf, u16::from(dst_local), value); Ok(()) } @@ -1073,13 +1119,13 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { fn exec_global_set<T: InternalValue>(&mut self, global_index: u32) { let global_addr = self.module.resolve_global_addr(global_index); - let value = self.store.value_stack.pop::<T>().into(); + let value = <T>::stack_pop(&mut self.store.value_stack).into(); self.store.state.set_global_val(global_addr, value); } fn exec_global_set_32(&mut self, global_index: u32) { let global_addr = self.module.resolve_global_addr(global_index); - let raw = self.store.value_stack.pop::<Value32>(); + let raw = <Value32>::stack_pop(&mut self.store.value_stack); let value = match self.store.state.get_global(global_addr).ty.ty { WasmType::I32 | WasmType::F32 => TinyWasmValue::Value32(raw), WasmType::RefExtern | WasmType::RefFunc => TinyWasmValue::ValueRef(ValueRef::from_raw(raw)), @@ -1092,7 +1138,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { self.store.value_stack.push(val) } fn exec_ref_is_null(&mut self) -> Result<(), Trap> { - let is_null = i32::from(self.store.value_stack.pop::<ValueRef>().is_null()); + let is_null = i32::from(<ValueRef>::stack_pop(&mut self.store.value_stack).is_null()); self.store.value_stack.push::<i32>(is_null) } @@ -1107,8 +1153,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { let mem = self.store.state.get_mem_mut(self.module.resolve_mem_addr(addr)); let is_64bit = mem.is_64bit(); let pages_delta = match is_64bit { - true => self.store.value_stack.pop::<i64>(), - false => i64::from(self.store.value_stack.pop::<i32>()), + true => <i64>::stack_pop(&mut self.store.value_stack), + false => i64::from(<i32>::stack_pop(&mut self.store.value_stack)), }; let size = mem.grow(pages_delta, self.store.engine.config().trap_on_oom())?.unwrap_or(-1); @@ -1121,9 +1167,9 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { } fn exec_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Result<(), Trap> { - let size: i32 = self.store.value_stack.pop(); - let src: i32 = self.store.value_stack.pop(); - let dst: i32 = self.store.value_stack.pop(); + let size = i32::stack_pop(&mut self.store.value_stack); + let src = i32::stack_pop(&mut self.store.value_stack); + let dst = i32::stack_pop(&mut self.store.value_stack); let dst_mem_addr = self.module.resolve_mem_addr(dst_mem); if dst_mem == src_mem { @@ -1139,14 +1185,14 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { Ok(()) } fn exec_memory_fill(&mut self, addr: u32) -> Result<(), Trap> { - let size: i32 = self.store.value_stack.pop(); - let val: i32 = self.store.value_stack.pop(); - let dst: i32 = self.store.value_stack.pop(); + let size = i32::stack_pop(&mut self.store.value_stack); + let val = i32::stack_pop(&mut self.store.value_stack); + let dst = i32::stack_pop(&mut self.store.value_stack); self.exec_memory_fill_impl(addr, dst, val as u8, size) } fn exec_memory_fill_imm(&mut self, addr: u32, val: u8, size: i32) -> Result<(), Trap> { - let dst: i32 = self.store.value_stack.pop(); + let dst = i32::stack_pop(&mut self.store.value_stack); self.exec_memory_fill_impl(addr, dst, val, size) } @@ -1164,9 +1210,9 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { } fn exec_memory_init(&mut self, data_index: u32, mem_index: u32) -> Result<(), Trap> { - let size: i32 = self.store.value_stack.pop(); - let offset: i32 = self.store.value_stack.pop(); - let dst: i32 = self.store.value_stack.pop(); + let size = i32::stack_pop(&mut self.store.value_stack); + let offset = i32::stack_pop(&mut self.store.value_stack); + let dst = i32::stack_pop(&mut self.store.value_stack); let data_addr = self.module.resolve_data_addr(data_index) as usize; let Some(data) = self.store.state.data.get(data_addr) else { @@ -1200,9 +1246,9 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { Ok(()) } fn exec_table_copy(&mut self, dst_table: u32, src_table: u32) -> Result<(), Trap> { - let size: i32 = self.store.value_stack.pop(); - let src: i32 = self.store.value_stack.pop(); - let dst: i32 = self.store.value_stack.pop(); + let size = i32::stack_pop(&mut self.store.value_stack); + let src = i32::stack_pop(&mut self.store.value_stack); + let dst = i32::stack_pop(&mut self.store.value_stack); let dst_table_addr = self.module.resolve_table_addr(dst_table); if dst_table == src_table { @@ -1224,8 +1270,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { ) -> Result<(), Trap> { let mem = self.store.state.get_mem(self.module.resolve_mem_addr(mem_addr)); let base = match mem.is_64bit() { - true => self.store.value_stack.pop::<i64>() as u64, - false => self.store.value_stack.pop::<i32>() as u32 as u64, + true => <i64>::stack_pop(&mut self.store.value_stack) as u64, + false => <i32>::stack_pop(&mut self.store.value_stack) as u32 as u64, }; let val = match mem.load::<LOAD_SIZE>(base, offset) { Ok(val) => val, @@ -1235,7 +1281,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { } }; let offset = lane as usize * LOAD_SIZE; - let mut imm = self.store.value_stack.pop::<Value128>().to_mem_bytes(); + let mut imm = <Value128>::stack_pop(&mut self.store.value_stack).to_mem_bytes(); imm[offset..offset + LOAD_SIZE].copy_from_slice(&val); self.store.value_stack.push(Value128(imm))?; Ok(()) @@ -1250,14 +1296,13 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { ) -> Result<(), Trap> { let mem = self.store.state.get_mem(self.module.resolve_mem_addr(mem_addr)); let base = match mem.is_64bit() { - true => self.store.value_stack.pop::<i64>() as u64, - false => self.store.value_stack.pop::<i32>() as u32 as u64, + true => <i64>::stack_pop(&mut self.store.value_stack) as u64, + false => <i32>::stack_pop(&mut self.store.value_stack) as u32 as u64, }; - match mem.load::<LOAD_SIZE>(base, offset) { + match LOAD::load(&*mem.inner, base, offset) { Ok(val) => { - let val = cast(LOAD::from_mem_bytes(val)); - self.store.value_stack.push(val)?; + self.store.value_stack.push(cast(val))?; Ok(()) } Err(e) => { @@ -1273,15 +1318,15 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { offset: u64, lane: u8, ) -> Result<(), Trap> { - let bytes = self.store.value_stack.pop::<Value128>().to_mem_bytes(); + let bytes = <Value128>::stack_pop(&mut self.store.value_stack).to_mem_bytes(); let lane_offset = lane as usize * N; let mut val = [0u8; N]; val.copy_from_slice(&bytes[lane_offset..lane_offset + N]); let mem_addr = self.module.resolve_mem_addr(mem_addr); let mem = self.store.state.get_mem_mut(mem_addr); let addr = match mem.is_64bit() { - true => self.store.value_stack.pop::<i64>() as u64, - false => self.store.value_stack.pop::<i32>() as u32 as u64, + true => <i64>::stack_pop(&mut self.store.value_stack) as u64, + false => <i32>::stack_pop(&mut self.store.value_stack) as u32 as u64, }; match mem.store(addr, offset, val) { Ok(()) => Ok(()), @@ -1298,14 +1343,14 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { offset: u64, cast: impl Fn(T) -> U, ) -> Result<(), Trap> { - let val = self.store.value_stack.pop::<T>(); + let val = <T>::stack_pop(&mut self.store.value_stack); let val = cast(val).to_mem_bytes(); let mem_addr = self.module.resolve_mem_addr(mem_addr); let mem = self.store.state.get_mem_mut(mem_addr); let addr = match mem.is_64bit() { - true => self.store.value_stack.pop::<i64>() as u64, - false => self.store.value_stack.pop::<i32>() as u32 as u64, + true => <i64>::stack_pop(&mut self.store.value_stack) as u64, + false => <i32>::stack_pop(&mut self.store.value_stack) as u32 as u64, }; match mem.store(addr, offset, val) { Ok(()) => Ok(()), @@ -1317,14 +1362,14 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { } fn exec_table_get(&mut self, table_index: u32) -> Result<(), Trap> { - let idx: i32 = self.store.value_stack.pop::<i32>(); + let idx: i32 = <i32>::stack_pop(&mut self.store.value_stack); let table = self.store.state.get_table(self.module.resolve_table_addr(table_index)); let v = table.get_wasm_val(idx as u32)?; self.store.value_stack.push_dyn(v.into()) } fn exec_table_set(&mut self, table_index: u32) -> Result<(), Trap> { - let val = self.store.value_stack.pop::<ValueRef>(); - let idx = self.store.value_stack.pop::<i32>() as u32; + let val = <ValueRef>::stack_pop(&mut self.store.value_stack); + let idx = <i32>::stack_pop(&mut self.store.value_stack) as u32; let table = self.store.state.get_table_mut(self.module.resolve_table_addr(table_index)); table.set(idx, val.addr().into()) } @@ -1333,9 +1378,9 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { self.store.value_stack.push(table.size()) } fn exec_table_init(&mut self, elem_index: u32, table_index: u32) -> Result<(), Trap> { - let size: i32 = self.store.value_stack.pop(); // n - let offset: i32 = self.store.value_stack.pop(); // s - let dst: i32 = self.store.value_stack.pop(); // d + let size = i32::stack_pop(&mut self.store.value_stack); // n + let offset = i32::stack_pop(&mut self.store.value_stack); // s + let dst = i32::stack_pop(&mut self.store.value_stack); // d let elem_addr = self.module.resolve_elem_addr(elem_index) as usize; let elem = self.store.state.elements.get(elem_addr).ok_or_else(|| Trap::Other("element not found"))?; @@ -1369,8 +1414,8 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { fn exec_table_grow(&mut self, table_index: u32) -> Result<(), Trap> { let table = self.store.state.get_table_mut(self.module.resolve_table_addr(table_index)); let sz = table.size(); - let n = self.store.value_stack.pop::<i32>(); - let val = self.store.value_stack.pop::<ValueRef>(); + let n = <i32>::stack_pop(&mut self.store.value_stack); + let val = <ValueRef>::stack_pop(&mut self.store.value_stack); match table.grow(n, val.addr().into()) { Ok(()) => self.store.value_stack.push(sz), Err(_) => self.store.value_stack.push(-1_i32), @@ -1379,9 +1424,9 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { fn exec_table_fill(&mut self, table_index: u32) -> Result<(), Trap> { let table = self.store.state.get_table_mut(self.module.resolve_table_addr(table_index)); - let n = self.store.value_stack.pop::<i32>(); - let val = self.store.value_stack.pop::<ValueRef>(); - let i = self.store.value_stack.pop::<i32>(); + let n = <i32>::stack_pop(&mut self.store.value_stack); + let val = <ValueRef>::stack_pop(&mut self.store.value_stack); + let i = <i32>::stack_pop(&mut self.store.value_stack); if i + n > table.size() { cold_path(); diff --git a/crates/tinywasm/src/interpreter/num_helpers.rs b/crates/tinywasm/src/interpreter/num_helpers.rs index 88fa691..9651ce0 100644 --- a/crates/tinywasm/src/interpreter/num_helpers.rs +++ b/crates/tinywasm/src/interpreter/num_helpers.rs @@ -31,7 +31,7 @@ macro_rules! checked_conv_float { }; // Conversion with an intermediate unsigned type and error checking (three types) ($from:tt, $intermediate:tt, $to:tt, $self:expr) => {{ - let v = $self.store.value_stack.pop::<$from>(); + let v = <$from>::stack_pop(&mut $self.store.value_stack); let (min, max) = float_min_max!($from, $intermediate); if unlikely(v.is_nan()) { return Err(crate::Trap::InvalidConversionToInt); diff --git a/crates/tinywasm/src/interpreter/simd/mod.rs b/crates/tinywasm/src/interpreter/simd/mod.rs index d3cdd98..55e342a 100644 --- a/crates/tinywasm/src/interpreter/simd/mod.rs +++ b/crates/tinywasm/src/interpreter/simd/mod.rs @@ -34,6 +34,11 @@ impl MemValue<16> for Value128 { fn to_mem_bytes(self) -> [u8; 16] { self.0 } + + #[inline(always)] + fn load(mem: &dyn crate::LinearMemory, base: u64, offset: u64) -> core::result::Result<Self, crate::Trap> { + Ok(Self(mem.read_128(base, offset)?)) + } } impl From<Value128> for i128 { diff --git a/crates/tinywasm/src/interpreter/stack/call_stack.rs b/crates/tinywasm/src/interpreter/stack/call_stack.rs index 03cf5fe..a307b50 100644 --- a/crates/tinywasm/src/interpreter/stack/call_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/call_stack.rs @@ -54,7 +54,7 @@ impl CallStack { } } -#[derive(Clone, Copy, Default)] +#[derive(Clone, Copy)] #[cfg_attr(feature = "debug", derive(Debug))] pub(crate) struct CallFrame { pub(crate) instr_ptr: u32, diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index 631fd00..9c5b553 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -1,8 +1,8 @@ use alloc::vec::Vec; use core::hint::cold_path; -use tinywasm_types::{ExternRef, FuncRef, LocalAddr, ValueCounts, WasmType, WasmValue}; +use tinywasm_types::{ExternRef, FuncRef, ValueCounts, WasmType, WasmValue}; -use super::{CallFrame, StackBase}; +use super::StackBase; use crate::{ Result, Trap, engine::{Config, StackConfig}, @@ -39,7 +39,11 @@ impl<T: Copy + Default> Stack<T> { #[inline(always)] pub(crate) fn push(&mut self, value: T) -> Result<(), Trap> { - self.ensure_capacity_for(self.data.len() + 1)?; + if !self.ensure_capacity_for(self.data.len() + 1) { + cold_path(); + return Err(Trap::ValueStackOverflow); + } + self.data.push(value); Ok(()) } @@ -96,36 +100,47 @@ impl<T: Copy + Default> Stack<T> { #[inline(always)] pub(crate) fn enter_locals(&mut self, param_count: usize, local_count: usize) -> Result<u32, Trap> { - debug_assert!(param_count <= local_count && param_count <= self.data.len()); + debug_assert!(param_count <= local_count); + debug_assert!(param_count <= self.data.len()); - let start = self.data.len() - param_count; + let len = self.data.len(); + let start = len - param_count; let end = start + local_count; - self.ensure_capacity_for(end)?; - self.data.resize(end, T::default()); + if end > self.data.capacity() { + cold_path(); + if end > self.max_size || !self.dynamic { + return Err(Trap::ValueStackOverflow); + } + let cap = self.data.capacity(); + let target = end.max(cap.max(1).saturating_mul(2)).min(self.max_size); + if self.data.try_reserve_exact(target - len).is_err() { + return Err(Trap::ValueStackOverflow); + } + } + + self.data.resize(end, T::default()); Ok(start as u32) } - #[inline(always)] - fn ensure_capacity_for(&mut self, required_len: usize) -> Result<(), Trap> { - if required_len <= self.data.capacity() { - return Ok(()); - } + fn ensure_capacity_for(&mut self, required_len: usize) -> bool { + let cap = self.data.capacity(); - if required_len > self.max_size || !self.dynamic { + if required_len > cap { cold_path(); - return Err(Trap::ValueStackOverflow); - } - let target_capacity = required_len.max(self.data.capacity().max(1).saturating_mul(2)).min(self.max_size); - match self.data.try_reserve(target_capacity.saturating_sub(self.data.len())) { - Ok(()) => {} - Err(_) => { - cold_path(); - return Err(Trap::ValueStackOverflow); + if required_len > self.max_size || !self.dynamic { + return false; + } + let doubled = cap.max(1).saturating_mul(2); + let target = required_len.max(doubled).min(self.max_size); + let additional = target - cap; + if self.data.try_reserve_exact(additional).is_err() { + return false; } } - Ok(()) + + true } #[inline(always)] @@ -175,39 +190,13 @@ impl ValueStack { } #[inline(always)] - pub(crate) fn peek<T: InternalValue>(&self) -> T { - T::stack_peek(self) - } - - #[inline(always)] - pub(crate) fn pop<T: InternalValue>(&mut self) -> T { - T::stack_pop(self) - } - - #[inline(always)] pub(crate) fn push<T: InternalValue>(&mut self, value: T) -> Result<(), Trap> { T::stack_push(self, value) } - #[inline(always)] - pub(crate) fn drop<T: InternalValue>(&mut self) { - T::stack_pop(self); - } - - #[inline(always)] - pub(crate) fn select<T: InternalValue>(&mut self) -> Result<(), Trap> { - let cond: i32 = self.pop(); - let val2: T = self.pop(); - if cond == 0 { - self.drop::<T>(); - self.push(val2)?; - } - Ok(()) - } - #[inline] pub(crate) fn select_multi(&mut self, counts: ValueCounts) { - let condition = self.pop::<i32>() != 0; + let condition = i32::stack_pop(self) != 0; self.stack_32.select_many(counts.c32 as usize, condition); self.stack_64.select_many(counts.c64 as usize, condition); self.stack_128.select_many(counts.c128 as usize, condition); @@ -220,6 +209,7 @@ impl ValueStack { val_types.into_iter().map(|val_type| self.pop_wasmvalue(*val_type)) } + #[inline(always)] pub(crate) fn enter_locals(&mut self, params: &ValueCounts, locals: &ValueCounts) -> Result<StackBase, Trap> { let locals_base32 = self.stack_32.enter_locals(params.c32 as usize, locals.c32 as usize)?; let locals_base64 = self.stack_64.enter_locals(params.c64 as usize, locals.c64 as usize)?; @@ -227,22 +217,13 @@ impl ValueStack { Ok(StackBase { s32: locals_base32, s64: locals_base64, s128: locals_base128 }) } + #[inline(always)] pub(crate) fn truncate_keep_counts(&mut self, base: StackBase, keep: ValueCounts) { self.stack_32.truncate_keep(base.s32 as usize, keep.c32 as usize); self.stack_64.truncate_keep(base.s64 as usize, keep.c64 as usize); self.stack_128.truncate_keep(base.s128 as usize, keep.c128 as usize); } - #[inline] - pub(crate) fn local_get<T: InternalValue>(&self, frame: &CallFrame, index: LocalAddr) -> T { - T::local_get(self, frame, index) - } - - #[inline] - pub(crate) fn local_set<T: InternalValue>(&mut self, frame: &CallFrame, index: LocalAddr, value: T) { - T::local_set(self, frame, index, value); - } - pub(crate) fn push_dyn(&mut self, value: TinyWasmValue) -> Result<(), Trap> { match value { TinyWasmValue::Value32(v) => self.stack_32.push(v)?, @@ -255,13 +236,13 @@ impl ValueStack { pub(crate) fn pop_wasmvalue(&mut self, val_type: WasmType) -> WasmValue { match val_type { - WasmType::I32 => WasmValue::I32(self.pop()), - WasmType::I64 => WasmValue::I64(self.pop()), - WasmType::F32 => WasmValue::F32(self.pop()), - WasmType::F64 => WasmValue::F64(self.pop()), - WasmType::RefExtern => WasmValue::RefExtern(ExternRef::from_raw(self.pop::<ValueRef>().raw())), - WasmType::RefFunc => WasmValue::RefFunc(FuncRef::from_raw(self.pop::<ValueRef>().raw())), - WasmType::V128 => WasmValue::V128(self.pop::<Value128>().into()), + WasmType::I32 => WasmValue::I32(i32::stack_pop(self)), + WasmType::I64 => WasmValue::I64(i64::stack_pop(self)), + WasmType::F32 => WasmValue::F32(f32::stack_pop(self)), + WasmType::F64 => WasmValue::F64(f64::stack_pop(self)), + WasmType::RefExtern => WasmValue::RefExtern(ExternRef::from_raw(ValueRef::stack_pop(self).raw())), + WasmType::RefFunc => WasmValue::RefFunc(FuncRef::from_raw(ValueRef::stack_pop(self).raw())), + WasmType::V128 => WasmValue::V128(Value128::stack_pop(self).into()), } } diff --git a/crates/tinywasm/src/interpreter/values.rs b/crates/tinywasm/src/interpreter/values.rs index ac5ca07..271cc27 100644 --- a/crates/tinywasm/src/interpreter/values.rs +++ b/crates/tinywasm/src/interpreter/values.rs @@ -147,47 +147,85 @@ mod sealed { pub(crate) trait InternalValue: sealed::Sealed + Into<TinyWasmValue> + Copy + Default { fn stack_push(stack: &mut ValueStack, value: Self) -> Result<(), crate::Trap>; - fn local_get(stack: &ValueStack, frame: &CallFrame, index: LocalAddr) -> Self; - fn local_set(stack: &mut ValueStack, frame: &CallFrame, index: LocalAddr, value: Self); fn stack_pop(stack: &mut ValueStack) -> Self; fn stack_peek(stack: &ValueStack) -> Self; + fn stack_select(stack: &mut ValueStack) -> Result<(), crate::Trap>; + fn local_get(stack: &ValueStack, frame: &CallFrame, index: LocalAddr) -> Self; + fn local_set(stack: &mut ValueStack, frame: &CallFrame, index: LocalAddr, value: Self); + fn local_copy(stack: &mut ValueStack, frame: &CallFrame, from: LocalAddr, to: LocalAddr); } macro_rules! impl_internalvalue { - ($( $variant:ident, $stack:ident, $stack_base:ident, $outer:ty, $to_value:expr, $to_stack:expr, $from_stack:expr )*) => { + ( + $( + $variant:ident, $stack:ident, $stack_base:ident, $outer:ty, + |$to_value_v:ident| $to_value:expr, + |$to_stack_v:ident| $to_stack:expr, + |$from_stack_v:ident| $from_stack:expr + )* + ) => { $( impl sealed::Sealed for $outer {} impl From<$outer> for TinyWasmValue { + #[inline(always)] fn from(value: $outer) -> Self { - TinyWasmValue::$variant($to_value(value)) + let $to_value_v = value; + TinyWasmValue::$variant($to_value) } } impl InternalValue for $outer { #[inline(always)] fn stack_push(stack: &mut ValueStack, value: Self) -> Result<(), crate::Trap> { - stack.$stack.push($to_stack(value)) + let $to_stack_v = value; + stack.$stack.push($to_stack) } #[inline(always)] fn local_get(stack: &ValueStack, frame: &CallFrame, index: LocalAddr) -> Self { - $from_stack(*stack.$stack.get(frame.locals_base.$stack_base as usize + index as usize)) + let $from_stack_v = *stack.$stack.get(frame.locals_base.$stack_base as usize + index as usize); + $from_stack } #[inline(always)] fn local_set(stack: &mut ValueStack, frame: &CallFrame, index: LocalAddr, value: Self) { - stack.$stack.set(frame.locals_base.$stack_base as usize + index as usize, $to_stack(value)); + let $to_stack_v = value; + stack.$stack.set( + frame.locals_base.$stack_base as usize + index as usize, + $to_stack, + ); + } + + #[inline(always)] + fn local_copy(stack: &mut ValueStack, frame: &CallFrame, from: LocalAddr, to: LocalAddr) { + let val = stack.$stack.get(frame.locals_base.$stack_base as usize + from as usize); + stack.$stack.set(frame.locals_base.$stack_base as usize + to as usize, *val); } #[inline(always)] fn stack_pop(stack: &mut ValueStack) -> Self { - $from_stack(stack.$stack.pop()) + let $from_stack_v = stack.$stack.pop(); + $from_stack } #[inline(always)] fn stack_peek(stack: &ValueStack) -> Self { - $from_stack(*stack.$stack.last()) + let $from_stack_v = *stack.$stack.last(); + $from_stack + } + + #[inline(always)] + fn stack_select(stack: &mut ValueStack) -> Result<(), crate::Trap> { + let cond = stack.stack_32.pop() as i32; + let val2 = stack.$stack.pop(); + + if cond == 0 { + Self::stack_pop(stack); + stack.$stack.push(val2)?; + } + + Ok(()) } } )* @@ -195,12 +233,12 @@ macro_rules! impl_internalvalue { } impl_internalvalue! { - Value32, stack_32, s32, u32, |v| v, |v| v, |v| v - Value64, stack_64, s64, u64, |v| v, |v| v, |v| v - Value32, stack_32, s32, i32, |v: i32| v as u32, |v: i32| v as u32, |v: u32| v as i32 - Value64, stack_64, s64, i64, |v: i64| v as u64, |v: i64| v as u64, |v: u64| v as i64 - Value32, stack_32, s32, f32, f32::to_bits, f32::to_bits, f32::from_bits - Value64, stack_64, s64, f64, f64::to_bits, f64::to_bits, f64::from_bits - ValueRef, stack_32, s32, ValueRef, |v| v, |v: ValueRef| v.raw(), |v: u32| ValueRef(v) - Value128, stack_128, s128, Value128, |v| v, |v| v, |v| v + Value32, stack_32, s32, u32, |v| v, |v| v, |v| v + Value64, stack_64, s64, u64, |v| v, |v| v, |v| v + Value32, stack_32, s32, i32, |v| v as u32, |v| v as u32, |v| v as i32 + Value64, stack_64, s64, i64, |v| v as u64, |v| v as u64, |v| v as i64 + Value32, stack_32, s32, f32, |v| f32::to_bits(v), |v| f32::to_bits(v), |v| f32::from_bits(v) + Value64, stack_64, s64, f64, |v| f64::to_bits(v), |v| f64::to_bits(v), |v| f64::from_bits(v) + ValueRef, stack_32, s32, ValueRef, |v| v, |v| v.raw(), |v| ValueRef(v) + Value128, stack_128, s128, Value128, |v| v, |v| v, |v| v } diff --git a/crates/tinywasm/src/store/memory/mod.rs b/crates/tinywasm/src/store/memory/mod.rs index 37a8798..50ad520 100644 --- a/crates/tinywasm/src/store/memory/mod.rs +++ b/crates/tinywasm/src/store/memory/mod.rs @@ -376,10 +376,12 @@ pub(crate) trait MemValue<const N: usize>: Copy + Default { /// Load a value from memory fn from_mem_bytes(bytes: [u8; N]) -> Self; + + fn load(mem: &dyn LinearMemory, base: u64, offset: u64) -> core::result::Result<Self, crate::Trap>; } macro_rules! impl_mem_traits { - ($($ty:ty, $size:expr),*) => { + ($($ty:ty, $size:expr, $read:ident),* $(,)?) => { $( impl MemValue<$size> for $ty { #[inline(always)] @@ -391,12 +393,58 @@ macro_rules! impl_mem_traits { fn to_mem_bytes(self) -> [u8; $size] { self.to_le_bytes() } + + #[inline(always)] + fn load( + mem: &dyn LinearMemory, + base: u64, + offset: u64, + ) -> core::result::Result<Self, crate::Trap> { + Ok(Self::from_mem_bytes(mem.$read(base, offset)?)) + } } )* + }; +} + +impl MemValue<1> for u8 { + #[inline(always)] + fn from_mem_bytes(bytes: [u8; 1]) -> Self { + bytes[0] + } + + #[inline(always)] + fn to_mem_bytes(self) -> [u8; 1] { + [self] + } + + #[inline(always)] + fn load(mem: &dyn LinearMemory, base: u64, offset: u64) -> core::result::Result<Self, crate::Trap> { + mem.read_8(base, offset) + } +} + +impl MemValue<1> for i8 { + #[inline(always)] + fn from_mem_bytes(bytes: [u8; 1]) -> Self { + i8::from_le_bytes(bytes) + } + + #[inline(always)] + fn to_mem_bytes(self) -> [u8; 1] { + self.to_le_bytes() + } + + #[inline(always)] + fn load(mem: &dyn LinearMemory, base: u64, offset: u64) -> core::result::Result<Self, crate::Trap> { + Ok(mem.read_8(base, offset)? as i8) } } -impl_mem_traits!(u8, 1, i8, 1, u16, 2, i16, 2, u32, 4, i32, 4, f32, 4, u64, 8, i64, 8, f64, 8); +impl_mem_traits!( + u16, 2, read_16, i16, 2, read_16, u32, 4, read_32, i32, 4, read_32, f32, 4, read_32, u64, 8, read_64, i64, 8, + read_64, f64, 8, read_64, +); fn memory_oob(offset: usize, len: usize, max: usize) -> crate::Trap { crate::Trap::MemoryOutOfBounds { offset, len, max } diff --git a/crates/tinywasm/src/store/memory/vec.rs b/crates/tinywasm/src/store/memory/vec.rs index f556564..d84890b 100644 --- a/crates/tinywasm/src/store/memory/vec.rs +++ b/crates/tinywasm/src/store/memory/vec.rs @@ -120,25 +120,49 @@ impl LinearMemory for VecMemory { #[inline(always)] fn read_16(&self, base: u64, offset: u64) -> core::result::Result<[u8; 2], crate::Trap> { let addr = checked_effective_addr::<2>(self.data.len(), base, offset)?; - Ok(self.data[addr..addr + 2].try_into().unwrap_or_else(|_| unreachable!())) + match self.data[addr..addr + 2].try_into() { + Ok(bytes) => Ok(bytes), + Err(_) => { + cold_path(); + unreachable!(); + } + } } #[inline(always)] fn read_32(&self, base: u64, offset: u64) -> core::result::Result<[u8; 4], crate::Trap> { let addr = checked_effective_addr::<4>(self.data.len(), base, offset)?; - Ok(self.data[addr..addr + 4].try_into().unwrap_or_else(|_| unreachable!())) + match self.data[addr..addr + 4].try_into() { + Ok(bytes) => Ok(bytes), + Err(_) => { + cold_path(); + unreachable!(); + } + } } #[inline(always)] fn read_64(&self, base: u64, offset: u64) -> core::result::Result<[u8; 8], crate::Trap> { let addr = checked_effective_addr::<8>(self.data.len(), base, offset)?; - Ok(self.data[addr..addr + 8].try_into().unwrap_or_else(|_| unreachable!())) + match self.data[addr..addr + 8].try_into() { + Ok(bytes) => Ok(bytes), + Err(_) => { + cold_path(); + unreachable!(); + } + } } #[inline(always)] fn read_128(&self, base: u64, offset: u64) -> core::result::Result<[u8; 16], crate::Trap> { let addr = checked_effective_addr::<16>(self.data.len(), base, offset)?; - Ok(self.data[addr..addr + 16].try_into().unwrap_or_else(|_| unreachable!())) + match self.data[addr..addr + 16].try_into() { + Ok(bytes) => Ok(bytes), + Err(_) => { + cold_path(); + unreachable!(); + } + } } #[inline(always)] diff --git a/crates/tinywasm/tests/testsuite/util.rs b/crates/tinywasm/tests/testsuite/util.rs index 61c163e..61dfc97 100644 --- a/crates/tinywasm/tests/testsuite/util.rs +++ b/crates/tinywasm/tests/testsuite/util.rs @@ -1,11 +1,35 @@ use std::panic::{self, AssertUnwindSafe}; +use std::time::Duration; use eyre::{Result, bail, eyre}; -use tinywasm::ModuleInstance; +use tinywasm::{ExecProgress, ModuleInstance}; use tinywasm_types::{ExternRef, FuncRef, Module, ModuleInstanceAddr, WasmType, WasmValue}; use wasm_testsuite::wast; use wasm_testsuite::wast::{QuoteWat, core::AbstractHeapType}; +const TEST_TIME_SLICE: Duration = Duration::from_millis(10); +const TEST_MAX_SUSPENSIONS: u32 = 100; + +fn exec_with_budget( + func: &tinywasm::Function, + store: &mut tinywasm::Store, + args: &[tinywasm_types::WasmValue], +) -> Result<Vec<tinywasm_types::WasmValue>, tinywasm::Error> { + let mut exec = func.call_resumable(store, args)?; + + for _ in 0..TEST_MAX_SUSPENSIONS { + match exec.resume_with_time_budget(TEST_TIME_SLICE)? { + ExecProgress::Completed(values) => return Ok(values), + ExecProgress::Suspended => {} + } + } + + Err(tinywasm::Error::Other(format!( + "testsuite execution timed out after {} time slices of {:?}", + TEST_MAX_SUSPENSIONS, TEST_TIME_SLICE + ))) +} + pub fn try_downcast_panic(panic: Box<dyn std::any::Any + Send>) -> String { let info = panic.downcast_ref::<panic::PanicHookInfo>().or(None).map(ToString::to_string).clone(); let info_string = panic.downcast_ref::<String>().cloned(); @@ -28,7 +52,7 @@ pub fn exec_fn_instance( }; let func = instance.func_untyped(store, name)?; - func.call(store, args) + exec_with_budget(&func, store, args) } pub fn exec_fn( @@ -43,7 +67,8 @@ pub fn exec_fn( let mut store = tinywasm::Store::default(); let instance = ModuleInstance::instantiate(&mut store, module, imports)?; - instance.func_untyped(&store, name)?.call(&mut store, args) + let func = instance.func_untyped(&store, name)?; + exec_with_budget(&func, &mut store, args) } pub fn catch_unwind_silent<R>(f: impl FnOnce() -> R) -> std::thread::Result<R> { diff --git a/crates/types/src/instructions.rs b/crates/types/src/instructions.rs index 48158de..a1bc0fe 100644 --- a/crates/types/src/instructions.rs +++ b/crates/types/src/instructions.rs @@ -132,7 +132,7 @@ pub enum Instruction { BinOpLocalConstTee32(BinOp, LocalAddr, i32, LocalAddr), BinOpLocalConstTee64(BinOp, LocalAddr, i64, LocalAddr), BinOpLocalConstTee128(BinOp128, LocalAddr, ConstIdx, LocalAddr), - SetLocalConst32(LocalAddr, i32), SetLocalConst64(LocalAddr, i64), + SetLocalConst32(LocalAddr, i32), SetLocalConst64(LocalAddr, i64), SetLocalConst128(LocalAddr, ConstIdx), StoreLocalLocal32(MemoryArg, u8, u8), StoreLocalLocal64(MemoryArg, u8, u8), StoreLocalLocal128(MemoryArg, u8, u8), @@ -158,6 +158,10 @@ pub enum Instruction { JumpIfNonZero32(u32), JumpIfZero64(u32), JumpIfNonZero64(u32), + JumpIfLocalZero32 { target_ip: u32, local: LocalAddr }, + JumpIfLocalNonZero32 { target_ip: u32, local: LocalAddr }, + JumpIfLocalZero64 { target_ip: u32, local: LocalAddr }, + JumpIfLocalNonZero64 { target_ip: u32, local: LocalAddr }, JumpCmpStackConst32 { target_ip: u32, imm: i32, op: CmpOp }, JumpCmpStackConst64 { target_ip: u32, imm: i64, op: CmpOp }, JumpCmpLocalConst32 { target_ip: u32, local: LocalAddr, imm: i32, op: CmpOp }, @@ -219,10 +223,8 @@ pub enum Instruction { MemoryGrow(MemAddr), // > Constants - I32Const(i32), - I64Const(i64), - F32Const(f32), - F64Const(f64), + Const32(i32), + Const64(i64), // > Reference Types RefNull(WasmType), @@ -290,7 +292,7 @@ pub enum Instruction { V128Store(MemoryArg), V128Store8Lane(MemoryArg, u8), V128Store16Lane(MemoryArg, u8), V128Store32Lane(MemoryArg, u8), V128Store64Lane(MemoryArg, u8), I8x16Shuffle(ConstIdx), - V128Const(ConstIdx), + Const128(ConstIdx), I8x16ExtractLaneS(u8), I8x16ExtractLaneU(u8), I8x16ReplaceLane(u8), I16x8ExtractLaneS(u8), I16x8ExtractLaneU(u8), I16x8ReplaceLane(u8), diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index 3eb14ba..29eb709 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -418,6 +418,7 @@ pub struct WasmFunction { pub data: WasmFunctionData, pub locals: ValueCounts, pub params: ValueCounts, + pub results: ValueCounts, pub ty: Arc<FuncType>, } |