diff options
Diffstat (limited to 'crates/parser/src')
| -rw-r--r-- | crates/parser/src/lib.rs | 46 | ||||
| -rw-r--r-- | crates/parser/src/macros.rs | 117 | ||||
| -rw-r--r-- | crates/parser/src/module.rs | 15 | ||||
| -rw-r--r-- | crates/parser/src/optimize.rs | 794 | ||||
| -rw-r--r-- | crates/parser/src/visit.rs | 8 |
5 files changed, 785 insertions, 195 deletions
diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index 7b3dfa2..949afa2 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -47,11 +47,22 @@ pub use tinywasm_types::Module; pub struct ParserOptions { /// Whether to optimize local memory allocation by skipping allocation of unused local memories. pub optimize_local_memory_allocation: bool, + /// Whether to run the peephole rewrite optimizer. + pub optimize_rewrite: bool, + /// Whether to remove `Nop` and `MergeBarrier` instructions after rewriting. + pub optimize_remove_nop: bool, + /// Whether to invert conditional branches over an unconditional jump. + pub optimize_branch_inversion: bool, } impl Default for ParserOptions { fn default() -> Self { - Self { optimize_local_memory_allocation: true } + Self { + optimize_local_memory_allocation: true, + optimize_rewrite: true, + optimize_remove_nop: true, + optimize_branch_inversion: false, + } } } @@ -66,6 +77,39 @@ impl ParserOptions { pub const fn optimize_local_memory_allocation(&self) -> bool { self.optimize_local_memory_allocation } + + /// Enable or disable the peephole rewrite optimizer. + pub const fn with_rewrite_optimization(mut self, enabled: bool) -> Self { + self.optimize_rewrite = enabled; + self + } + + /// Returns whether the peephole rewrite optimizer is enabled. + pub const fn optimize_rewrite(&self) -> bool { + self.optimize_rewrite + } + + /// Enable or disable `Nop`/`MergeBarrier` removal after rewriting. + pub const fn with_nop_removal_optimization(mut self, enabled: bool) -> Self { + self.optimize_remove_nop = enabled; + self + } + + /// Returns whether `Nop`/`MergeBarrier` removal is enabled. + pub const fn optimize_remove_nop(&self) -> bool { + self.optimize_remove_nop + } + + /// Enable or disable the optimization that inverts conditional branches over an unconditional jump. + pub const fn with_branch_inversion_optimization(mut self, enabled: bool) -> Self { + self.optimize_branch_inversion = enabled; + self + } + + /// Returns whether conditional branch inversion optimization is enabled. + pub const fn optimize_branch_inversion(&self) -> bool { + self.optimize_branch_inversion + } } /// A WebAssembly parser diff --git a/crates/parser/src/macros.rs b/crates/parser/src/macros.rs index 97f0080..a6eab61 100644 --- a/crates/parser/src/macros.rs +++ b/crates/parser/src/macros.rs @@ -216,5 +216,120 @@ pub(crate) mod optimize { }; } - pub(crate) use {replace, rewrite}; + macro_rules! define_local_source_resolver { + ( + $name:ident, + get = $get:ident, + tee = $tee:ident, + set = $set:ident, + copy = $copy:ident, + binop_local_local_tee = $lltee:ident, + binop_local_local_set = $llset:ident, + binop_local_const_tee = $lctee:ident, + binop_local_const_set = $lcset:ident + $(, load_local_tee = $loadtee:ident, load_local_set = $loadset:ident)? + ) => { + fn $name(instrs: &mut [Instruction], read: usize, instr: Instruction) -> Option<(Instruction, u16)> { + Some(match instr { + Instruction::$get(local) => (Instruction::Nop, local), + Instruction::$tee(local) => { + let replacement = if let Some([(prev_idx, Instruction::$get(src))]) = previous_non_nop::<1>(instrs, read) { + instrs[prev_idx] = Instruction::Nop; + if src == local { Instruction::Nop } else { Instruction::$copy(src, local) } + } else { + Instruction::$set(local) + }; + (replacement, local) + } + Instruction::$lltee(op, a, b, local) => (Instruction::$llset(op, a, b, local), local), + Instruction::$lctee(op, src, c, local) => (Instruction::$lcset(op, src, c, local), local), + $(Instruction::$loadtee(memarg, addr, local) => (Instruction::$loadset(memarg, addr, local), local.into()),)? + _ => return None, + }) + } + }; + } + + macro_rules! fold_local_binop { + ( + $instrs:ident, $read:expr, $dst:expr, + source = $source:ident, + op = $op:ident, + const = $const:ident, + local_local = $local_local:ident, + local_const = $local_const:expr + ) => {{ + if let Some([(lhs_idx, lhs_src), (rhs_idx, rhs_src), (op_idx, raw_op)]) = + previous_non_nop::<3>($instrs, $read) + && let Some((lhs_instr, lhs)) = $source($instrs, lhs_idx, lhs_src) + && let Some(op) = $op(raw_op) + { + if let Some((rhs_instr, rhs)) = $source($instrs, rhs_idx, rhs_src) { + $instrs[lhs_idx] = lhs_instr; + $instrs[rhs_idx] = rhs_instr; + $instrs[op_idx] = Instruction::Nop; + $instrs[$read] = Instruction::$local_local(op, lhs, rhs, $dst); + } else if let Some(imm) = $const(rhs_src, raw_op) { + $instrs[lhs_idx] = lhs_instr; + $instrs[rhs_idx] = Instruction::Nop; + $instrs[op_idx] = Instruction::Nop; + $instrs[$read] = $local_const($dst, lhs, op, imm); + } + } + }}; + } + + macro_rules! rewrite_local_set_direct { + ( + $instrs:ident, $read:ident, $dst:expr, + get = $get:ident, + copy = $copy:ident, + binop_local_local = $ll:ident, + binop_local_local_set = $llset:ident, + binop_local_const = $lc:ident, + binop_local_const_set = $lcset:expr + $(, const_instr = $const_instr:ident, set_local_const = $set_local_const:ident)? + ) => {{ + rewrite!($instrs, $read, [$get(src)] => if src == $dst { Instruction::Nop } else { Instruction::$copy(src, $dst) }); + $(rewrite!($instrs, $read, [$const_instr(c)] => Instruction::$set_local_const($dst, c));)? + rewrite!($instrs, $read, [$ll(op, a, b)] => Instruction::$llset(op, a, b, $dst)); + rewrite!($instrs, $read, [$lc(op, src, c)] => { replace!($instrs, $read, 1 => $lcset($dst, src, op, c)); }); + }}; + } + + macro_rules! rewrite_local_tee_direct { + ( + $instrs:ident, $read:ident, $dst:expr, + get = $get:ident, + binop_local_local = $ll:ident, + binop_local_local_tee = $lltee:ident, + binop_local_const = $lc:ident, + binop_local_const_tee = $lctee:ident + ) => {{ + rewrite!($instrs, $read, [$get(src)] if (src == $dst) => [Instruction::$get(src), Instruction::Nop]); + rewrite!($instrs, $read, [$ll(op, a, b)] => Instruction::$lltee(op, a, b, $dst)); + rewrite!($instrs, $read, [$lc(op, src, c)] => Instruction::$lctee(op, src, c, $dst)); + }}; + } + + macro_rules! rewrite_drop_tee_direct { + ( + $instrs:ident, $read:ident, + tee = $tee:ident, + set = $set:ident, + binop_local_local_tee = $lltee:ident, + binop_local_local_set = $llset:ident, + binop_local_const_tee = $lctee:ident, + binop_local_const_set = $lcset:ident + ) => {{ + rewrite!($instrs, $read, [$tee(local)] => [Instruction::$set(local), Instruction::Nop]); + rewrite!($instrs, $read, [$lltee(op, a, b, dst)] => Instruction::$llset(op, a, b, dst)); + rewrite!($instrs, $read, [$lctee(op, src, c, dst)] => Instruction::$lcset(op, src, c, dst)); + }}; + } + + pub(crate) use { + define_local_source_resolver, fold_local_binop, replace, rewrite, rewrite_drop_tee_direct, + rewrite_local_set_direct, rewrite_local_tee_direct, + }; } diff --git a/crates/parser/src/module.rs b/crates/parser/src/module.rs index 164127f..548280e 100644 --- a/crates/parser/src/module.rs +++ b/crates/parser/src/module.rs @@ -200,17 +200,26 @@ impl ModuleReader { { let ty = self.func_types.get(ty_idx as usize).expect("No func type for func, this is a bug").clone(); let params = ValueCounts::from_iter(ty.params()); + let results = ValueCounts::from_iter(ty.results()); let self_func = (imported_func_count + func_idx) as u32; let local_mem_alloc = optimize_local_memory_allocation && local_memory_allocation != LocalMemoryAllocation::Eager; - let optimized = - optimize::optimize_instructions(instructions, &mut data, self_func, import_mem_count, local_mem_alloc); + let optimized = optimize::optimize_instructions( + instructions, + &mut data, + options, + self_func, + import_mem_count, + local_mem_alloc, + ); if optimized.uses_local_memory { local_memory_allocation = LocalMemoryAllocation::Eager; } - funcs.push(WasmFunction { instructions: optimized.instructions.into(), data, locals, params, ty }.into()); + funcs.push( + WasmFunction { instructions: optimized.instructions.into(), data, locals, params, results, ty }.into(), + ); } Ok(ModuleInner { diff --git a/crates/parser/src/optimize.rs b/crates/parser/src/optimize.rs index 51d0944..baaaf66 100644 --- a/crates/parser/src/optimize.rs +++ b/crates/parser/src/optimize.rs @@ -1,6 +1,7 @@ +use crate::ParserOptions; use crate::macros::optimize::*; use alloc::vec::Vec; -use tinywasm_types::{BinOp, BinOp128, CmpOp, Instruction, WasmFunctionData}; +use tinywasm_types::{BinOp, BinOp128, CmpOp, ConstIdx, Instruction, WasmFunctionData}; pub(crate) struct OptimizeResult { pub(crate) instructions: Vec<Instruction>, @@ -10,12 +11,27 @@ pub(crate) struct OptimizeResult { pub(crate) fn optimize_instructions( mut instructions: Vec<Instruction>, function_data: &mut WasmFunctionData, + options: &ParserOptions, self_func_addr: u32, imported_memory_count: u32, track_local_memory_usage: bool, ) -> OptimizeResult { - let uses_local_memory = rewrite(&mut instructions, self_func_addr, imported_memory_count, track_local_memory_usage); - remove_nop(&mut instructions, function_data); + let uses_local_memory = if options.optimize_rewrite() { + rewrite( + &mut instructions, + self_func_addr, + imported_memory_count, + track_local_memory_usage, + options.optimize_branch_inversion(), + ) + } else { + track_local_memory_usage + && instructions.iter().any(|instr| instr.memory_addr().is_some_and(|mem| mem >= imported_memory_count)) + }; + + if options.optimize_remove_nop() { + remove_nop(&mut instructions, function_data); + } OptimizeResult { instructions, uses_local_memory } } @@ -24,6 +40,7 @@ fn rewrite( self_func_addr: u32, imported_memory_count: u32, track_local_memory_usage: bool, + optimize_branch_inversion: bool, ) -> bool { use Instruction::*; let mut uses_local_memory = false; @@ -38,71 +55,71 @@ fn rewrite( instr @ (I32Add | I32Mul | I32And | I32Or | I32Xor) => { let Some(op) = int_bin_op_32(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); - rewrite!(instrs, i, [LocalGet32(local), I32Const(c)] => BinOpLocalConst32(op, local, c)); - rewrite!(instrs, i, [I32Const(c), LocalGet32(local)] => BinOpLocalConst32(op, local, c)); + rewrite!(instrs, i, [LocalGet32(local), Const32(c)] => BinOpLocalConst32(op, local, c)); + rewrite!(instrs, i, [Const32(c), LocalGet32(local)] => BinOpLocalConst32(op, local, c)); if matches!(op, BinOp::IAdd) { - rewrite!(instrs, i, [I32Const(c)] => AddConst32(c)); + rewrite!(instrs, i, [Const32(c)] => AddConst32(c)); } } instr @ (I32Sub | I32Shl | I32ShrS | I32ShrU | I32Rotl | I32Rotr) => { let Some(op) = int_bin_op_32(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); - rewrite!(instrs, i, [LocalGet32(local), I32Const(c)] => BinOpLocalConst32(op, local, c)); + rewrite!(instrs, i, [LocalGet32(local), Const32(c)] => BinOpLocalConst32(op, local, c)); } instr @ (I64Add | I64Mul | I64And | I64Or | I64Xor) => { let Some(op) = int_bin_op_64(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); - rewrite!(instrs, i, [LocalGet64(local), I64Const(c)] => BinOpLocalConst64(op, local, c)); - rewrite!(instrs, i, [I64Const(c), LocalGet64(local)] => BinOpLocalConst64(op, local, c)); + rewrite!(instrs, i, [LocalGet64(local), Const64(c)] => BinOpLocalConst64(op, local, c)); + rewrite!(instrs, i, [Const64(c), LocalGet64(local)] => BinOpLocalConst64(op, local, c)); if matches!(op, BinOp::IAdd) { - rewrite!(instrs, i, [I64Const(c)] => AddConst64(c)); + rewrite!(instrs, i, [Const64(c)] => AddConst64(c)); } } instr @ (I64Sub | I64Shl | I64ShrS | I64ShrU | I64Rotl | I64Rotr) => { let Some(op) = int_bin_op_64(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); - rewrite!(instrs, i, [LocalGet64(local), I64Const(c)] => BinOpLocalConst64(op, local, c)); + rewrite!(instrs, i, [LocalGet64(local), Const64(c)] => BinOpLocalConst64(op, local, c)); } instr @ (F32Add | F32Mul | F32Min | F32Max) => { let Some(op) = float_bin_op_32(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); - rewrite!(instrs, i, [LocalGet32(local), F32Const(c)] => BinOpLocalConst32(op, local, f32_const_bits(c))); - rewrite!(instrs, i, [F32Const(c), LocalGet32(local)] => BinOpLocalConst32(op, local, f32_const_bits(c))); + rewrite!(instrs, i, [LocalGet32(local), Const32(c)] => BinOpLocalConst32(op, local, c)); + rewrite!(instrs, i, [Const32(c), LocalGet32(local)] => BinOpLocalConst32(op, local, c)); } instr @ (F32Sub | F32Div | F32Copysign) => { let Some(op) = float_bin_op_32(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); - rewrite!(instrs, i, [LocalGet32(local), F32Const(c)] => BinOpLocalConst32(op, local, f32_const_bits(c))); + rewrite!(instrs, i, [LocalGet32(local), Const32(c)] => BinOpLocalConst32(op, local, c)); } instr @ (F64Add | F64Mul | F64Min | F64Max) => { let Some(op) = float_bin_op_64(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); - rewrite!(instrs, i, [LocalGet64(local), F64Const(c)] => BinOpLocalConst64(op, local, f64_const_bits(c))); - rewrite!(instrs, i, [F64Const(c), LocalGet64(local)] => BinOpLocalConst64(op, local, f64_const_bits(c))); + rewrite!(instrs, i, [LocalGet64(local), Const64(c)] => BinOpLocalConst64(op, local, c)); + rewrite!(instrs, i, [Const64(c), LocalGet64(local)] => BinOpLocalConst64(op, local, c)); } instr @ (F64Sub | F64Div | F64Copysign) => { let Some(op) = float_bin_op_64(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); - rewrite!(instrs, i, [LocalGet64(local), F64Const(c)] => BinOpLocalConst64(op, local, f64_const_bits(c))); + rewrite!(instrs, i, [LocalGet64(local), Const64(c)] => BinOpLocalConst64(op, local, c)); } instr @ (V128And | V128Or | V128Xor | I64x2Add | I64x2Mul) => { let Some(op) = bin_op_128(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet128(a), LocalGet128(b)] => BinOpLocalLocal128(op, a, b)); - rewrite!(instrs, i, [LocalGet128(local), V128Const(c)] => BinOpLocalConst128(op, local, c)); - rewrite!(instrs, i, [V128Const(c), LocalGet128(local)] => BinOpLocalConst128(op, local, c)); + rewrite!(instrs, i, [LocalGet128(local), Const128(c)] => BinOpLocalConst128(op, local, c)); + rewrite!(instrs, i, [Const128(c), LocalGet128(local)] => BinOpLocalConst128(op, local, c)); } V128AndNot => { rewrite!(instrs, i, [LocalGet128(a), LocalGet128(b)] => BinOpLocalLocal128(BinOp128::AndNot, a, b)); - rewrite!(instrs, i, [LocalGet128(local), V128Const(c)] => BinOpLocalConst128(BinOp128::AndNot, local, c)); + rewrite!(instrs, i, [LocalGet128(local), Const128(c)] => BinOpLocalConst128(BinOp128::AndNot, local, c)); } - I32Store(memarg) => { + I32Store(memarg) | F32Store(memarg) => { rewrite!(instrs, i, [LocalGet32(addr_local), LocalGet32(value_local)] if (let (Ok(addr_local), Ok(value_local)) = (u8::try_from(addr_local), u8::try_from(value_local))) => StoreLocalLocal32(memarg, addr_local, value_local) ); } - I64Store(memarg) => { + I64Store(memarg) | F64Store(memarg) => { rewrite!(instrs, i, [LocalGet32(addr_local), LocalGet64(value_local)] if (let (Ok(addr_local), Ok(value_local)) = (u8::try_from(addr_local), u8::try_from(value_local))) => @@ -116,48 +133,48 @@ fn rewrite( StoreLocalLocal128(memarg, addr_local, value_local) ); } - I32Load(memarg) => { + I32Load(memarg) | F32Load(memarg) => { rewrite!(instrs, i, [LocalGet32(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) => LoadLocal32(memarg, addr_local) ); } MemoryFill(mem) => { - rewrite!(instrs, i, [I32Const(val), I32Const(size)] => MemoryFillImm(mem, val as u8, size)) + rewrite!(instrs, i, [Const32(val), Const32(size)] => MemoryFillImm(mem, val as u8, size)) } LocalGet32(dst) => rewrite!(instrs, i, [LocalSet32(src)] if (src == dst) => [LocalTee32(src), Nop]), LocalGet64(dst) => rewrite!(instrs, i, [LocalSet64(src)] if (src == dst) => [LocalTee64(src), Nop]), LocalGet128(dst) => rewrite!(instrs, i, [LocalSet128(src)] if (src == dst) => [LocalTee128(src), Nop]), LocalSet32(dst) => { - if let Some([(lhs_idx, lhs_src), (rhs_idx, rhs_src), (op_idx, raw_op)]) = previous_non_nop_3(instrs, i) - && let Some((lhs_instr, lhs)) = stack_source_local_32(lhs_src) - && let Some(op) = scalar_bin_op_32(raw_op) - { - if let Some((rhs_instr, rhs)) = stack_source_local_32(rhs_src) { - instrs[lhs_idx] = lhs_instr; - instrs[rhs_idx] = rhs_instr; - instrs[op_idx] = Nop; - instrs[i] = BinOpLocalLocalSet32(op, lhs, rhs, dst); - } else if let Some(imm) = scalar_const_32(rhs_src, raw_op) { - instrs[lhs_idx] = lhs_instr; - instrs[rhs_idx] = Nop; - instrs[op_idx] = Nop; - instrs[i] = match (dst == lhs, op) { - (true, BinOp::IAdd) => IncLocal32(dst, imm), - (true, BinOp::ISub) => IncLocal32(dst, imm.wrapping_neg()), - _ => BinOpLocalConstSet32(op, lhs, imm, dst), - }; + fold_local_binop!( + instrs, i, dst, + source = resolve_local_source_32, + op = scalar_bin_op_32, + const = scalar_const_32, + local_local = BinOpLocalLocalSet32, + local_const = |dst, lhs, op, imm| match (dst == lhs, op) { + (true, BinOp::IAdd) => Instruction::IncLocal32(dst, imm), + (true, BinOp::ISub) => Instruction::IncLocal32(dst, imm.wrapping_neg()), + _ => Instruction::BinOpLocalConstSet32(op, lhs, imm, dst), } - } - rewrite!(instrs, i, [LocalGet32(src)] => if src == dst { Nop } else { LocalCopy32(src, dst) }); - rewrite!(instrs, i, [I32Const(c)] => SetLocalConst32(dst, c)); - rewrite!(instrs, i, [F32Const(c)] => SetLocalConst32(dst, i32::from_ne_bytes(c.to_bits().to_ne_bytes()))); - rewrite!(instrs, i, [BinOpLocalLocal32(op, a, b)] => BinOpLocalLocalSet32(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConst32(op, src, c)] => match (dst == src, op) { - (true, BinOp::IAdd) => IncLocal32(dst, c), - (true, BinOp::ISub) => IncLocal32(dst, c.wrapping_neg()), - _ => BinOpLocalConstSet32(op, src, c, dst), - }); + ); + rewrite_local_set_direct!( + instrs, + i, + dst, + get = LocalGet32, + copy = LocalCopy32, + binop_local_local = BinOpLocalLocal32, + binop_local_local_set = BinOpLocalLocalSet32, + binop_local_const = BinOpLocalConst32, + binop_local_const_set = |dst, src, op, c| match (dst == src, op) { + (true, BinOp::IAdd) => IncLocal32(dst, c), + (true, BinOp::ISub) => IncLocal32(dst, c.wrapping_neg()), + _ => BinOpLocalConstSet32(op, src, c, dst), + }, + const_instr = Const32, + set_local_const = SetLocalConst32 + ); rewrite!(instrs, i, [LoadLocal32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => LoadLocalSet32(memarg, addr, dst)); rewrite!(instrs, i, [LocalGet32(addr), I32Load(memarg)] if @@ -166,32 +183,58 @@ fn rewrite( ); } LocalSet64(dst) => { - rewrite!(instrs, i, [LocalGet64(src)] => if src == dst { Nop } else { LocalCopy64(src, dst) }); - rewrite!(instrs, i, - [LocalTee64(src), I64Const(c), instr] if (let Some(op) = int_bin_op_64(instr)) => - [LocalSet64(src), Nop, Nop, match (dst == src, op) { + fold_local_binop!( + instrs, i, dst, + source = resolve_local_source_64, + op = scalar_bin_op_64, + const = scalar_const_64, + local_local = BinOpLocalLocalSet64, + local_const = |dst, lhs, op, imm| match (dst == lhs, op) { + (true, BinOp::IAdd) => Instruction::IncLocal64(dst, imm), + (true, BinOp::ISub) => Instruction::IncLocal64(dst, imm.wrapping_neg()), + _ => Instruction::BinOpLocalConstSet64(op, lhs, imm, dst), + } + ); + rewrite_local_set_direct!( + instrs, + i, + dst, + get = LocalGet64, + copy = LocalCopy64, + binop_local_local = BinOpLocalLocal64, + binop_local_local_set = BinOpLocalLocalSet64, + binop_local_const = BinOpLocalConst64, + binop_local_const_set = |dst, src, op, c| match (dst == src, op) { (true, BinOp::IAdd) => IncLocal64(dst, c), (true, BinOp::ISub) => IncLocal64(dst, c.wrapping_neg()), _ => BinOpLocalConstSet64(op, src, c, dst), - }] + }, + const_instr = Const64, + set_local_const = SetLocalConst64 ); - rewrite!(instrs, i, - [LocalTee64(src), F64Const(c), instr] if (let Some(op) = float_bin_op_64(instr)) => - [LocalSet64(src), Nop, Nop, BinOpLocalConstSet64(op, src, f64_const_bits(c), dst)] - ); - rewrite!(instrs, i, [I64Const(c)] => SetLocalConst64(dst, c)); - rewrite!(instrs, i, [F64Const(c)] => SetLocalConst64(dst, i64::from_ne_bytes(c.to_bits().to_ne_bytes()))); - rewrite!(instrs, i, [BinOpLocalLocal64(op, a, b)] => BinOpLocalLocalSet64(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConst64(op, src, c)] => match (dst == src, op) { - (true, BinOp::IAdd) => IncLocal64(dst, c), - (true, BinOp::ISub) => IncLocal64(dst, c.wrapping_neg()), - _ => BinOpLocalConstSet64(op, src, c, dst), - }); } LocalSet128(dst) => { - rewrite!(instrs, i, [LocalGet128(src)] => if src == dst { Nop } else { LocalCopy128(src, dst) }); - rewrite!(instrs, i, [BinOpLocalLocal128(op, a, b)] => BinOpLocalLocalSet128(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConst128(op, src, c)] => BinOpLocalConstSet128(op, src, c, dst)); + fold_local_binop!( + instrs, i, dst, + source = resolve_local_source_128, + op = bin_op_128, + const = const_128, + local_local = BinOpLocalLocalSet128, + local_const = |dst, lhs, op, imm| Instruction::BinOpLocalConstSet128(op, lhs, imm, dst) + ); + rewrite_local_set_direct!( + instrs, + i, + dst, + get = LocalGet128, + copy = LocalCopy128, + binop_local_local = BinOpLocalLocal128, + binop_local_local_set = BinOpLocalLocalSet128, + binop_local_const = BinOpLocalConst128, + binop_local_const_set = |dst, src, op, c| BinOpLocalConstSet128(op, src, c, dst), + const_instr = Const128, + set_local_const = SetLocalConst128 + ); rewrite!(instrs, i, [LocalGet32(addr), V128Load(memarg)] if (let (Ok(addr), Ok(dst)) = (u8::try_from(addr), u8::try_from(dst))) => @@ -199,153 +242,353 @@ fn rewrite( ); } LocalTee32(dst) => { - if let Some([(lhs_idx, lhs_src), (rhs_idx, rhs_src), (op_idx, raw_op)]) = previous_non_nop_3(instrs, i) - && let Some((lhs_instr, lhs)) = stack_source_local_32(lhs_src) - && let Some(op) = scalar_bin_op_32(raw_op) - { - if let Some((rhs_instr, rhs)) = stack_source_local_32(rhs_src) { - instrs[lhs_idx] = lhs_instr; - instrs[rhs_idx] = rhs_instr; - instrs[op_idx] = Nop; - instrs[i] = BinOpLocalLocalTee32(op, lhs, rhs, dst); - } else if let Some(imm) = scalar_const_32(rhs_src, raw_op) { - instrs[lhs_idx] = lhs_instr; - instrs[rhs_idx] = Nop; - instrs[op_idx] = Nop; - instrs[i] = BinOpLocalConstTee32(op, lhs, imm, dst); - } - } - rewrite!(instrs, i, [LocalGet32(src)] if (src == dst) => [LocalGet32(src), Nop]); - rewrite!(instrs, i, [BinOpLocalLocal32(op, a, b)] => BinOpLocalLocalTee32(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConst32(op, src, c)] => BinOpLocalConstTee32(op, src, c, dst)); - rewrite!(instrs, i, [I32Const(c), I32And] => AndConstTee32(c, dst)); - rewrite!(instrs, i, [I32Const(c), I32Sub] => SubConstTee32(c, dst)); + fold_local_binop!( + instrs, i, dst, + source = resolve_local_source_32, + op = scalar_bin_op_32, + const = scalar_const_32, + local_local = BinOpLocalLocalTee32, + local_const = |dst, lhs, op, imm| Instruction::BinOpLocalConstTee32(op, lhs, imm, dst) + ); + rewrite_local_tee_direct!( + instrs, + i, + dst, + get = LocalGet32, + binop_local_local = BinOpLocalLocal32, + binop_local_local_tee = BinOpLocalLocalTee32, + binop_local_const = BinOpLocalConst32, + binop_local_const_tee = BinOpLocalConstTee32 + ); + rewrite!(instrs, i, [Const32(c), I32And] => AndConstTee32(c, dst)); + rewrite!(instrs, i, [Const32(c), I32Sub] => SubConstTee32(c, dst)); rewrite!(instrs, i, [LocalGet32(addr), I32Load(memarg)] if (let (Ok(addr), Ok(dst)) = (u8::try_from(addr), u8::try_from(dst))) => LoadLocalTee32(memarg, addr, dst) ); rewrite!(instrs, i, + [LocalGet32(addr), F32Load(memarg)] if + (let (Ok(addr), Ok(dst)) = (u8::try_from(addr), u8::try_from(dst))) => + LoadLocalTee32(memarg, addr, dst) + ); + rewrite!(instrs, i, [LoadLocal32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => LoadLocalTee32(memarg, addr, dst) ); } LocalTee64(dst) => { - rewrite!(instrs, i, [LocalGet64(src)] if (src == dst) => [LocalGet64(src), Nop]); - rewrite!(instrs, i, [BinOpLocalLocal64(op, a, b)] => BinOpLocalLocalTee64(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConst64(op, src, c)] => BinOpLocalConstTee64(op, src, c, dst)); - rewrite!(instrs, i, [I64Const(c), I64And] => AndConstTee64(c, dst)); - rewrite!(instrs, i, [I64Const(c), I64Sub] => SubConstTee64(c, dst)); + fold_local_binop!( + instrs, i, dst, + source = resolve_local_source_64, + op = scalar_bin_op_64, + const = scalar_const_64, + local_local = BinOpLocalLocalTee64, + local_const = |dst, lhs, op, imm| Instruction::BinOpLocalConstTee64(op, lhs, imm, dst) + ); + rewrite_local_tee_direct!( + instrs, + i, + dst, + get = LocalGet64, + binop_local_local = BinOpLocalLocal64, + binop_local_local_tee = BinOpLocalLocalTee64, + binop_local_const = BinOpLocalConst64, + binop_local_const_tee = BinOpLocalConstTee64 + ); + rewrite!(instrs, i, [Const64(c), I64And] => AndConstTee64(c, dst)); + rewrite!(instrs, i, [Const64(c), I64Sub] => SubConstTee64(c, dst)); } LocalTee128(dst) => { - rewrite!(instrs, i, [LocalGet128(src)] if (src == dst) => [LocalGet128(src), Nop]); - rewrite!(instrs, i, [BinOpLocalLocal128(op, a, b)] => BinOpLocalLocalTee128(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConst128(op, src, c)] => BinOpLocalConstTee128(op, src, c, dst)); + fold_local_binop!( + instrs, i, dst, + source = resolve_local_source_128, + op = bin_op_128, + const = const_128, + local_local = BinOpLocalLocalTee128, + local_const = |dst, lhs, op, imm| Instruction::BinOpLocalConstTee128(op, lhs, imm, dst) + ); + rewrite_local_tee_direct!( + instrs, + i, + dst, + get = LocalGet128, + binop_local_local = BinOpLocalLocal128, + binop_local_local_tee = BinOpLocalLocalTee128, + binop_local_const = BinOpLocalConst128, + binop_local_const_tee = BinOpLocalConstTee128 + ); rewrite!(instrs, i, [LocalGet32(addr), V128Load(memarg)] if (let (Ok(addr), Ok(dst)) = (u8::try_from(addr), u8::try_from(dst))) => LoadLocalTee128(memarg, addr, dst) ); } - Drop32 => { - rewrite!(instrs, i, [LocalTee32(local)] => [LocalSet32(local), Nop]); - rewrite!(instrs, i, [BinOpLocalLocalTee32(op, a, b, dst)] => BinOpLocalLocalSet32(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConstTee32(op, src, c, dst)] => BinOpLocalConstSet32(op, src, c, dst)); - } - Drop64 => { - rewrite!(instrs, i, [LocalTee64(local)] => [LocalSet64(local), Nop]); - rewrite!(instrs, i, [BinOpLocalLocalTee64(op, a, b, dst)] => BinOpLocalLocalSet64(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConstTee64(op, src, c, dst)] => BinOpLocalConstSet64(op, src, c, dst)); - } - Drop128 => { - rewrite!(instrs, i, [LocalTee128(local)] => [LocalSet128(local), Nop]); - rewrite!(instrs, i, [BinOpLocalLocalTee128(op, a, b, dst)] => BinOpLocalLocalSet128(op, a, b, dst)); - rewrite!(instrs, i, [BinOpLocalConstTee128(op, src, c, dst)] => BinOpLocalConstSet128(op, src, c, dst)); + Drop32 => rewrite_drop_tee_direct!( + instrs, + i, + tee = LocalTee32, + set = LocalSet32, + binop_local_local_tee = BinOpLocalLocalTee32, + binop_local_local_set = BinOpLocalLocalSet32, + binop_local_const_tee = BinOpLocalConstTee32, + binop_local_const_set = BinOpLocalConstSet32 + ), + Drop64 => rewrite_drop_tee_direct!( + instrs, + i, + tee = LocalTee64, + set = LocalSet64, + binop_local_local_tee = BinOpLocalLocalTee64, + binop_local_local_set = BinOpLocalLocalSet64, + binop_local_const_tee = BinOpLocalConstTee64, + binop_local_const_set = BinOpLocalConstSet64 + ), + Drop128 => rewrite_drop_tee_direct!( + instrs, + i, + tee = LocalTee128, + set = LocalSet128, + binop_local_local_tee = BinOpLocalLocalTee128, + binop_local_local_set = BinOpLocalLocalSet128, + binop_local_const_tee = BinOpLocalConstTee128, + binop_local_const_set = BinOpLocalConstSet128 + ), + Jump(ip) => { + let target = resolve_jump_target(instrs, ip); + if target == next_non_nop(instrs, i + 1) as u32 { + instrs[i] = Nop; + } else if target != ip { + instrs[i] = Jump(target); + } } JumpIfZero(ip) => { + let target = resolve_jump_target(instrs, ip); + rewrite!(instrs, i, [LocalGet32(local), I32Eqz] => { + replace!(instrs, i, 2 => [Nop, Nop, JumpIfLocalNonZero32 { target_ip: target, local }]); + continue; + }); rewrite!(instrs, i, [I32Eqz] => { - replace!(instrs, i, 1 => [Nop, JumpIfNonZero32(ip)]); + replace!(instrs, i, 1 => [Nop, JumpIfNonZero32(target)]); + continue; + }); + rewrite!(instrs, i, [LocalGet32(local)] => JumpIfLocalZero32 { target_ip: target, local }); + rewrite!(instrs, i, [LocalGet64(local), I64Eqz] => { + replace!(instrs, i, 2 => [Nop, Nop, JumpIfLocalNonZero64 { target_ip: target, local }]); continue; }); rewrite!(instrs, i, [I64Eqz] => { - replace!(instrs, i, 1 => [Nop, JumpIfNonZero64(ip)]); + replace!(instrs, i, 1 => [Nop, JumpIfNonZero64(target)]); continue; }); rewrite!(instrs, i, - [LocalGet32(local), I32Const(imm), cmp] if (let Some(op) = cmp_op(cmp)) => - JumpCmpLocalConst32 { target_ip: ip, local, imm, op: inverse_cmp_op(op) } + [LocalGet32(local), Const32(imm), cmp] if (let Some(op) = cmp_op(cmp)) => + match (imm, inverse_cmp_op(op)) { + (0, CmpOp::Eq) => JumpIfLocalZero32 { target_ip: target, local }, + (0, CmpOp::Ne) => JumpIfLocalNonZero32 { target_ip: target, local }, + (imm, op) => JumpCmpLocalConst32 { target_ip: target, local, imm, op }, + } ); rewrite!(instrs, i, - [LocalGet64(local), I64Const(imm), cmp] if + [LocalGet64(local), Const64(imm), cmp] if (let Some(op) = cmp_op_64(cmp) && let Ok(imm) = i32::try_from(imm)) => - JumpCmpLocalConst64 { target_ip: ip, local, imm, op: inverse_cmp_op(op) } + match (imm, inverse_cmp_op(op)) { + (0, CmpOp::Eq) => JumpIfLocalZero64 { target_ip: target, local }, + (0, CmpOp::Ne) => JumpIfLocalNonZero64 { target_ip: target, local }, + (imm, op) => JumpCmpLocalConst64 { target_ip: target, local, imm, op }, + } ); rewrite!(instrs, i, [LocalGet32(left), LocalGet32(right), cmp] if (let Some(op) = cmp_op(cmp)) => - JumpCmpLocalLocal32 { target_ip: ip, left, right, op: inverse_cmp_op(op) } + JumpCmpLocalLocal32 { target_ip: target, left, right, op: inverse_cmp_op(op) } ); rewrite!(instrs, i, [LocalGet64(left), LocalGet64(right), cmp] if (let Some(op) = cmp_op_64(cmp)) => - JumpCmpLocalLocal64 { target_ip: ip, left, right, op: inverse_cmp_op(op) } + JumpCmpLocalLocal64 { target_ip: target, left, right, op: inverse_cmp_op(op) } ); - rewrite!(instrs, i, [I32Const(imm), cmp] if (let Some(op) = cmp_op(cmp)) => match (imm, inverse_cmp_op(op)) { - (0, CmpOp::Eq) => JumpIfZero32(ip), - (0, CmpOp::Ne) => JumpIfNonZero32(ip), - (imm, op) => JumpCmpStackConst32 { target_ip: ip, imm, op }, + rewrite!(instrs, i, [Const32(imm), cmp] if (let Some(op) = cmp_op(cmp)) => match (imm, inverse_cmp_op(op)) { + (0, CmpOp::Eq) => JumpIfZero32(target), + (0, CmpOp::Ne) => JumpIfNonZero32(target), + (imm, op) => JumpCmpStackConst32 { target_ip: target, imm, op }, }); - rewrite!(instrs, i, [I64Const(imm), cmp] if (let Some(op) = cmp_op_64(cmp)) => match (imm, inverse_cmp_op(op)) { - (0, CmpOp::Eq) => JumpIfZero64(ip), - (0, CmpOp::Ne) => JumpIfNonZero64(ip), - (imm, op) => JumpCmpStackConst64 { target_ip: ip, imm, op }, + rewrite!(instrs, i, [Const64(imm), cmp] if (let Some(op) = cmp_op_64(cmp)) => match (imm, inverse_cmp_op(op)) { + (0, CmpOp::Eq) => JumpIfZero64(target), + (0, CmpOp::Ne) => JumpIfNonZero64(target), + (imm, op) => JumpCmpStackConst64 { target_ip: target, imm, op }, }); + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + if let JumpIfZero(current) = &mut instrs[i] { + *current = target; + } } JumpIfNonZero(ip) => { + let target = resolve_jump_target(instrs, ip); + rewrite!(instrs, i, [LocalGet32(local), I32Eqz] => { + replace!(instrs, i, 2 => [Nop, Nop, JumpIfLocalZero32 { target_ip: target, local }]); + continue; + }); rewrite!(instrs, i, [I32Eqz] => { - replace!(instrs, i, 1 => [Nop, JumpIfZero32(ip)]); + replace!(instrs, i, 1 => [Nop, JumpIfZero32(target)]); + continue; + }); + rewrite!(instrs, i, [LocalGet32(local)] => JumpIfLocalNonZero32 { target_ip: target, local }); + rewrite!(instrs, i, [LocalGet64(local), I64Eqz] => { + replace!(instrs, i, 2 => [Nop, Nop, JumpIfLocalZero64 { target_ip: target, local }]); continue; }); rewrite!(instrs, i, [I64Eqz] => { - replace!(instrs, i, 1 => [Nop, JumpIfZero64(ip)]); + replace!(instrs, i, 1 => [Nop, JumpIfZero64(target)]); continue; }); rewrite!(instrs, i, - [LocalGet32(local), I32Const(imm), cmp] if (let Some(op) = cmp_op(cmp)) => - JumpCmpLocalConst32 { target_ip: ip, local, imm, op } + [LocalGet32(local), Const32(imm), cmp] if (let Some(op) = cmp_op(cmp)) => + match (imm, op) { + (0, CmpOp::Eq) => JumpIfLocalZero32 { target_ip: target, local }, + (0, CmpOp::Ne) => JumpIfLocalNonZero32 { target_ip: target, local }, + (imm, op) => JumpCmpLocalConst32 { target_ip: target, local, imm, op }, + } ); rewrite!(instrs, i, - [LocalGet64(local), I64Const(imm), cmp] if + [LocalGet64(local), Const64(imm), cmp] if (let Some(op) = cmp_op_64(cmp) && let Ok(imm) = i32::try_from(imm)) => - JumpCmpLocalConst64 { target_ip: ip, local, imm, op } + match (imm, op) { + (0, CmpOp::Eq) => JumpIfLocalZero64 { target_ip: target, local }, + (0, CmpOp::Ne) => JumpIfLocalNonZero64 { target_ip: target, local }, + (imm, op) => JumpCmpLocalConst64 { target_ip: target, local, imm, op }, + } ); rewrite!(instrs, i, [LocalGet32(left), LocalGet32(right), cmp] if (let Some(op) = cmp_op(cmp)) => - JumpCmpLocalLocal32 { target_ip: ip, left, right, op } + JumpCmpLocalLocal32 { target_ip: target, left, right, op } ); rewrite!(instrs, i, [LocalGet64(left), LocalGet64(right), cmp] if (let Some(op) = cmp_op_64(cmp)) => - JumpCmpLocalLocal64 { target_ip: ip, left, right, op } + JumpCmpLocalLocal64 { target_ip: target, left, right, op } ); - rewrite!(instrs, i, [I32Const(imm), cmp] if (let Some(op) = cmp_op(cmp)) => match (imm, op) { - (0, CmpOp::Eq) => JumpIfZero32(ip), - (0, CmpOp::Ne) => JumpIfNonZero32(ip), - (imm, op) => JumpCmpStackConst32 { target_ip: ip, imm, op }, + rewrite!(instrs, i, [Const32(imm), cmp] if (let Some(op) = cmp_op(cmp)) => match (imm, op) { + (0, CmpOp::Eq) => JumpIfZero32(target), + (0, CmpOp::Ne) => JumpIfNonZero32(target), + (imm, op) => JumpCmpStackConst32 { target_ip: target, imm, op }, }); - rewrite!(instrs, i, [I64Const(imm), cmp] if (let Some(op) = cmp_op_64(cmp)) => match (imm, op) { - (0, CmpOp::Eq) => JumpIfZero64(ip), - (0, CmpOp::Ne) => JumpIfNonZero64(ip), - (imm, op) => JumpCmpStackConst64 { target_ip: ip, imm, op }, + rewrite!(instrs, i, [Const64(imm), cmp] if (let Some(op) = cmp_op_64(cmp)) => match (imm, op) { + (0, CmpOp::Eq) => JumpIfZero64(target), + (0, CmpOp::Ne) => JumpIfNonZero64(target), + (imm, op) => JumpCmpStackConst64 { target_ip: target, imm, op }, }); + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + if let JumpIfNonZero(current) = &mut instrs[i] { + *current = target; + } + } + JumpIfZero32(ip) => { + let target = resolve_jump_target(instrs, ip); + rewrite!(instrs, i, [LocalGet32(local)] => JumpIfLocalZero32 { target_ip: target, local }); + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + if let JumpIfZero32(current) = &mut instrs[i] { + *current = target; + } + } + JumpIfNonZero32(ip) => { + let target = resolve_jump_target(instrs, ip); + rewrite!(instrs, i, [LocalGet32(local)] => JumpIfLocalNonZero32 { target_ip: target, local }); + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + if let JumpIfNonZero32(current) = &mut instrs[i] { + *current = target; + } + } + JumpIfZero64(ip) => { + let target = resolve_jump_target(instrs, ip); + rewrite!(instrs, i, [LocalGet64(local)] => JumpIfLocalZero64 { target_ip: target, local }); + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + if let JumpIfZero64(current) = &mut instrs[i] { + *current = target; + } + } + JumpIfNonZero64(ip) => { + let target = resolve_jump_target(instrs, ip); + rewrite!(instrs, i, [LocalGet64(local)] => JumpIfLocalNonZero64 { target_ip: target, local }); + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + if let JumpIfNonZero64(current) = &mut instrs[i] { + *current = target; + } + } + JumpCmpStackConst32 { target_ip, imm: 0, op } => { + match op { + CmpOp::Eq => instrs[i] = JumpIfZero32(target_ip), + CmpOp::Ne => instrs[i] = JumpIfNonZero32(target_ip), + _ => {} + } + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + } + JumpCmpStackConst64 { target_ip, imm: 0, op } => { + match op { + CmpOp::Eq => instrs[i] = JumpIfZero64(target_ip), + CmpOp::Ne => instrs[i] = JumpIfNonZero64(target_ip), + _ => {} + } + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + } + JumpCmpLocalConst32 { target_ip, local, imm: 0, op } => { + match op { + CmpOp::Eq => instrs[i] = JumpIfLocalZero32 { target_ip, local }, + CmpOp::Ne => instrs[i] = JumpIfLocalNonZero32 { target_ip, local }, + _ => {} + } + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + } + JumpCmpLocalConst64 { target_ip, local, imm: 0, op } => { + match op { + CmpOp::Eq => instrs[i] = JumpIfLocalZero64 { target_ip, local }, + CmpOp::Ne => instrs[i] = JumpIfLocalNonZero64 { target_ip, local }, + _ => {} + } + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); + } + JumpCmpStackConst32 { .. } + | JumpCmpStackConst64 { .. } + | JumpCmpLocalConst32 { .. } + | JumpCmpLocalConst64 { .. } + | JumpCmpLocalLocal32 { .. } + | JumpCmpLocalLocal64 { .. } + | JumpIfLocalZero32 { .. } + | JumpIfLocalNonZero32 { .. } + | JumpIfLocalZero64 { .. } + | JumpIfLocalNonZero64 { .. } => { + if optimize_branch_inversion { + invert_conditional_over_jump(instrs, i); + } + canonicalize_jump_like(instrs, i); } - JumpCmpStackConst32 { target_ip, imm: 0, op } => match op { - CmpOp::Eq => instrs[i] = JumpIfZero32(target_ip), - CmpOp::Ne => instrs[i] = JumpIfNonZero32(target_ip), - _ => {} - }, - JumpCmpStackConst64 { target_ip, imm: 0, op } => match op { - CmpOp::Eq => instrs[i] = JumpIfZero64(target_ip), - CmpOp::Ne => instrs[i] = JumpIfNonZero64(target_ip), - _ => {} - }, _ => {} } @@ -437,31 +680,71 @@ fn scalar_bin_op_32(instr: Instruction) -> Option<BinOp> { int_bin_op_32(instr).or_else(|| float_bin_op_32(instr)) } -fn stack_source_local_32(instr: Instruction) -> Option<(Instruction, u16)> { - Some(match instr { - Instruction::LocalGet32(local) => (Instruction::Nop, local), - Instruction::LocalTee32(local) => (Instruction::LocalSet32(local), local), - Instruction::BinOpLocalLocalTee32(op, a, b, local) => { - (Instruction::BinOpLocalLocalSet32(op, a, b, local), local) - } - Instruction::BinOpLocalConstTee32(op, src, c, local) => { - (Instruction::BinOpLocalConstSet32(op, src, c, local), local) - } - Instruction::LoadLocalTee32(memarg, addr, local) => { - (Instruction::LoadLocalSet32(memarg, addr, local), local.into()) - } - _ => return None, - }) +fn scalar_bin_op_64(instr: Instruction) -> Option<BinOp> { + int_bin_op_64(instr).or_else(|| float_bin_op_64(instr)) } fn scalar_const_32(instr: Instruction, op_instr: Instruction) -> Option<i32> { match instr { - Instruction::I32Const(c) if int_bin_op_32(op_instr).is_some() => Some(c), - Instruction::F32Const(c) if float_bin_op_32(op_instr).is_some() => Some(f32_const_bits(c)), + Instruction::Const32(c) if int_bin_op_32(op_instr).is_some() || float_bin_op_32(op_instr).is_some() => Some(c), + _ => None, + } +} + +fn scalar_const_64(instr: Instruction, op_instr: Instruction) -> Option<i64> { + match instr { + Instruction::Const64(c) if int_bin_op_64(op_instr).is_some() || float_bin_op_64(op_instr).is_some() => Some(c), _ => None, } } +fn const_128(instr: Instruction, op_instr: Instruction) -> Option<ConstIdx> { + match instr { + Instruction::Const128(c) if bin_op_128(op_instr).is_some() => Some(c), + _ => None, + } +} + +define_local_source_resolver!( + resolve_local_source_32, + get = LocalGet32, + tee = LocalTee32, + set = LocalSet32, + copy = LocalCopy32, + binop_local_local_tee = BinOpLocalLocalTee32, + binop_local_local_set = BinOpLocalLocalSet32, + binop_local_const_tee = BinOpLocalConstTee32, + binop_local_const_set = BinOpLocalConstSet32, + load_local_tee = LoadLocalTee32, + load_local_set = LoadLocalSet32 +); + +define_local_source_resolver!( + resolve_local_source_64, + get = LocalGet64, + tee = LocalTee64, + set = LocalSet64, + copy = LocalCopy64, + binop_local_local_tee = BinOpLocalLocalTee64, + binop_local_local_set = BinOpLocalLocalSet64, + binop_local_const_tee = BinOpLocalConstTee64, + binop_local_const_set = BinOpLocalConstSet64 +); + +define_local_source_resolver!( + resolve_local_source_128, + get = LocalGet128, + tee = LocalTee128, + set = LocalSet128, + copy = LocalCopy128, + binop_local_local_tee = BinOpLocalLocalTee128, + binop_local_local_set = BinOpLocalLocalSet128, + binop_local_const_tee = BinOpLocalConstTee128, + binop_local_const_set = BinOpLocalConstSet128, + load_local_tee = LoadLocalTee128, + load_local_set = LoadLocalSet128 +); + fn bin_op_128(instr: Instruction) -> Option<BinOp128> { Some(match instr { Instruction::V128And => BinOp128::And, @@ -474,14 +757,6 @@ fn bin_op_128(instr: Instruction) -> Option<BinOp128> { }) } -fn f32_const_bits(value: f32) -> i32 { - i32::from_ne_bytes(value.to_bits().to_ne_bytes()) -} - -fn f64_const_bits(value: f64) -> i64 { - i64::from_ne_bytes(value.to_bits().to_ne_bytes()) -} - fn cmp_op_64(instr: Instruction) -> Option<CmpOp> { Some(match instr { Instruction::I64Eq => CmpOp::Eq, @@ -513,8 +788,8 @@ fn inverse_cmp_op(op: CmpOp) -> CmpOp { } } -fn previous_non_nop_3(instrs: &[Instruction], read: usize) -> Option<[(usize, Instruction); 3]> { - let mut out = [(0usize, Instruction::Nop); 3]; +fn previous_non_nop<const N: usize>(instrs: &[Instruction], read: usize) -> Option<[(usize, Instruction); N]> { + let mut out = [(0usize, Instruction::Nop); N]; let mut filled = 0usize; for idx in (0..read).rev() { @@ -526,9 +801,9 @@ fn previous_non_nop_3(instrs: &[Instruction], read: usize) -> Option<[(usize, In continue; } - out[2 - filled] = (idx, instr); + out[N - 1 - filled] = (idx, instr); filled += 1; - if filled == 3 { + if filled == N { return Some(out); } } @@ -536,6 +811,149 @@ fn previous_non_nop_3(instrs: &[Instruction], read: usize) -> Option<[(usize, In None } +fn next_non_nop(instrs: &[Instruction], mut idx: usize) -> usize { + while idx < instrs.len() && matches!(instrs[idx], Instruction::Nop | Instruction::MergeBarrier) { + idx += 1; + } + idx +} + +fn resolve_jump_target(instrs: &[Instruction], target: u32) -> u32 { + let mut idx = next_non_nop(instrs, target as usize); + let mut steps = 0usize; + + while idx < instrs.len() && steps < instrs.len() { + match instrs[idx] { + Instruction::Jump(next) => { + idx = next_non_nop(instrs, next as usize); + steps += 1; + } + _ => break, + } + } + + idx as u32 +} + +fn jump_target(instr: Instruction) -> Option<u32> { + Some(match instr { + Instruction::Jump(ip) + | Instruction::JumpIfZero(ip) + | Instruction::JumpIfNonZero(ip) + | Instruction::JumpIfZero32(ip) + | Instruction::JumpIfNonZero32(ip) + | Instruction::JumpIfZero64(ip) + | Instruction::JumpIfNonZero64(ip) => ip, + Instruction::JumpCmpStackConst32 { target_ip, .. } + | Instruction::JumpCmpStackConst64 { target_ip, .. } + | Instruction::JumpIfLocalZero32 { target_ip, .. } + | Instruction::JumpIfLocalNonZero32 { target_ip, .. } + | Instruction::JumpIfLocalZero64 { target_ip, .. } + | Instruction::JumpIfLocalNonZero64 { target_ip, .. } + | Instruction::JumpCmpLocalConst32 { target_ip, .. } + | Instruction::JumpCmpLocalConst64 { target_ip, .. } + | Instruction::JumpCmpLocalLocal32 { target_ip, .. } + | Instruction::JumpCmpLocalLocal64 { target_ip, .. } => target_ip, + _ => return None, + }) +} + +fn set_jump_target(instr: &mut Instruction, target: u32) { + match instr { + Instruction::Jump(ip) + | Instruction::JumpIfZero(ip) + | Instruction::JumpIfNonZero(ip) + | Instruction::JumpIfZero32(ip) + | Instruction::JumpIfNonZero32(ip) + | Instruction::JumpIfZero64(ip) + | Instruction::JumpIfNonZero64(ip) + | Instruction::JumpCmpStackConst32 { target_ip: ip, .. } + | Instruction::JumpCmpStackConst64 { target_ip: ip, .. } + | Instruction::JumpIfLocalZero32 { target_ip: ip, .. } + | Instruction::JumpIfLocalNonZero32 { target_ip: ip, .. } + | Instruction::JumpIfLocalZero64 { target_ip: ip, .. } + | Instruction::JumpIfLocalNonZero64 { target_ip: ip, .. } + | Instruction::JumpCmpLocalConst32 { target_ip: ip, .. } + | Instruction::JumpCmpLocalConst64 { target_ip: ip, .. } + | Instruction::JumpCmpLocalLocal32 { target_ip: ip, .. } + | Instruction::JumpCmpLocalLocal64 { target_ip: ip, .. } => *ip = target, + _ => {} + } +} + +fn invert_jump(instr: Instruction, target: u32) -> Option<Instruction> { + Some(match instr { + Instruction::JumpCmpStackConst32 { imm, op, .. } => { + Instruction::JumpCmpStackConst32 { target_ip: target, imm, op: inverse_cmp_op(op) } + } + Instruction::JumpCmpStackConst64 { imm, op, .. } => { + Instruction::JumpCmpStackConst64 { target_ip: target, imm, op: inverse_cmp_op(op) } + } + Instruction::JumpCmpLocalConst32 { local, imm, op, .. } => { + Instruction::JumpCmpLocalConst32 { target_ip: target, local, imm, op: inverse_cmp_op(op) } + } + Instruction::JumpCmpLocalConst64 { local, imm, op, .. } => { + Instruction::JumpCmpLocalConst64 { target_ip: target, local, imm, op: inverse_cmp_op(op) } + } + Instruction::JumpCmpLocalLocal32 { left, right, op, .. } => { + Instruction::JumpCmpLocalLocal32 { target_ip: target, left, right, op: inverse_cmp_op(op) } + } + Instruction::JumpCmpLocalLocal64 { left, right, op, .. } => { + Instruction::JumpCmpLocalLocal64 { target_ip: target, left, right, op: inverse_cmp_op(op) } + } + _ => return None, + }) +} + +fn canonicalize_jump_like(instrs: &mut [Instruction], idx: usize) { + let Some(target) = jump_target(instrs[idx]) else { + return; + }; + + let target = resolve_jump_target(instrs, target); + if matches!(instrs[idx], Instruction::Jump(_)) && target == next_non_nop(instrs, idx + 1) as u32 { + instrs[idx] = Instruction::Nop; + } else { + set_jump_target(&mut instrs[idx], target); + } +} + +fn invert_conditional_over_jump(instrs: &mut [Instruction], idx: usize) { + let Some(target) = jump_target(instrs[idx]) else { + return; + }; + if matches!(instrs[idx], Instruction::Jump(_)) { + return; + } + + let target_idx = next_non_nop(instrs, target as usize); + if target_idx >= instrs.len() || target_idx <= idx + 1 { + return; + } + + let Some(jump_idx) = ((idx + 1)..target_idx) + .rev() + .find(|&candidate| !matches!(instrs[candidate], Instruction::Nop | Instruction::MergeBarrier)) + else { + return; + }; + + let Instruction::Jump(exit_target) = instrs[jump_idx] else { + return; + }; + if next_non_nop(instrs, jump_idx + 1) != target_idx { + return; + } + + let exit_target = resolve_jump_target(instrs, exit_target); + let Some(inverted) = invert_jump(instrs[idx], exit_target) else { + return; + }; + + instrs[idx] = inverted; + instrs[jump_idx] = Instruction::Nop; +} + fn remove_nop(instructions: &mut Vec<Instruction>, function_data: &mut WasmFunctionData) { let old_len = instructions.len(); if old_len == 0 { @@ -574,6 +992,10 @@ fn remove_nop(instructions: &mut Vec<Instruction>, function_data: &mut WasmFunct | Instruction::JumpIfNonZero32(ip) | Instruction::JumpIfZero64(ip) | Instruction::JumpIfNonZero64(ip) + | Instruction::JumpIfLocalZero32 { target_ip: ip, .. } + | Instruction::JumpIfLocalNonZero32 { target_ip: ip, .. } + | Instruction::JumpIfLocalZero64 { target_ip: ip, .. } + | Instruction::JumpIfLocalNonZero64 { target_ip: ip, .. } | Instruction::JumpCmpStackConst32 { target_ip: ip, .. } | Instruction::JumpCmpStackConst64 { target_ip: ip, .. } | Instruction::JumpCmpLocalConst32 { target_ip: ip, .. } diff --git a/crates/parser/src/visit.rs b/crates/parser/src/visit.rs index 5e1ca15..9a094ed 100644 --- a/crates/parser/src/visit.rs +++ b/crates/parser/src/visit.rs @@ -108,7 +108,7 @@ impl<'a, R: WasmModuleResources> wasmparser::VisitOperator<'a> for FunctionBuild define_operands! { // basic instructions - visit_global_get(GlobalGet, u32), visit_i32_const(I32Const, i32), visit_i64_const(I64Const, i64), visit_return(Return), + visit_global_get(GlobalGet, u32), visit_i32_const(Const32, i32), visit_i64_const(Const64, i64), visit_return(Return), visit_call(Call, u32), visit_call_indirect(CallIndirect, u32, u32), visit_return_call_indirect(ReturnCallIndirect, u32, u32), visit_return_call(ReturnCall, u32), visit_memory_size(MemorySize, u32), visit_memory_grow(MemoryGrow, u32), visit_unreachable(Unreachable), visit_nop(Nop), visit_i32_eqz(I32Eqz), visit_i32_eq(I32Eq), visit_i32_ne(I32Ne), visit_i32_lt_s(I32LtS), visit_i32_lt_u(I32LtU), @@ -360,11 +360,11 @@ impl<'a, R: WasmModuleResources> wasmparser::VisitOperator<'a> for FunctionBuild } fn visit_f32_const(&mut self, val: wasmparser::Ieee32) -> Self::Output { - self.instructions.push(Instruction::F32Const(f32::from_bits(val.bits()))); + self.instructions.push(Instruction::Const32(i32::from_ne_bytes(val.bits().to_ne_bytes()))); } fn visit_f64_const(&mut self, val: wasmparser::Ieee64) -> Self::Output { - self.instructions.push(Instruction::F64Const(f64::from_bits(val.bits()))); + self.instructions.push(Instruction::Const64(i64::from_ne_bytes(val.bits().to_ne_bytes()))); } fn visit_table_copy(&mut self, dst_table: u32, src_table: u32) -> Self::Output { @@ -485,7 +485,7 @@ impl<R: WasmModuleResources> wasmparser::VisitSimdOperator<'_> for FunctionBuild } fn visit_v128_const(&mut self, value: wasmparser::V128) -> Self::Output { - self.instructions.push(Instruction::V128Const(self.data.v128_constants.len() as u32)); + self.instructions.push(Instruction::Const128(self.data.v128_constants.len() as u32)); self.data.v128_constants.push(*value.bytes()); } } |