diff options
| -rw-r--r-- | crates/parser/src/conversion.rs | 52 | ||||
| -rw-r--r-- | crates/parser/src/error.rs | 4 | ||||
| -rw-r--r-- | crates/parser/src/lib.rs | 53 | ||||
| -rw-r--r-- | crates/parser/src/macros.rs | 297 | ||||
| -rw-r--r-- | crates/parser/src/module.rs | 237 | ||||
| -rw-r--r-- | crates/parser/src/optimize.rs | 369 | ||||
| -rw-r--r-- | crates/parser/src/parallel.rs | 34 | ||||
| -rw-r--r-- | crates/parser/src/visit.rs | 1295 | ||||
| -rw-r--r-- | crates/tinywasm/src/interpreter/executor.rs | 20 | ||||
| -rw-r--r-- | crates/tinywasm/tests/wasm-custom/mixed-width-branch-shaping.wast | 16 | ||||
| -rw-r--r-- | crates/types/src/instructions.rs | 22 |
11 files changed, 1454 insertions, 945 deletions
diff --git a/crates/parser/src/conversion.rs b/crates/parser/src/conversion.rs index 99cf1ba..34dcdfd 100644 --- a/crates/parser/src/conversion.rs +++ b/crates/parser/src/conversion.rs @@ -136,42 +136,44 @@ pub(crate) fn convert_module_export(export: wasmparser::Export<'_>) -> Result<Ex pub(crate) fn convert_module_code( func: wasmparser::FunctionBody<'_>, - mut validator: FuncValidator<ValidatorResources>, + mut validator: Option<FuncValidator<ValidatorResources>>, reader_allocs: OperatorsReaderAllocations, -) -> Result<(FunctionCode, FuncValidatorAllocations, OperatorsReaderAllocations)> { + metadata: &crate::visit::ModuleMetadata, + ty_idx: u32, +) -> Result<(FunctionCode, Option<FuncValidatorAllocations>, OperatorsReaderAllocations)> { let locals_reader = func.get_locals_reader()?; - let count = locals_reader.get_count(); let pos = locals_reader.original_position(); - - // maps a local's address to the index in the type's locals array - let mut local_addr_map = Vec::with_capacity(count as usize); - let mut local_counts = ValueCounts::default(); + let signature = metadata.signature(ty_idx)?.clone(); + let mut local_types = signature.params.clone(); for (i, local) in locals_reader.into_iter().enumerate() { let local = local?; - validator.define_locals(pos + i, local.0, local.1)?; + if let Some(validator) = validator.as_mut() { + validator.define_locals(pos + i, local.0, local.1)?; + } + let size = crate::visit::OperandSize::from(local.1); + let count = usize::try_from(local.0) + .map_err(|_| crate::ParseError::Other("local declaration count is too large".into()))?; + local_types.reserve(count); + local_types.extend(core::iter::repeat_n(size, count)); } - for i in 0..validator.len_locals() { - match validator.get_local_type(i) { - Some(wasmparser::ValType::I32 | wasmparser::ValType::F32 | wasmparser::ValType::Ref(_)) => { - local_addr_map.push(local_counts.c32); - local_counts.c32 += 1; - } - Some(wasmparser::ValType::I64 | wasmparser::ValType::F64) => { - local_addr_map.push(local_counts.c64); - local_counts.c64 += 1; - } - Some(wasmparser::ValType::V128) => { - local_addr_map.push(local_counts.c128); - local_counts.c128 += 1; - } - None => return Err(crate::ParseError::UnsupportedOperator("Unknown local type".to_string())), - } + // maps a local's address to the index in the type's locals array + let mut local_addr_map = Vec::with_capacity(local_types.len()); + let mut local_counts = ValueCounts::default(); + + for ty in &local_types { + let (count, error) = match ty { + crate::visit::OperandSize::S32 => (&mut local_counts.c32, "too many 32-bit locals"), + crate::visit::OperandSize::S64 => (&mut local_counts.c64, "too many 64-bit locals"), + crate::visit::OperandSize::S128 => (&mut local_counts.c128, "too many 128-bit locals"), + }; + local_addr_map.push(*count); + *count = count.checked_add(1).ok_or_else(|| crate::ParseError::Other(error.into()))?; } let (body, data, validator_allocs, reader_allocs) = - process_operators_and_validate(validator, func, local_addr_map, reader_allocs)?; + process_operators_and_validate(validator, func, local_types, local_addr_map, metadata, ty_idx, reader_allocs)?; Ok(( FunctionCode { instructions: body, data, locals: local_counts, uses_local_memory: false }, validator_allocs, diff --git a/crates/parser/src/error.rs b/crates/parser/src/error.rs index 7b5dc34..e4d8abd 100644 --- a/crates/parser/src/error.rs +++ b/crates/parser/src/error.rs @@ -60,8 +60,8 @@ impl Display for ParseError { impl core::error::Error for ParseError {} -impl From<wasmparser::BinaryReaderError> for ParseError { - fn from(value: wasmparser::BinaryReaderError) -> Self { +impl From<wasmparser::Error> for ParseError { + fn from(value: wasmparser::Error) -> Self { Self::ParseError { message: value.message().to_string(), offset: value.offset() } } } diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index 42fddf6..cc305a5 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -49,12 +49,12 @@ pub use tinywasm_types::Module; #[non_exhaustive] #[derive(Debug, Clone)] pub struct ParserOptions { + /// Whether to validate modules while parsing. + pub validation: bool, /// Whether to optimize local memory allocation by skipping allocation of unused local memories. pub optimize_local_memory_allocation: bool, /// Whether to run the peephole rewrite optimizer. pub optimize_rewrite: bool, - /// Whether to remove `Nop` and `MergeBarrier` instructions after rewriting. - pub optimize_remove_nop: bool, #[cfg(parallel_parser)] /// Number of threads to use for parallel parsing. @@ -70,9 +70,9 @@ pub struct ParserOptions { impl Default for ParserOptions { fn default() -> Self { Self { + validation: true, optimize_local_memory_allocation: true, optimize_rewrite: true, - optimize_remove_nop: true, #[cfg(parallel_parser)] parser_threads: None, } @@ -80,6 +80,17 @@ impl Default for ParserOptions { } impl ParserOptions { + /// Enable or disable WebAssembly validation. + pub const fn with_validation(mut self, enabled: bool) -> Self { + self.validation = enabled; + self + } + + /// Returns whether WebAssembly validation is enabled. + pub const fn validation(&self) -> bool { + self.validation + } + /// Enable or disable the optimization that skips allocating unused local memories. pub const fn with_local_memory_allocation_optimization(mut self, enabled: bool) -> Self { self.optimize_local_memory_allocation = enabled; @@ -102,17 +113,6 @@ impl ParserOptions { self.optimize_rewrite } - /// Enable or disable `Nop`/`MergeBarrier` removal after rewriting. - pub const fn with_nop_removal_optimization(mut self, enabled: bool) -> Self { - self.optimize_remove_nop = enabled; - self - } - - /// Returns whether `Nop`/`MergeBarrier` removal is enabled. - pub const fn optimize_remove_nop(&self) -> bool { - self.optimize_remove_nop - } - #[cfg(parallel_parser)] /// Set the number of threads for parallel parsing. /// @@ -188,18 +188,18 @@ impl Parser { /// Parse a [`Module`] from bytes pub fn parse_module_bytes(&self, wasm: impl AsRef<[u8]>) -> Result<Module> { let wasm = wasm.as_ref(); - let mut validator = Self::create_validator(self.options.clone()); + let mut validator = self.options.validation().then(|| Self::create_validator(self.options.clone())); let mut reader = ModuleReader::default(); for payload in wasmparser::Parser::new(0).parse_all(wasm) { match payload? { wasmparser::Payload::CodeSectionStart { count, range, size } => { - reader.begin_code_section(count, range, size, &mut validator, &self.options)?; + reader.begin_code_section(count, range, size, validator.as_mut(), &self.options)?; } wasmparser::Payload::CodeSectionEntry(function) => { - reader.process_borrowed_code_section_entry(function, &mut validator, &self.options)?; + reader.process_borrowed_code_section_entry(function, validator.as_mut(), &self.options)?; } - payload => reader.process_payload(payload, &mut validator)?, + payload => reader.process_payload(payload, validator.as_mut())?, } } @@ -222,7 +222,7 @@ impl Parser { #[cfg(feature = "std")] /// Parse a [`Module`] from a stream. Requires `std` feature. pub fn parse_module_stream(&self, mut stream: impl std::io::Read) -> Result<Module> { - let mut validator = Self::create_validator(self.options.clone()); + let mut validator = self.options.validation().then(|| Self::create_validator(self.options.clone())); let mut reader = ModuleReader::default(); let mut buffer = alloc::vec::Vec::new(); let mut parser = wasmparser::Parser::new(0); @@ -240,8 +240,13 @@ impl Parser { match payload { wasmparser::Payload::CodeSectionStart { count, range, size } => { - let defer = - reader.begin_code_section(count, range.clone(), size, &mut validator, &self.options)?; + let defer = reader.begin_code_section( + count, + range.clone(), + size, + validator.as_mut(), + &self.options, + )?; #[cfg(parallel_parser)] if defer { @@ -252,10 +257,10 @@ impl Parser { let _ = defer; } wasmparser::Payload::CodeSectionEntry(function) => { - reader.process_inline_code_section_entry(function, &mut validator, &self.options)?; + reader.process_inline_code_section_entry(function, validator.as_mut(), &self.options)?; } payload => { - reader.process_payload(payload, &mut validator)?; + reader.process_payload(payload, validator.as_mut())?; } } buffer.drain(..consumed); @@ -274,7 +279,7 @@ impl Parser { } let section_bytes = alloc::sync::Arc::<[u8]>::from(buffer[..section_size].to_vec()); - reader.queue_owned_code_section(count, body_offset, section_bytes, &mut validator)?; + reader.queue_owned_code_section(count, body_offset, section_bytes, validator.as_mut())?; parser.skip_section(); buffer.drain(..section_size); continue; diff --git a/crates/parser/src/macros.rs b/crates/parser/src/macros.rs index 8a5330a..ff216a5 100644 --- a/crates/parser/src/macros.rs +++ b/crates/parser/src/macros.rs @@ -2,12 +2,13 @@ pub(crate) mod visit { macro_rules! validate_then_visit { ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident ($($ann:tt)*))*) => {$( fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { - self.0.$visit($($($arg.clone()),*)?); - let validation = self.0.validator.visitor(self.0.position).$visit($($($arg),*)?); - if let Err(e) = validation { - cold_path(); - self.0.record_error(crate::ParseError::ParseError { message: e.to_string(), offset: self.0.position }); + if let Some(validator) = self.validator.as_mut() { + if let Err(e) = validator.visitor(self.position).$visit($($($arg.clone()),*)?) { + core::hint::cold_path(); + return Err(crate::ParseError::ParseError { message: e.to_string(), offset: self.position }); + } } + self.builder.$visit($($($arg),*)?) } )*}; } @@ -15,64 +16,103 @@ pub(crate) mod visit { macro_rules! validate_then_visit_simd { ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident ($($ann:tt)*))*) => {$( fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { - self.0.$visit($($($arg),*)?); - let validation = self.0.validator.simd_visitor(self.0.position).$visit($($($arg),*)?); - if let Err(e) = validation { - cold_path(); - self.0.record_error(crate::ParseError::ParseError { message: e.to_string(), offset: self.0.position }); + if let Some(validator) = self.validator.as_mut() { + if let Err(e) = validator.simd_visitor(self.position).$visit($($($arg.clone()),*)?) { + core::hint::cold_path(); + return Err(crate::ParseError::ParseError { message: e.to_string(), offset: self.position }); + } } + self.builder.$visit($($($arg),*)?) } )*}; } - macro_rules! define_operand { - ($name:ident($instr:expr, $ty:ty)) => { - fn $name(&mut self, arg: $ty) -> Self::Output { - self.instructions.push($instr(arg).into()); - } + macro_rules! lowering_ops { + () => {}; + ($kind:ident $inputs:tt => $outputs:tt { + $($visit:ident $(($($arg:ident: $ty:ty),+))? => $instr:ident),* $(,)? + } $($rest:tt)*) => { + $(lowering_ops!(@$kind $inputs => $outputs $visit $(($($arg: $ty),+))? => $instr);)* + lowering_ops!($($rest)*); + }; + (effect $inputs:tt => $outputs:tt { $($visit:ident),* $(,)? } $($rest:tt)*) => { + $(lowering_ops!(@effect $inputs => $outputs $visit);)* + lowering_ops!($($rest)*); }; - ($name:ident($instr:expr, $ty:ty, $ty2:ty)) => { - fn $name(&mut self, arg: $ty, arg2: $ty2) -> Self::Output { - self.instructions.push($instr(arg, arg2).into()); + (@fixed [$($input:ident),*] => [$($output:ident),*] + $visit:ident $(($($arg:ident: $ty:ty),+))? => $instr:ident + ) => { + fn $visit(&mut self $(, $($arg: $ty),+)?) -> Self::Output { + lowering_ops!(@emit self fixed [$($input),*] => [$($output),*] + Instruction::$instr $(($($arg),+))?.into()) } }; - - ($name:ident($instr:expr)) => { - fn $name(&mut self) -> Self::Output { - self.instructions.push($instr.into()); + (@memory [$($input:ident),*] => [$($output:ident),*] + $visit:ident $(($lane:ident: $ty:ty))? => $instr:ident + ) => { + fn $visit(&mut self, memarg: wasmparser::MemArg $(, $lane: $ty)?) -> Self::Output { + let address = self.metadata.memory_size(memarg.memory)?; + lowering_ops!(@emit self address(address) [$($input),*] => [$($output),*] + Instruction::$instr(MemoryArg::new(memarg.offset, memarg.memory) $(, $lane)?).into()) } }; - } - - macro_rules! define_operands { - ($($name:ident($instr:ident $(,$ty:ty)*)),*) => {$( - define_operand!($name(Instruction::$instr $(,$ty)*)); - )*}; - } - - macro_rules! define_mem_operands { - ($($name:ident($instr:ident)),*) => {$( - fn $name(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.instructions.push(Instruction::$instr(MemoryArg::new(memarg.offset, memarg.memory))); + (@global $inputs:tt => $outputs:tt $($operator:tt)*) => { + lowering_ops!(@resolved global_size $inputs => $outputs $($operator)*); + }; + (@memory_index $inputs:tt => $outputs:tt $($operator:tt)*) => { + lowering_ops!(@resolved memory_size $inputs => $outputs $($operator)*); + }; + (@table $inputs:tt => $outputs:tt $($operator:tt)*) => { + lowering_ops!(@resolved table_size $inputs => $outputs $($operator)*); + }; + (@resolved $resolver:ident [$($input:ident),*] => [$($output:ident),*] + $visit:ident($index:ident: $ty:ty) => $instr:ident + ) => { + fn $visit(&mut self, $index: $ty) -> Self::Output { + let address = self.metadata.$resolver($index)?; + lowering_ops!(@emit self address(address) [$($input),*] => [$($output),*] + Instruction::$instr($index).into()) } - )*}; - } - - macro_rules! define_mem_operands_simd { - ($($name:ident($instr:ident)),*) => {$( - fn $name(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.instructions.push(Instruction::$instr(MemoryArg::new(memarg.offset, memarg.memory)).into()); + }; + (@resolved $resolver:ident [$($input:ident),*] => [$($output:ident),*] + $visit:ident($arg:ident: $arg_ty:ty, $index:ident: $index_ty:ty) => $instr:ident + ) => { + fn $visit(&mut self, $arg: $arg_ty, $index: $index_ty) -> Self::Output { + let address = self.metadata.$resolver($index)?; + lowering_ops!(@emit self address(address) [$($input),*] => [$($output),*] + Instruction::$instr($arg, $index).into()) } - )*}; - } - - macro_rules! define_mem_operands_simd_lane { - ($($name:ident($instr:ident)),*) => {$( - fn $name(&mut self, memarg: wasmparser::MemArg, lane: u8) -> Self::Output { - self.instructions.push(Instruction::$instr(MemoryArg::new(memarg.offset, memarg.memory), lane).into()); + }; + (@effect [$($input:ident),*] => [$($output:ident),*] $visit:ident) => { + fn $visit(&mut self) -> Self::Output { + self.apply_effect(&[$(lowering_ops!(@size $input)),*], &[$(lowering_ops!(@size $output)),*]) } - )*}; + }; + (@terminating [$($input:ident),*] => [$($output:ident),*] $visit:ident => $instr:ident) => { + fn $visit(&mut self) -> Self::Output { + self.mark_unreachable(); + lowering_ops!(@emit self fixed [$($input),*] => [$($output),*] Instruction::$instr) + } + }; + + (@emit $self:ident fixed [$($input:ident),*] => [$($output:ident),*] $instruction:expr) => { + $self.emit( + &[$(lowering_ops!(@size $input)),*], + &[$(lowering_ops!(@size $output)),*], + $instruction, + ) + }; + (@emit $self:ident address($address:ident) [$($input:ident),*] => [$($output:ident),*] $instruction:expr) => { + $self.emit( + &[$(lowering_ops!(@size $input, $address)),*], + &[$(lowering_ops!(@size $output, $address)),*], + $instruction, + ) + }; + + (@size Addr, $address:ident) => { $address }; + (@size $size:ident $(, $address:ident)?) => { OperandSize::$size }; } macro_rules! impl_visit_operator { @@ -92,100 +132,58 @@ pub(crate) mod visit { (@@$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident ($($ann:tt)*)) => { fn $visit(&mut self $($(,_: $argty)*)?) -> Self::Output { - self.unsupported(stringify!($visit)) + Err(crate::ParseError::UnsupportedOperator(stringify!($visit).to_string())) } }; } - pub(crate) use { - define_mem_operands, define_mem_operands_simd, define_mem_operands_simd_lane, define_operand, define_operands, - impl_visit_operator, validate_then_visit, validate_then_visit_simd, - }; + pub(crate) use {impl_visit_operator, lowering_ops, validate_then_visit, validate_then_visit_simd}; } pub(crate) mod optimize { macro_rules! replace { - ($instructions:ident, $read:ident, $consumed:literal => [$($out:expr),+ $(,)?]) => {{ + ($instructions:ident, $read:ident, $consumed:expr => [$($out:expr),+ $(,)?]) => {{ const { assert!($consumed >= 1 && $consumed <= 3); assert!([$(stringify!($out)),+].len() <= $consumed + 1); } let replacements = [$($out),+]; - let replacement_start = $read + 1 - replacements.len(); - $instructions[$read - $consumed..replacement_start].fill(Instruction::Nop); - $instructions[replacement_start..=$read].copy_from_slice(&replacements); + let start = $read - $consumed; + $instructions[start..start + replacements.len()].copy_from_slice(&replacements); + $instructions.truncate(start + replacements.len()); + #[allow(unused_assignments)] + { $read = $instructions.len() - 1; } }}; - ($instructions:ident, $read:ident, $consumed:literal => $out:expr) => { + ($instructions:ident, $read:ident, $consumed:expr => $out:expr) => { replace!($instructions, $read, $consumed => [$out]); }; } macro_rules! rewrite { - ($instructions:ident, $read:ident, [$a:pat] if ($($guard:tt)+) => [$($out:expr),+ $(,)?]) => { - rewrite!($instructions, $read, [$a] if ($($guard)+) => { replace!($instructions, $read, 1 => [$($out),+]); }) - }; - ($instructions:ident, $read:ident, [$a:pat, $b:pat] if ($($guard:tt)+) => [$($out:expr),+ $(,)?]) => { - rewrite!($instructions, $read, [$a, $b] if ($($guard)+) => { replace!($instructions, $read, 2 => [$($out),+]); }) - }; - ($instructions:ident, $read:ident, [$a:pat, $b:pat, $c:pat] if ($($guard:tt)+) => [$($out:expr),+ $(,)?]) => { - rewrite!($instructions, $read, [$a, $b, $c] if ($($guard)+) => { replace!($instructions, $read, 3 => [$($out),+]); }) - }; - ($instructions:ident, $read:ident, [$a:pat] => [$($out:expr),+ $(,)?]) => { - rewrite!($instructions, $read, [$a] => { replace!($instructions, $read, 1 => [$($out),+]); }) - }; - ($instructions:ident, $read:ident, [$a:pat, $b:pat] => [$($out:expr),+ $(,)?]) => { - rewrite!($instructions, $read, [$a, $b] => { replace!($instructions, $read, 2 => [$($out),+]); }) - }; - ($instructions:ident, $read:ident, [$a:pat, $b:pat, $c:pat] => [$($out:expr),+ $(,)?]) => { - rewrite!($instructions, $read, [$a, $b, $c] => { replace!($instructions, $read, 3 => [$($out),+]); }) + ($instructions:ident, $read:ident, [$($pattern:pat),+] $(if ($($guard:tt)+))? => [$($out:expr),+ $(,)?]) => { + rewrite!($instructions, $read, [$($pattern),+] $(if ($($guard)+))? => { + replace!($instructions, $read, [$(stringify!($pattern)),+].len() => [$($out),+]); + }) }; - ($instructions:ident, $read:ident, [$a:pat] if ($($guard:tt)+) => $body:block $(,)?) => { - if $read > 0 && let $a = $instructions[$read - 1] && $($guard)+ { - $body - } - }; - ($instructions:ident, $read:ident, [$a:pat, $b:pat] if ($($guard:tt)+) => $body:block $(,)?) => { - if $read > 1 && let ($a, $b) = ($instructions[$read - 2], $instructions[$read - 1]) && $($guard)+ { - $body - } - }; - ($instructions:ident, $read:ident, [$a:pat, $b:pat, $c:pat] if ($($guard:tt)+) => $body:block $(,)?) => { - if $read > 2 && let ($a, $b, $c) = ($instructions[$read - 3], $instructions[$read - 2], $instructions[$read - 1]) && $($guard)+ { - $body - } - }; - ($instructions:ident, $read:ident, [$a:pat] => $body:block $(,)?) => { - if $read > 0 && let $a = $instructions[$read - 1] { - $body - } - }; - ($instructions:ident, $read:ident, [$a:pat, $b:pat] => $body:block $(,)?) => { - if $read > 1 && let ($a, $b) = ($instructions[$read - 2], $instructions[$read - 1]) { - $body - } - }; - ($instructions:ident, $read:ident, [$a:pat, $b:pat, $c:pat] => $body:block $(,)?) => { - if $read > 2 && let ($a, $b, $c) = ($instructions[$read - 3], $instructions[$read - 2], $instructions[$read - 1]) { - $body + ($instructions:ident, $read:ident, [$($pattern:pat),+] $(if ($($guard:tt)+))? => $body:block $(,)?) => {{ + const CONSUMED: usize = [$(stringify!($pattern)),+].len(); + if !$instructions.tail_rewritten + && $read < $instructions.len() + && $read >= $instructions.block_start + CONSUMED + { + let previous: [Instruction; CONSUMED] = $instructions[$read - CONSUMED..$read].try_into().unwrap(); + if let [$($pattern),+] = previous $( + && $($guard)+ + )? { + $instructions.tail_rewritten = true; + $body + } } - }; - ($instructions:ident, $read:ident, [$a:pat] if ($($guard:tt)+) => $out:expr $(,)?) => { - rewrite!($instructions, $read, [$a] if ($($guard)+) => { replace!($instructions, $read, 1 => $out); }) - }; - ($instructions:ident, $read:ident, [$a:pat, $b:pat] if ($($guard:tt)+) => $out:expr $(,)?) => { - rewrite!($instructions, $read, [$a, $b] if ($($guard)+) => { replace!($instructions, $read, 2 => $out); }) - }; - ($instructions:ident, $read:ident, [$a:pat, $b:pat, $c:pat] if ($($guard:tt)+) => $out:expr $(,)?) => { - rewrite!($instructions, $read, [$a, $b, $c] if ($($guard)+) => { replace!($instructions, $read, 3 => $out); }) - }; - ($instructions:ident, $read:ident, [$a:pat] => $out:expr $(,)?) => { - rewrite!($instructions, $read, [$a] => { replace!($instructions, $read, 1 => $out); }) - }; - ($instructions:ident, $read:ident, [$a:pat, $b:pat] => $out:expr $(,)?) => { - rewrite!($instructions, $read, [$a, $b] => { replace!($instructions, $read, 2 => $out); }) - }; - ($instructions:ident, $read:ident, [$a:pat, $b:pat, $c:pat] => $out:expr $(,)?) => { - rewrite!($instructions, $read, [$a, $b, $c] => { replace!($instructions, $read, 3 => $out); }) + }}; + ($instructions:ident, $read:ident, [$($pattern:pat),+] $(if ($($guard:tt)+))? => $out:expr $(,)?) => { + rewrite!($instructions, $read, [$($pattern),+] $(if ($($guard)+))? => { + replace!($instructions, $read, [$(stringify!($pattern)),+].len() => $out); + }) }; } @@ -201,13 +199,13 @@ pub(crate) mod optimize { binop_local_const_set = $lcset:ident $(, load_local_tee = $loadtee:ident, load_local_set = $loadset:ident)? ) => { - fn $name(instr: Instruction) -> Option<(Instruction, u16)> { + fn $name(instr: Instruction) -> Option<(Option<Instruction>, u16)> { Some(match instr { - Instruction::$get(local) => (Instruction::Nop, local), - Instruction::$tee(local) => (Instruction::$set(local), local), - Instruction::$lltee(op, a, b, local) => (Instruction::$llset(op, a, b, local), local), - Instruction::$lctee(op, src, c, local) => (Instruction::$lcset(op, src, c, local), local), - $(Instruction::$loadtee(memarg, addr, local) => (Instruction::$loadset(memarg, addr, local), local.into()),)? + Instruction::$get(local) => (None, local), + Instruction::$tee(local) => (Some(Instruction::$set(local)), local), + Instruction::$lltee(op, a, b, local) => (Some(Instruction::$llset(op, a, b, local)), local), + Instruction::$lctee(op, src, c, local) => (Some(Instruction::$lcset(op, src, c, local)), local), + $(Instruction::$loadtee(memarg, addr, local) => (Some(Instruction::$loadset(memarg, addr, local)), local.into()),)? _ => return None, }) } @@ -223,21 +221,28 @@ pub(crate) mod optimize { local_local = $local_local:ident, local_const = $local_const:expr ) => {{ - if let Some([(lhs_idx, lhs_src), (rhs_idx, rhs_src), (op_idx, raw_op)]) = - previous_non_nop::<3>($instrs, $read) + if !$instrs.tail_rewritten + && $read < $instrs.len() + && $read >= $instrs.block_start + 3 + && let [lhs_src, rhs_src, raw_op] = [$instrs[$read - 3], $instrs[$read - 2], $instrs[$read - 1]] && let Some((lhs_instr, lhs)) = $source(lhs_src) && let Some(op) = $op(raw_op) { if let Some((rhs_instr, rhs)) = $source(rhs_src) { - $instrs[lhs_idx] = lhs_instr; - $instrs[rhs_idx] = rhs_instr; - $instrs[op_idx] = Instruction::Nop; - $instrs[$read] = Instruction::$local_local(op, lhs, rhs, $dst); + if rhs_instr.is_none() || rhs != lhs { + $instrs.tail_rewritten = true; + $instrs.truncate($read - 3); + $instrs.extend(lhs_instr); + $instrs.extend(rhs_instr); + $instrs.push(Instruction::$local_local(op, lhs, rhs, $dst)); + $read = $instrs.len() - 1; + } } else if let Some(imm) = $const(rhs_src, raw_op) { - $instrs[lhs_idx] = lhs_instr; - $instrs[rhs_idx] = Instruction::Nop; - $instrs[op_idx] = Instruction::Nop; - $instrs[$read] = $local_const($dst, lhs, op, imm); + $instrs.tail_rewritten = true; + $instrs.truncate($read - 3); + $instrs.extend(lhs_instr); + $instrs.push($local_const($dst, lhs, op, imm)); + $read = $instrs.len() - 1; } } }}; @@ -254,7 +259,17 @@ pub(crate) mod optimize { binop_local_const_set = $lcset:expr $(, const_instr = $const_instr:ident, set_local_const = $set_local_const:ident)? ) => {{ - rewrite!($instrs, $read, [$get(src)] => if src == $dst { Instruction::Nop } else { Instruction::$copy(src, $dst) }); + rewrite!($instrs, $read, [$get(src)] if (src != $dst) => Instruction::$copy(src, $dst)); + if !$instrs.tail_rewritten + && $read < $instrs.len() + && $read > $instrs.block_start + && let Instruction::$get(src) = $instrs[$read - 1] + && src == $dst + { + $instrs.tail_rewritten = true; + $instrs.truncate($read - 1); + $read = $instrs.len(); + } $(rewrite!($instrs, $read, [$const_instr(c)] => Instruction::$set_local_const($dst, c));)? rewrite!($instrs, $read, [$ll(op, a, b)] => Instruction::$llset(op, a, b, $dst)); rewrite!($instrs, $read, [$lc(op, src, c)] => { replace!($instrs, $read, 1 => $lcset($dst, src, op, c)); }); @@ -270,7 +285,7 @@ pub(crate) mod optimize { binop_local_const = $lc:ident, binop_local_const_tee = $lctee:ident ) => {{ - rewrite!($instrs, $read, [$get(src)] if (src == $dst) => [Instruction::$get(src), Instruction::Nop]); + rewrite!($instrs, $read, [$get(src)] if (src == $dst) => Instruction::$get(src)); rewrite!($instrs, $read, [$ll(op, a, b)] => Instruction::$lltee(op, a, b, $dst)); rewrite!($instrs, $read, [$lc(op, src, c)] => Instruction::$lctee(op, src, c, $dst)); }}; @@ -286,7 +301,7 @@ pub(crate) mod optimize { binop_local_const_tee = $lctee:ident, binop_local_const_set = $lcset:ident ) => {{ - rewrite!($instrs, $read, [$tee(local)] => [Instruction::$set(local), Instruction::Nop]); + rewrite!($instrs, $read, [$tee(local)] => Instruction::$set(local)); rewrite!($instrs, $read, [$lltee(op, a, b, dst)] => Instruction::$llset(op, a, b, dst)); rewrite!($instrs, $read, [$lctee(op, src, c, dst)] => Instruction::$lcset(op, src, c, dst)); }}; diff --git a/crates/parser/src/module.rs b/crates/parser/src/module.rs index 4030a72..9273d44 100644 --- a/crates/parser/src/module.rs +++ b/crates/parser/src/module.rs @@ -13,21 +13,13 @@ pub(crate) struct FunctionCode { pub uses_local_memory: bool, } -pub(crate) fn imported_func_count(imports: &[Import]) -> usize { - imports.iter().filter(|i| matches!(&i.kind, ImportKind::Function(_))).count() -} - -pub(crate) fn imported_memory_count(imports: &[Import]) -> u32 { - imports.iter().filter(|i| matches!(&i.kind, ImportKind::Memory(_))).count() as u32 -} - pub(crate) fn optimize_function_code( mut code: FunctionCode, options: &ParserOptions, function_results: ValueCounts, self_func_addr: u32, imported_memory_count: u32, -) -> FunctionCode { +) -> Result<FunctionCode> { let optimized = optimize::optimize_instructions( code.instructions, &mut code.data, @@ -35,17 +27,18 @@ pub(crate) fn optimize_function_code( function_results, self_func_addr, imported_memory_count, - ); + )?; code.instructions = optimized.instructions; code.uses_local_memory = optimized.uses_local_memory; - code + Ok(code) } #[derive(Default)] pub(crate) struct ModuleReader<'a> { func_validator_allocations: Option<FuncValidatorAllocations>, operators_reader_allocations: Option<OperatorsReaderAllocations>, + translation_metadata: Option<Arc<crate::visit::ModuleMetadata>>, has_code_section: bool, marker: PhantomData<&'a [u8]>, @@ -54,6 +47,7 @@ pub(crate) struct ModuleReader<'a> { pub(crate) start_func: Option<u32>, pub(crate) func_types: Arc<[Arc<FuncType>]>, pub(crate) code_type_addrs: Box<[u32]>, + code_results: Box<[ValueCounts]>, pub(crate) exports: Arc<[Export]>, pub(crate) code: Vec<FunctionCode>, pub(crate) globals: Box<[Global]>, @@ -63,19 +57,33 @@ pub(crate) struct ModuleReader<'a> { pub(crate) data: Box<[Data]>, pub(crate) elements: Box<[Element]>, pub(crate) end_reached: bool, + imported_func_count: usize, + imported_memory_count: u32, #[cfg(parallel_parser)] pending_functions: Option<Vec<crate::parallel::PendingFunction<'a>>>, } impl<'a> ModuleReader<'a> { - fn function_results(&self, ordinal: usize) -> ValueCounts { - let ty_idx = self.code_type_addrs[ordinal]; - let ty = self.func_types.get(ty_idx as usize).expect("No func type for func, this is a bug"); - ValueCounts::from_iter(ty.results()) + fn translation_metadata(&mut self) -> &crate::visit::ModuleMetadata { + if self.translation_metadata.is_none() { + self.translation_metadata = Some(Arc::new(crate::visit::ModuleMetadata::new( + &self.func_types, + &self.code_type_addrs, + &self.imports, + &self.globals, + &self.memory_types, + &self.table_types, + ))); + } + self.translation_metadata.as_deref().unwrap() } - pub(crate) fn process_payload(&mut self, payload: Payload<'_>, validator: &mut Validator) -> Result<()> { + pub(crate) fn process_payload( + &mut self, + payload: Payload<'_>, + mut validator: Option<&mut Validator>, + ) -> Result<()> { fn check_section(section: &str, duplicate: bool) -> Result<()> { debug!("found {section} section"); if duplicate { @@ -86,7 +94,9 @@ impl<'a> ModuleReader<'a> { match payload { Payload::Version { num, encoding, range } => { - validator.version(num, encoding, &range)?; + if let Some(validator) = validator.as_mut() { + validator.version(num, encoding, &range)?; + } self.version = Some(num); if let wasmparser::Encoding::Component = encoding { return Err(ParseError::InvalidEncoding(encoding)); @@ -94,40 +104,54 @@ impl<'a> ModuleReader<'a> { } Payload::StartSection { func, range } => { check_section("start", self.start_func.is_some())?; - validator.start_section(func, &range)?; + if let Some(validator) = validator.as_mut() { + validator.start_section(func, &range)?; + } self.start_func = Some(func); } Payload::TypeSection(reader) => { check_section("type", !self.func_types.is_empty())?; - validator.type_section(&reader)?; + if let Some(validator) = validator.as_mut() { + validator.type_section(&reader)?; + } self.func_types = reader.into_iter().map(|t| convert_module_type(t?)).collect::<Result<_>>()?; } Payload::GlobalSection(reader) => { check_section("global", !self.globals.is_empty())?; - validator.global_section(&reader)?; + if let Some(validator) = validator.as_mut() { + validator.global_section(&reader)?; + } self.globals = convert_module_globals(reader)?; } Payload::TableSection(reader) => { check_section("table", !self.table_types.is_empty())?; - validator.table_section(&reader)?; + if let Some(validator) = validator.as_mut() { + validator.table_section(&reader)?; + } self.table_types = reader.into_iter().map(|table| convert_module_table(table?)).collect::<Result<_>>()?; } Payload::MemorySection(reader) => { check_section("memory", !self.memory_types.is_empty())?; - validator.memory_section(&reader)?; + if let Some(validator) = validator.as_mut() { + validator.memory_section(&reader)?; + } self.memory_types = reader.into_iter().map(|memory| Ok(convert_module_memory(memory?))).collect::<Result<_>>()?; } Payload::ElementSection(reader) => { debug!("Found element section"); - validator.element_section(&reader)?; + if let Some(validator) = validator.as_mut() { + validator.element_section(&reader)?; + } self.elements = reader.into_iter().map(|element| convert_module_element(element?)).collect::<Result<_>>()?; } Payload::DataSection(reader) => { check_section("data", !self.data.is_empty())?; - validator.data_section(&reader)?; + if let Some(validator) = validator.as_mut() { + validator.data_section(&reader)?; + } self.data = reader.into_iter().map(|data| convert_module_data(data?)).collect::<Result<_>>()?; } Payload::DataCountSection { count, range } => { @@ -135,22 +159,51 @@ impl<'a> ModuleReader<'a> { if !self.data.is_empty() { return Err(ParseError::UnsupportedSection("Data count section after data section".into())); } - validator.data_count_section(count, &range)?; + if let Some(validator) = validator.as_mut() { + validator.data_count_section(count, &range)?; + } } Payload::FunctionSection(reader) => { check_section("function", !self.code_type_addrs.is_empty())?; - validator.function_section(&reader)?; - self.code_type_addrs = reader.into_iter().map(|f| Ok(f?)).collect::<Result<_>>()?; + if let Some(validator) = validator.as_mut() { + validator.function_section(&reader)?; + } + let mut type_addrs = Vec::with_capacity(reader.count() as usize); + let mut results = Vec::with_capacity(reader.count() as usize); + for ty_idx in reader { + let ty_idx = ty_idx?; + let ty = self + .func_types + .get(ty_idx as usize) + .ok_or_else(|| ParseError::Other(format!("function type index out of bounds: {ty_idx}")))?; + type_addrs.push(ty_idx); + results.push(ValueCounts::from_iter(ty.results())); + } + self.code_type_addrs = type_addrs.into_boxed_slice(); + self.code_results = results.into_boxed_slice(); } Payload::ImportSection(reader) => { check_section("import", !self.imports.is_empty())?; - validator.import_section(&reader)?; - self.imports = - reader.into_imports().map(|import| convert_module_import(import?)).collect::<Result<_>>()?; + if let Some(validator) = validator.as_mut() { + validator.import_section(&reader)?; + } + let mut imports = Vec::with_capacity(reader.count() as usize); + for import in reader.into_imports() { + let import = convert_module_import(import?)?; + match import.kind { + ImportKind::Function(_) => self.imported_func_count += 1, + ImportKind::Memory(_) => self.imported_memory_count += 1, + _ => {} + } + imports.push(import); + } + self.imports = imports.into_boxed_slice(); } Payload::ExportSection(reader) => { check_section("export", !self.exports.is_empty())?; - validator.export_section(&reader)?; + if let Some(validator) = validator.as_mut() { + validator.export_section(&reader)?; + } self.exports = reader.into_iter().map(|e| convert_module_export(e?)).collect::<Result<_>>()?; } Payload::End(offset) => { @@ -159,7 +212,9 @@ impl<'a> ModuleReader<'a> { return Err(ParseError::DuplicateSection("End section".into())); } - validator.end(offset)?; + if let Some(validator) = validator.as_mut() { + validator.end(offset)?; + } self.end_reached = true; } Payload::CustomSection(_reader) => { @@ -181,7 +236,7 @@ impl<'a> ModuleReader<'a> { count: u32, range: Range<usize>, size: u32, - validator: &mut Validator, + validator: Option<&mut Validator>, options: &ParserOptions, ) -> Result<bool> { debug!("Found code section ({count} functions)"); @@ -191,7 +246,9 @@ impl<'a> ModuleReader<'a> { self.has_code_section = true; self.code.reserve(count as usize); - validator.code_section_start(&range)?; + if let Some(validator) = validator { + validator.code_section_start(&range)?; + } #[cfg(parallel_parser)] { @@ -213,29 +270,38 @@ impl<'a> ModuleReader<'a> { pub(crate) fn process_inline_code_section_entry( &mut self, function: wasmparser::FunctionBody<'_>, - validator: &mut Validator, + validator: Option<&mut Validator>, options: &ParserOptions, ) -> Result<()> { debug!("Found code section entry"); - let func_validator_allocs = self.func_validator_allocations.take().unwrap_or_default(); + let func_validator_allocs = self.func_validator_allocations.take(); let operators_reader_allocs = self.operators_reader_allocations.take().unwrap_or_default(); - let func_to_validate = validator.code_section_entry(&function)?; - let func_validator = func_to_validate.into_validator(func_validator_allocs); + let func_validator = validator + .map(|validator| validator.code_section_entry(&function)) + .transpose()? + .map(|func| func.into_validator(func_validator_allocs.unwrap_or_default())); + + let ordinal = self.code.len(); + let ty_idx = *self + .code_type_addrs + .get(ordinal) + .ok_or_else(|| ParseError::Other("code entry has no function signature".into()))?; + let metadata = self.translation_metadata(); let (code, func_validator_allocs, operators_reader_allocs) = - convert_module_code(function, func_validator, operators_reader_allocs)?; + convert_module_code(function, func_validator, operators_reader_allocs, metadata, ty_idx)?; self.code.push(optimize_function_code( code, options, - self.function_results(self.code.len()), - (imported_func_count(&self.imports) + self.code.len()) as u32, - imported_memory_count(&self.imports), - )); + self.code_results[self.code.len()], + (self.imported_func_count + self.code.len()) as u32, + self.imported_memory_count, + )?); - self.func_validator_allocations = Some(func_validator_allocs); + self.func_validator_allocations = func_validator_allocs; self.operators_reader_allocations = Some(operators_reader_allocs); Ok(()) } @@ -243,23 +309,15 @@ impl<'a> ModuleReader<'a> { pub(crate) fn process_borrowed_code_section_entry( &mut self, function: wasmparser::FunctionBody<'a>, - validator: &mut Validator, + validator: Option<&mut Validator>, options: &ParserOptions, ) -> Result<()> { debug!("Found code section entry"); #[cfg(parallel_parser)] - if let Some(pending) = self.pending_functions.as_mut() { - let func_to_validate = validator.code_section_entry(&function)?; - let ordinal = self.code.len() + pending.len(); - let ty_idx = self.code_type_addrs[ordinal]; - pending.push(crate::parallel::PendingFunction { - ordinal, - ty_idx, - func_to_validate, - body: crate::parallel::FunctionBodyInput::Borrowed(function), - }); - return Ok(()); + if self.pending_functions.is_some() { + let func_to_validate = validator.map(|validator| validator.code_section_entry(&function)).transpose()?; + return self.queue_function(crate::parallel::FunctionBodyInput::Borrowed(function), func_to_validate); } self.process_inline_code_section_entry(function, validator, options) @@ -271,32 +329,23 @@ impl<'a> ModuleReader<'a> { count: u32, body_offset: usize, section_bytes: Arc<[u8]>, - validator: &mut Validator, + mut validator: Option<&mut Validator>, ) -> Result<()> { - let code_len = self.code.len(); - let pending = self - .pending_functions - .as_mut() - .ok_or_else(|| ParseError::Other("owned code section queued without pending storage".into()))?; - let mut reader = wasmparser::BinaryReader::new(§ion_bytes, body_offset); for _ in 0..count { let body_reader = reader.read_reader()?; let body_range = body_reader.range(); let function = wasmparser::FunctionBody::new(body_reader); - let func_to_validate = validator.code_section_entry(&function)?; - let ordinal = code_len + pending.len(); - let ty_idx = self.code_type_addrs[ordinal]; - pending.push(crate::parallel::PendingFunction { - ordinal, - ty_idx, - func_to_validate, - body: crate::parallel::FunctionBodyInput::Owned(crate::parallel::OwnedFunctionBody { + let func_to_validate = + validator.as_mut().map(|validator| validator.code_section_entry(&function)).transpose()?; + self.queue_function( + crate::parallel::FunctionBodyInput::Owned(crate::parallel::OwnedFunctionBody { section_bytes: section_bytes.clone(), body_range: (body_range.start - body_offset)..(body_range.end - body_offset), body_offset: body_range.start, }), - }); + func_to_validate, + )?; } if reader.bytes_remaining() != 0 { @@ -310,18 +359,40 @@ impl<'a> ModuleReader<'a> { } #[cfg(parallel_parser)] + fn queue_function( + &mut self, + body: crate::parallel::FunctionBodyInput<'a>, + func_to_validate: Option<wasmparser::FuncToValidate<wasmparser::ValidatorResources>>, + ) -> Result<()> { + let ordinal = self.code.len() + self.pending_functions.as_ref().map_or(0, Vec::len); + let results = *self + .code_results + .get(ordinal) + .ok_or_else(|| ParseError::Other("code entry has no function signature".into()))?; + let ty_idx = *self + .code_type_addrs + .get(ordinal) + .ok_or_else(|| ParseError::Other("code entry has no function signature".into()))?; + let job = crate::parallel::PendingFunction { ordinal, results, func_to_validate, ty_idx, body }; + self.pending_functions + .as_mut() + .ok_or_else(|| ParseError::Other("function queued without pending storage".into()))? + .push(job); + Ok(()) + } + + #[cfg(parallel_parser)] pub(crate) fn process_pending_functions(&mut self, options: &ParserOptions) -> Result<()> { let Some(pending) = self.pending_functions.take().filter(|pending| !pending.is_empty()) else { return Ok(()); }; - self.code.extend(crate::parallel::process_pending( - pending, - options, - &self.func_types, - imported_func_count(&self.imports), - imported_memory_count(&self.imports), - )?); + let imported_func_count = self.imported_func_count; + let imported_memory_count = self.imported_memory_count; + let metadata = self.translation_metadata(); + let code = + crate::parallel::process_pending(pending, metadata, options, imported_func_count, imported_memory_count)?; + self.code.extend(code); Ok(()) } @@ -339,7 +410,7 @@ impl<'a> ModuleReader<'a> { return Err(ParseError::Other("Code and code type address count mismatch".to_string())); } - let import_mem_count = imported_memory_count(&self.imports); + let import_mem_count = self.imported_memory_count; let has_local_mem_export = self.exports.iter().any(|export| export.kind == ExternalKind::Memory && export.index >= import_mem_count); let has_active_data_segment_on_local_memory = self.data.iter().any(|data| match &data.kind { @@ -371,10 +442,10 @@ impl<'a> ModuleReader<'a> { .code .into_iter() .zip(self.code_type_addrs) - .map(|(code, ty_idx)| { - let ty = self.func_types.get(ty_idx as usize).expect("No func type for func, this is a bug").clone(); + .zip(self.code_results) + .map(|((code, ty_idx), results)| { + let ty = self.func_types.get(ty_idx as usize).expect("function type was checked while parsing").clone(); let params = ValueCounts::from_iter(ty.params()); - let results = ValueCounts::from_iter(ty.results()); if code.uses_local_memory { local_memory_allocation = LocalMemoryAllocation::Eager; } diff --git a/crates/parser/src/optimize.rs b/crates/parser/src/optimize.rs index e94e517..a26142c 100644 --- a/crates/parser/src/optimize.rs +++ b/crates/parser/src/optimize.rs @@ -1,6 +1,7 @@ -use crate::ParserOptions; use crate::macros::optimize::*; +use crate::{ParseError, ParserOptions, Result}; use alloc::vec::Vec; +use core::ops::{Deref, DerefMut}; use tinywasm_types::{BinOp, BinOp128, CmpOp, ConstIdx, Instruction, ValueCounts, WasmFunctionData}; pub(crate) struct OptimizeResult { @@ -8,34 +9,56 @@ pub(crate) struct OptimizeResult { pub(crate) uses_local_memory: bool, } +struct CompactOutput { + instructions: Vec<Instruction>, + block_start: usize, + tail_rewritten: bool, +} + +impl Deref for CompactOutput { + type Target = Vec<Instruction>; + + fn deref(&self) -> &Self::Target { + &self.instructions + } +} + +impl DerefMut for CompactOutput { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.instructions + } +} + pub(crate) fn optimize_instructions( - mut instructions: Vec<Instruction>, + instructions: Vec<Instruction>, function_data: &mut WasmFunctionData, options: &ParserOptions, function_results: ValueCounts, self_func_addr: u32, imported_memory_count: u32, -) -> OptimizeResult { - let uses_local_memory = if options.optimize_rewrite() { - rewrite(&mut instructions, function_results, self_func_addr, imported_memory_count) +) -> Result<OptimizeResult> { + let (mut instructions, old_to_new) = if options.optimize_rewrite() { + let boundaries = target_boundaries(&instructions, function_data)?; + let (instructions, old_to_new) = rewrite(instructions, &boundaries, function_results, self_func_addr); + (instructions, Some(old_to_new)) } else { - instructions.iter().any(|instr| instr.memory_addr().is_some_and(|mem| mem >= imported_memory_count)) + (instructions, None) }; - - if options.optimize_remove_nop() { - remove_nop(&mut instructions, function_data); - } - OptimizeResult { instructions, uses_local_memory } + let uses_local_memory = finalize(&mut instructions, function_data, old_to_new.as_deref(), imported_memory_count)?; + Ok(OptimizeResult { instructions, uses_local_memory }) } fn rewrite( - instrs: &mut [Instruction], + source: Vec<Instruction>, + boundaries: &[bool], function_results: ValueCounts, self_func_addr: u32, - imported_memory_count: u32, -) -> bool { +) -> (Vec<Instruction>, Vec<u32>) { use Instruction::*; - let mut uses_local_memory = false; + let mut instrs = + CompactOutput { instructions: Vec::with_capacity(source.len()), block_start: 0, tail_rewritten: false }; + let mut old_to_new = alloc::vec![0; source.len() + 1]; + let mut after_terminator = false; let return_instr = match function_results { ValueCounts { c32: 0, c64: 0, c128: 0 } => Some(ReturnVoid), ValueCounts { c32: 1, c64: 0, c128: 0 } => Some(Return32), @@ -44,11 +67,24 @@ fn rewrite( _ => None, }; - for i in 0..instrs.len() { + for (old_idx, instr) in source.iter().copied().enumerate() { + if boundaries[old_idx] || after_terminator { + instrs.block_start = instrs.len(); + } + old_to_new[old_idx] = instrs.len() as u32; + instrs.tail_rewritten = false; + instrs.push(instr); + let mut i = instrs.len() - 1; match instrs[i] { - LocalCopy32(a, b) if a == b => instrs[i] = Nop, - LocalCopy64(a, b) if a == b => instrs[i] = Nop, - LocalCopy128(a, b) if a == b => instrs[i] = Nop, + LocalCopy32(a, b) if a == b => { + instrs.pop(); + } + LocalCopy64(a, b) if a == b => { + instrs.pop(); + } + LocalCopy128(a, b) if a == b => { + instrs.pop(); + } Call(addr) if addr == self_func_addr => instrs[i] = CallSelf, ReturnCall(addr) if addr == self_func_addr => instrs[i] = ReturnCallSelf, Return if let Some(return_instr) = return_instr => instrs[i] = return_instr, @@ -57,20 +93,20 @@ fn rewrite( rewrite!(instrs, i, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); rewrite!(instrs, i, [LocalGet32(local), Const32(c)] => BinOpLocalConst32(op, local, c)); rewrite!(instrs, i, [Const32(c), LocalGet32(local)] => BinOpLocalConst32(op, local, c)); - rewrite!(instrs, i, [GlobalGet(global)] => [Nop, BinOpStackGlobal32(op, global)]); + rewrite!(instrs, i, [GlobalGet(global)] => BinOpStackGlobal32(op, global)); if matches!(op, BinOp::IAdd) { rewrite!(instrs, i, [Const32(c)] => AddConst32(c)); - rewrite!(instrs, i, [I32Add] => [Nop, I32Add3]); + rewrite!(instrs, i, [I32Add] => I32Add3); } } instr @ (I32Sub | I32Shl | I32ShrS | I32ShrU | I32Rotl | I32Rotr) => { let Some(op) = int_bin_op(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet32(a), LocalGet32(b)] => BinOpLocalLocal32(op, a, b)); rewrite!(instrs, i, [LocalGet32(local), Const32(c)] => BinOpLocalConst32(op, local, c)); - rewrite!(instrs, i, [GlobalGet(global)] => [Nop, BinOpStackGlobal32(op, global)]); + rewrite!(instrs, i, [GlobalGet(global)] => BinOpStackGlobal32(op, global)); if matches!(op, BinOp::IShrS) { - rewrite!(instrs, i, [BinOpLocalConst32(BinOp::IShl, local, 8), Const32(8)] => [Nop, LocalGet32(local), I32Extend8S]); - rewrite!(instrs, i, [BinOpLocalConst32(BinOp::IShl, local, 16), Const32(16)] => [Nop, LocalGet32(local), I32Extend16S]); + rewrite!(instrs, i, [BinOpLocalConst32(BinOp::IShl, local, 8), Const32(8)] => [LocalGet32(local), I32Extend8S]); + rewrite!(instrs, i, [BinOpLocalConst32(BinOp::IShl, local, 16), Const32(16)] => [LocalGet32(local), I32Extend16S]); } } instr @ (I64Add | I64Mul | I64And | I64Or | I64Xor) => { @@ -78,21 +114,21 @@ fn rewrite( rewrite!(instrs, i, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); rewrite!(instrs, i, [LocalGet64(local), Const64(c)] => BinOpLocalConst64(op, local, c)); rewrite!(instrs, i, [Const64(c), LocalGet64(local)] => BinOpLocalConst64(op, local, c)); - rewrite!(instrs, i, [GlobalGet(global)] => [Nop, BinOpStackGlobal64(op, global)]); + rewrite!(instrs, i, [GlobalGet(global)] => BinOpStackGlobal64(op, global)); if matches!(op, BinOp::IAdd) { rewrite!(instrs, i, [Const64(c)] => AddConst64(c)); - rewrite!(instrs, i, [I64Add] => [Nop, I64Add3]); + rewrite!(instrs, i, [I64Add] => I64Add3); } } instr @ (I64Sub | I64Shl | I64ShrS | I64ShrU | I64Rotl | I64Rotr) => { let Some(op) = int_bin_op(instr) else { unreachable!() }; rewrite!(instrs, i, [LocalGet64(a), LocalGet64(b)] => BinOpLocalLocal64(op, a, b)); rewrite!(instrs, i, [LocalGet64(local), Const64(c)] => BinOpLocalConst64(op, local, c)); - rewrite!(instrs, i, [GlobalGet(global)] => [Nop, BinOpStackGlobal64(op, global)]); + rewrite!(instrs, i, [GlobalGet(global)] => BinOpStackGlobal64(op, global)); if matches!(op, BinOp::IShrS) { - rewrite!(instrs, i, [BinOpLocalConst64(BinOp::IShl, local, 8), Const64(8)] => [Nop, LocalGet64(local), I64Extend8S]); - rewrite!(instrs, i, [BinOpLocalConst64(BinOp::IShl, local, 16), Const64(16)] => [Nop, LocalGet64(local), I64Extend16S]); - rewrite!(instrs, i, [BinOpLocalConst64(BinOp::IShl, local, 32), Const64(32)] => [Nop, LocalGet64(local), I64Extend32S]); + rewrite!(instrs, i, [BinOpLocalConst64(BinOp::IShl, local, 8), Const64(8)] => [LocalGet64(local), I64Extend8S]); + rewrite!(instrs, i, [BinOpLocalConst64(BinOp::IShl, local, 16), Const64(16)] => [LocalGet64(local), I64Extend16S]); + rewrite!(instrs, i, [BinOpLocalConst64(BinOp::IShl, local, 32), Const64(32)] => [LocalGet64(local), I64Extend32S]); } } instr @ (F32Add | F32Mul | F32Min | F32Max) => { @@ -128,7 +164,7 @@ fn rewrite( rewrite!(instrs, i, [LocalGet128(local), Const128(c)] => BinOpLocalConst128(BinOp128::AndNot, local, c)); } I32Store(memarg) | F32Store(memarg) => { - rewrite!(instrs, i, [F32Mul, F32Add] => [Nop, Nop, FMaStoreF32(memarg)]); + rewrite!(instrs, i, [F32Mul, F32Add] => FMaStoreF32(memarg)); rewrite!(instrs, i, [LocalGet32(addr_local), LocalGet32(value_local)] if (let (Ok(addr_local), Ok(value_local)) = (u8::try_from(addr_local), u8::try_from(value_local))) => @@ -136,7 +172,7 @@ fn rewrite( ); } I64Store(memarg) | F64Store(memarg) => { - rewrite!(instrs, i, [F64Mul, F64Add] => [Nop, Nop, FMaStoreF64(memarg)]); + rewrite!(instrs, i, [F64Mul, F64Add] => FMaStoreF64(memarg)); rewrite!(instrs, i, [LocalGet32(addr_local), LocalGet64(value_local)] if (let (Ok(addr_local), Ok(value_local)) = (u8::try_from(addr_local), u8::try_from(value_local))) => @@ -159,9 +195,9 @@ fn rewrite( MemoryFill(mem) => { rewrite!(instrs, i, [Const32(val), Const32(size)] => MemoryFillImm(mem, val as u8, size)) } - LocalGet32(dst) => rewrite!(instrs, i, [LocalSet32(src)] if (src == dst) => [LocalTee32(src), Nop]), - LocalGet64(dst) => rewrite!(instrs, i, [LocalSet64(src)] if (src == dst) => [LocalTee64(src), Nop]), - LocalGet128(dst) => rewrite!(instrs, i, [LocalSet128(src)] if (src == dst) => [LocalTee128(src), Nop]), + LocalGet32(dst) => rewrite!(instrs, i, [LocalSet32(src)] if (src == dst) => LocalTee32(src)), + LocalGet64(dst) => rewrite!(instrs, i, [LocalSet64(src)] if (src == dst) => LocalTee64(src)), + LocalGet128(dst) => rewrite!(instrs, i, [LocalSet128(src)] if (src == dst) => LocalTee128(src)), LocalSet32(dst) => { fold_local_binop!( instrs, i, dst, @@ -175,8 +211,8 @@ fn rewrite( _ => Instruction::BinOpLocalConstSet32(op, lhs, imm, dst), } ); - rewrite!(instrs, i, [I32Mul, LocalGet32(acc), I32Add] if (acc == dst) => [Nop, Nop, Nop, MulAccLocal32(dst)]); - rewrite!(instrs, i, [F32Mul, LocalGet32(acc), F32Add] if (acc == dst) => [Nop, Nop, Nop, FMulAccLocal32(dst)]); + rewrite!(instrs, i, [I32Mul, LocalGet32(acc), I32Add] if (acc == dst) => MulAccLocal32(dst)); + rewrite!(instrs, i, [F32Mul, LocalGet32(acc), F32Add] if (acc == dst) => FMulAccLocal32(dst)); rewrite_local_set_direct!( instrs, i, @@ -214,8 +250,8 @@ fn rewrite( _ => Instruction::BinOpLocalConstSet64(op, lhs, imm, dst), } ); - rewrite!(instrs, i, [I64Mul, LocalGet64(acc), I64Add] if (acc == dst) => [Nop, Nop, Nop, MulAccLocal64(dst)]); - rewrite!(instrs, i, [F64Mul, LocalGet64(acc), F64Add] if (acc == dst) => [Nop, Nop, Nop, FMulAccLocal64(dst)]); + rewrite!(instrs, i, [I64Mul, LocalGet64(acc), I64Add] if (acc == dst) => MulAccLocal64(dst)); + rewrite!(instrs, i, [F64Mul, LocalGet64(acc), F64Add] if (acc == dst) => FMulAccLocal64(dst)); rewrite_local_set_direct!( instrs, i, @@ -376,41 +412,41 @@ fn rewrite( binop_local_const_set = BinOpLocalConstSet128 ), Jump(ip) => { - let target = resolve_jump_target(instrs, ip); - let exit = next_non_nop(instrs, i + 1) as u32; - let body = next_non_nop(instrs, target as usize + 1) as u32; + let target = resolve_jump_target(&source, ip); + let exit = old_idx as u32 + 1; + let body = target + 1; - match instrs[target as usize] { - JumpCmpLocalLocal32 { target_ip, left, right, op } - if resolve_jump_target(instrs, target_ip) == exit && body > target => + match source.get(target as usize).copied() { + Some(JumpCmpLocalLocal32 { target_ip, left, right, op }) + if resolve_jump_target(&source, target_ip) == exit && body > target => { instrs[i] = JumpCmpLocalLocal32 { target_ip: body, left, right, op: inverse_cmp_op(op) }; } - JumpCmpLocalLocal64 { target_ip, left, right, op } - if resolve_jump_target(instrs, target_ip) == exit && body > target => + Some(JumpCmpLocalLocal64 { target_ip, left, right, op }) + if resolve_jump_target(&source, target_ip) == exit && body > target => { instrs[i] = JumpCmpLocalLocal64 { target_ip: body, left, right, op: inverse_cmp_op(op) }; } - _ => canonicalize_jump_like_with_target(instrs, i, target), + _ => canonicalize_jump_like_with_target(&mut instrs, i, target, exit), } } JumpIfZero32(ip) => { - let target = resolve_jump_target(instrs, ip); + let target = resolve_jump_target(&source, ip); rewrite!(instrs, i, [LocalGet32(local), I32Eqz] => { - replace!(instrs, i, 2 => [Nop, Nop, JumpIfLocalNonZero32 { target_ip: target, local }]); + replace!(instrs, i, 2 => JumpIfLocalNonZero32 { target_ip: target, local }); continue; }); rewrite!(instrs, i, [I32Eqz] => { - replace!(instrs, i, 1 => [Nop, JumpIfNonZero32(target)]); + replace!(instrs, i, 1 => JumpIfNonZero32(target)); continue; }); rewrite!(instrs, i, [LocalGet32(local)] => JumpIfLocalZero32 { target_ip: target, local }); rewrite!(instrs, i, [LocalGet64(local), I64Eqz] => { - replace!(instrs, i, 2 => [Nop, Nop, JumpIfLocalNonZero64 { target_ip: target, local }]); + replace!(instrs, i, 2 => JumpIfLocalNonZero64 { target_ip: target, local }); continue; }); rewrite!(instrs, i, [I64Eqz] => { - replace!(instrs, i, 1 => [Nop, JumpIfNonZero64(target)]); + replace!(instrs, i, 1 => JumpIfNonZero64(target)); continue; }); rewrite!(instrs, i, @@ -448,25 +484,25 @@ fn rewrite( (0, CmpOp::Ne) => JumpIfNonZero64(target), (imm, op) => JumpCmpStackConst64 { target_ip: target, imm, op }, }); - canonicalize_jump_like_with_target(instrs, i, target); + canonicalize_jump_like_with_target(&mut instrs, i, target, old_idx as u32 + 1); } JumpIfNonZero32(ip) => { - let target = resolve_jump_target(instrs, ip); + let target = resolve_jump_target(&source, ip); rewrite!(instrs, i, [LocalGet32(local), I32Eqz] => { - replace!(instrs, i, 2 => [Nop, Nop, JumpIfLocalZero32 { target_ip: target, local }]); + replace!(instrs, i, 2 => JumpIfLocalZero32 { target_ip: target, local }); continue; }); rewrite!(instrs, i, [I32Eqz] => { - replace!(instrs, i, 1 => [Nop, JumpIfZero32(target)]); + replace!(instrs, i, 1 => JumpIfZero32(target)); continue; }); rewrite!(instrs, i, [LocalGet32(local)] => JumpIfLocalNonZero32 { target_ip: target, local }); rewrite!(instrs, i, [LocalGet64(local), I64Eqz] => { - replace!(instrs, i, 2 => [Nop, Nop, JumpIfLocalZero64 { target_ip: target, local }]); + replace!(instrs, i, 2 => JumpIfLocalZero64 { target_ip: target, local }); continue; }); rewrite!(instrs, i, [I64Eqz] => { - replace!(instrs, i, 1 => [Nop, JumpIfZero64(target)]); + replace!(instrs, i, 1 => JumpIfZero64(target)); continue; }); rewrite!(instrs, i, @@ -504,17 +540,17 @@ fn rewrite( (0, CmpOp::Ne) => JumpIfNonZero64(target), (imm, op) => JumpCmpStackConst64 { target_ip: target, imm, op }, }); - canonicalize_jump_like_with_target(instrs, i, target); + canonicalize_jump_like_with_target(&mut instrs, i, target, old_idx as u32 + 1); } JumpIfZero64(ip) => { - let target = resolve_jump_target(instrs, ip); + let target = resolve_jump_target(&source, ip); rewrite!(instrs, i, [LocalGet64(local)] => JumpIfLocalZero64 { target_ip: target, local }); - canonicalize_jump_like_with_target(instrs, i, target); + canonicalize_jump_like_with_target(&mut instrs, i, target, old_idx as u32 + 1); } JumpIfNonZero64(ip) => { - let target = resolve_jump_target(instrs, ip); + let target = resolve_jump_target(&source, ip); rewrite!(instrs, i, [LocalGet64(local)] => JumpIfLocalNonZero64 { target_ip: target, local }); - canonicalize_jump_like_with_target(instrs, i, target); + canonicalize_jump_like_with_target(&mut instrs, i, target, old_idx as u32 + 1); } JumpCmpStackConst32 { target_ip, imm: 0, op } => { match op { @@ -522,7 +558,7 @@ fn rewrite( CmpOp::Ne => instrs[i] = JumpIfNonZero32(target_ip), _ => {} } - canonicalize_jump_like(instrs, i); + canonicalize_jump_like(&source, &mut instrs, i, old_idx as u32 + 1); } JumpCmpStackConst64 { target_ip, imm: 0, op } => { match op { @@ -530,7 +566,7 @@ fn rewrite( CmpOp::Ne => instrs[i] = JumpIfNonZero64(target_ip), _ => {} } - canonicalize_jump_like(instrs, i); + canonicalize_jump_like(&source, &mut instrs, i, old_idx as u32 + 1); } JumpCmpLocalConst32 { target_ip, local, imm: 0, op } => { match op { @@ -538,7 +574,7 @@ fn rewrite( CmpOp::Ne => instrs[i] = JumpIfLocalNonZero32 { target_ip, local }, _ => {} } - canonicalize_jump_like(instrs, i); + canonicalize_jump_like(&source, &mut instrs, i, old_idx as u32 + 1); } JumpCmpLocalConst64 { target_ip, local, imm: 0, op } => { match op { @@ -546,7 +582,7 @@ fn rewrite( CmpOp::Ne => instrs[i] = JumpIfLocalNonZero64 { target_ip, local }, _ => {} } - canonicalize_jump_like(instrs, i); + canonicalize_jump_like(&source, &mut instrs, i, old_idx as u32 + 1); } JumpCmpStackConst32 { .. } | JumpCmpStackConst64 { .. } @@ -558,17 +594,16 @@ fn rewrite( | JumpIfLocalNonZero32 { .. } | JumpIfLocalZero64 { .. } | JumpIfLocalNonZero64 { .. } => { - canonicalize_jump_like(instrs, i); + canonicalize_jump_like(&source, &mut instrs, i, old_idx as u32 + 1); } _ => {} } - if !uses_local_memory { - uses_local_memory = instrs[i].memory_addr().is_some_and(|mem| mem >= imported_memory_count); - } + after_terminator = is_unconditional_terminator(instr); } - uses_local_memory + old_to_new[source.len()] = instrs.len() as u32; + (instrs.instructions, old_to_new) } fn cmp_op(instr: Instruction) -> Option<CmpOp> { @@ -706,57 +741,21 @@ fn inverse_cmp_op(op: CmpOp) -> CmpOp { } } -const PREVIOUS_NON_NOP_BACKTRACK_LIMIT: usize = 32; - -fn previous_non_nop<const N: usize>(instrs: &[Instruction], read: usize) -> Option<[(usize, Instruction); N]> { - let mut out = [(0usize, Instruction::Nop); N]; - let mut filled = 0usize; - let start = read.saturating_sub(PREVIOUS_NON_NOP_BACKTRACK_LIMIT); - - for idx in (start..read).rev() { - let instr = instrs[idx]; - if matches!(instr, Instruction::MergeBarrier) { - return None; - } - if matches!(instr, Instruction::Nop) { - continue; - } - - out[N - 1 - filled] = (idx, instr); - filled += 1; - if filled == N { - return Some(out); - } - } - - None -} - -fn next_non_nop(instrs: &[Instruction], mut idx: usize) -> usize { - while idx < instrs.len() && matches!(instrs[idx], Instruction::Nop | Instruction::MergeBarrier) { - idx += 1; - } - idx -} - fn resolve_jump_target(instrs: &[Instruction], target: u32) -> u32 { - let mut idx = next_non_nop(instrs, target as usize); + let mut idx = target as usize; let mut steps = 0usize; - while idx < instrs.len() && steps < instrs.len() { - match instrs[idx] { - Instruction::Jump(next) => { - idx = next_non_nop(instrs, next as usize); - steps += 1; - } - _ => break, - } + while let Some(Instruction::Jump(next)) = instrs.get(idx) + && steps < instrs.len() + { + idx = *next as usize; + steps += 1; } idx as u32 } -fn instruction_target_mut(instr: &mut Instruction, include_branch_table: bool) -> Option<&mut u32> { +fn instruction_target_mut(instr: &mut Instruction) -> Option<&mut u32> { Some(match instr { Instruction::Jump(ip) | Instruction::JumpIfZero32(ip) @@ -773,68 +772,116 @@ fn instruction_target_mut(instr: &mut Instruction, include_branch_table: bool) - | Instruction::JumpCmpLocalConst64 { target_ip: ip, .. } | Instruction::JumpCmpLocalLocal32 { target_ip: ip, .. } | Instruction::JumpCmpLocalLocal64 { target_ip: ip, .. } => ip, - Instruction::BranchTable(ip, _, _) if include_branch_table => ip, + Instruction::BranchTable(ip, _, _) => ip, _ => return None, }) } -fn canonicalize_jump_like(instrs: &mut [Instruction], idx: usize) { - let Some(target) = instruction_target_mut(&mut instrs[idx], false).map(|target| *target) else { +fn instruction_target(instr: &Instruction) -> Option<u32> { + let mut instr = *instr; + instruction_target_mut(&mut instr).copied() +} + +fn canonicalize_jump_like(source: &[Instruction], instrs: &mut Vec<Instruction>, idx: usize, fallthrough: u32) { + let Some(target) = instruction_target(&instrs[idx]) else { return; }; - canonicalize_jump_like_with_target(instrs, idx, resolve_jump_target(instrs, target)); + canonicalize_jump_like_with_target(instrs, idx, resolve_jump_target(source, target), fallthrough); } -fn canonicalize_jump_like_with_target(instrs: &mut [Instruction], idx: usize, target: u32) { - if matches!(instrs[idx], Instruction::Jump(_)) && target == next_non_nop(instrs, idx + 1) as u32 { - instrs[idx] = Instruction::Nop; - } else if let Some(ip) = instruction_target_mut(&mut instrs[idx], false) { +fn canonicalize_jump_like_with_target(instrs: &mut Vec<Instruction>, idx: usize, target: u32, fallthrough: u32) { + if matches!(instrs[idx], Instruction::Jump(_)) && target == fallthrough { + instrs.truncate(idx); + } else if let Some(ip) = instruction_target_mut(&mut instrs[idx]) { *ip = target; } } -fn remove_nop(instructions: &mut Vec<Instruction>, function_data: &mut WasmFunctionData) { - let old_len = instructions.len(); - if old_len == 0 { - return; - } - - let mut removed_before = Vec::with_capacity(old_len + 1); - removed_before.push(0u32); - instructions.iter().for_each(|instr| { - let removed = removed_before.last().copied().unwrap_or(0) - + u32::from(matches!(instr, Instruction::Nop | Instruction::MergeBarrier)); - removed_before.push(removed); - }); +fn is_unconditional_terminator(instr: Instruction) -> bool { + matches!( + instr, + Instruction::Unreachable + | Instruction::Jump(_) + | Instruction::BranchTable(..) + | Instruction::Return + | Instruction::ReturnVoid + | Instruction::Return32 + | Instruction::Return64 + | Instruction::Return128 + | Instruction::ReturnCall(_) + | Instruction::ReturnCallSelf + | Instruction::ReturnCallIndirect(..) + ) +} - let removed_total = removed_before[old_len]; - if removed_total == 0 { - return; +fn target_boundaries(instructions: &[Instruction], function_data: &WasmFunctionData) -> Result<Vec<bool>> { + let mut boundaries = alloc::vec![false; instructions.len() + 1]; + for instr in instructions { + if let Some(target) = instruction_target(instr) { + let boundary = boundaries + .get_mut(target as usize) + .ok_or_else(|| ParseError::Other(alloc::format!("instruction target out of bounds: {target}")))?; + *boundary = true; + } + if let Instruction::BranchTable(_, start, count) = *instr { + let end = + start.checked_add(count).ok_or_else(|| ParseError::Other("branch table range overflow".into()))?; + let targets = function_data + .branch_table_targets + .get(start as usize..end as usize) + .ok_or_else(|| ParseError::Other("branch table range out of bounds".into()))?; + for &target in targets { + let boundary = boundaries + .get_mut(target as usize) + .ok_or_else(|| ParseError::Other(alloc::format!("branch table target out of bounds: {target}")))?; + *boundary = true; + } + } } + Ok(boundaries) +} - let compacted_len = old_len as u32 - removed_total; - - function_data.branch_table_targets.iter_mut().for_each(|ip| { - let old_target = *ip as usize; - if old_target <= old_len { - *ip -= removed_before[old_target]; - debug_assert!(*ip < compacted_len, "remapped jump target points past end of function"); +/// Remaps rewritten targets, then validates targets and ranges while detecting local memory use. +fn finalize( + instructions: &mut [Instruction], + function_data: &mut WasmFunctionData, + old_to_new: Option<&[u32]>, + imported_memory_count: u32, +) -> Result<bool> { + let len = instructions.len() as u32; + for target in &mut function_data.branch_table_targets { + if let Some(old_to_new) = old_to_new { + *target = *old_to_new + .get(*target as usize) + .ok_or_else(|| ParseError::Other(alloc::format!("instruction target out of bounds: {target}")))?; } - }); - - instructions.retain_mut(|instr| { - let Some(ip) = instruction_target_mut(instr, true) else { - return !matches!(instr, Instruction::Nop | Instruction::MergeBarrier); - }; - - let old_target = *ip as usize; - if old_target > old_len { - return !matches!(instr, Instruction::Nop | Instruction::MergeBarrier); + if *target >= len { + return Err(ParseError::Other(alloc::format!("branch table target out of bounds: {target}"))); } + } - *ip -= removed_before[old_target]; - debug_assert!(*ip < compacted_len, "remapped jump target points past end of function"); - !matches!(instr, Instruction::Nop | Instruction::MergeBarrier) - }); + let mut uses_local_memory = false; + for instr in instructions { + if let Some(target) = instruction_target_mut(instr) { + if let Some(old_to_new) = old_to_new { + *target = *old_to_new + .get(*target as usize) + .ok_or_else(|| ParseError::Other(alloc::format!("instruction target out of bounds: {target}")))?; + } + if *target >= len { + return Err(ParseError::Other(alloc::format!("instruction target out of bounds: {target}"))); + } + } + if let Instruction::BranchTable(_, start, count) = *instr { + let end = + start.checked_add(count).ok_or_else(|| ParseError::Other("branch table range overflow".into()))?; + function_data + .branch_table_targets + .get(start as usize..end as usize) + .ok_or_else(|| ParseError::Other("branch table range out of bounds".into()))?; + } + uses_local_memory |= instr.memory_addr().is_some_and(|mem| mem >= imported_memory_count); + } + Ok(uses_local_memory) } diff --git a/crates/parser/src/parallel.rs b/crates/parser/src/parallel.rs index 8833970..137eb2f 100644 --- a/crates/parser/src/parallel.rs +++ b/crates/parser/src/parallel.rs @@ -3,7 +3,7 @@ use crate::{ParseError, ParserOptions, Result, conversion}; use alloc::sync::Arc; use alloc::vec::Vec; use core::ops::Range; -use tinywasm_types::{FuncType, ValueCounts}; +use tinywasm_types::ValueCounts; use wasmparser::{FuncValidatorAllocations, ValidatorResources}; pub(crate) enum FunctionBodyInput<'a> { @@ -21,8 +21,9 @@ pub(crate) struct OwnedFunctionBody { pub(crate) struct PendingFunction<'a> { pub ordinal: usize, + pub results: ValueCounts, pub ty_idx: u32, - pub func_to_validate: wasmparser::FuncToValidate<ValidatorResources>, + pub func_to_validate: Option<wasmparser::FuncToValidate<ValidatorResources>>, pub body: FunctionBodyInput<'a>, } @@ -60,37 +61,38 @@ fn body_len(body: &FunctionBodyInput<'_>) -> usize { fn process_function_job( job: PendingFunction<'_>, + metadata: &crate::visit::ModuleMetadata, options: &ParserOptions, - func_types: &[Arc<FuncType>], imported_func_count: usize, imported_memory_count: u32, ) -> Result<(usize, FunctionCode)> { - let validator = job.func_to_validate.into_validator(FuncValidatorAllocations::default()); + let validator = job.func_to_validate.map(|func| func.into_validator(FuncValidatorAllocations::default())); let (code, _, _) = match job.body { - FunctionBodyInput::Borrowed(func) => conversion::convert_module_code(func, validator, Default::default())?, + FunctionBodyInput::Borrowed(func) => { + conversion::convert_module_code(func, validator, Default::default(), metadata, job.ty_idx)? + } FunctionBodyInput::Owned(body) => { let reader = wasmparser::BinaryReader::new(&body.section_bytes[body.body_range], body.body_offset); let func = wasmparser::FunctionBody::new(reader); - conversion::convert_module_code(func, validator, Default::default())? + conversion::convert_module_code(func, validator, Default::default(), metadata, job.ty_idx)? } }; - let ty = func_types.get(job.ty_idx as usize).expect("No func type for func, this is a bug"); let code = optimize_function_code( code, options, - ValueCounts::from_iter(ty.results()), + job.results, (imported_func_count + job.ordinal) as u32, imported_memory_count, - ); + )?; Ok((job.ordinal, code)) } pub(crate) fn process_pending( pending: Vec<PendingFunction<'_>>, + metadata: &crate::visit::ModuleMetadata, options: &ParserOptions, - func_types: &[Arc<FuncType>], imported_func_count: usize, imported_memory_count: u32, ) -> Result<Vec<FunctionCode>> { @@ -99,7 +101,7 @@ pub(crate) fn process_pending( let mut codes = small_jobs .into_iter() - .map(|job| process_function_job(job, options, func_types, imported_func_count, imported_memory_count)) + .map(|job| process_function_job(job, metadata, options, imported_func_count, imported_memory_count)) .collect::<Result<Vec<_>>>()?; let num_workers = worker_count(options, large_jobs.len()); @@ -107,7 +109,7 @@ pub(crate) fn process_pending( codes.extend( large_jobs .into_iter() - .map(|job| process_function_job(job, options, func_types, imported_func_count, imported_memory_count)) + .map(|job| process_function_job(job, metadata, options, imported_func_count, imported_memory_count)) .collect::<Result<Vec<_>>>()?, ); } else { @@ -136,13 +138,7 @@ pub(crate) fn process_pending( chunk .into_iter() .map(|job| { - process_function_job( - job, - options, - func_types, - imported_func_count, - imported_memory_count, - ) + process_function_job(job, metadata, options, imported_func_count, imported_memory_count) }) .collect::<Vec<_>>() }) diff --git a/crates/parser/src/visit.rs b/crates/parser/src/visit.rs index 4e941ad..0ca9494 100644 --- a/crates/parser/src/visit.rs +++ b/crates/parser/src/visit.rs @@ -1,40 +1,96 @@ -use core::hint::cold_path; - use crate::{Result, conversion::convert_heaptype, macros::visit::*}; use alloc::string::ToString; +use alloc::sync::Arc; use alloc::vec::Vec; -use tinywasm_types::{Instruction, MemoryArg, WasmFunctionData}; +use tinywasm_types::{ + FuncType, Global, Import, ImportKind, Instruction, MemoryArch, MemoryArg, MemoryType, TableType, ValueCounts, + WasmFunctionData, WasmType, +}; use wasmparser::{ - FrameKind, FuncValidator, FuncValidatorAllocations, FunctionBody, OperatorsReader, OperatorsReaderAllocations, - VisitOperator, VisitSimdOperator, WasmModuleResources, + FuncValidator, FuncValidatorAllocations, FunctionBody, OperatorsReader, OperatorsReaderAllocations, + ValidatorResources, VisitOperator, VisitSimdOperator, }; #[derive(Debug, Clone, Copy)] enum BlockKind { + Function, Block, Loop, If, } -#[derive(Debug, Clone, Copy)] -enum OperandSize { +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum OperandSize { S32, S64, S128, } -#[derive(Debug, Clone, Copy, Default)] -struct StackBase { - s32: u16, - s64: u16, - s128: u16, +impl OperandSize { + fn choose<T>(self, s32: T, s64: T, s128: T) -> T { + match self { + Self::S32 => s32, + Self::S64 => s64, + Self::S128 => s128, + } + } +} + +impl From<wasmparser::ValType> for OperandSize { + fn from(ty: wasmparser::ValType) -> Self { + match ty { + wasmparser::ValType::I32 | wasmparser::ValType::F32 | wasmparser::ValType::Ref(_) => Self::S32, + wasmparser::ValType::I64 | wasmparser::ValType::F64 => Self::S64, + wasmparser::ValType::V128 => Self::S128, + } + } +} + +impl From<&WasmType> for OperandSize { + fn from(ty: &WasmType) -> Self { + match ty { + WasmType::I32 | WasmType::F32 | WasmType::RefFunc | WasmType::RefExtern => Self::S32, + WasmType::I64 | WasmType::F64 => Self::S64, + WasmType::V128 => Self::S128, + } + } } -struct LoweringCtx { +impl From<MemoryArch> for OperandSize { + fn from(arch: MemoryArch) -> Self { + match arch { + MemoryArch::I32 => Self::S32, + MemoryArch::I64 => Self::S64, + } + } +} + +struct ControlFrame { kind: BlockKind, has_else: bool, start_ip: usize, branch_jumps: Vec<usize>, + height: usize, + base: ValueCounts, + params: Vec<OperandSize>, + results: Vec<OperandSize>, + unreachable: bool, + entry_unreachable: bool, + end_reachable: bool, +} + +#[derive(Clone)] +pub(crate) struct Signature { + pub params: Vec<OperandSize>, + results: Vec<OperandSize>, +} + +pub(crate) struct ModuleMetadata { + signatures: Vec<Signature>, + functions: Vec<u32>, + globals: Vec<OperandSize>, + memories: Vec<OperandSize>, + tables: Vec<OperandSize>, } #[derive(Default)] @@ -43,26 +99,130 @@ struct FunctionDataBuilder { branch_table_targets: Vec<u32>, } -impl FunctionDataBuilder { - fn finish(self) -> WasmFunctionData { - WasmFunctionData { - v128_constants: self.v128_constants.into_boxed_slice(), - branch_table_targets: self.branch_table_targets.into_boxed_slice(), +pub(crate) struct FunctionBuilder<'a> { + instructions: Vec<Instruction>, + data: FunctionDataBuilder, + control_stack: Vec<ControlFrame>, + operand_stack: Vec<OperandSize>, + lane_counts: ValueCounts, + metadata: &'a ModuleMetadata, + local_types: Vec<OperandSize>, + local_addr_map: Vec<u16>, +} + +impl<'a> FunctionBuilder<'a> { + pub(crate) fn new( + metadata: &'a ModuleMetadata, + signature: Signature, + local_types: Vec<OperandSize>, + local_addr_map: Vec<u16>, + body_size: usize, + ) -> Self { + Self { + local_types, + local_addr_map, + metadata, + instructions: Vec::with_capacity(body_size.min(1024)), + data: FunctionDataBuilder::default(), + control_stack: alloc::vec![ControlFrame { + kind: BlockKind::Function, + has_else: false, + start_ip: 0, + branch_jumps: Vec::new(), + height: 0, + base: ValueCounts::default(), + params: Vec::new(), + results: signature.results, + unreachable: false, + entry_unreachable: false, + end_reachable: false, + }], + operand_stack: Vec::new(), + lane_counts: ValueCounts::default(), } } } -struct ValidateThenVisit<'a, R: WasmModuleResources>(&'a mut FunctionBuilder<R>); -fn operand_size(ty: wasmparser::ValType) -> OperandSize { - match ty { - wasmparser::ValType::I32 | wasmparser::ValType::F32 | wasmparser::ValType::Ref(_) => OperandSize::S32, - wasmparser::ValType::I64 | wasmparser::ValType::F64 => OperandSize::S64, - wasmparser::ValType::V128 => OperandSize::S128, +struct ValidateThenVisit<'a, 'm> { + validator: Option<&'a mut FuncValidator<ValidatorResources>>, + builder: &'a mut FunctionBuilder<'m>, + position: usize, +} + +impl ModuleMetadata { + pub(crate) fn new( + types: &[Arc<FuncType>], + code_type_addrs: &[u32], + imports: &[Import], + globals: &[Global], + memories: &[MemoryType], + tables: &[TableType], + ) -> Self { + let mut functions = Vec::with_capacity(imports.len() + code_type_addrs.len()); + let mut global_sizes = Vec::with_capacity(imports.len() + globals.len()); + let mut memory_sizes = Vec::with_capacity(imports.len() + memories.len()); + let mut table_sizes = Vec::with_capacity(imports.len() + tables.len()); + + for import in imports { + match &import.kind { + ImportKind::Function(ty) => functions.push(*ty), + ImportKind::Global(ty) => global_sizes.push(OperandSize::from(&ty.ty)), + ImportKind::Memory(ty) => memory_sizes.push(OperandSize::from(ty.arch())), + ImportKind::Table(_) => table_sizes.push(OperandSize::S32), + } + } + + functions.extend_from_slice(code_type_addrs); + global_sizes.extend(globals.iter().map(|global| OperandSize::from(&global.ty.ty))); + memory_sizes.extend(memories.iter().map(|ty| OperandSize::from(ty.arch()))); + table_sizes.extend(tables.iter().map(|_| OperandSize::S32)); + + let signatures = types + .iter() + .map(|ty| Signature { + params: ty.params().iter().map(OperandSize::from).collect(), + results: ty.results().iter().map(OperandSize::from).collect(), + }) + .collect(); + Self { signatures, functions, globals: global_sizes, memories: memory_sizes, tables: table_sizes } + } + + pub(crate) fn signature(&self, idx: u32) -> Result<&Signature> { + self.signatures + .get(idx as usize) + .ok_or_else(|| crate::ParseError::Other(alloc::format!("type index out of bounds: {idx}"))) + } + + fn function_signature(&self, idx: u32) -> Result<&Signature> { + let ty = *self + .functions + .get(idx as usize) + .ok_or_else(|| crate::ParseError::Other(alloc::format!("function index out of bounds: {idx}")))?; + self.signature(ty) + } + + fn global_size(&self, idx: u32) -> Result<OperandSize> { + Self::indexed_size(&self.globals, "global", idx) + } + + fn memory_size(&self, idx: u32) -> Result<OperandSize> { + Self::indexed_size(&self.memories, "memory", idx) + } + + fn table_size(&self, idx: u32) -> Result<OperandSize> { + Self::indexed_size(&self.tables, "table", idx) + } + + fn indexed_size(sizes: &[OperandSize], entity: &str, idx: u32) -> Result<OperandSize> { + sizes + .get(idx as usize) + .copied() + .ok_or_else(|| crate::ParseError::Other(alloc::format!("{entity} index out of bounds: {idx}"))) } } -impl<'a, R: WasmModuleResources> VisitOperator<'a> for ValidateThenVisit<'_, R> { - type Output = (); +impl<'a> VisitOperator<'a> for ValidateThenVisit<'_, '_> { + type Output = Result<()>; wasmparser::for_each_visit_operator!(validate_then_visit); @@ -71,49 +231,49 @@ impl<'a, R: WasmModuleResources> VisitOperator<'a> for ValidateThenVisit<'_, R> } } -impl<R: WasmModuleResources> VisitSimdOperator<'_> for ValidateThenVisit<'_, R> { +impl VisitSimdOperator<'_> for ValidateThenVisit<'_, '_> { wasmparser::for_each_visit_simd_operator!(validate_then_visit_simd); } pub(crate) fn process_operators_and_validate( - validator: FuncValidator<impl WasmModuleResources>, + mut validator: Option<FuncValidator<ValidatorResources>>, body: FunctionBody<'_>, + local_types: Vec<OperandSize>, local_addr_map: Vec<u16>, + metadata: &ModuleMetadata, + ty_idx: u32, allocs: OperatorsReaderAllocations, -) -> Result<(Vec<Instruction>, WasmFunctionData, FuncValidatorAllocations, OperatorsReaderAllocations)> { +) -> Result<(Vec<Instruction>, WasmFunctionData, Option<FuncValidatorAllocations>, OperatorsReaderAllocations)> { + let body_size = body.as_bytes().len(); let reader = body.get_binary_reader_for_operators()?; let mut reader = OperatorsReader::new_with_allocs(reader, allocs); - let mut builder = FunctionBuilder::new(validator, local_addr_map); + let signature = metadata.signature(ty_idx)?.clone(); + let mut builder = FunctionBuilder::new(metadata, signature, local_types, local_addr_map, body_size); while !reader.eof() { - builder.position = reader.original_position(); - if let Err(e) = reader.visit_operator(&mut ValidateThenVisit(&mut builder)) { - cold_path(); - return Err(crate::ParseError::ParseError { message: e.to_string(), offset: builder.position }); + let position = reader.original_position(); + let res = reader + .visit_operator(&mut ValidateThenVisit { validator: validator.as_mut(), builder: &mut builder, position }) + .map_err(|e| crate::ParseError::ParseError { message: e.to_string(), offset: position }); + + if let Err(e) = res.flatten() { + core::hint::cold_path(); + return Err(e); } } reader.finish()?; - if let Some(error) = builder.error { - return Err(error); - } - - Ok((builder.instructions, builder.data.finish(), builder.validator.into_allocations(), reader.into_allocations())) -} - -pub(crate) struct FunctionBuilder<R> { - validator: FuncValidator<R>, - position: usize, - instructions: Vec<Instruction>, - data: FunctionDataBuilder, - ctx_stack: Vec<LoweringCtx>, - local_addr_map: Vec<u16>, - error: Option<crate::ParseError>, + let validator_allocations = validator.map(FuncValidator::into_allocations); + let data = WasmFunctionData { + v128_constants: builder.data.v128_constants.into_boxed_slice(), + branch_table_targets: builder.data.branch_table_targets.into_boxed_slice(), + }; + Ok((builder.instructions, data, validator_allocations, reader.into_allocations())) } -impl<'a, R: WasmModuleResources> wasmparser::VisitOperator<'a> for FunctionBuilder<R> { - type Output = (); +impl<'a> wasmparser::VisitOperator<'a> for FunctionBuilder<'_> { + type Output = Result<()>; fn simd_visitor(&mut self) -> Option<&mut dyn VisitSimdOperator<'a, Output = Self::Output>> { Some(self) @@ -121,236 +281,297 @@ impl<'a, R: WasmModuleResources> wasmparser::VisitOperator<'a> for FunctionBuild wasmparser::for_each_visit_operator!(impl_visit_operator); - define_mem_operands! { - visit_i32_load(I32Load), visit_i64_load(I64Load), visit_f32_load(F32Load), visit_f64_load(F64Load), - visit_i32_load8_s(I32Load8S), visit_i32_load8_u(I32Load8U), visit_i32_load16_s(I32Load16S), - visit_i32_load16_u(I32Load16U), visit_i64_load8_s(I64Load8S), visit_i64_load8_u(I64Load8U), - visit_i64_load16_s(I64Load16S), visit_i64_load16_u(I64Load16U), visit_i64_load32_s(I64Load32S), - visit_i64_load32_u(I64Load32U), visit_f32_store(F32Store), visit_f64_store(F64Store), visit_i32_store8(I32Store8), - visit_i32_store16(I32Store16), visit_i64_store8(I64Store8), visit_i64_store16(I64Store16), - visit_i64_store32(I64Store32), visit_i32_store(I32Store), visit_i64_store(I64Store) + lowering_ops! { + memory [Addr] => [S32] { + visit_i32_load => I32Load, visit_f32_load => F32Load, visit_i32_load8_s => I32Load8S, + visit_i32_load8_u => I32Load8U, visit_i32_load16_s => I32Load16S, + visit_i32_load16_u => I32Load16U, + } + memory [Addr] => [S64] { + visit_i64_load => I64Load, visit_f64_load => F64Load, visit_i64_load8_s => I64Load8S, + visit_i64_load8_u => I64Load8U, visit_i64_load16_s => I64Load16S, + visit_i64_load16_u => I64Load16U, visit_i64_load32_s => I64Load32S, + visit_i64_load32_u => I64Load32U, + } + memory [Addr, S32] => [] { + visit_f32_store => F32Store, visit_i32_store8 => I32Store8, + visit_i32_store16 => I32Store16, visit_i32_store => I32Store, + } + memory [Addr, S64] => [] { + visit_f64_store => F64Store, visit_i64_store8 => I64Store8, + visit_i64_store16 => I64Store16, visit_i64_store32 => I64Store32, visit_i64_store => I64Store, + } + fixed [] => [] { visit_data_drop(segment: u32) => DataDrop, visit_elem_drop(segment: u32) => ElemDrop } + fixed [] => [S32] { visit_i32_const(value: i32) => Const32, visit_ref_func(function: u32) => RefFunc } + fixed [] => [S64] { visit_i64_const(value: i64) => Const64 } + fixed [S32] => [S32] { + visit_i32_eqz => I32Eqz, visit_ref_is_null => RefIsNull, visit_i32_clz => I32Clz, + visit_i32_ctz => I32Ctz, visit_i32_popcnt => I32Popcnt, visit_i32_extend8_s => I32Extend8S, + visit_i32_extend16_s => I32Extend16S, visit_i32_trunc_f32_s => I32TruncF32S, + visit_i32_trunc_f32_u => I32TruncF32U, visit_f32_convert_i32_s => F32ConvertI32S, + visit_f32_convert_i32_u => F32ConvertI32U, visit_i32_trunc_sat_f32_s => I32TruncSatF32S, + visit_i32_trunc_sat_f32_u => I32TruncSatF32U, visit_f32_abs => F32Abs, visit_f32_neg => F32Neg, + visit_f32_ceil => F32Ceil, visit_f32_floor => F32Floor, visit_f32_trunc => F32Trunc, + visit_f32_nearest => F32Nearest, visit_f32_sqrt => F32Sqrt, + } + fixed [S64] => [S64] { + visit_i64_clz => I64Clz, visit_i64_ctz => I64Ctz, visit_i64_popcnt => I64Popcnt, + visit_i64_extend8_s => I64Extend8S, visit_i64_extend16_s => I64Extend16S, + visit_i64_extend32_s => I64Extend32S, visit_i64_trunc_f64_s => I64TruncF64S, + visit_i64_trunc_f64_u => I64TruncF64U, visit_f64_convert_i64_s => F64ConvertI64S, + visit_f64_convert_i64_u => F64ConvertI64U, visit_i64_trunc_sat_f64_s => I64TruncSatF64S, + visit_i64_trunc_sat_f64_u => I64TruncSatF64U, visit_f64_abs => F64Abs, visit_f64_neg => F64Neg, + visit_f64_ceil => F64Ceil, visit_f64_floor => F64Floor, visit_f64_trunc => F64Trunc, + visit_f64_nearest => F64Nearest, visit_f64_sqrt => F64Sqrt, + } + fixed [S64] => [S32] { + visit_i64_eqz => I64Eqz, visit_i32_wrap_i64 => I32WrapI64, visit_i32_trunc_f64_s => I32TruncF64S, + visit_i32_trunc_f64_u => I32TruncF64U, visit_f32_convert_i64_s => F32ConvertI64S, + visit_f32_convert_i64_u => F32ConvertI64U, visit_f32_demote_f64 => F32DemoteF64, + visit_i32_trunc_sat_f64_s => I32TruncSatF64S, visit_i32_trunc_sat_f64_u => I32TruncSatF64U, + } + fixed [S32] => [S64] { + visit_i64_extend_i32_s => I64ExtendI32S, visit_i64_extend_i32_u => I64ExtendI32U, + visit_i64_trunc_f32_s => I64TruncF32S, visit_i64_trunc_f32_u => I64TruncF32U, + visit_f64_convert_i32_s => F64ConvertI32S, visit_f64_convert_i32_u => F64ConvertI32U, + visit_f64_promote_f32 => F64PromoteF32, visit_i64_trunc_sat_f32_s => I64TruncSatF32S, + visit_i64_trunc_sat_f32_u => I64TruncSatF32U, + } + fixed [S32, S32] => [S32] { + visit_i32_eq => I32Eq, visit_i32_ne => I32Ne, visit_i32_lt_s => I32LtS, visit_i32_lt_u => I32LtU, + visit_i32_gt_s => I32GtS, visit_i32_gt_u => I32GtU, visit_i32_le_s => I32LeS, + visit_i32_le_u => I32LeU, visit_i32_ge_s => I32GeS, visit_i32_ge_u => I32GeU, + visit_f32_eq => F32Eq, visit_f32_ne => F32Ne, visit_f32_lt => F32Lt, visit_f32_gt => F32Gt, + visit_f32_le => F32Le, visit_f32_ge => F32Ge, visit_i32_add => I32Add, visit_i32_sub => I32Sub, + visit_i32_mul => I32Mul, visit_i32_div_s => I32DivS, visit_i32_div_u => I32DivU, + visit_i32_rem_s => I32RemS, visit_i32_rem_u => I32RemU, visit_i32_and => I32And, + visit_i32_or => I32Or, visit_i32_xor => I32Xor, visit_i32_shl => I32Shl, visit_i32_shr_s => I32ShrS, + visit_i32_shr_u => I32ShrU, visit_i32_rotl => I32Rotl, visit_i32_rotr => I32Rotr, + visit_f32_add => F32Add, visit_f32_sub => F32Sub, visit_f32_mul => F32Mul, visit_f32_div => F32Div, + visit_f32_min => F32Min, visit_f32_max => F32Max, visit_f32_copysign => F32Copysign, + } + fixed [S64, S64] => [S32] { + visit_i64_eq => I64Eq, visit_i64_ne => I64Ne, visit_i64_lt_s => I64LtS, visit_i64_lt_u => I64LtU, + visit_i64_gt_s => I64GtS, visit_i64_gt_u => I64GtU, visit_i64_le_s => I64LeS, + visit_i64_le_u => I64LeU, visit_i64_ge_s => I64GeS, visit_i64_ge_u => I64GeU, + visit_f64_eq => F64Eq, visit_f64_ne => F64Ne, visit_f64_lt => F64Lt, visit_f64_gt => F64Gt, + visit_f64_le => F64Le, visit_f64_ge => F64Ge, + } + fixed [S64, S64] => [S64] { + visit_i64_add => I64Add, visit_i64_sub => I64Sub, visit_i64_mul => I64Mul, + visit_i64_div_s => I64DivS, visit_i64_div_u => I64DivU, visit_i64_rem_s => I64RemS, + visit_i64_rem_u => I64RemU, visit_i64_and => I64And, visit_i64_or => I64Or, visit_i64_xor => I64Xor, + visit_i64_shl => I64Shl, visit_i64_shr_s => I64ShrS, visit_i64_shr_u => I64ShrU, + visit_i64_rotl => I64Rotl, visit_i64_rotr => I64Rotr, visit_f64_add => F64Add, + visit_f64_sub => F64Sub, visit_f64_mul => F64Mul, visit_f64_div => F64Div, + visit_f64_min => F64Min, visit_f64_max => F64Max, visit_f64_copysign => F64Copysign, + } + fixed [S64, S64, S64, S64] => [S64, S64] { visit_i64_add128 => I64Add128, visit_i64_sub128 => I64Sub128 } + fixed [S64, S64] => [S64, S64] { visit_i64_mul_wide_s => I64MulWideS, visit_i64_mul_wide_u => I64MulWideU } + effect [] => [] { visit_nop } + effect [S32] => [S32] { visit_f32_reinterpret_i32, visit_i32_reinterpret_f32 } + effect [S64] => [S64] { visit_f64_reinterpret_i64, visit_i64_reinterpret_f64 } + terminating [] => [] { visit_unreachable => Unreachable, visit_return => Return } + global [] => [Addr] { visit_global_get(global_index: u32) => GlobalGet } + memory_index [] => [Addr] { visit_memory_size(memory: u32) => MemorySize } + memory_index [Addr] => [Addr] { visit_memory_grow(memory: u32) => MemoryGrow } + memory_index [Addr, S32, Addr] => [] { + visit_memory_init(data_index: u32, memory: u32) => MemoryInit, + visit_memory_fill(memory: u32) => MemoryFill, + } + table [Addr] => [S32] { visit_table_get(table: u32) => TableGet } + table [Addr, S32] => [] { visit_table_set(table: u32) => TableSet } + table [] => [Addr] { visit_table_size(table: u32) => TableSize } + table [S32, Addr] => [Addr] { visit_table_grow(table: u32) => TableGrow } + table [Addr, S32, Addr] => [] { visit_table_fill(table: u32) => TableFill } + table [Addr, S32, S32] => [] { visit_table_init(elem_index: u32, table: u32) => TableInit } } - define_operands! { - // basic instructions - visit_global_get(GlobalGet, u32), visit_i32_const(Const32, i32), visit_i64_const(Const64, i64), visit_return(Return), - visit_call(Call, u32), visit_call_indirect(CallIndirect, u32, u32), visit_return_call_indirect(ReturnCallIndirect, u32, u32), - visit_return_call(ReturnCall, u32), visit_memory_size(MemorySize, u32), visit_memory_grow(MemoryGrow, u32), visit_unreachable(Unreachable), - visit_nop(Nop), visit_i32_eqz(I32Eqz), visit_i32_eq(I32Eq), visit_i32_ne(I32Ne), visit_i32_lt_s(I32LtS), visit_i32_lt_u(I32LtU), - visit_i32_gt_s(I32GtS), visit_i32_gt_u(I32GtU), visit_i32_le_s(I32LeS), visit_i32_le_u(I32LeU), visit_i32_ge_s(I32GeS), - visit_i32_ge_u(I32GeU), visit_i64_eqz(I64Eqz), visit_i64_eq(I64Eq), visit_i64_ne(I64Ne), visit_i64_lt_s(I64LtS), visit_i64_lt_u(I64LtU), - visit_i64_gt_s(I64GtS), visit_i64_gt_u(I64GtU), visit_i64_le_s(I64LeS), visit_i64_le_u(I64LeU), visit_i64_ge_s(I64GeS), visit_i64_ge_u(I64GeU), - visit_f32_eq(F32Eq), visit_f32_ne(F32Ne), visit_f32_lt(F32Lt), visit_f32_gt(F32Gt), visit_f32_le(F32Le), visit_f32_ge(F32Ge), visit_f64_eq(F64Eq), - visit_f64_ne(F64Ne), visit_f64_lt(F64Lt), visit_f64_gt(F64Gt), visit_f64_le(F64Le), visit_f64_ge(F64Ge), visit_i32_clz(I32Clz), visit_i32_ctz(I32Ctz), - visit_i32_popcnt(I32Popcnt), visit_i32_sub(I32Sub), visit_i32_mul(I32Mul), visit_i32_div_s(I32DivS), visit_i32_div_u(I32DivU), visit_i32_rem_s(I32RemS), - visit_i32_rem_u(I32RemU), visit_i32_and(I32And), visit_i32_or(I32Or), visit_i32_xor(I32Xor), visit_i32_shl(I32Shl), visit_i32_shr_s(I32ShrS), - visit_i32_shr_u(I32ShrU), visit_i32_rotl(I32Rotl), visit_i32_rotr(I32Rotr), visit_i64_clz(I64Clz), visit_i64_ctz(I64Ctz), visit_i64_popcnt(I64Popcnt), - visit_i64_sub(I64Sub), visit_i64_mul(I64Mul), visit_i64_div_s(I64DivS), visit_i64_div_u(I64DivU), visit_i64_rem_s(I64RemS), visit_i64_rem_u(I64RemU), - visit_i64_and(I64And), visit_i64_or(I64Or), visit_i64_xor(I64Xor), visit_i64_shl(I64Shl), visit_i64_shr_s(I64ShrS), visit_i64_shr_u(I64ShrU), - visit_i64_rotr(I64Rotr), visit_f32_abs(F32Abs), visit_f32_neg(F32Neg), visit_f32_ceil(F32Ceil), visit_f32_floor(F32Floor), visit_f32_trunc(F32Trunc), - visit_f32_nearest(F32Nearest), visit_f32_sqrt(F32Sqrt), visit_f32_add(F32Add), visit_f32_sub(F32Sub), visit_f32_mul(F32Mul), visit_f32_div(F32Div), - visit_f32_min(F32Min), visit_f32_max(F32Max), visit_f32_copysign(F32Copysign), visit_f64_abs(F64Abs), visit_f64_neg(F64Neg), visit_f64_ceil(F64Ceil), - visit_f64_floor(F64Floor), visit_f64_trunc(F64Trunc), visit_f64_nearest(F64Nearest), visit_f64_sqrt(F64Sqrt), visit_f64_add(F64Add), visit_f64_sub(F64Sub), - visit_f64_mul(F64Mul), visit_f64_div(F64Div), visit_f64_min(F64Min), visit_f64_max(F64Max), visit_f64_copysign(F64Copysign), visit_i32_wrap_i64(I32WrapI64), - visit_i32_trunc_f32_s(I32TruncF32S), visit_i32_trunc_f32_u(I32TruncF32U), visit_i32_trunc_f64_s(I32TruncF64S), visit_i32_trunc_f64_u(I32TruncF64U), - visit_i64_extend_i32_s(I64ExtendI32S), visit_i64_extend_i32_u(I64ExtendI32U), visit_i64_trunc_f32_s(I64TruncF32S), visit_i64_trunc_f32_u(I64TruncF32U), - visit_i64_trunc_f64_s(I64TruncF64S), visit_i64_trunc_f64_u(I64TruncF64U), visit_f32_convert_i32_s(F32ConvertI32S), visit_f32_convert_i32_u(F32ConvertI32U), - visit_f32_convert_i64_s(F32ConvertI64S), visit_f32_convert_i64_u(F32ConvertI64U), visit_f32_demote_f64(F32DemoteF64), visit_f64_convert_i32_s(F64ConvertI32S), - visit_f64_convert_i32_u(F64ConvertI32U), visit_f64_convert_i64_s(F64ConvertI64S), visit_f64_convert_i64_u(F64ConvertI64U), visit_f64_promote_f32(F64PromoteF32), - visit_i32_add(I32Add), visit_i64_add(I64Add), visit_i64_rotl(I64Rotl), - - // sign_extension - visit_i32_extend8_s(I32Extend8S), visit_i32_extend16_s(I32Extend16S), visit_i64_extend8_s(I64Extend8S), visit_i64_extend16_s(I64Extend16S), - visit_i64_extend32_s(I64Extend32S), - - // Non-trapping Float-to-int Conversions - visit_i32_trunc_sat_f32_s(I32TruncSatF32S), visit_i32_trunc_sat_f32_u(I32TruncSatF32U), visit_i32_trunc_sat_f64_s(I32TruncSatF64S), - visit_i32_trunc_sat_f64_u(I32TruncSatF64U), visit_i64_trunc_sat_f32_s(I64TruncSatF32S), visit_i64_trunc_sat_f32_u(I64TruncSatF32U), - visit_i64_trunc_sat_f64_s(I64TruncSatF64S), visit_i64_trunc_sat_f64_u(I64TruncSatF64U), + fn visit_call(&mut self, function_index: u32) -> Self::Output { + let signature = self.metadata.function_signature(function_index)?.clone(); + self.emit(&signature.params, &signature.results, Instruction::Call(function_index)) + } - // Reference Types - visit_ref_func(RefFunc, u32), visit_table_fill(TableFill, u32), visit_table_get(TableGet, u32), visit_table_set(TableSet, u32), - visit_table_grow(TableGrow, u32), visit_table_size(TableSize, u32), visit_ref_is_null(RefIsNull), + fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { + let signature = self.metadata.signature(type_index)?.clone(); + let mut inputs = signature.params; + inputs.push(self.metadata.table_size(table_index)?); + self.emit(&inputs, &signature.results, Instruction::CallIndirect(type_index, table_index)) + } - // Bulk Memory - visit_memory_init(MemoryInit, u32, u32), visit_memory_fill(MemoryFill, u32), visit_table_init(TableInit, u32, u32), - visit_data_drop(DataDrop, u32), visit_elem_drop(ElemDrop, u32), + fn visit_return_call(&mut self, function_index: u32) -> Self::Output { + let signature = self.metadata.function_signature(function_index)?.clone(); + self.apply_effect(&signature.params, &[])?; + self.mark_unreachable(); + self.instructions.push(Instruction::ReturnCall(function_index)); + Ok(()) + } - // Wide Arithmetic - visit_i64_add128(I64Add128), visit_i64_sub128(I64Sub128), visit_i64_mul_wide_s(I64MulWideS), visit_i64_mul_wide_u(I64MulWideU) + fn visit_return_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { + let signature = self.metadata.signature(type_index)?.clone(); + let mut inputs = signature.params; + inputs.push(self.metadata.table_size(table_index)?); + self.apply_effect(&inputs, &[])?; + self.mark_unreachable(); + self.instructions.push(Instruction::ReturnCallIndirect(type_index, table_index)); + Ok(()) } fn visit_global_set(&mut self, global_index: u32) -> Self::Output { - if let Some(Some(t)) = self.validator.get_operand_type(0) { - self.instructions.push(match operand_size(t) { - OperandSize::S32 => Instruction::GlobalSet32(global_index), - OperandSize::S64 => Instruction::GlobalSet64(global_index), - OperandSize::S128 => Instruction::GlobalSet128(global_index), - }) - } + let size = self.metadata.global_size(global_index)?; + let instruction = size.choose( + Instruction::GlobalSet32(global_index), + Instruction::GlobalSet64(global_index), + Instruction::GlobalSet128(global_index), + ); + self.emit(&[size], &[], instruction) } fn visit_drop(&mut self) -> Self::Output { - if let Some(Some(t)) = self.validator.get_operand_type(0) { - self.instructions.push(match operand_size(t) { - OperandSize::S32 => Instruction::Drop32, - OperandSize::S64 => Instruction::Drop64, - OperandSize::S128 => Instruction::Drop128, - }) - } + let size = self.operand_stack.last().copied().unwrap_or(OperandSize::S32); + let instruction = size.choose(Instruction::Drop32, Instruction::Drop64, Instruction::Drop128); + self.emit(&[size], &[], instruction) } fn visit_select(&mut self) -> Self::Output { - match self.validator.get_operand_type(1) { - Some(Some(t)) => self.visit_typed_select(t), - _ => self.visit_unreachable(), - }; + let size = self.operand_stack.iter().rev().nth(1).copied().unwrap_or(OperandSize::S32); + let instruction = size.choose(Instruction::Select32, Instruction::Select64, Instruction::Select128); + self.emit(&[size, size, OperandSize::S32], &[size], instruction) } fn visit_local_get(&mut self, idx: u32) -> Self::Output { - let resolved_idx = self.local_addr_map[idx as usize]; - if let Some(t) = self.validator.get_local_type(idx) { - self.instructions.push(match operand_size(t) { - OperandSize::S32 => Instruction::LocalGet32(resolved_idx), - OperandSize::S64 => Instruction::LocalGet64(resolved_idx), - OperandSize::S128 => Instruction::LocalGet128(resolved_idx), - }); - } + let (size, local_idx) = self.local(idx)?; + let instruction = size.choose( + Instruction::LocalGet32(local_idx), + Instruction::LocalGet64(local_idx), + Instruction::LocalGet128(local_idx), + ); + self.emit(&[], &[size], instruction) } fn visit_local_set(&mut self, idx: u32) -> Self::Output { - let resolved_idx = self.local_addr_map[idx as usize]; - if let Some(Some(t)) = self.validator.get_operand_type(0) { - self.instructions.push(match operand_size(t) { - OperandSize::S32 => Instruction::LocalSet32(resolved_idx), - OperandSize::S64 => Instruction::LocalSet64(resolved_idx), - OperandSize::S128 => Instruction::LocalSet128(resolved_idx), - }) - } + let (size, local_idx) = self.local(idx)?; + let instruction = size.choose( + Instruction::LocalSet32(local_idx), + Instruction::LocalSet64(local_idx), + Instruction::LocalSet128(local_idx), + ); + self.emit(&[size], &[], instruction) } fn visit_local_tee(&mut self, idx: u32) -> Self::Output { - let resolved_idx = self.local_addr_map[idx as usize]; - if let Some(Some(t)) = self.validator.get_operand_type(0) { - let size = operand_size(t); - let last = self.instructions.last(); - let src = match (size, last) { - (OperandSize::S32, Some(Instruction::LocalGet32(src))) => Some(*src), - (OperandSize::S64, Some(Instruction::LocalGet64(src))) => Some(*src), - (OperandSize::S128, Some(Instruction::LocalGet128(src))) => Some(*src), - _ => None, + let (size, local_idx) = self.local(idx)?; + self.apply_effect(&[size], &[size])?; + let src = match (size, self.instructions.last()) { + (OperandSize::S32, Some(Instruction::LocalGet32(src))) => Some(*src), + (OperandSize::S64, Some(Instruction::LocalGet64(src))) => Some(*src), + (OperandSize::S128, Some(Instruction::LocalGet128(src))) => Some(*src), + _ => None, + }; + if let Some(src) = src { + self.instructions.pop(); + let instructions = match size { + OperandSize::S32 => [Instruction::LocalCopy32(src, local_idx), Instruction::LocalGet32(local_idx)], + OperandSize::S64 => [Instruction::LocalCopy64(src, local_idx), Instruction::LocalGet64(local_idx)], + OperandSize::S128 => [Instruction::LocalCopy128(src, local_idx), Instruction::LocalGet128(local_idx)], }; - - if let Some(src) = src { - self.instructions.pop(); - match size { - OperandSize::S32 => { - self.instructions.push(Instruction::LocalCopy32(src, resolved_idx)); - self.instructions.push(Instruction::LocalGet32(resolved_idx)); - } - OperandSize::S64 => { - self.instructions.push(Instruction::LocalCopy64(src, resolved_idx)); - self.instructions.push(Instruction::LocalGet64(resolved_idx)); - } - OperandSize::S128 => { - self.instructions.push(Instruction::LocalCopy128(src, resolved_idx)); - self.instructions.push(Instruction::LocalGet128(resolved_idx)); - } - } - } else { - self.instructions.push(match size { - OperandSize::S32 => Instruction::LocalTee32(resolved_idx), - OperandSize::S64 => Instruction::LocalTee64(resolved_idx), - OperandSize::S128 => Instruction::LocalTee128(resolved_idx), - }) - } + self.instructions.extend(instructions); + } else { + self.instructions.push(size.choose( + Instruction::LocalTee32(local_idx), + Instruction::LocalTee64(local_idx), + Instruction::LocalTee128(local_idx), + )); } + Ok(()) } - fn visit_block(&mut self, _blockty: wasmparser::BlockType) -> Self::Output { - let start_ip = self.instructions.len(); - self.ctx_stack.push(LoweringCtx { - kind: BlockKind::Block, - has_else: false, - start_ip, - branch_jumps: Vec::new(), - }); + fn visit_block(&mut self, blockty: wasmparser::BlockType) -> Self::Output { + self.push_control(BlockKind::Block, blockty, None) } - fn visit_loop(&mut self, _ty: wasmparser::BlockType) -> Self::Output { - if !matches!(self.instructions.last(), Some(Instruction::Nop | Instruction::MergeBarrier)) { - self.instructions.push(Instruction::MergeBarrier); // prevent superinstructions from merging across block boundaries - } - let start_ip = self.instructions.len(); - self.ctx_stack.push(LoweringCtx { kind: BlockKind::Loop, has_else: false, start_ip, branch_jumps: Vec::new() }); + fn visit_loop(&mut self, ty: wasmparser::BlockType) -> Self::Output { + self.push_control(BlockKind::Loop, ty, None) } - fn visit_if(&mut self, _ty: wasmparser::BlockType) -> Self::Output { + fn visit_if(&mut self, ty: wasmparser::BlockType) -> Self::Output { + self.pop_expect(OperandSize::S32)?; self.instructions.push(Instruction::JumpIfZero32(0)); - self.ctx_stack.push(LoweringCtx { - kind: BlockKind::If, - has_else: false, - start_ip: self.instructions.len(), - branch_jumps: alloc::vec![self.instructions.len() - 1], - }); + self.push_control(BlockKind::If, ty, Some(self.instructions.len() - 1)) } fn visit_else(&mut self) -> Self::Output { - let last_if = self.ctx_stack.last().filter(|ctx| matches!(ctx.kind, BlockKind::If)); - if let Some(cond_jump_ip) = last_if.map(|ctx| ctx.branch_jumps[0]) { - let jump_ip = self.instructions.len(); - self.instructions.push(Instruction::Jump(0)); - if let Some(ctx) = self.ctx_stack.last_mut() { - ctx.has_else = true; - ctx.branch_jumps.push(jump_ip); - self.patch_jump(cond_jump_ip, self.instructions.len()); - if !matches!(self.instructions.last(), Some(Instruction::Nop | Instruction::MergeBarrier)) { - self.instructions.push(Instruction::MergeBarrier); // prevent superinstructions from merging across block boundaries - } - }; + let (cond_jump_ip, height, base, params, entry_unreachable) = { + let ctx = self + .control_stack + .last_mut() + .filter(|ctx| matches!(ctx.kind, BlockKind::If)) + .ok_or_else(|| crate::ParseError::Other("else without matching if".into()))?; + ctx.end_reachable |= !ctx.unreachable; + ctx.has_else = true; + (ctx.branch_jumps[0], ctx.height, ctx.base, ctx.params.clone(), ctx.entry_unreachable) }; + let jump_ip = self.instructions.len(); + self.instructions.push(Instruction::Jump(0)); + self.control_stack.last_mut().unwrap().branch_jumps.push(jump_ip); + self.patch_jump(cond_jump_ip, self.instructions.len()); + self.reset_stack(height, base); + self.push_sizes(¶ms)?; + self.control_stack.last_mut().unwrap().unreachable = entry_unreachable; + Ok(()) } fn visit_end(&mut self) -> Self::Output { - if let Some(ctx) = self.ctx_stack.pop() { - self.patch_end_jumps(ctx, self.instructions.len()); - if !matches!(self.instructions.last(), Some(Instruction::Nop | Instruction::MergeBarrier)) { - self.instructions.push(Instruction::MergeBarrier); // prevent superinstructions from merging across block boundaries - } - } else { + let ctx = + self.control_stack.pop().ok_or_else(|| crate::ParseError::Other("end without control frame".into()))?; + if matches!(ctx.kind, BlockKind::Function) { self.instructions.push(Instruction::Return); + } else { + let reachable = !ctx.entry_unreachable + && (!ctx.unreachable || ctx.end_reachable || matches!(ctx.kind, BlockKind::If) && !ctx.has_else); + self.reset_stack(ctx.height, ctx.base); + self.push_sizes(&ctx.results)?; + if let Some(parent) = self.control_stack.last_mut() { + parent.unreachable = !reachable; + } + self.patch_end_jumps(ctx, self.instructions.len()); } + Ok(()) } fn visit_br(&mut self, depth: u32) -> Self::Output { - self.emit_dropkeep_to_label(depth); - self.emit_branch_jump_or_return(depth); + self.emit_dropkeep_to_label(depth)?; + self.emit_branch_jump_or_return(depth)?; + self.mark_unreachable(); + Ok(()) } fn visit_br_if(&mut self, depth: u32) -> Self::Output { + self.pop_expect(OperandSize::S32)?; let cond_jump_ip = self.instructions.len(); self.instructions.push(Instruction::JumpIfZero32(0)); let branch_side_start = self.instructions.len(); - self.emit_dropkeep_to_label(depth); + self.emit_dropkeep_to_label(depth)?; if self.instructions.len() == branch_side_start - && let Some(ctx_idx) = self.get_ctx_idx(depth) + && let Ok(ctx_idx) = self.get_ctx_idx(depth) + && !matches!(self.control_stack[ctx_idx].kind, BlockKind::Function) { self.instructions[cond_jump_ip] = Instruction::JumpIfNonZero32(0); - self.ctx_stack[ctx_idx].branch_jumps.push(cond_jump_ip); - return; + self.control_stack[ctx_idx].branch_jumps.push(cond_jump_ip); + self.control_stack[ctx_idx].end_reachable = true; + return Ok(()); } - self.emit_branch_jump_or_return(depth); + self.emit_branch_jump_or_return(depth)?; self.patch_jump(cond_jump_ip, self.instructions.len()); + Ok(()) } fn visit_br_table(&mut self, targets: wasmparser::BrTable<'_>) -> Self::Output { - let ts = targets - .targets() - .collect::<Result<Vec<_>, wasmparser::BinaryReaderError>>() - .expect("visit_br_table: BrTable targets are invalid"); + let ts = targets.targets().collect::<Result<Vec<_>, wasmparser::Error>>()?; + self.pop_expect(OperandSize::S32)?; let default_depth = targets.default(); let len = ts.len() as u32; @@ -373,19 +594,34 @@ impl<'a, R: WasmModuleResources> wasmparser::VisitOperator<'a> for FunctionBuild continue; } - let (pad_start, jump_or_ret_ip, is_return) = self.emit_br_table_pad(depth); + let pad_start = self.instructions.len(); + let (jump_or_ret_ip, is_return) = if self.is_unreachable() { + self.instructions.push(Instruction::Return); + (pad_start, true) + } else { + let frame = &self.control_stack[self.get_ctx_idx(depth)?]; + let base = frame.base; + let label_types = if matches!(frame.kind, BlockKind::Loop) { &frame.params } else { &frame.results }; + self.emit_dropkeep(base, Self::value_counts(label_types)); + let jump_ip = self.instructions.len(); + self.instructions.push(Instruction::Jump(0)); + (jump_ip, false) + }; pads.push(PadInfo { depth, pad_start, jump_or_ret_ip, is_return }); } for &depth in &target_depths { - let pad = pads.iter().find(|pad| pad.depth == depth).expect("visit_br_table: missing branch table target"); + let pad = pads + .iter() + .find(|pad| pad.depth == depth) + .ok_or_else(|| crate::ParseError::Other("missing branch table target".into()))?; self.data.branch_table_targets.push(pad.pad_start as u32); } let default_pad = pads .iter() .find(|pad| pad.depth == default_depth) - .expect("visit_br_table: missing default branch table target"); + .ok_or_else(|| crate::ParseError::Other("missing default branch table target".into()))?; if let Instruction::BranchTable(default_ip, _, _) = &mut self.instructions[header_ip] { *default_ip = default_pad.pad_start as u32; } @@ -394,53 +630,66 @@ impl<'a, R: WasmModuleResources> wasmparser::VisitOperator<'a> for FunctionBuild if pad.is_return { continue; } - self.patch_branch_jump_or_return(pad.depth, pad.jump_or_ret_ip); + let ctx_idx = self.get_ctx_idx(pad.depth)?; + if matches!(self.control_stack[ctx_idx].kind, BlockKind::Function) { + self.instructions[pad.jump_or_ret_ip] = Instruction::Return; + } else if matches!(self.control_stack[ctx_idx].kind, BlockKind::Loop) { + self.patch_jump(pad.jump_or_ret_ip, self.control_stack[ctx_idx].start_ip); + } else { + self.control_stack[ctx_idx].branch_jumps.push(pad.jump_or_ret_ip); + self.control_stack[ctx_idx].end_reachable = true; + } } + self.mark_unreachable(); + Ok(()) } fn visit_f32_const(&mut self, val: wasmparser::Ieee32) -> Self::Output { - self.instructions.push(Instruction::Const32(val.bits() as i32)); + self.emit(&[], &[OperandSize::S32], Instruction::Const32(val.bits() as i32)) } fn visit_f64_const(&mut self, val: wasmparser::Ieee64) -> Self::Output { - self.instructions.push(Instruction::Const64(val.bits() as i64)); + self.emit(&[], &[OperandSize::S64], Instruction::Const64(val.bits() as i64)) } fn visit_table_copy(&mut self, dst_table: u32, src_table: u32) -> Self::Output { - self.instructions.push(Instruction::TableCopy { dst_table, src_table }); + self.metadata.table_size(dst_table)?; + self.metadata.table_size(src_table)?; + self.emit( + &[OperandSize::S32, OperandSize::S32, OperandSize::S32], + &[], + Instruction::TableCopy { dst_table, src_table }, + ) } fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output { - self.instructions.push(Instruction::MemoryCopy { dst_mem, src_mem }); + let dst = self.metadata.memory_size(dst_mem)?; + let src = self.metadata.memory_size(src_mem)?; + let len = if dst == OperandSize::S32 || src == OperandSize::S32 { OperandSize::S32 } else { OperandSize::S64 }; + self.emit(&[dst, src, len], &[], Instruction::MemoryCopy { dst_mem, src_mem }) } // Reference Types fn visit_ref_null(&mut self, ty: wasmparser::HeapType) -> Self::Output { - match convert_heaptype(ty) { - Ok(ty) => self.instructions.push(Instruction::RefNull(ty)), - Err(err) => { - self.error.get_or_insert(err); - } - }; + let instruction = Instruction::RefNull(convert_heaptype(ty)?); + self.emit(&[], &[OperandSize::S32], instruction) } fn visit_typed_select_multi(&mut self, tys: Vec<wasmparser::ValType>) -> Self::Output { - let (c32, c64, c128) = Self::label_keep_counts(&tys); - self.instructions.push(Instruction::SelectMulti(tinywasm_types::ValueCounts { c32, c64, c128 })); + let sizes: Vec<_> = tys.into_iter().map(OperandSize::from).collect(); + let counts = Self::value_counts(&sizes); + self.emit( + &[sizes.as_slice(), sizes.as_slice(), &[OperandSize::S32]].concat(), + &sizes, + Instruction::SelectMulti(counts), + ) } fn visit_typed_select(&mut self, ty: wasmparser::ValType) -> Self::Output { - self.instructions.push(match operand_size(ty) { - OperandSize::S32 => Instruction::Select32, - OperandSize::S64 => Instruction::Select64, - OperandSize::S128 => Instruction::Select128, - }); + let size = OperandSize::from(ty); + let instruction = size.choose(Instruction::Select32, Instruction::Select64, Instruction::Select128); + self.emit(&[size, size, OperandSize::S32], &[size], instruction) } - - fn visit_f32_reinterpret_i32(&mut self) -> Self::Output {} - fn visit_f64_reinterpret_i64(&mut self) -> Self::Output {} - fn visit_i32_reinterpret_f32(&mut self) -> Self::Output {} - fn visit_i64_reinterpret_f64(&mut self) -> Self::Output {} } macro_rules! impl_visit_simd_operator { @@ -451,167 +700,330 @@ macro_rules! impl_visit_simd_operator { (@@simd $($rest:tt)* ) => {}; (@@relaxed_simd $($rest:tt)* ) => {}; (@@$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident ($($ann:tt)*)) => { - fn $visit(&mut self $($(,$arg: $argty)*)?) { - self.unsupported(stringify!($visit)) + fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { + Err(crate::ParseError::UnsupportedOperator(stringify!($visit).to_string())) } }; } -impl<R: WasmModuleResources> wasmparser::VisitSimdOperator<'_> for FunctionBuilder<R> { +impl wasmparser::VisitSimdOperator<'_> for FunctionBuilder<'_> { wasmparser::for_each_visit_simd_operator!(impl_visit_simd_operator); - // simd - define_mem_operands_simd! { - visit_v128_load(V128Load), visit_v128_load8x8_s(V128Load8x8S), visit_v128_load8x8_u(V128Load8x8U), visit_v128_load16x4_s(V128Load16x4S), visit_v128_load16x4_u(V128Load16x4U), visit_v128_load32x2_s(V128Load32x2S), visit_v128_load32x2_u(V128Load32x2U), visit_v128_load8_splat(V128Load8Splat), visit_v128_load16_splat(V128Load16Splat), visit_v128_load32_splat(V128Load32Splat), visit_v128_load64_splat(V128Load64Splat), visit_v128_load32_zero(V128Load32Zero), visit_v128_load64_zero(V128Load64Zero), visit_v128_store(V128Store) - } - - define_mem_operands_simd_lane! { - visit_v128_load8_lane(V128Load8Lane), visit_v128_load16_lane(V128Load16Lane), visit_v128_load32_lane(V128Load32Lane), visit_v128_load64_lane(V128Load64Lane), - visit_v128_store8_lane(V128Store8Lane), visit_v128_store16_lane(V128Store16Lane), visit_v128_store32_lane(V128Store32Lane), visit_v128_store64_lane(V128Store64Lane) - } - - define_operands! { - visit_v128_not(V128Not), visit_v128_and(V128And), visit_v128_andnot(V128AndNot), visit_v128_or(V128Or), visit_v128_xor(V128Xor), visit_v128_bitselect(V128Bitselect), visit_v128_any_true(V128AnyTrue), - visit_i8x16_splat(I8x16Splat), visit_i8x16_swizzle(I8x16Swizzle), visit_i8x16_eq(I8x16Eq), visit_i8x16_ne(I8x16Ne), visit_i8x16_lt_s(I8x16LtS), visit_i8x16_lt_u(I8x16LtU), visit_i8x16_gt_s(I8x16GtS), visit_i8x16_gt_u(I8x16GtU), visit_i8x16_le_s(I8x16LeS), visit_i8x16_le_u(I8x16LeU), visit_i8x16_ge_s(I8x16GeS), visit_i8x16_ge_u(I8x16GeU), - visit_i16x8_splat(I16x8Splat), visit_i16x8_eq(I16x8Eq), visit_i16x8_ne(I16x8Ne), visit_i16x8_lt_s(I16x8LtS), visit_i16x8_lt_u(I16x8LtU), visit_i16x8_gt_s(I16x8GtS), visit_i16x8_gt_u(I16x8GtU), visit_i16x8_le_s(I16x8LeS), visit_i16x8_le_u(I16x8LeU), visit_i16x8_ge_s(I16x8GeS), visit_i16x8_ge_u(I16x8GeU), - visit_i32x4_splat(I32x4Splat), visit_i32x4_eq(I32x4Eq), visit_i32x4_ne(I32x4Ne), visit_i32x4_lt_s(I32x4LtS), visit_i32x4_lt_u(I32x4LtU), visit_i32x4_gt_s(I32x4GtS), visit_i32x4_gt_u(I32x4GtU), visit_i32x4_le_s(I32x4LeS), visit_i32x4_le_u(I32x4LeU), visit_i32x4_ge_s(I32x4GeS), visit_i32x4_ge_u(I32x4GeU), - visit_i64x2_splat(I64x2Splat), visit_i64x2_eq(I64x2Eq), visit_i64x2_ne(I64x2Ne), visit_i64x2_lt_s(I64x2LtS), visit_i64x2_gt_s(I64x2GtS), visit_i64x2_le_s(I64x2LeS), visit_i64x2_ge_s(I64x2GeS), - visit_f32x4_splat(F32x4Splat), visit_f32x4_eq(F32x4Eq), visit_f32x4_ne(F32x4Ne), visit_f32x4_lt(F32x4Lt), visit_f32x4_gt(F32x4Gt), visit_f32x4_le(F32x4Le), visit_f32x4_ge(F32x4Ge), - visit_f64x2_splat(F64x2Splat), visit_f64x2_eq(F64x2Eq), visit_f64x2_ne(F64x2Ne), visit_f64x2_lt(F64x2Lt), visit_f64x2_gt(F64x2Gt), visit_f64x2_le(F64x2Le), visit_f64x2_ge(F64x2Ge), - visit_i8x16_abs(I8x16Abs), visit_i8x16_neg(I8x16Neg), visit_i8x16_all_true(I8x16AllTrue), visit_i8x16_bitmask(I8x16Bitmask), visit_i8x16_shl(I8x16Shl), visit_i8x16_shr_s(I8x16ShrS), visit_i8x16_shr_u(I8x16ShrU), visit_i8x16_add(I8x16Add), visit_i8x16_sub(I8x16Sub), visit_i8x16_min_s(I8x16MinS), visit_i8x16_min_u(I8x16MinU), visit_i8x16_max_s(I8x16MaxS), visit_i8x16_max_u(I8x16MaxU), - visit_i16x8_abs(I16x8Abs), visit_i16x8_neg(I16x8Neg), visit_i16x8_all_true(I16x8AllTrue), visit_i16x8_bitmask(I16x8Bitmask), visit_i16x8_shl(I16x8Shl), visit_i16x8_shr_s(I16x8ShrS), visit_i16x8_shr_u(I16x8ShrU), visit_i16x8_add(I16x8Add), visit_i16x8_sub(I16x8Sub), visit_i16x8_min_s(I16x8MinS), visit_i16x8_min_u(I16x8MinU), visit_i16x8_max_s(I16x8MaxS), visit_i16x8_max_u(I16x8MaxU), - visit_i32x4_abs(I32x4Abs), visit_i32x4_neg(I32x4Neg), visit_i32x4_all_true(I32x4AllTrue), visit_i32x4_bitmask(I32x4Bitmask), visit_i32x4_shl(I32x4Shl), visit_i32x4_shr_s(I32x4ShrS), visit_i32x4_shr_u(I32x4ShrU), visit_i32x4_add(I32x4Add), visit_i32x4_sub(I32x4Sub), visit_i32x4_min_s(I32x4MinS), visit_i32x4_min_u(I32x4MinU), visit_i32x4_max_s(I32x4MaxS), visit_i32x4_max_u(I32x4MaxU), - visit_i64x2_abs(I64x2Abs), visit_i64x2_neg(I64x2Neg), visit_i64x2_all_true(I64x2AllTrue), visit_i64x2_bitmask(I64x2Bitmask), visit_i64x2_shl(I64x2Shl), visit_i64x2_shr_s(I64x2ShrS), visit_i64x2_shr_u(I64x2ShrU), visit_i64x2_add(I64x2Add), visit_i64x2_sub(I64x2Sub), visit_i64x2_mul(I64x2Mul), - visit_i8x16_narrow_i16x8_s(I8x16NarrowI16x8S), visit_i8x16_narrow_i16x8_u(I8x16NarrowI16x8U), visit_i8x16_add_sat_s(I8x16AddSatS), visit_i8x16_add_sat_u(I8x16AddSatU), visit_i8x16_sub_sat_s(I8x16SubSatS), visit_i8x16_sub_sat_u(I8x16SubSatU), visit_i8x16_avgr_u(I8x16AvgrU), - visit_i16x8_narrow_i32x4_s(I16x8NarrowI32x4S), visit_i16x8_narrow_i32x4_u(I16x8NarrowI32x4U), visit_i16x8_add_sat_s(I16x8AddSatS), visit_i16x8_add_sat_u(I16x8AddSatU), visit_i16x8_sub_sat_s(I16x8SubSatS), visit_i16x8_sub_sat_u(I16x8SubSatU), visit_i16x8_avgr_u(I16x8AvgrU), - visit_i16x8_extadd_pairwise_i8x16_s(I16x8ExtAddPairwiseI8x16S), visit_i16x8_extadd_pairwise_i8x16_u(I16x8ExtAddPairwiseI8x16U), visit_i16x8_mul(I16x8Mul), - visit_i32x4_extadd_pairwise_i16x8_s(I32x4ExtAddPairwiseI16x8S), visit_i32x4_extadd_pairwise_i16x8_u(I32x4ExtAddPairwiseI16x8U), visit_i32x4_mul(I32x4Mul), - visit_i16x8_extmul_low_i8x16_s(I16x8ExtMulLowI8x16S), visit_i16x8_extmul_low_i8x16_u(I16x8ExtMulLowI8x16U), visit_i16x8_extmul_high_i8x16_s(I16x8ExtMulHighI8x16S), visit_i16x8_extmul_high_i8x16_u(I16x8ExtMulHighI8x16U), - visit_i32x4_extmul_low_i16x8_s(I32x4ExtMulLowI16x8S), visit_i32x4_extmul_low_i16x8_u(I32x4ExtMulLowI16x8U), visit_i32x4_extmul_high_i16x8_s(I32x4ExtMulHighI16x8S), visit_i32x4_extmul_high_i16x8_u(I32x4ExtMulHighI16x8U), - visit_i64x2_extmul_low_i32x4_s(I64x2ExtMulLowI32x4S), visit_i64x2_extmul_low_i32x4_u(I64x2ExtMulLowI32x4U), visit_i64x2_extmul_high_i32x4_s(I64x2ExtMulHighI32x4S), visit_i64x2_extmul_high_i32x4_u(I64x2ExtMulHighI32x4U), - visit_i16x8_extend_low_i8x16_s(I16x8ExtendLowI8x16S), visit_i16x8_extend_low_i8x16_u(I16x8ExtendLowI8x16U), visit_i16x8_extend_high_i8x16_s(I16x8ExtendHighI8x16S), visit_i16x8_extend_high_i8x16_u(I16x8ExtendHighI8x16U), - visit_i32x4_extend_low_i16x8_s(I32x4ExtendLowI16x8S), visit_i32x4_extend_low_i16x8_u(I32x4ExtendLowI16x8U), visit_i32x4_extend_high_i16x8_s(I32x4ExtendHighI16x8S), visit_i32x4_extend_high_i16x8_u(I32x4ExtendHighI16x8U), - visit_i64x2_extend_low_i32x4_s(I64x2ExtendLowI32x4S), visit_i64x2_extend_low_i32x4_u(I64x2ExtendLowI32x4U), visit_i64x2_extend_high_i32x4_s(I64x2ExtendHighI32x4S), visit_i64x2_extend_high_i32x4_u(I64x2ExtendHighI32x4U), - visit_i8x16_popcnt(I8x16Popcnt), visit_i16x8_q15mulr_sat_s(I16x8Q15MulrSatS), visit_i32x4_dot_i16x8_s(I32x4DotI16x8S), - visit_f32x4_ceil(F32x4Ceil), visit_f32x4_floor(F32x4Floor), visit_f32x4_trunc(F32x4Trunc), visit_f32x4_nearest(F32x4Nearest), visit_f32x4_abs(F32x4Abs), visit_f32x4_neg(F32x4Neg), visit_f32x4_sqrt(F32x4Sqrt), visit_f32x4_add(F32x4Add), visit_f32x4_sub(F32x4Sub), visit_f32x4_mul(F32x4Mul), visit_f32x4_div(F32x4Div), visit_f32x4_min(F32x4Min), visit_f32x4_max(F32x4Max), visit_f32x4_pmin(F32x4PMin), visit_f32x4_pmax(F32x4PMax), - visit_f64x2_ceil(F64x2Ceil), visit_f64x2_floor(F64x2Floor), visit_f64x2_trunc(F64x2Trunc), visit_f64x2_nearest(F64x2Nearest), visit_f64x2_abs(F64x2Abs), visit_f64x2_neg(F64x2Neg), visit_f64x2_sqrt(F64x2Sqrt), visit_f64x2_add(F64x2Add), visit_f64x2_sub(F64x2Sub), visit_f64x2_mul(F64x2Mul), visit_f64x2_div(F64x2Div), visit_f64x2_min(F64x2Min), visit_f64x2_max(F64x2Max), visit_f64x2_pmin(F64x2PMin), visit_f64x2_pmax(F64x2PMax), - visit_i32x4_trunc_sat_f32x4_s(I32x4TruncSatF32x4S), visit_i32x4_trunc_sat_f32x4_u(I32x4TruncSatF32x4U), - visit_f32x4_convert_i32x4_s(F32x4ConvertI32x4S), visit_f32x4_convert_i32x4_u(F32x4ConvertI32x4U), - visit_i32x4_trunc_sat_f64x2_s_zero(I32x4TruncSatF64x2SZero), visit_i32x4_trunc_sat_f64x2_u_zero(I32x4TruncSatF64x2UZero), - visit_f64x2_convert_low_i32x4_s(F64x2ConvertLowI32x4S), visit_f64x2_convert_low_i32x4_u(F64x2ConvertLowI32x4U), - visit_f32x4_demote_f64x2_zero(F32x4DemoteF64x2Zero), visit_f64x2_promote_low_f32x4(F64x2PromoteLowF32x4), - - visit_i8x16_relaxed_swizzle(I8x16RelaxedSwizzle), - visit_i32x4_relaxed_trunc_f32x4_s(I32x4RelaxedTruncF32x4S), visit_i32x4_relaxed_trunc_f32x4_u(I32x4RelaxedTruncF32x4U), - visit_i32x4_relaxed_trunc_f64x2_s_zero(I32x4RelaxedTruncF64x2SZero), visit_i32x4_relaxed_trunc_f64x2_u_zero(I32x4RelaxedTruncF64x2UZero), - visit_f32x4_relaxed_madd(F32x4RelaxedMadd), visit_f32x4_relaxed_nmadd(F32x4RelaxedNmadd), - visit_f64x2_relaxed_madd(F64x2RelaxedMadd), visit_f64x2_relaxed_nmadd(F64x2RelaxedNmadd), - visit_i8x16_relaxed_laneselect(I8x16RelaxedLaneselect), visit_i16x8_relaxed_laneselect(I16x8RelaxedLaneselect), - visit_i32x4_relaxed_laneselect(I32x4RelaxedLaneselect), visit_i64x2_relaxed_laneselect(I64x2RelaxedLaneselect), - visit_f32x4_relaxed_min(F32x4RelaxedMin), visit_f32x4_relaxed_max(F32x4RelaxedMax), - visit_f64x2_relaxed_min(F64x2RelaxedMin), visit_f64x2_relaxed_max(F64x2RelaxedMax), - visit_i16x8_relaxed_q15mulr_s(I16x8RelaxedQ15mulrS), - visit_i16x8_relaxed_dot_i8x16_i7x16_s(I16x8RelaxedDotI8x16I7x16S), - visit_i32x4_relaxed_dot_i8x16_i7x16_add_s(I32x4RelaxedDotI8x16I7x16AddS), - - visit_i8x16_extract_lane_s(I8x16ExtractLaneS, u8), visit_i8x16_extract_lane_u(I8x16ExtractLaneU, u8), visit_i8x16_replace_lane(I8x16ReplaceLane, u8), - visit_i16x8_extract_lane_s(I16x8ExtractLaneS, u8), visit_i16x8_extract_lane_u(I16x8ExtractLaneU, u8), visit_i16x8_replace_lane(I16x8ReplaceLane, u8), - visit_i32x4_extract_lane(I32x4ExtractLane, u8), visit_i32x4_replace_lane(I32x4ReplaceLane, u8), - visit_i64x2_extract_lane(I64x2ExtractLane, u8), visit_i64x2_replace_lane(I64x2ReplaceLane, u8), - visit_f32x4_extract_lane(F32x4ExtractLane, u8), visit_f32x4_replace_lane(F32x4ReplaceLane, u8), - visit_f64x2_extract_lane(F64x2ExtractLane, u8), visit_f64x2_replace_lane(F64x2ReplaceLane, u8) + lowering_ops! { + memory [Addr] => [S128] { + visit_v128_load => V128Load, visit_v128_load8x8_s => V128Load8x8S, + visit_v128_load8x8_u => V128Load8x8U, visit_v128_load16x4_s => V128Load16x4S, + visit_v128_load16x4_u => V128Load16x4U, visit_v128_load32x2_s => V128Load32x2S, + visit_v128_load32x2_u => V128Load32x2U, visit_v128_load8_splat => V128Load8Splat, + visit_v128_load16_splat => V128Load16Splat, visit_v128_load32_splat => V128Load32Splat, + visit_v128_load64_splat => V128Load64Splat, visit_v128_load32_zero => V128Load32Zero, + visit_v128_load64_zero => V128Load64Zero, + } + memory [Addr, S128] => [] { visit_v128_store => V128Store } + memory [Addr, S128] => [S128] { + visit_v128_load8_lane(lane: u8) => V128Load8Lane, + visit_v128_load16_lane(lane: u8) => V128Load16Lane, + visit_v128_load32_lane(lane: u8) => V128Load32Lane, + visit_v128_load64_lane(lane: u8) => V128Load64Lane, + } + memory [Addr, S128] => [] { + visit_v128_store8_lane(lane: u8) => V128Store8Lane, + visit_v128_store16_lane(lane: u8) => V128Store16Lane, + visit_v128_store32_lane(lane: u8) => V128Store32Lane, + visit_v128_store64_lane(lane: u8) => V128Store64Lane, + } + fixed [S32] => [S128] { + visit_i8x16_splat => I8x16Splat, visit_i16x8_splat => I16x8Splat, + visit_i32x4_splat => I32x4Splat, visit_f32x4_splat => F32x4Splat, + } + fixed [S64] => [S128] { visit_i64x2_splat => I64x2Splat, visit_f64x2_splat => F64x2Splat } + fixed [S128] => [S32] { + visit_v128_any_true => V128AnyTrue, visit_i8x16_all_true => I8x16AllTrue, + visit_i8x16_bitmask => I8x16Bitmask, visit_i16x8_all_true => I16x8AllTrue, + visit_i16x8_bitmask => I16x8Bitmask, visit_i32x4_all_true => I32x4AllTrue, + visit_i32x4_bitmask => I32x4Bitmask, visit_i64x2_all_true => I64x2AllTrue, + visit_i64x2_bitmask => I64x2Bitmask, visit_i8x16_extract_lane_s(lane: u8) => I8x16ExtractLaneS, + visit_i8x16_extract_lane_u(lane: u8) => I8x16ExtractLaneU, + visit_i16x8_extract_lane_s(lane: u8) => I16x8ExtractLaneS, + visit_i16x8_extract_lane_u(lane: u8) => I16x8ExtractLaneU, + visit_i32x4_extract_lane(lane: u8) => I32x4ExtractLane, + visit_f32x4_extract_lane(lane: u8) => F32x4ExtractLane, + } + fixed [S128] => [S64] { + visit_i64x2_extract_lane(lane: u8) => I64x2ExtractLane, + visit_f64x2_extract_lane(lane: u8) => F64x2ExtractLane, + } + fixed [S128, S32] => [S128] { + visit_i8x16_shl => I8x16Shl, visit_i8x16_shr_s => I8x16ShrS, + visit_i8x16_shr_u => I8x16ShrU, visit_i16x8_shl => I16x8Shl, visit_i16x8_shr_s => I16x8ShrS, + visit_i16x8_shr_u => I16x8ShrU, visit_i32x4_shl => I32x4Shl, visit_i32x4_shr_s => I32x4ShrS, + visit_i32x4_shr_u => I32x4ShrU, visit_i64x2_shl => I64x2Shl, visit_i64x2_shr_s => I64x2ShrS, + visit_i64x2_shr_u => I64x2ShrU, visit_i8x16_replace_lane(lane: u8) => I8x16ReplaceLane, + visit_i16x8_replace_lane(lane: u8) => I16x8ReplaceLane, + visit_i32x4_replace_lane(lane: u8) => I32x4ReplaceLane, + visit_f32x4_replace_lane(lane: u8) => F32x4ReplaceLane, + } + fixed [S128, S64] => [S128] { + visit_i64x2_replace_lane(lane: u8) => I64x2ReplaceLane, + visit_f64x2_replace_lane(lane: u8) => F64x2ReplaceLane, + } + fixed [S128] => [S128] { + visit_v128_not => V128Not, visit_i8x16_abs => I8x16Abs, visit_i8x16_neg => I8x16Neg, + visit_i16x8_abs => I16x8Abs, visit_i16x8_neg => I16x8Neg, visit_i32x4_abs => I32x4Abs, + visit_i32x4_neg => I32x4Neg, visit_i64x2_abs => I64x2Abs, visit_i64x2_neg => I64x2Neg, + visit_i16x8_extadd_pairwise_i8x16_s => I16x8ExtAddPairwiseI8x16S, + visit_i16x8_extadd_pairwise_i8x16_u => I16x8ExtAddPairwiseI8x16U, + visit_i32x4_extadd_pairwise_i16x8_s => I32x4ExtAddPairwiseI16x8S, + visit_i32x4_extadd_pairwise_i16x8_u => I32x4ExtAddPairwiseI16x8U, + visit_i16x8_extend_low_i8x16_s => I16x8ExtendLowI8x16S, + visit_i16x8_extend_low_i8x16_u => I16x8ExtendLowI8x16U, + visit_i16x8_extend_high_i8x16_s => I16x8ExtendHighI8x16S, + visit_i16x8_extend_high_i8x16_u => I16x8ExtendHighI8x16U, + visit_i32x4_extend_low_i16x8_s => I32x4ExtendLowI16x8S, + visit_i32x4_extend_low_i16x8_u => I32x4ExtendLowI16x8U, + visit_i32x4_extend_high_i16x8_s => I32x4ExtendHighI16x8S, + visit_i32x4_extend_high_i16x8_u => I32x4ExtendHighI16x8U, + visit_i64x2_extend_low_i32x4_s => I64x2ExtendLowI32x4S, + visit_i64x2_extend_low_i32x4_u => I64x2ExtendLowI32x4U, + visit_i64x2_extend_high_i32x4_s => I64x2ExtendHighI32x4S, + visit_i64x2_extend_high_i32x4_u => I64x2ExtendHighI32x4U, visit_i8x16_popcnt => I8x16Popcnt, + visit_f32x4_ceil => F32x4Ceil, visit_f32x4_floor => F32x4Floor, visit_f32x4_trunc => F32x4Trunc, + visit_f32x4_nearest => F32x4Nearest, visit_f32x4_abs => F32x4Abs, visit_f32x4_neg => F32x4Neg, + visit_f32x4_sqrt => F32x4Sqrt, visit_f64x2_ceil => F64x2Ceil, visit_f64x2_floor => F64x2Floor, + visit_f64x2_trunc => F64x2Trunc, visit_f64x2_nearest => F64x2Nearest, visit_f64x2_abs => F64x2Abs, + visit_f64x2_neg => F64x2Neg, visit_f64x2_sqrt => F64x2Sqrt, + visit_i32x4_trunc_sat_f32x4_s => I32x4TruncSatF32x4S, + visit_i32x4_trunc_sat_f32x4_u => I32x4TruncSatF32x4U, + visit_f32x4_convert_i32x4_s => F32x4ConvertI32x4S, + visit_f32x4_convert_i32x4_u => F32x4ConvertI32x4U, + visit_i32x4_trunc_sat_f64x2_s_zero => I32x4TruncSatF64x2SZero, + visit_i32x4_trunc_sat_f64x2_u_zero => I32x4TruncSatF64x2UZero, + visit_f64x2_convert_low_i32x4_s => F64x2ConvertLowI32x4S, + visit_f64x2_convert_low_i32x4_u => F64x2ConvertLowI32x4U, + visit_f32x4_demote_f64x2_zero => F32x4DemoteF64x2Zero, + visit_f64x2_promote_low_f32x4 => F64x2PromoteLowF32x4, + visit_i32x4_relaxed_trunc_f32x4_s => I32x4RelaxedTruncF32x4S, + visit_i32x4_relaxed_trunc_f32x4_u => I32x4RelaxedTruncF32x4U, + visit_i32x4_relaxed_trunc_f64x2_s_zero => I32x4RelaxedTruncF64x2SZero, + visit_i32x4_relaxed_trunc_f64x2_u_zero => I32x4RelaxedTruncF64x2UZero, + } + fixed [S128, S128] => [S128] { + visit_v128_and => V128And, visit_v128_andnot => V128AndNot, visit_v128_or => V128Or, + visit_v128_xor => V128Xor, visit_i8x16_swizzle => I8x16Swizzle, visit_i8x16_eq => I8x16Eq, + visit_i8x16_ne => I8x16Ne, visit_i8x16_lt_s => I8x16LtS, visit_i8x16_lt_u => I8x16LtU, + visit_i8x16_gt_s => I8x16GtS, visit_i8x16_gt_u => I8x16GtU, visit_i8x16_le_s => I8x16LeS, + visit_i8x16_le_u => I8x16LeU, visit_i8x16_ge_s => I8x16GeS, visit_i8x16_ge_u => I8x16GeU, + visit_i16x8_eq => I16x8Eq, visit_i16x8_ne => I16x8Ne, visit_i16x8_lt_s => I16x8LtS, + visit_i16x8_lt_u => I16x8LtU, visit_i16x8_gt_s => I16x8GtS, visit_i16x8_gt_u => I16x8GtU, + visit_i16x8_le_s => I16x8LeS, visit_i16x8_le_u => I16x8LeU, visit_i16x8_ge_s => I16x8GeS, + visit_i16x8_ge_u => I16x8GeU, visit_i32x4_eq => I32x4Eq, visit_i32x4_ne => I32x4Ne, + visit_i32x4_lt_s => I32x4LtS, visit_i32x4_lt_u => I32x4LtU, visit_i32x4_gt_s => I32x4GtS, + visit_i32x4_gt_u => I32x4GtU, visit_i32x4_le_s => I32x4LeS, visit_i32x4_le_u => I32x4LeU, + visit_i32x4_ge_s => I32x4GeS, visit_i32x4_ge_u => I32x4GeU, visit_i64x2_eq => I64x2Eq, + visit_i64x2_ne => I64x2Ne, visit_i64x2_lt_s => I64x2LtS, visit_i64x2_gt_s => I64x2GtS, + visit_i64x2_le_s => I64x2LeS, visit_i64x2_ge_s => I64x2GeS, visit_f32x4_eq => F32x4Eq, + visit_f32x4_ne => F32x4Ne, visit_f32x4_lt => F32x4Lt, visit_f32x4_gt => F32x4Gt, + visit_f32x4_le => F32x4Le, visit_f32x4_ge => F32x4Ge, visit_f64x2_eq => F64x2Eq, + visit_f64x2_ne => F64x2Ne, visit_f64x2_lt => F64x2Lt, visit_f64x2_gt => F64x2Gt, + visit_f64x2_le => F64x2Le, visit_f64x2_ge => F64x2Ge, visit_i8x16_add => I8x16Add, + visit_i8x16_sub => I8x16Sub, visit_i8x16_min_s => I8x16MinS, visit_i8x16_min_u => I8x16MinU, + visit_i8x16_max_s => I8x16MaxS, visit_i8x16_max_u => I8x16MaxU, + visit_i8x16_narrow_i16x8_s => I8x16NarrowI16x8S, + visit_i8x16_narrow_i16x8_u => I8x16NarrowI16x8U, visit_i8x16_add_sat_s => I8x16AddSatS, + visit_i8x16_add_sat_u => I8x16AddSatU, visit_i8x16_sub_sat_s => I8x16SubSatS, + visit_i8x16_sub_sat_u => I8x16SubSatU, visit_i8x16_avgr_u => I8x16AvgrU, + visit_i16x8_add => I16x8Add, visit_i16x8_sub => I16x8Sub, visit_i16x8_min_s => I16x8MinS, + visit_i16x8_min_u => I16x8MinU, visit_i16x8_max_s => I16x8MaxS, visit_i16x8_max_u => I16x8MaxU, + visit_i16x8_narrow_i32x4_s => I16x8NarrowI32x4S, + visit_i16x8_narrow_i32x4_u => I16x8NarrowI32x4U, visit_i16x8_add_sat_s => I16x8AddSatS, + visit_i16x8_add_sat_u => I16x8AddSatU, visit_i16x8_sub_sat_s => I16x8SubSatS, + visit_i16x8_sub_sat_u => I16x8SubSatU, visit_i16x8_avgr_u => I16x8AvgrU, + visit_i16x8_mul => I16x8Mul, visit_i32x4_add => I32x4Add, visit_i32x4_sub => I32x4Sub, + visit_i32x4_min_s => I32x4MinS, visit_i32x4_min_u => I32x4MinU, visit_i32x4_max_s => I32x4MaxS, + visit_i32x4_max_u => I32x4MaxU, visit_i32x4_mul => I32x4Mul, visit_i64x2_add => I64x2Add, + visit_i64x2_sub => I64x2Sub, visit_i64x2_mul => I64x2Mul, + visit_i16x8_extmul_low_i8x16_s => I16x8ExtMulLowI8x16S, + visit_i16x8_extmul_low_i8x16_u => I16x8ExtMulLowI8x16U, + visit_i16x8_extmul_high_i8x16_s => I16x8ExtMulHighI8x16S, + visit_i16x8_extmul_high_i8x16_u => I16x8ExtMulHighI8x16U, + visit_i32x4_extmul_low_i16x8_s => I32x4ExtMulLowI16x8S, + visit_i32x4_extmul_low_i16x8_u => I32x4ExtMulLowI16x8U, + visit_i32x4_extmul_high_i16x8_s => I32x4ExtMulHighI16x8S, + visit_i32x4_extmul_high_i16x8_u => I32x4ExtMulHighI16x8U, + visit_i64x2_extmul_low_i32x4_s => I64x2ExtMulLowI32x4S, + visit_i64x2_extmul_low_i32x4_u => I64x2ExtMulLowI32x4U, + visit_i64x2_extmul_high_i32x4_s => I64x2ExtMulHighI32x4S, + visit_i64x2_extmul_high_i32x4_u => I64x2ExtMulHighI32x4U, + visit_i16x8_q15mulr_sat_s => I16x8Q15MulrSatS, visit_i32x4_dot_i16x8_s => I32x4DotI16x8S, + visit_f32x4_add => F32x4Add, visit_f32x4_sub => F32x4Sub, visit_f32x4_mul => F32x4Mul, + visit_f32x4_div => F32x4Div, visit_f32x4_min => F32x4Min, visit_f32x4_max => F32x4Max, + visit_f32x4_pmin => F32x4PMin, visit_f32x4_pmax => F32x4PMax, visit_f64x2_add => F64x2Add, + visit_f64x2_sub => F64x2Sub, visit_f64x2_mul => F64x2Mul, visit_f64x2_div => F64x2Div, + visit_f64x2_min => F64x2Min, visit_f64x2_max => F64x2Max, visit_f64x2_pmin => F64x2PMin, + visit_f64x2_pmax => F64x2PMax, visit_i8x16_relaxed_swizzle => I8x16RelaxedSwizzle, + visit_f32x4_relaxed_min => F32x4RelaxedMin, visit_f32x4_relaxed_max => F32x4RelaxedMax, + visit_f64x2_relaxed_min => F64x2RelaxedMin, visit_f64x2_relaxed_max => F64x2RelaxedMax, + visit_i16x8_relaxed_q15mulr_s => I16x8RelaxedQ15mulrS, + visit_i16x8_relaxed_dot_i8x16_i7x16_s => I16x8RelaxedDotI8x16I7x16S, + } + fixed [S128, S128, S128] => [S128] { + visit_v128_bitselect => V128Bitselect, + visit_f32x4_relaxed_madd => F32x4RelaxedMadd, visit_f32x4_relaxed_nmadd => F32x4RelaxedNmadd, + visit_f64x2_relaxed_madd => F64x2RelaxedMadd, visit_f64x2_relaxed_nmadd => F64x2RelaxedNmadd, + visit_i8x16_relaxed_laneselect => I8x16RelaxedLaneselect, + visit_i16x8_relaxed_laneselect => I16x8RelaxedLaneselect, + visit_i32x4_relaxed_laneselect => I32x4RelaxedLaneselect, + visit_i64x2_relaxed_laneselect => I64x2RelaxedLaneselect, + visit_i32x4_relaxed_dot_i8x16_i7x16_add_s => I32x4RelaxedDotI8x16I7x16AddS, + } } fn visit_i8x16_shuffle(&mut self, lanes: [u8; 16]) -> Self::Output { - self.instructions.push(Instruction::I8x16Shuffle(self.data.v128_constants.len() as u32)); + self.emit( + &[OperandSize::S128, OperandSize::S128], + &[OperandSize::S128], + Instruction::I8x16Shuffle(self.data.v128_constants.len() as u32), + )?; self.data.v128_constants.push(lanes); + Ok(()) } fn visit_v128_const(&mut self, value: wasmparser::V128) -> Self::Output { - self.instructions.push(Instruction::Const128(self.data.v128_constants.len() as u32)); + self.emit(&[], &[OperandSize::S128], Instruction::Const128(self.data.v128_constants.len() as u32))?; self.data.v128_constants.push(*value.bytes()); + Ok(()) } } -impl<R: WasmModuleResources> FunctionBuilder<R> { - pub(crate) fn new(validator: FuncValidator<R>, local_addr_map: Vec<u16>) -> Self { - Self { - position: 0, - validator, - local_addr_map, - instructions: Vec::with_capacity(1024), - data: FunctionDataBuilder::default(), - ctx_stack: Vec::with_capacity(256), - error: None, - } +impl FunctionBuilder<'_> { + fn is_unreachable(&self) -> bool { + self.control_stack.last().is_none_or(|frame| frame.unreachable) } - fn record_error(&mut self, error: crate::ParseError) { - if self.error.is_none() { - self.error = Some(error); - } + fn get_ctx_idx(&self, depth: u32) -> Result<usize> { + self.control_stack + .len() + .checked_sub(depth as usize + 1) + .ok_or_else(|| crate::ParseError::Other(alloc::format!("branch depth out of bounds: {depth}"))) } - fn stack_base_at_frame(&self, depth: usize) -> StackBase { - let Some(frame) = self.validator.get_control_frame(depth) else { - return StackBase::default(); - }; - let mut base = StackBase::default(); - let stack_height = self.validator.operand_stack_height() as usize; - for i in 0..frame.height { - let depth_from_top = stack_height - 1 - i; - if let Some(Some(ty)) = self.validator.get_operand_type(depth_from_top) { - match operand_size(ty) { - OperandSize::S32 => base.s32 += 1, - OperandSize::S64 => base.s64 += 1, - OperandSize::S128 => base.s128 += 1, - } - } + fn local(&self, idx: u32) -> Result<(OperandSize, u16)> { + let size = *self + .local_types + .get(idx as usize) + .ok_or_else(|| crate::ParseError::Other(alloc::format!("local index out of bounds: {idx}")))?; + let addr = *self + .local_addr_map + .get(idx as usize) + .ok_or_else(|| crate::ParseError::Other(alloc::format!("local address missing: {idx}")))?; + Ok((size, addr)) + } + + /// Pushes logical operands while maintaining the lane counts used by `DropKeep`. + fn push_sizes(&mut self, sizes: &[OperandSize]) -> Result<()> { + for &size in sizes { + let count = match size { + OperandSize::S32 => &mut self.lane_counts.c32, + OperandSize::S64 => &mut self.lane_counts.c64, + OperandSize::S128 => &mut self.lane_counts.c128, + }; + *count = count + .checked_add(1) + .ok_or_else(|| crate::ParseError::Other("logical operand lane count is too large".into()))?; + self.operand_stack.push(size); } + Ok(()) + } - base + /// Pops an operand, allowing a polymorphic value at an unreachable frame base. + fn pop_expect(&mut self, expected: OperandSize) -> Result<()> { + let frame_height = self.control_stack.last().map_or(0, |frame| frame.height); + if self.operand_stack.len() == frame_height && self.is_unreachable() { + return Ok(()); + } + let actual = self + .operand_stack + .pop() + .ok_or_else(|| crate::ParseError::Other("logical operand stack underflow".into()))?; + if actual != expected { + return Err(crate::ParseError::Other("logical operand width mismatch".into())); + } + match actual { + OperandSize::S32 => self.lane_counts.c32 -= 1, + OperandSize::S64 => self.lane_counts.c64 -= 1, + OperandSize::S128 => self.lane_counts.c128 -= 1, + } + Ok(()) } - fn unsupported(&mut self, name: &str) { - self.record_error(crate::ParseError::UnsupportedOperator(name.to_string())); + /// Applies a declared logical stack effect in WebAssembly operand order. + fn apply_effect(&mut self, inputs: &[OperandSize], outputs: &[OperandSize]) -> Result<()> { + inputs.iter().rev().try_for_each(|&size| self.pop_expect(size))?; + self.push_sizes(outputs)?; + Ok(()) } - fn is_unreachable(&self) -> bool { - self.validator.get_control_frame(0).is_none_or(|f| f.unreachable) + /// Applies an instruction's stack effect before adding it to the bytecode. + fn emit(&mut self, inputs: &[OperandSize], outputs: &[OperandSize], instruction: Instruction) -> Result<()> { + self.apply_effect(inputs, outputs)?; + self.instructions.push(instruction); + Ok(()) } - fn get_ctx_idx(&self, depth: u32) -> Option<usize> { - let len = self.ctx_stack.len(); - let idx = len.checked_sub(depth as usize + 1)?; - Some(idx) + /// Restores both logical operand order and lane counts to a control-frame base. + fn reset_stack(&mut self, height: usize, base: ValueCounts) { + self.operand_stack.truncate(height); + self.lane_counts = base; } - fn emit_dropkeep(&mut self, base: StackBase, c32: u16, c64: u16, c128: u16) { - if base.s32 == 0 && c32 == 0 && base.s64 == 0 && c64 == 0 && base.s128 == 0 && c128 == 0 { - return; + /// Marks the current path unreachable and restores its entry stack. + fn mark_unreachable(&mut self) { + if let Some(frame) = self.control_stack.last_mut() { + frame.unreachable = true; + let height = frame.height; + let base = frame.base; + self.reset_stack(height, base); } + } - let fits_u8 = base.s32 <= u8::MAX as u16 - && c32 <= u8::MAX as u16 - && base.s64 <= u8::MAX as u16 - && c64 <= u8::MAX as u16 - && base.s128 <= u8::MAX as u16 - && c128 <= u8::MAX as u16; + /// Enters a control frame with its parameters restored above the saved base. + fn push_control(&mut self, kind: BlockKind, ty: wasmparser::BlockType, initial_jump: Option<usize>) -> Result<()> { + let signature = match ty { + wasmparser::BlockType::Empty => Signature { params: Vec::new(), results: Vec::new() }, + wasmparser::BlockType::Type(ty) => { + Signature { params: Vec::new(), results: alloc::vec![OperandSize::from(ty)] } + } + wasmparser::BlockType::FuncType(idx) => self.metadata.signature(idx)?.clone(), + }; + for &size in signature.params.iter().rev() { + self.pop_expect(size)?; + } + let height = self.operand_stack.len(); + let base = self.lane_counts; + self.push_sizes(&signature.params)?; + let entry_unreachable = self.is_unreachable(); + self.control_stack.push(ControlFrame { + kind, + has_else: false, + start_ip: self.instructions.len(), + branch_jumps: initial_jump.into_iter().collect(), + height, + base, + params: signature.params, + results: signature.results, + unreachable: entry_unreachable, + entry_unreachable, + end_reachable: false, + }); + Ok(()) + } - if fits_u8 { - self.instructions.push(Instruction::DropKeep { - base32: base.s32, - keep32: c32 as u8, - base64: base.s64, - keep64: c64 as u8, - base128: base.s128, - keep128: c128 as u8, - }); - } else { - self.instructions.push(Instruction::DropKeep32(base.s32, c32)); - self.instructions.push(Instruction::DropKeep64(base.s64, c64)); - self.instructions.push(Instruction::DropKeep128(base.s128, c128)); + /// Emits the stack-shaping instruction required by a branch. + fn emit_dropkeep(&mut self, base: ValueCounts, keep: ValueCounts) { + if base.is_empty() && keep.is_empty() { + return; } + self.instructions.push(Instruction::DropKeep((base, keep).into())); } fn patch_jump(&mut self, jump_ip: usize, target: usize) { @@ -623,117 +1035,58 @@ impl<R: WasmModuleResources> FunctionBuilder<R> { } } - fn label_keep_counts(label_types: &[wasmparser::ValType]) -> (u16, u16, u16) { - let (mut c32, mut c64, mut c128) = (0, 0, 0); - for &ty in label_types { - match operand_size(ty) { - OperandSize::S32 => c32 += 1, - OperandSize::S64 => c64 += 1, - OperandSize::S128 => c128 += 1, - } - } - - (c32, c64, c128) - } - - fn label_keep_counts_for_frame(&self, frame: &wasmparser::Frame) -> (u16, u16, u16) { - match &frame.block_type { - wasmparser::BlockType::Empty => (0, 0, 0), - wasmparser::BlockType::Type(ty) => match frame.kind { - FrameKind::Loop => (0, 0, 0), - _ => Self::label_keep_counts(core::slice::from_ref(ty)), - }, - wasmparser::BlockType::FuncType(idx) => { - let sub_type = self.validator.resources().sub_type_at(*idx); - let func_ty = match sub_type { - Some(st) => st.composite_type.unwrap_func(), - None => return (0, 0, 0), - }; - match frame.kind { - FrameKind::Loop => Self::label_keep_counts(func_ty.params()), - _ => Self::label_keep_counts(func_ty.results()), - } + fn value_counts(sizes: &[OperandSize]) -> ValueCounts { + let mut counts = ValueCounts::default(); + for size in sizes { + match size { + OperandSize::S32 => counts.c32 += 1, + OperandSize::S64 => counts.c64 += 1, + OperandSize::S128 => counts.c128 += 1, } } + counts } - fn emit_dropkeep_to_label(&mut self, label_depth: u32) { + /// Shapes stack lanes to the values consumed by a branch target. + fn emit_dropkeep_to_label(&mut self, label_depth: u32) -> Result<()> { if self.is_unreachable() { - return; - } - - let Some(frame) = self.validator.get_control_frame(label_depth as usize) else { - return; - }; - - let base = self.stack_base_at_frame(label_depth as usize); - let (c32, c64, c128) = self.label_keep_counts_for_frame(frame); - - self.emit_dropkeep(base, c32, c64, c128); - } - - fn emit_branch_jump_or_return(&mut self, depth: u32) { - if let Some(ctx_idx) = self.get_ctx_idx(depth) { - let jump_ip = self.instructions.len(); - self.instructions.push(Instruction::Jump(0)); - self.ctx_stack[ctx_idx].branch_jumps.push(jump_ip); - } else { - self.instructions.push(Instruction::Return); + return Ok(()); } + let frame = &self.control_stack[self.get_ctx_idx(label_depth)?]; + let base = frame.base; + let label_types = if matches!(frame.kind, BlockKind::Loop) { &frame.params } else { &frame.results }; + self.emit_dropkeep(base, Self::value_counts(label_types)); + Ok(()) } - fn emit_br_table_pad(&mut self, depth: u32) -> (usize, usize, bool) { - let pad_start = self.instructions.len(); - let frame = if self.is_unreachable() { None } else { self.validator.get_control_frame(depth as usize) }; - let Some(frame) = frame else { - let ip = self.instructions.len(); - self.instructions.push(Instruction::Return); - return (pad_start, ip, true); - }; - - let base = self.stack_base_at_frame(depth as usize); - let (c32, c64, c128) = self.label_keep_counts_for_frame(frame); - self.emit_dropkeep(base, c32, c64, c128); - - let jump_ip = self.instructions.len(); - self.instructions.push(Instruction::Jump(0)); - (pad_start, jump_ip, false) - } - - fn patch_branch_jump_or_return(&mut self, depth: u32, jump_ip: usize) { - let Some(frame) = self.validator.get_control_frame(depth as usize) else { - self.instructions[jump_ip] = Instruction::Return; - return; - }; - let Some(ctx_idx) = self.get_ctx_idx(depth) else { - self.instructions[jump_ip] = Instruction::Return; - return; - }; - - match frame.kind { - FrameKind::Loop => self.patch_jump(jump_ip, self.ctx_stack[ctx_idx].start_ip), - _ => self.ctx_stack[ctx_idx].branch_jumps.push(jump_ip), + fn emit_branch_jump_or_return(&mut self, depth: u32) -> Result<()> { + let ctx_idx = self.get_ctx_idx(depth)?; + match self.control_stack[ctx_idx].kind { + BlockKind::Function => self.instructions.push(Instruction::Return), + BlockKind::Loop => self.instructions.push(Instruction::Jump(self.control_stack[ctx_idx].start_ip as u32)), + BlockKind::Block | BlockKind::If => { + self.control_stack[ctx_idx].branch_jumps.push(self.instructions.len()); + self.control_stack[ctx_idx].end_reachable = true; + self.instructions.push(Instruction::Jump(0)); + } } + Ok(()) } - fn patch_end_jumps(&mut self, ctx: LoweringCtx, end_ip: usize) { - match ctx.kind { - BlockKind::Block | BlockKind::Loop => { - let target = if matches!(ctx.kind, BlockKind::Loop) { ctx.start_ip } else { end_ip }; - for jump_ip in ctx.branch_jumps { - self.patch_jump(jump_ip, target); - } - } - BlockKind::If => { - if let Some((&cond_jump_ip, branch_jumps)) = ctx.branch_jumps.split_first() { - if !ctx.has_else { - self.patch_jump(cond_jump_ip, end_ip); - } - for &jump_ip in branch_jumps { - self.patch_jump(jump_ip, end_ip); - } - } + /// Resolves all jumps owned by a completed control frame. + fn patch_end_jumps(&mut self, ctx: ControlFrame, end_ip: usize) { + let target = if matches!(ctx.kind, BlockKind::Loop) { ctx.start_ip } else { end_ip }; + let mut jumps = ctx.branch_jumps.as_slice(); + if matches!(ctx.kind, BlockKind::If) + && let Some((cond_jump, branch_jumps)) = jumps.split_first() + { + if !ctx.has_else { + self.patch_jump(*cond_jump, end_ip); } + jumps = branch_jumps; + } + for &jump in jumps { + self.patch_jump(jump, target); } } } diff --git a/crates/tinywasm/src/interpreter/executor.rs b/crates/tinywasm/src/interpreter/executor.rs index b87c3b5..d7a0b27 100644 --- a/crates/tinywasm/src/interpreter/executor.rs +++ b/crates/tinywasm/src/interpreter/executor.rs @@ -251,8 +251,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { use tinywasm_types::Instruction::*; #[rustfmt::skip] match next { - Nop => {} - MergeBarrier | Unreachable => return Err(Trap::Unreachable), + Unreachable => return Err(Trap::Unreachable), Drop32 => { _ = Value32::stack_pop(&mut self.store.value_stack)}, Drop64 => { _ = Value64::stack_pop(&mut self.store.value_stack)}, Drop128 => { _ = Value128::stack_pop(&mut self.store.value_stack)}, @@ -281,10 +280,7 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { JumpCmpLocalConst64 { target_ip, local, imm, op } => if self.exec_jump_cmp_local_const_64(*target_ip, *local, *imm, *op) { return Ok(None) }, JumpCmpLocalLocal32 { target_ip, left, right, op } => if self.exec_jump_cmp_local_local_32(*target_ip, *left, *right, *op) { return Ok(None) }, JumpCmpLocalLocal64 { target_ip, left, right, op } => if self.exec_jump_cmp_local_local_64(*target_ip, *left, *right, *op) { return Ok(None) }, - DropKeep { base32, keep32, base64, keep64, base128, keep128 } => self.exec_drop_keep(*base32, *keep32, *base64, *keep64, *base128, *keep128), - DropKeep32(base, keep) => self.store.value_stack.stack_32.truncate_keep((self.cf.stack_base().s32 + *base as u32) as usize, *keep as usize), - DropKeep64(base, keep) => self.store.value_stack.stack_64.truncate_keep((self.cf.stack_base().s64 + *base as u32) as usize, *keep as usize), - DropKeep128(base, keep) => self.store.value_stack.stack_128.truncate_keep((self.cf.stack_base().s128 + *base as u32) as usize, *keep as usize), + DropKeep(drop_keep) => self.exec_drop_keep(*drop_keep), BranchTable(default_ip, start, len) => { self.exec_branch_table(*default_ip, *start, *len); return Ok(None); } Return => { if self.exec_return() { return Ok(Some(())); } return Ok(None); } ReturnVoid => { if self.exec_return_void() { return Ok(Some(())); } return Ok(None); } @@ -930,14 +926,12 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> { self.cf.instr_ptr = target_ip as usize; } - fn exec_drop_keep(&mut self, base32: u16, keep32: u8, base64: u16, keep64: u8, base128: u16, keep128: u8) { + fn exec_drop_keep(&mut self, drop_keep: DropKeep) { let mut base = self.cf.stack_base(); - base.s32 += base32 as u32; - base.s64 += base64 as u32; - base.s128 += base128 as u32; - self.store - .value_stack - .truncate_keep_counts(base, ValueCounts { c32: keep32 as u16, c64: keep64 as u16, c128: keep128 as u16 }); + base.s32 += drop_keep.base.c32 as u32; + base.s64 += drop_keep.base.c64 as u32; + base.s128 += drop_keep.base.c128 as u32; + self.store.value_stack.truncate_keep_counts(base, drop_keep.keep); } fn exec_call(&mut self, wasm_func: WasmFunctionInstance, func_addr: FuncAddr) -> Result<(), Trap> { diff --git a/crates/tinywasm/tests/wasm-custom/mixed-width-branch-shaping.wast b/crates/tinywasm/tests/wasm-custom/mixed-width-branch-shaping.wast new file mode 100644 index 0000000..010c1a0 --- /dev/null +++ b/crates/tinywasm/tests/wasm-custom/mixed-width-branch-shaping.wast @@ -0,0 +1,16 @@ +;; A branch preserves its mixed-width results while discarding lane-specific +;; temporaries above values that predate the block. +(module + (func (export "mixed") (result i64) + i64.const 40 + (block (result i64 i32) + i32.const 99 + i64.const 98 + v128.const i32x4 1 2 3 4 + i64.const 9 + i32.const 7 + br 0) + drop + i64.add)) + +(assert_return (invoke "mixed") (i64.const 49)) diff --git a/crates/types/src/instructions.rs b/crates/types/src/instructions.rs index c8ffe2c..c104ce0 100644 --- a/crates/types/src/instructions.rs +++ b/crates/types/src/instructions.rs @@ -11,6 +11,21 @@ pub struct MemoryArg { mem_addr: MemAddr, } +/// Stack lanes discarded and retained when branching to a control frame. +#[derive(Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "debug", derive(Debug))] +#[cfg_attr(feature = "archive", derive(serde::Serialize, serde::Deserialize))] +pub struct DropKeep { + pub base: ValueCounts, + pub keep: ValueCounts, +} + +impl From<(ValueCounts, ValueCounts)> for DropKeep { + fn from((base, keep): (ValueCounts, ValueCounts)) -> Self { + Self { base, keep } + } +} + impl MemoryArg { #[inline] pub const fn new(offset: u64, mem_addr: MemAddr) -> Self { @@ -159,8 +174,6 @@ pub enum Instruction { // > Control Instructions (jump-oriented, lowered from structured control during parsing) // See <https://webassembly.github.io/spec/core/binary/instructions.html#control-instructions> Unreachable, - Nop, - MergeBarrier, Jump(u32), JumpIfZero32(u32), JumpIfNonZero32(u32), @@ -176,10 +189,7 @@ pub enum Instruction { JumpCmpLocalConst64 { target_ip: u32, local: LocalAddr, imm: i32, op: CmpOp }, JumpCmpLocalLocal32 { target_ip: u32, left: LocalAddr, right: LocalAddr, op: CmpOp }, JumpCmpLocalLocal64 { target_ip: u32, left: LocalAddr, right: LocalAddr, op: CmpOp }, - DropKeep { base32: u16, keep32: u8, base64: u16, keep64: u8, base128: u16, keep128: u8 }, - DropKeep32(u16, u16), - DropKeep64(u16, u16), - DropKeep128(u16, u16), + DropKeep(DropKeep), BranchTable(u32, u32, u32), // (default_landing_pad_ip, branch_table_start, target_count) Return, ReturnVoid, |