chore: add more memory superinstructions

Signed-off-by: Henry <mail@henrygressmann.de>

4 files changed, 194 insertions, 14 deletions

diff --git a/crates/parser/src/optimize.rs b/crates/parser/src/optimize.rs
index a26142c..80dfa0d 100644
--- a/crates/parser/src/optimize.rs
+++ b/crates/parser/src/optimize.rs
@@ -164,6 +164,16 @@ fn rewrite(
                 rewrite!(instrs, i, [LocalGet128(local), Const128(c)] => BinOpLocalConst128(BinOp128::AndNot, local, c));
             }
             I32Store(memarg) | F32Store(memarg) => {
+                rewrite!(instrs, i,
+                    [LocalGet32(addr), LoadLocal32(load_memarg, load_addr), AddConst32(1)] if
+                    (load_memarg == memarg && addr == u16::from(load_addr)) =>
+                    IncMemoryLocal32(memarg, load_addr)
+                );
+                rewrite!(instrs, i,
+                    [LocalGet64(addr), LoadLocal32(load_memarg, load_addr), AddConst32(1)] if
+                    (load_memarg == memarg && addr == u16::from(load_addr)) =>
+                    IncMemoryLocal32(memarg, load_addr)
+                );
                 rewrite!(instrs, i, [F32Mul, F32Add] => FMaStoreF32(memarg));
                 rewrite!(instrs, i,
                     [LocalGet32(addr_local), LocalGet32(value_local)] if
@@ -172,6 +182,16 @@ fn rewrite(
                 );
             }
             I64Store(memarg) | F64Store(memarg) => {
+                rewrite!(instrs, i,
+                    [LocalGet32(addr), LoadLocal64(load_memarg, load_addr), AddConst64(1)] if
+                    (load_memarg == memarg && addr == u16::from(load_addr)) =>
+                    IncMemoryLocal64(memarg, load_addr)
+                );
+                rewrite!(instrs, i,
+                    [LocalGet64(addr), LoadLocal64(load_memarg, load_addr), AddConst64(1)] if
+                    (load_memarg == memarg && addr == u16::from(load_addr)) =>
+                    IncMemoryLocal64(memarg, load_addr)
+                );
                 rewrite!(instrs, i, [F64Mul, F64Add] => FMaStoreF64(memarg));
                 rewrite!(instrs, i,
                     [LocalGet32(addr_local), LocalGet64(value_local)] if
@@ -191,6 +211,60 @@ fn rewrite(
                     [LocalGet32(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) =>
                     LoadLocal32(memarg, addr_local)
                 );
+                rewrite!(instrs, i,
+                    [LocalGet64(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) =>
+                    LoadLocal32(memarg, addr_local)
+                );
+            }
+            I64Load(memarg) | F64Load(memarg) => {
+                rewrite!(instrs, i,
+                    [LocalGet32(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) =>
+                    LoadLocal64(memarg, addr_local)
+                );
+                rewrite!(instrs, i,
+                    [LocalGet64(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) =>
+                    LoadLocal64(memarg, addr_local)
+                );
+            }
+            I32Load8S(memarg) => {
+                rewrite!(instrs, i,
+                    [LocalGet32(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) =>
+                    LoadLocal8S32(memarg, addr_local)
+                );
+                rewrite!(instrs, i,
+                    [LocalGet64(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) =>
+                    LoadLocal8S32(memarg, addr_local)
+                );
+            }
+            I32Load8U(memarg) => {
+                rewrite!(instrs, i,
+                    [LocalGet32(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) =>
+                    LoadLocal8U32(memarg, addr_local)
+                );
+                rewrite!(instrs, i,
+                    [LocalGet64(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) =>
+                    LoadLocal8U32(memarg, addr_local)
+                );
+            }
+            I32Load16S(memarg) => {
+                rewrite!(instrs, i,
+                    [LocalGet32(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) =>
+                    LoadLocal16S32(memarg, addr_local)
+                );
+                rewrite!(instrs, i,
+                    [LocalGet64(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) =>
+                    LoadLocal16S32(memarg, addr_local)
+                );
+            }
+            I32Load16U(memarg) => {
+                rewrite!(instrs, i,
+                    [LocalGet32(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) =>
+                    LoadLocal16U32(memarg, addr_local)
+                );
+                rewrite!(instrs, i,
+                    [LocalGet64(addr_local)] if (let Ok(addr_local) = u8::try_from(addr_local)) =>
+                    LoadLocal16U32(memarg, addr_local)
+                );
             }
             MemoryFill(mem) => {
                 rewrite!(instrs, i, [Const32(val), Const32(size)] => MemoryFillImm(mem, val as u8, size))
@@ -231,6 +305,10 @@ fn rewrite(
                     set_local_const = SetLocalConst32
                 );
                 rewrite!(instrs, i, [LoadLocal32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => LoadLocalSet32(memarg, addr, dst));
+                rewrite!(instrs, i, [LoadLocal8S32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => LoadLocalSet8S32(memarg, addr, dst));
+                rewrite!(instrs, i, [LoadLocal8U32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => LoadLocalSet8U32(memarg, addr, dst));
+                rewrite!(instrs, i, [LoadLocal16S32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => LoadLocalSet16S32(memarg, addr, dst));
+                rewrite!(instrs, i, [LoadLocal16U32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) => LoadLocalSet16U32(memarg, addr, dst));
                 rewrite!(instrs, i,
                     [LocalGet32(addr), I32Load(memarg)] if
                     (let (Ok(addr), Ok(dst)) = (u8::try_from(addr), u8::try_from(dst))) =>
@@ -333,6 +411,22 @@ fn rewrite(
                     [LoadLocal32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) =>
                     LoadLocalTee32(memarg, addr, dst)
                 );
+                rewrite!(instrs, i,
+                    [LoadLocal8S32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) =>
+                    LoadLocalTee8S32(memarg, addr, dst)
+                );
+                rewrite!(instrs, i,
+                    [LoadLocal8U32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) =>
+                    LoadLocalTee8U32(memarg, addr, dst)
+                );
+                rewrite!(instrs, i,
+                    [LoadLocal16S32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) =>
+                    LoadLocalTee16S32(memarg, addr, dst)
+                );
+                rewrite!(instrs, i,
+                    [LoadLocal16U32(memarg, addr)] if (let Ok(dst) = u8::try_from(dst)) =>
+                    LoadLocalTee16U32(memarg, addr, dst)
+                );
             }
             LocalTee64(dst) => {
                 fold_local_binop!(
diff --git a/crates/tinywasm/src/interpreter/executor.rs b/crates/tinywasm/src/interpreter/executor.rs
index 8183b55..1c85f7a 100644
--- a/crates/tinywasm/src/interpreter/executor.rs
+++ b/crates/tinywasm/src/interpreter/executor.rs
@@ -334,14 +334,41 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> {
             SetLocalConst32(local_index, c) => i32::local_set(&mut self.store.value_stack, &self.cf, *local_index, *c),
             SetLocalConst64(local_index, c) => i64::local_set(&mut self.store.value_stack, &self.cf, *local_index, *c),
             SetLocalConst128(local_index, c) => Value128::local_set(&mut self.store.value_stack, &self.cf, *local_index, Value128(self.func.data.v128_const(*c))),
+            IncMemoryLocal32(m, addr_local) => self.exec_inc_memory_local::<i32, 4>(*m, *addr_local, |v| v.wrapping_add(1))?,
+            IncMemoryLocal64(m, addr_local) => self.exec_inc_memory_local::<i64, 8>(*m, *addr_local, |v| v.wrapping_add(1))?,
             StoreLocalLocal32(m, addr_local, value_local) => self.exec_store_local_local::<u32, 4>(*m, *addr_local, *value_local)?,
             StoreLocalLocal64(m, addr_local, value_local) => self.exec_store_local_local::<i64, 8>(*m, *addr_local, *value_local)?,
             StoreLocalLocal128(m, addr_local, value_local) => self.exec_store_local_local::<Value128, 16>(*m, *addr_local, *value_local)?,
             LoadLocal32(m, addr_local) => self.store.value_stack.push(self.exec_load_local_value::<i32, 4>(*m, *addr_local)?)?,
-            LoadLocalTee32(m, addr_local, dst_local) => self.exec_load_local_tee::<i32, 4>(*m, *addr_local, *dst_local)?,
-            LoadLocalSet32(m, addr_local, dst_local) => self.exec_load_local_set::<i32, 4>(*m, *addr_local, *dst_local)?,
-            LoadLocalTee128(m, addr_local, dst_local) => self.exec_load_local_tee::<Value128, 16>(*m, *addr_local, *dst_local)?,
-            LoadLocalSet128(m, addr_local, dst_local) => self.exec_load_local_set::<Value128, 16>(*m, *addr_local, *dst_local)?,
+            LoadLocal64(m, addr_local) => self.store.value_stack.push(self.exec_load_local_value::<i64, 8>(*m, *addr_local)?)?,
+            LoadLocal8S32(m, addr_local) => {
+                let value = self.exec_load_local_value::<i8, 1>(*m, *addr_local)?;
+                self.store.value_stack.push(i32::from(value))?;
+            }
+            LoadLocal8U32(m, addr_local) => {
+                let value = self.exec_load_local_value::<u8, 1>(*m, *addr_local)?;
+                self.store.value_stack.push(i32::from(value))?;
+            }
+            LoadLocal16S32(m, addr_local) => {
+                let value = self.exec_load_local_value::<i16, 2>(*m, *addr_local)?;
+                self.store.value_stack.push(i32::from(value))?;
+            }
+            LoadLocal16U32(m, addr_local) => {
+                let value = self.exec_load_local_value::<u16, 2>(*m, *addr_local)?;
+                self.store.value_stack.push(i32::from(value))?;
+            }
+            LoadLocalTee32(m, addr_local, dst_local) => self.exec_load_local_tee::<i32, 4, _>(*m, *addr_local, *dst_local, |v| v)?,
+            LoadLocalSet32(m, addr_local, dst_local) => self.exec_load_local_set::<i32, 4, _>(*m, *addr_local, *dst_local, |v| v)?,
+            LoadLocalTee8S32(m, addr_local, dst_local) => self.exec_load_local_tee::<i8, 1, _>(*m, *addr_local, *dst_local, i32::from)?,
+            LoadLocalTee8U32(m, addr_local, dst_local) => self.exec_load_local_tee::<u8, 1, _>(*m, *addr_local, *dst_local, i32::from)?,
+            LoadLocalTee16S32(m, addr_local, dst_local) => self.exec_load_local_tee::<i16, 2, _>(*m, *addr_local, *dst_local, i32::from)?,
+            LoadLocalTee16U32(m, addr_local, dst_local) => self.exec_load_local_tee::<u16, 2, _>(*m, *addr_local, *dst_local, i32::from)?,
+            LoadLocalSet8S32(m, addr_local, dst_local) => self.exec_load_local_set::<i8, 1, _>(*m, *addr_local, *dst_local, i32::from)?,
+            LoadLocalSet8U32(m, addr_local, dst_local) => self.exec_load_local_set::<u8, 1, _>(*m, *addr_local, *dst_local, i32::from)?,
+            LoadLocalSet16S32(m, addr_local, dst_local) => self.exec_load_local_set::<i16, 2, _>(*m, *addr_local, *dst_local, i32::from)?,
+            LoadLocalSet16U32(m, addr_local, dst_local) => self.exec_load_local_set::<u16, 2, _>(*m, *addr_local, *dst_local, i32::from)?,
+            LoadLocalTee128(m, addr_local, dst_local) => self.exec_load_local_tee::<Value128, 16, _>(*m, *addr_local, *dst_local, |v| v)?,
+            LoadLocalSet128(m, addr_local, dst_local) => self.exec_load_local_set::<Value128, 16, _>(*m, *addr_local, *dst_local, |v| v)?,
             AndConstTee32(c, local_index) => { stack_op!(unary i32, |v| v & *c); stack_op!(local_tee i32, local_index); }
             SubConstTee32(c, local_index) => { stack_op!(unary i32, |v| v.wrapping_sub(*c)); stack_op!(local_tee i32, local_index); }
             AndConstTee64(c, local_index) => { stack_op!(unary i64, |v| v & *c); stack_op!(local_tee i64, local_index); }
@@ -1151,6 +1178,28 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> {
     }
 
     #[inline(always)]
+    fn exec_inc_memory_local<T: MemValue<N>, const N: usize>(
+        &mut self,
+        memarg: MemoryArg,
+        addr_local: u8,
+        increment: impl FnOnce(T) -> T,
+    ) -> Result<(), Trap> {
+        let mem_addr = self.module.resolve_mem_addr(memarg.mem_addr());
+        let mem = self.store.state.get_mem(mem_addr);
+        let addr = if mem.is_64bit() {
+            let base = i64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as u64;
+            mem.effective_addr_64::<N>(base, memarg.offset())?
+        } else {
+            let base = u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local));
+            mem.effective_addr_32::<N>(base, memarg.offset())?
+        };
+
+        let mem = self.store.state.get_mem_mut(mem_addr);
+        let value = T::load_at(&*mem.inner, addr)?;
+        increment(value).store_at(&mut *mem.inner, addr)
+    }
+
+    #[inline(always)]
     fn exec_fma_store<
         T: InternalValue + MemValue<N> + core::ops::Add<Output = T> + core::ops::Mul<Output = T>,
         const N: usize,
@@ -1187,8 +1236,13 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> {
         addr_local: u8,
     ) -> Result<T, Trap> {
         let mem = self.store.state.get_mem(self.module.resolve_mem_addr(memarg.mem_addr()));
-        let base = u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local));
-        let addr = mem.effective_addr_32::<N>(base, memarg.offset())?;
+        let addr = if mem.is_64bit() {
+            let base = i64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as u64;
+            mem.effective_addr_64::<N>(base, memarg.offset())?
+        } else {
+            let base = u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local));
+            mem.effective_addr_32::<N>(base, memarg.offset())?
+        };
         match T::load_at(&*mem.inner, addr) {
             Ok(res) => Ok(res),
             Err(err) => {
@@ -1198,26 +1252,28 @@ impl<'store, const BUDGETED: bool> Executor<'store, BUDGETED> {
         }
     }
 
-    fn exec_load_local_tee<T: InternalValue + MemValue<N>, const N: usize>(
+    fn exec_load_local_tee<LOAD: MemValue<N>, const N: usize, TARGET: InternalValue>(
         &mut self,
         memarg: MemoryArg,
         addr_local: u8,
         dst_local: u8,
+        cast: impl Fn(LOAD) -> TARGET,
     ) -> Result<(), Trap> {
-        let value = self.exec_load_local_value::<T, N>(memarg, addr_local)?;
-        T::local_set(&mut self.store.value_stack, &self.cf, u16::from(dst_local), value);
+        let value = cast(self.exec_load_local_value::<LOAD, N>(memarg, addr_local)?);
+        TARGET::local_set(&mut self.store.value_stack, &self.cf, u16::from(dst_local), value);
         self.store.value_stack.push(value)?;
         Ok(())
     }
 
-    fn exec_load_local_set<T: InternalValue + MemValue<N>, const N: usize>(
+    fn exec_load_local_set<LOAD: MemValue<N>, const N: usize, TARGET: InternalValue>(
         &mut self,
         memarg: MemoryArg,
         addr_local: u8,
         dst_local: u8,
+        cast: impl Fn(LOAD) -> TARGET,
     ) -> Result<(), Trap> {
-        let value = self.exec_load_local_value::<T, N>(memarg, addr_local)?;
-        T::local_set(&mut self.store.value_stack, &self.cf, u16::from(dst_local), value);
+        let value = cast(self.exec_load_local_value::<LOAD, N>(memarg, addr_local)?);
+        TARGET::local_set(&mut self.store.value_stack, &self.cf, u16::from(dst_local), value);
         Ok(())
     }
 
diff --git a/crates/tinywasm/src/store/memory/mod.rs b/crates/tinywasm/src/store/memory/mod.rs
index 57783d2..aeaf3ba 100644
--- a/crates/tinywasm/src/store/memory/mod.rs
+++ b/crates/tinywasm/src/store/memory/mod.rs
@@ -375,7 +375,7 @@ macro_rules! impl_mem_traits {
 
                 #[inline(always)]
                 fn load_at(mem: &dyn LinearMemory, addr: usize) -> core::result::Result<Self, crate::Trap> {
-                    Ok(Self::from_mem_bytes(match mem.$read(addr) {
+                    Ok(Self::from_le_bytes(match mem.$read(addr) {
                         Ok(bytes) => bytes,
                         Err(trap) => {
                             cold_path();
diff --git a/crates/types/src/instructions.rs b/crates/types/src/instructions.rs
index c104ce0..a22279e 100644
--- a/crates/types/src/instructions.rs
+++ b/crates/types/src/instructions.rs
@@ -150,12 +150,27 @@ pub enum Instruction {
     BinOpStackGlobal32(BinOp, u32),
     BinOpStackGlobal64(BinOp, u32),
     SetLocalConst32(LocalAddr, i32), SetLocalConst64(LocalAddr, i64), SetLocalConst128(LocalAddr, ConstIdx),
+    IncMemoryLocal32(MemoryArg, u8),
+    IncMemoryLocal64(MemoryArg, u8),
     StoreLocalLocal32(MemoryArg, u8, u8),
     StoreLocalLocal64(MemoryArg, u8, u8),
     StoreLocalLocal128(MemoryArg, u8, u8),
     LoadLocal32(MemoryArg, u8),
+    LoadLocal64(MemoryArg, u8),
+    LoadLocal8S32(MemoryArg, u8),
+    LoadLocal8U32(MemoryArg, u8),
+    LoadLocal16S32(MemoryArg, u8),
+    LoadLocal16U32(MemoryArg, u8),
     LoadLocalTee32(MemoryArg, u8, u8),
     LoadLocalSet32(MemoryArg, u8, u8),
+    LoadLocalTee8S32(MemoryArg, u8, u8),
+    LoadLocalTee8U32(MemoryArg, u8, u8),
+    LoadLocalTee16S32(MemoryArg, u8, u8),
+    LoadLocalTee16U32(MemoryArg, u8, u8),
+    LoadLocalSet8S32(MemoryArg, u8, u8),
+    LoadLocalSet8U32(MemoryArg, u8, u8),
+    LoadLocalSet16S32(MemoryArg, u8, u8),
+    LoadLocalSet16U32(MemoryArg, u8, u8),
     LoadLocalTee128(MemoryArg, u8, u8),
     LoadLocalSet128(MemoryArg, u8, u8),
     AndConstTee32(i32, LocalAddr),
@@ -381,12 +396,27 @@ impl Instruction {
     #[inline]
     pub const fn memory_addr(&self) -> Option<MemAddr> {
         match self {
-            Self::StoreLocalLocal32(arg, ..)
+            Self::IncMemoryLocal32(arg, ..)
+            | Self::IncMemoryLocal64(arg, ..)
+            | Self::StoreLocalLocal32(arg, ..)
             | Self::StoreLocalLocal64(arg, ..)
             | Self::StoreLocalLocal128(arg, ..)
             | Self::LoadLocal32(arg, ..)
+            | Self::LoadLocal64(arg, ..)
+            | Self::LoadLocal8S32(arg, ..)
+            | Self::LoadLocal8U32(arg, ..)
+            | Self::LoadLocal16S32(arg, ..)
+            | Self::LoadLocal16U32(arg, ..)
             | Self::LoadLocalTee32(arg, ..)
             | Self::LoadLocalSet32(arg, ..)
+            | Self::LoadLocalTee8S32(arg, ..)
+            | Self::LoadLocalTee8U32(arg, ..)
+            | Self::LoadLocalTee16S32(arg, ..)
+            | Self::LoadLocalTee16U32(arg, ..)
+            | Self::LoadLocalSet8S32(arg, ..)
+            | Self::LoadLocalSet8U32(arg, ..)
+            | Self::LoadLocalSet16S32(arg, ..)
+            | Self::LoadLocalSet16U32(arg, ..)
             | Self::LoadLocalTee128(arg, ..)
             | Self::LoadLocalSet128(arg, ..)
             | Self::I32Load(arg)