feat: improve memory perf + basic bulk memory access

Signed-off-by: Henry Gressmann <mail@henrygressmann.de>

12 files changed, 395 insertions, 147 deletions

diff --git a/crates/parser/src/conversion.rs b/crates/parser/src/conversion.rs
index 835842f..79069e1 100644
--- a/crates/parser/src/conversion.rs
+++ b/crates/parser/src/conversion.rs
@@ -332,6 +332,12 @@ pub fn process_operators<'a>(
             GlobalSet { global_index } => Instruction::GlobalSet(global_index),
             MemorySize { mem, mem_byte } => Instruction::MemorySize(mem, mem_byte),
             MemoryGrow { mem, mem_byte } => Instruction::MemoryGrow(mem, mem_byte),
+
+            MemoryCopy { dst_mem, src_mem } => Instruction::MemoryCopy(src_mem, dst_mem),
+            MemoryFill { mem } => Instruction::MemoryFill(mem),
+            MemoryInit { data_index, mem } => Instruction::MemoryInit(data_index, mem),
+            DataDrop { data_index } => Instruction::DataDrop(data_index),
+
             I32Const { value } => Instruction::I32Const(value),
             I64Const { value } => Instruction::I64Const(value),
             F32Const { value } => Instruction::F32Const(f32::from_bits(value.bits())),
diff --git a/crates/tinywasm/Cargo.toml b/crates/tinywasm/Cargo.toml
index f1a3242..a9e24bf 100644
--- a/crates/tinywasm/Cargo.toml
+++ b/crates/tinywasm/Cargo.toml
@@ -33,6 +33,7 @@ default=["std", "parser", "logging"]
 logging=["log", "tinywasm-types/logging", "tinywasm-parser?/logging"]
 std=["tinywasm-parser?/std", "tinywasm-types/std"]
 parser=["tinywasm-parser"]
+unsafe=[]
 
 [[test]]
 name="generate-charts"
diff --git a/crates/tinywasm/src/lib.rs b/crates/tinywasm/src/lib.rs
index bc64ac4..d786ab8 100644
--- a/crates/tinywasm/src/lib.rs
+++ b/crates/tinywasm/src/lib.rs
@@ -1,11 +1,11 @@
 #![no_std]
-#![forbid(unsafe_code)]
 #![doc(test(
     no_crate_inject,
     attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_assignments, unused_variables))
 ))]
 #![warn(missing_docs, missing_debug_implementations, rust_2018_idioms, unreachable_pub)]
 #![cfg_attr(nightly, feature(error_in_core))]
+#![cfg_attr(not(feature = "unsafe"), deny(unsafe_code))]
 
 //! A tiny WebAssembly Runtime written in Rust
 //!
diff --git a/crates/tinywasm/src/runtime/interpreter/macros.rs b/crates/tinywasm/src/runtime/interpreter/macros.rs
index eba866c..a769b12 100644
--- a/crates/tinywasm/src/runtime/interpreter/macros.rs
+++ b/crates/tinywasm/src/runtime/interpreter/macros.rs
@@ -14,14 +14,14 @@ macro_rules! mem_load {
         // TODO: there could be a lot of performance improvements here
         let mem_idx = $module.resolve_mem_addr($arg.mem_addr);
         let mem = $store.get_mem(mem_idx as usize)?;
+        let mem_ref = mem.borrow_mut();
 
         let addr = $stack.values.pop()?.raw_value();
-
         let addr = $arg.offset.checked_add(addr).ok_or_else(|| {
             Error::Trap(crate::Trap::MemoryOutOfBounds {
                 offset: $arg.offset as usize,
                 len: core::mem::size_of::<$load_type>(),
-                max: mem.borrow().max_pages(),
+                max: mem_ref.max_pages(),
             })
         })?;
 
@@ -29,18 +29,14 @@ macro_rules! mem_load {
             Error::Trap(crate::Trap::MemoryOutOfBounds {
                 offset: $arg.offset as usize,
                 len: core::mem::size_of::<$load_type>(),
-                max: mem.borrow().max_pages(),
+                max: mem_ref.max_pages(),
             })
         })?;
 
-        let val: [u8; core::mem::size_of::<$load_type>()] = {
-            let mem = mem.borrow_mut();
-            let val = mem.load(addr, $arg.align as usize, core::mem::size_of::<$load_type>())?;
-            val.try_into().expect("slice with incorrect length")
-        };
-
-        let loaded_value = <$load_type>::from_le_bytes(val);
-        $stack.values.push((loaded_value as $target_type).into());
+        const LEN: usize = core::mem::size_of::<$load_type>();
+        let val = mem_ref.load_as::<LEN, $load_type>(addr, $arg.align as usize)?;
+        // let loaded_value = mem_ref.load_as::<$load_type>(addr, $arg.align as usize)?;
+        $stack.values.push((val as $target_type).into());
     }};
 }
 
diff --git a/crates/tinywasm/src/runtime/interpreter/mod.rs b/crates/tinywasm/src/runtime/interpreter/mod.rs
index fdbe697..3c3b776 100644
--- a/crates/tinywasm/src/runtime/interpreter/mod.rs
+++ b/crates/tinywasm/src/runtime/interpreter/mod.rs
@@ -23,7 +23,7 @@ use macros::*;
 use traits::*;
 
 impl InterpreterRuntime {
-    #[inline(always)] // a small 2-3% performance improvement in some cases
+    // #[inline(always)] // a small 2-3% performance improvement in some cases
     pub(crate) fn exec(&self, store: &mut Store, stack: &mut Stack) -> Result<()> {
         // The current call frame, gets updated inside of exec_one
         let mut cf = stack.call_stack.pop()?;
@@ -388,6 +388,42 @@ fn exec_one(cf: &mut CallFrame, stack: &mut Stack, store: &mut Store, module: &M
             }
         }
 
+        // Bulk memory operations
+        MemoryCopy(from, to) => {
+            let size = stack.values.pop_t::<i32>()?;
+            let src = stack.values.pop_t::<i32>()?;
+            let dst = stack.values.pop_t::<i32>()?;
+
+            let mem = store.get_mem(module.resolve_mem_addr(*from) as usize)?;
+            let mut mem = mem.borrow_mut();
+
+            if from == to {
+                // copy within the same memory
+                mem.copy_within(dst as usize, src as usize, size as usize)?;
+            } else {
+                // copy between two memories
+                let mem2 = store.get_mem(module.resolve_mem_addr(*to) as usize)?;
+                let mut mem2 = mem2.borrow_mut();
+                mem2.copy_from_slice(dst as usize, mem.load(src as usize, 0, size as usize)?)?;
+            }
+        }
+
+        MemoryFill(addr) => {
+            let size = stack.values.pop_t::<i32>()?;
+            let val = stack.values.pop_t::<i32>()?;
+            let dst = stack.values.pop_t::<i32>()?;
+
+            let mem = store.get_mem(module.resolve_mem_addr(*addr) as usize)?;
+            let mut mem = mem.borrow_mut();
+            mem.fill(dst as usize, size as usize, val as u8)?;
+        }
+
+        // MemoryInit(data_index, mem_index) => {}
+        // DataDrop(data_index) => {
+        //     // let data_idx = module.resolve_data_addr(*data_index);
+        //     // let data = store.get_data(data_idx as usize)?;
+        //     // data.borrow_mut().drop()?;
+        // }
         I32Store(arg) => mem_store!(i32, arg, stack, store, module),
         I64Store(arg) => mem_store!(i64, arg, stack, store, module),
         F32Store(arg) => mem_store!(f32, arg, stack, store, module),
diff --git a/crates/tinywasm/src/runtime/stack/call_stack.rs b/crates/tinywasm/src/runtime/stack/call_stack.rs
index dbe420d..dcbfcac 100644
--- a/crates/tinywasm/src/runtime/stack/call_stack.rs
+++ b/crates/tinywasm/src/runtime/stack/call_stack.rs
@@ -58,7 +58,6 @@ pub(crate) struct CallFrame {
 }
 
 impl CallFrame {
-    // TOOD: perf: this is called a lot, and it's a bit slow
     /// Push a new label to the label stack and ensure the stack has the correct values
     pub(crate) fn enter_label(&mut self, label_frame: LabelFrame, stack: &mut super::ValueStack) {
         if label_frame.params > 0 {
@@ -77,14 +76,14 @@ impl CallFrame {
         // will increment it by 1 since we're changing the "current" instr_ptr
         match break_to.ty {
             BlockType::Loop => {
+                // this is a loop, so we want to jump back to the start of the loop
                 self.instr_ptr = break_to.instr_ptr;
 
+                // We also want to push the params to the stack
+                value_stack.break_to(break_to.stack_ptr, break_to.params);
+
                 // check if we're breaking to the loop
                 if break_to_relative != 0 {
-                    // this is a loop, so we want to jump back to the start of the loop
-                    // We also want to push the params to the stack
-                    value_stack.break_to(break_to.stack_ptr, break_to.params);
-
                     // we also want to trim the label stack to the loop (but not including the loop)
                     self.labels.truncate(self.labels.len() - break_to_relative as usize);
                     return Some(());
diff --git a/crates/tinywasm/src/runtime/stack/value_stack.rs b/crates/tinywasm/src/runtime/stack/value_stack.rs
index 3c5f48b..9b8f82d 100644
--- a/crates/tinywasm/src/runtime/stack/value_stack.rs
+++ b/crates/tinywasm/src/runtime/stack/value_stack.rs
@@ -1,7 +1,6 @@
 use core::ops::Range;
 
 use crate::{cold, runtime::RawWasmValue, unlikely, Error, Result};
-use alloc::vec;
 use alloc::vec::Vec;
 use tinywasm_types::{ValType, WasmValue};
 
@@ -15,7 +14,7 @@ pub(crate) struct ValueStack {
 
 impl Default for ValueStack {
     fn default() -> Self {
-        Self { stack: vec![RawWasmValue::default(); MIN_VALUE_STACK_SIZE] }
+        Self { stack: Vec::with_capacity(MIN_VALUE_STACK_SIZE) }
     }
 }
 
diff --git a/crates/tinywasm/src/runtime/value.rs b/crates/tinywasm/src/runtime/value.rs
index 329a60b..5341361 100644
--- a/crates/tinywasm/src/runtime/value.rs
+++ b/crates/tinywasm/src/runtime/value.rs
@@ -18,6 +18,7 @@ impl Debug for RawWasmValue {
 }
 
 impl RawWasmValue {
+    #[inline(always)]
     pub fn raw_value(&self) -> u64 {
         self.0
     }
diff --git a/crates/tinywasm/src/store/memory.rs b/crates/tinywasm/src/store/memory.rs
new file mode 100644
index 0000000..a087b1d
--- /dev/null
+++ b/crates/tinywasm/src/store/memory.rs
@@ -0,0 +1,326 @@
+use alloc::vec;
+use alloc::vec::Vec;
+use tinywasm_types::{MemoryType, ModuleInstanceAddr};
+
+use crate::{cold, unlikely, Error, Result};
+
+pub(crate) const PAGE_SIZE: usize = 65536;
+pub(crate) const MAX_PAGES: usize = 65536;
+pub(crate) const MAX_SIZE: u64 = PAGE_SIZE as u64 * MAX_PAGES as u64;
+
+/// A WebAssembly Memory Instance
+///
+/// See <https://webassembly.github.io/spec/core/exec/runtime.html#memory-instances>
+#[derive(Debug)]
+pub(crate) struct MemoryInstance {
+    pub(crate) kind: MemoryType,
+    pub(crate) data: Vec<u8>,
+    pub(crate) page_count: usize,
+    pub(crate) _owner: ModuleInstanceAddr, // index into store.module_instances
+}
+
+impl MemoryInstance {
+    pub(crate) fn new(kind: MemoryType, owner: ModuleInstanceAddr) -> Self {
+        assert!(kind.page_count_initial <= kind.page_count_max.unwrap_or(MAX_PAGES as u64));
+        log::debug!("initializing memory with {} pages", kind.page_count_initial);
+
+        Self {
+            kind,
+            data: vec![0; PAGE_SIZE * kind.page_count_initial as usize],
+            page_count: kind.page_count_initial as usize,
+            _owner: owner,
+        }
+    }
+
+    pub(crate) fn store(&mut self, addr: usize, _align: usize, data: &[u8], len: usize) -> Result<()> {
+        let Some(end) = addr.checked_add(len) else {
+            cold();
+            return Err(Error::Trap(crate::Trap::MemoryOutOfBounds {
+                offset: addr,
+                len: data.len(),
+                max: self.data.len(),
+            }));
+        };
+
+        if unlikely(end > self.data.len() || end < addr) {
+            return Err(Error::Trap(crate::Trap::MemoryOutOfBounds {
+                offset: addr,
+                len: data.len(),
+                max: self.data.len(),
+            }));
+        }
+
+        // WebAssembly doesn't require alignment for stores
+        #[cfg(not(feature = "unsafe"))]
+        self.data[addr..end].copy_from_slice(data);
+
+        #[cfg(feature = "unsafe")]
+        // SAFETY: we checked that `end` is in bounds above, this is the same as `copy_from_slice`
+        // src must is for reads of count * size_of::<T>() bytes.
+        // dst must is for writes of count * size_of::<T>() bytes.
+        // Both src and dst are properly aligned.
+        // The region of memory beginning at src does not overlap with the region of memory beginning at dst with the same size.
+        unsafe {
+            core::ptr::copy_nonoverlapping(data.as_ptr(), self.data[addr..end].as_mut_ptr(), len);
+        }
+
+        Ok(())
+    }
+
+    pub(crate) fn max_pages(&self) -> usize {
+        self.kind.page_count_max.unwrap_or(MAX_PAGES as u64) as usize
+    }
+
+    pub(crate) fn load(&self, addr: usize, _align: usize, len: usize) -> Result<&[u8]> {
+        let Some(end) = addr.checked_add(len) else {
+            cold();
+            return Err(Error::Trap(crate::Trap::MemoryOutOfBounds { offset: addr, len, max: self.data.len() }));
+        };
+
+        if unlikely(end > self.data.len() || end < addr) {
+            return Err(Error::Trap(crate::Trap::MemoryOutOfBounds { offset: addr, len, max: self.data.len() }));
+        }
+
+        Ok(&self.data[addr..end])
+    }
+
+    // this is a workaround since we can't use generic const expressions yet (https://github.com/rust-lang/rust/issues/76560)
+    pub(crate) fn load_as<const SIZE: usize, T: MemLoadable<SIZE>>(&self, addr: usize, _align: usize) -> Result<T> {
+        let Some(end) = addr.checked_add(SIZE) else {
+            cold();
+            return Err(Error::Trap(crate::Trap::MemoryOutOfBounds { offset: addr, len: SIZE, max: self.max_pages() }));
+        };
+
+        if unlikely(end > self.data.len()) {
+            return Err(Error::Trap(crate::Trap::MemoryOutOfBounds { offset: addr, len: SIZE, max: self.data.len() }));
+        }
+
+        #[cfg(feature = "unsafe")]
+        // WebAssembly doesn't require alignment for loads
+        // SAFETY: we checked that `end` is in bounds above. All types that implement `Into<RawWasmValue>` are valid
+        // to load from unaligned addresses.
+        let val = unsafe { core::ptr::read_unaligned(self.data[addr..end].as_ptr() as *const T) };
+
+        #[cfg(not(feature = "unsafe"))]
+        let val = T::from_le_bytes(self.data[addr..end].try_into().expect("slice size mismatch"));
+
+        Ok(val)
+    }
+
+    pub(crate) fn page_count(&self) -> usize {
+        self.page_count
+    }
+
+    pub(crate) fn fill(&mut self, addr: usize, len: usize, val: u8) -> Result<()> {
+        let end = addr
+            .checked_add(len)
+            .ok_or_else(|| Error::Trap(crate::Trap::MemoryOutOfBounds { offset: addr, len, max: self.data.len() }))?;
+        if unlikely(end > self.data.len()) {
+            return Err(Error::Trap(crate::Trap::MemoryOutOfBounds { offset: addr, len, max: self.data.len() }));
+        }
+
+        self.data[addr..end].fill(val);
+        Ok(())
+    }
+
+    pub(crate) fn copy_from_slice(&mut self, dst: usize, src: &[u8]) -> Result<()> {
+        let end = dst.checked_add(src.len()).ok_or_else(|| {
+            Error::Trap(crate::Trap::MemoryOutOfBounds { offset: dst, len: src.len(), max: self.data.len() })
+        })?;
+        if unlikely(end > self.data.len()) {
+            return Err(Error::Trap(crate::Trap::MemoryOutOfBounds {
+                offset: dst,
+                len: src.len(),
+                max: self.data.len(),
+            }));
+        }
+
+        self.data[dst..end].copy_from_slice(src);
+        Ok(())
+    }
+
+    pub(crate) fn copy_within(&mut self, dst: usize, src: usize, len: usize) -> Result<()> {
+        // Calculate the end of the source slice
+        let src_end = src
+            .checked_add(len)
+            .ok_or_else(|| Error::Trap(crate::Trap::MemoryOutOfBounds { offset: src, len, max: self.data.len() }))?;
+        if src_end > self.data.len() {
+            return Err(Error::Trap(crate::Trap::MemoryOutOfBounds { offset: src, len, max: self.data.len() }));
+        }
+
+        // Calculate the end of the destination slice
+        let dst_end = dst
+            .checked_add(len)
+            .ok_or_else(|| Error::Trap(crate::Trap::MemoryOutOfBounds { offset: dst, len, max: self.data.len() }))?;
+        if dst_end > self.data.len() {
+            return Err(Error::Trap(crate::Trap::MemoryOutOfBounds { offset: dst, len, max: self.data.len() }));
+        }
+
+        // Perform the copy
+        self.data.copy_within(src..src_end, dst);
+        Ok(())
+    }
+
+    pub(crate) fn grow(&mut self, pages_delta: i32) -> Option<i32> {
+        let current_pages = self.page_count();
+        let new_pages = current_pages as i64 + pages_delta as i64;
+
+        if new_pages < 0 || new_pages > MAX_PAGES as i64 {
+            return None;
+        }
+
+        if new_pages as usize > self.max_pages() {
+            log::info!("memory size out of bounds: {}", new_pages);
+            return None;
+        }
+
+        let new_size = new_pages as usize * PAGE_SIZE;
+        if new_size as u64 > MAX_SIZE {
+            return None;
+        }
+
+        // Zero initialize the new pages
+        self.data.resize(new_size, 0);
+        self.page_count = new_pages as usize;
+
+        log::debug!("memory was {} pages", current_pages);
+        log::debug!("memory grown by {} pages", pages_delta);
+        log::debug!("memory grown to {} pages", self.page_count);
+
+        Some(current_pages.try_into().expect("memory size out of bounds, this should have been caught earlier"))
+    }
+}
+
+#[allow(unsafe_code)]
+/// A trait for types that can be loaded from memory
+///
+/// # Safety
+/// Only implemented for primitive types, unsafe to not allow it for other types.
+/// Only actually unsafe to implement if the `unsafe` feature is enabled since there might be
+/// UB for loading things things like packed structs
+pub(crate) unsafe trait MemLoadable<const T: usize>: Sized + Copy {
+    /// Load a value from memory
+    fn from_le_bytes(bytes: [u8; T]) -> Self;
+    /// Load a value from memory
+    fn from_be_bytes(bytes: [u8; T]) -> Self;
+}
+
+#[allow(unsafe_code)]
+unsafe impl MemLoadable<1> for u8 {
+    fn from_le_bytes(bytes: [u8; 1]) -> Self {
+        bytes[0]
+    }
+    fn from_be_bytes(bytes: [u8; 1]) -> Self {
+        bytes[0]
+    }
+}
+
+#[allow(unsafe_code)]
+unsafe impl MemLoadable<2> for u16 {
+    fn from_le_bytes(bytes: [u8; 2]) -> Self {
+        Self::from_le_bytes(bytes)
+    }
+    fn from_be_bytes(bytes: [u8; 2]) -> Self {
+        Self::from_be_bytes(bytes)
+    }
+}
+
+#[allow(unsafe_code)]
+unsafe impl MemLoadable<4> for u32 {
+    fn from_le_bytes(bytes: [u8; 4]) -> Self {
+        Self::from_le_bytes(bytes)
+    }
+    fn from_be_bytes(bytes: [u8; 4]) -> Self {
+        Self::from_be_bytes(bytes)
+    }
+}
+
+#[allow(unsafe_code)]
+unsafe impl MemLoadable<8> for u64 {
+    fn from_le_bytes(bytes: [u8; 8]) -> Self {
+        Self::from_le_bytes(bytes)
+    }
+    fn from_be_bytes(bytes: [u8; 8]) -> Self {
+        Self::from_be_bytes(bytes)
+    }
+}
+
+#[allow(unsafe_code)]
+unsafe impl MemLoadable<16> for u128 {
+    fn from_le_bytes(bytes: [u8; 16]) -> Self {
+        Self::from_le_bytes(bytes)
+    }
+    fn from_be_bytes(bytes: [u8; 16]) -> Self {
+        Self::from_be_bytes(bytes)
+    }
+}
+
+#[allow(unsafe_code)]
+unsafe impl MemLoadable<1> for i8 {
+    fn from_le_bytes(bytes: [u8; 1]) -> Self {
+        bytes[0] as i8
+    }
+    fn from_be_bytes(bytes: [u8; 1]) -> Self {
+        bytes[0] as i8
+    }
+}
+
+#[allow(unsafe_code)]
+unsafe impl MemLoadable<2> for i16 {
+    fn from_le_bytes(bytes: [u8; 2]) -> Self {
+        Self::from_le_bytes(bytes)
+    }
+    fn from_be_bytes(bytes: [u8; 2]) -> Self {
+        Self::from_be_bytes(bytes)
+    }
+}
+
+#[allow(unsafe_code)]
+unsafe impl MemLoadable<4> for i32 {
+    fn from_le_bytes(bytes: [u8; 4]) -> Self {
+        Self::from_le_bytes(bytes)
+    }
+    fn from_be_bytes(bytes: [u8; 4]) -> Self {
+        Self::from_be_bytes(bytes)
+    }
+}
+
+#[allow(unsafe_code)]
+unsafe impl MemLoadable<8> for i64 {
+    fn from_le_bytes(bytes: [u8; 8]) -> Self {
+        Self::from_le_bytes(bytes)
+    }
+    fn from_be_bytes(bytes: [u8; 8]) -> Self {
+        Self::from_be_bytes(bytes)
+    }
+}
+
+#[allow(unsafe_code)]
+unsafe impl MemLoadable<16> for i128 {
+    fn from_le_bytes(bytes: [u8; 16]) -> Self {
+        Self::from_le_bytes(bytes)
+    }
+    fn from_be_bytes(bytes: [u8; 16]) -> Self {
+        Self::from_be_bytes(bytes)
+    }
+}
+
+#[allow(unsafe_code)]
+unsafe impl MemLoadable<4> for f32 {
+    fn from_le_bytes(bytes: [u8; 4]) -> Self {
+        Self::from_le_bytes(bytes)
+    }
+    fn from_be_bytes(bytes: [u8; 4]) -> Self {
+        Self::from_be_bytes(bytes)
+    }
+}
+
+#[allow(unsafe_code)]
+unsafe impl MemLoadable<8> for f64 {
+    fn from_le_bytes(bytes: [u8; 8]) -> Self {
+        Self::from_le_bytes(bytes)
+    }
+    fn from_be_bytes(bytes: [u8; 8]) -> Self {
+        Self::from_be_bytes(bytes)
+    }
+}
diff --git a/crates/tinywasm/src/store.rs b/crates/tinywasm/src/store/mod.rs
index a4f9b70..f89b931 100644
--- a/crates/tinywasm/src/store.rs
+++ b/crates/tinywasm/src/store/mod.rs
@@ -11,6 +11,9 @@ use crate::{
     Error, Function, ModuleInstance, Result, Trap,
 };
 
+mod memory;
+pub(crate) use memory::*;
+
 // global store id counter
 static STORE_ID: AtomicUsize = AtomicUsize::new(0);
 
@@ -562,131 +565,6 @@ impl TableInstance {
     }
 }
 
-pub(crate) const PAGE_SIZE: usize = 65536;
-pub(crate) const MAX_PAGES: usize = 65536;
-pub(crate) const MAX_SIZE: u64 = PAGE_SIZE as u64 * MAX_PAGES as u64;
-
-/// A WebAssembly Memory Instance
-///
-/// See <https://webassembly.github.io/spec/core/exec/runtime.html#memory-instances>
-#[derive(Debug)]
-pub(crate) struct MemoryInstance {
-    pub(crate) kind: MemoryType,
-    pub(crate) data: Vec<u8>,
-    pub(crate) page_count: usize,
-    pub(crate) _owner: ModuleInstanceAddr, // index into store.module_instances
-}
-
-impl MemoryInstance {
-    pub(crate) fn new(kind: MemoryType, owner: ModuleInstanceAddr) -> Self {
-        assert!(kind.page_count_initial <= kind.page_count_max.unwrap_or(MAX_PAGES as u64));
-        log::debug!("initializing memory with {} pages", kind.page_count_initial);
-
-        Self {
-            kind,
-            data: vec![0; PAGE_SIZE * kind.page_count_initial as usize],
-            page_count: kind.page_count_initial as usize,
-            _owner: owner,
-        }
-    }
-
-    pub(crate) fn store(&mut self, addr: usize, _align: usize, data: &[u8], len: usize) -> Result<()> {
-        let end = addr.checked_add(len).ok_or_else(|| {
-            Error::Trap(crate::Trap::MemoryOutOfBounds { offset: addr, len: data.len(), max: self.data.len() })
-        })?;
-
-        if end > self.data.len() || end < addr {
-            return Err(Error::Trap(crate::Trap::MemoryOutOfBounds {
-                offset: addr,
-                len: data.len(),
-                max: self.data.len(),
-            }));
-        }
-
-        // WebAssembly doesn't require alignment for stores
-        self.data[addr..end].copy_from_slice(data);
-        Ok(())
-    }
-
-    pub(crate) fn max_pages(&self) -> usize {
-        self.kind.page_count_max.unwrap_or(MAX_PAGES as u64) as usize
-    }
-
-    pub(crate) fn load(&self, addr: usize, _align: usize, len: usize) -> Result<&[u8]> {
-        let end = addr
-            .checked_add(len)
-            .ok_or_else(|| Error::Trap(crate::Trap::MemoryOutOfBounds { offset: addr, len, max: self.max_pages() }))?;
-
-        if end > self.data.len() {
-            return Err(Error::Trap(crate::Trap::MemoryOutOfBounds { offset: addr, len, max: self.data.len() }));
-        }
-
-        // WebAssembly doesn't require alignment for loads
-        Ok(&self.data[addr..end])
-    }
-
-    pub(crate) fn page_count(&self) -> usize {
-        self.page_count
-    }
-
-    pub(crate) fn fill(&mut self, addr: usize, len: usize, val: u8) -> Result<()> {
-        let end = addr
-            .checked_add(len)
-            .ok_or_else(|| Error::Trap(crate::Trap::MemoryOutOfBounds { offset: addr, len, max: self.data.len() }))?;
-        if end > self.data.len() {
-            return Err(Error::Trap(crate::Trap::MemoryOutOfBounds { offset: addr, len, max: self.data.len() }));
-        }
-        self.data[addr..end].fill(val);
-        Ok(())
-    }
-
-    pub(crate) fn copy_within(&mut self, dst: usize, src: usize, len: usize) -> Result<()> {
-        let end = src
-            .checked_add(len)
-            .ok_or_else(|| Error::Trap(crate::Trap::MemoryOutOfBounds { offset: src, len, max: self.data.len() }))?;
-        if end > self.data.len() || end < src {
-            return Err(Error::Trap(crate::Trap::MemoryOutOfBounds { offset: src, len, max: self.data.len() }));
-        }
-        let end = dst
-            .checked_add(len)
-            .ok_or_else(|| Error::Trap(crate::Trap::MemoryOutOfBounds { offset: dst, len, max: self.data.len() }))?;
-        if end > self.data.len() || end < dst {
-            return Err(Error::Trap(crate::Trap::MemoryOutOfBounds { offset: dst, len, max: self.data.len() }));
-        }
-        self.data[dst..end].copy_within(src..end, len);
-        Ok(())
-    }
-
-    pub(crate) fn grow(&mut self, pages_delta: i32) -> Option<i32> {
-        let current_pages = self.page_count();
-        let new_pages = current_pages as i64 + pages_delta as i64;
-
-        if new_pages < 0 || new_pages > MAX_PAGES as i64 {
-            return None;
-        }
-
-        if new_pages as usize > self.max_pages() {
-            log::info!("memory size out of bounds: {}", new_pages);
-            return None;
-        }
-
-        let new_size = new_pages as usize * PAGE_SIZE;
-        if new_size as u64 > MAX_SIZE {
-            return None;
-        }
-
-        // Zero initialize the new pages
-        self.data.resize(new_size, 0);
-        self.page_count = new_pages as usize;
-
-        log::debug!("memory was {} pages", current_pages);
-        log::debug!("memory grown by {} pages", pages_delta);
-        log::debug!("memory grown to {} pages", self.page_count);
-
-        Some(current_pages.try_into().expect("memory size out of bounds, this should have been caught earlier"))
-    }
-}
-
 /// A WebAssembly Global Instance
 ///
 /// See <https://webassembly.github.io/spec/core/exec/runtime.html#global-instances>
diff --git a/crates/tinywasm/tests/generated/2.0.csv b/crates/tinywasm/tests/generated/2.0.csv
index b5ea4cc..404fe16 100644
--- a/crates/tinywasm/tests/generated/2.0.csv
+++ b/crates/tinywasm/tests/generated/2.0.csv
@@ -1,2 +1,2 @@
 0.3.0,26722,1161,[{"name":"address.wast","passed":260,"failed":0},{"name":"align.wast","passed":156,"failed":0},{"name":"binary-leb128.wast","passed":91,"failed":0},{"name":"binary.wast","passed":112,"failed":0},{"name":"block.wast","passed":223,"failed":0},{"name":"br.wast","passed":97,"failed":0},{"name":"br_if.wast","passed":118,"failed":0},{"name":"br_table.wast","passed":174,"failed":0},{"name":"bulk.wast","passed":8,"failed":109},{"name":"call.wast","passed":91,"failed":0},{"name":"call_indirect.wast","passed":170,"failed":0},{"name":"comments.wast","passed":8,"failed":0},{"name":"const.wast","passed":778,"failed":0},{"name":"conversions.wast","passed":619,"failed":0},{"name":"custom.wast","passed":11,"failed":0},{"name":"data.wast","passed":61,"failed":0},{"name":"elem.wast","passed":98,"failed":1},{"name":"endianness.wast","passed":69,"failed":0},{"name":"exports.wast","passed":96,"failed":0},{"name":"f32.wast","passed":2514,"failed":0},{"name":"f32_bitwise.wast","passed":364,"failed":0},{"name":"f32_cmp.wast","passed":2407,"failed":0},{"name":"f64.wast","passed":2514,"failed":0},{"name":"f64_bitwise.wast","passed":364,"failed":0},{"name":"f64_cmp.wast","passed":2407,"failed":0},{"name":"fac.wast","passed":8,"failed":0},{"name":"float_exprs.wast","passed":900,"failed":0},{"name":"float_literals.wast","passed":163,"failed":0},{"name":"float_memory.wast","passed":90,"failed":0},{"name":"float_misc.wast","passed":441,"failed":0},{"name":"forward.wast","passed":5,"failed":0},{"name":"func.wast","passed":172,"failed":0},{"name":"func_ptrs.wast","passed":36,"failed":0},{"name":"global.wast","passed":110,"failed":0},{"name":"i32.wast","passed":460,"failed":0},{"name":"i64.wast","passed":416,"failed":0},{"name":"if.wast","passed":241,"failed":0},{"name":"imports.wast","passed":171,"failed":12},{"name":"inline-module.wast","passed":1,"failed":0},{"name":"int_exprs.wast","passed":108,"failed":0},{"name":"int_literals.wast","passed":51,"failed":0},{"name":"labels.wast","passed":29,"failed":0},{"name":"left-to-right.wast","passed":96,"failed":0},{"name":"linking.wast","passed":132,"failed":0},{"name":"load.wast","passed":97,"failed":0},{"name":"local_get.wast","passed":36,"failed":0},{"name":"local_set.wast","passed":53,"failed":0},{"name":"local_tee.wast","passed":97,"failed":0},{"name":"loop.wast","passed":120,"failed":0},{"name":"memory.wast","passed":79,"failed":0},{"name":"memory_copy.wast","passed":3928,"failed":522},{"name":"memory_fill.wast","passed":64,"failed":36},{"name":"memory_grow.wast","passed":96,"failed":0},{"name":"memory_init.wast","passed":177,"failed":63},{"name":"memory_redundancy.wast","passed":8,"failed":0},{"name":"memory_size.wast","passed":42,"failed":0},{"name":"memory_trap.wast","passed":182,"failed":0},{"name":"names.wast","passed":486,"failed":0},{"name":"nop.wast","passed":88,"failed":0},{"name":"ref_func.wast","passed":9,"failed":8},{"name":"ref_is_null.wast","passed":4,"failed":12},{"name":"ref_null.wast","passed":1,"failed":2},{"name":"return.wast","passed":84,"failed":0},{"name":"select.wast","passed":148,"failed":0},{"name":"skip-stack-guard-page.wast","passed":11,"failed":0},{"name":"stack.wast","passed":7,"failed":0},{"name":"start.wast","passed":20,"failed":0},{"name":"store.wast","passed":68,"failed":0},{"name":"switch.wast","passed":28,"failed":0},{"name":"table-sub.wast","passed":2,"failed":0},{"name":"table.wast","passed":19,"failed":0},{"name":"table_copy.wast","passed":1613,"failed":115},{"name":"table_fill.wast","passed":23,"failed":22},{"name":"table_get.wast","passed":10,"failed":6},{"name":"table_grow.wast","passed":21,"failed":29},{"name":"table_init.wast","passed":594,"failed":186},{"name":"table_set.wast","passed":19,"failed":7},{"name":"table_size.wast","passed":8,"failed":31},{"name":"token.wast","passed":58,"failed":0},{"name":"traps.wast","passed":36,"failed":0},{"name":"type.wast","passed":3,"failed":0},{"name":"unreachable.wast","passed":64,"failed":0},{"name":"unreached-invalid.wast","passed":118,"failed":0},{"name":"unreached-valid.wast","passed":7,"failed":0},{"name":"unwind.wast","passed":50,"failed":0},{"name":"utf8-custom-section-id.wast","passed":176,"failed":0},{"name":"utf8-import-field.wast","passed":176,"failed":0},{"name":"utf8-import-module.wast","passed":176,"failed":0},{"name":"utf8-invalid-encoding.wast","passed":176,"failed":0}]
-0.4.0-alpha.0,26861,1022,[{"name":"address.wast","passed":260,"failed":0},{"name":"align.wast","passed":156,"failed":0},{"name":"binary-leb128.wast","passed":91,"failed":0},{"name":"binary.wast","passed":112,"failed":0},{"name":"block.wast","passed":223,"failed":0},{"name":"br.wast","passed":97,"failed":0},{"name":"br_if.wast","passed":118,"failed":0},{"name":"br_table.wast","passed":174,"failed":0},{"name":"bulk.wast","passed":8,"failed":109},{"name":"call.wast","passed":91,"failed":0},{"name":"call_indirect.wast","passed":170,"failed":0},{"name":"comments.wast","passed":8,"failed":0},{"name":"const.wast","passed":778,"failed":0},{"name":"conversions.wast","passed":619,"failed":0},{"name":"custom.wast","passed":11,"failed":0},{"name":"data.wast","passed":61,"failed":0},{"name":"elem.wast","passed":99,"failed":0},{"name":"endianness.wast","passed":69,"failed":0},{"name":"exports.wast","passed":96,"failed":0},{"name":"f32.wast","passed":2514,"failed":0},{"name":"f32_bitwise.wast","passed":364,"failed":0},{"name":"f32_cmp.wast","passed":2407,"failed":0},{"name":"f64.wast","passed":2514,"failed":0},{"name":"f64_bitwise.wast","passed":364,"failed":0},{"name":"f64_cmp.wast","passed":2407,"failed":0},{"name":"fac.wast","passed":8,"failed":0},{"name":"float_exprs.wast","passed":900,"failed":0},{"name":"float_literals.wast","passed":163,"failed":0},{"name":"float_memory.wast","passed":90,"failed":0},{"name":"float_misc.wast","passed":441,"failed":0},{"name":"forward.wast","passed":5,"failed":0},{"name":"func.wast","passed":172,"failed":0},{"name":"func_ptrs.wast","passed":36,"failed":0},{"name":"global.wast","passed":110,"failed":0},{"name":"i32.wast","passed":460,"failed":0},{"name":"i64.wast","passed":416,"failed":0},{"name":"if.wast","passed":241,"failed":0},{"name":"imports.wast","passed":183,"failed":0},{"name":"inline-module.wast","passed":1,"failed":0},{"name":"int_exprs.wast","passed":108,"failed":0},{"name":"int_literals.wast","passed":51,"failed":0},{"name":"labels.wast","passed":29,"failed":0},{"name":"left-to-right.wast","passed":96,"failed":0},{"name":"linking.wast","passed":132,"failed":0},{"name":"load.wast","passed":97,"failed":0},{"name":"local_get.wast","passed":36,"failed":0},{"name":"local_set.wast","passed":53,"failed":0},{"name":"local_tee.wast","passed":97,"failed":0},{"name":"loop.wast","passed":120,"failed":0},{"name":"memory.wast","passed":79,"failed":0},{"name":"memory_copy.wast","passed":3928,"failed":522},{"name":"memory_fill.wast","passed":64,"failed":36},{"name":"memory_grow.wast","passed":96,"failed":0},{"name":"memory_init.wast","passed":177,"failed":63},{"name":"memory_redundancy.wast","passed":8,"failed":0},{"name":"memory_size.wast","passed":42,"failed":0},{"name":"memory_trap.wast","passed":182,"failed":0},{"name":"names.wast","passed":486,"failed":0},{"name":"nop.wast","passed":88,"failed":0},{"name":"ref_func.wast","passed":9,"failed":8},{"name":"ref_is_null.wast","passed":4,"failed":12},{"name":"ref_null.wast","passed":1,"failed":2},{"name":"return.wast","passed":84,"failed":0},{"name":"select.wast","passed":148,"failed":0},{"name":"skip-stack-guard-page.wast","passed":11,"failed":0},{"name":"stack.wast","passed":7,"failed":0},{"name":"start.wast","passed":20,"failed":0},{"name":"store.wast","passed":68,"failed":0},{"name":"switch.wast","passed":28,"failed":0},{"name":"table-sub.wast","passed":2,"failed":0},{"name":"table.wast","passed":19,"failed":0},{"name":"table_copy.wast","passed":1613,"failed":115},{"name":"table_fill.wast","passed":23,"failed":22},{"name":"table_get.wast","passed":10,"failed":6},{"name":"table_grow.wast","passed":21,"failed":29},{"name":"table_init.wast","passed":720,"failed":60},{"name":"table_set.wast","passed":19,"failed":7},{"name":"table_size.wast","passed":8,"failed":31},{"name":"token.wast","passed":58,"failed":0},{"name":"traps.wast","passed":36,"failed":0},{"name":"type.wast","passed":3,"failed":0},{"name":"unreachable.wast","passed":64,"failed":0},{"name":"unreached-invalid.wast","passed":118,"failed":0},{"name":"unreached-valid.wast","passed":7,"failed":0},{"name":"unwind.wast","passed":50,"failed":0},{"name":"utf8-custom-section-id.wast","passed":176,"failed":0},{"name":"utf8-import-field.wast","passed":176,"failed":0},{"name":"utf8-import-module.wast","passed":176,"failed":0},{"name":"utf8-invalid-encoding.wast","passed":176,"failed":0}]
+0.4.0-alpha.0,27464,419,[{"name":"address.wast","passed":260,"failed":0},{"name":"align.wast","passed":156,"failed":0},{"name":"binary-leb128.wast","passed":91,"failed":0},{"name":"binary.wast","passed":112,"failed":0},{"name":"block.wast","passed":223,"failed":0},{"name":"br.wast","passed":97,"failed":0},{"name":"br_if.wast","passed":118,"failed":0},{"name":"br_table.wast","passed":174,"failed":0},{"name":"bulk.wast","passed":53,"failed":64},{"name":"call.wast","passed":91,"failed":0},{"name":"call_indirect.wast","passed":170,"failed":0},{"name":"comments.wast","passed":8,"failed":0},{"name":"const.wast","passed":778,"failed":0},{"name":"conversions.wast","passed":619,"failed":0},{"name":"custom.wast","passed":11,"failed":0},{"name":"data.wast","passed":61,"failed":0},{"name":"elem.wast","passed":99,"failed":0},{"name":"endianness.wast","passed":69,"failed":0},{"name":"exports.wast","passed":96,"failed":0},{"name":"f32.wast","passed":2514,"failed":0},{"name":"f32_bitwise.wast","passed":364,"failed":0},{"name":"f32_cmp.wast","passed":2407,"failed":0},{"name":"f64.wast","passed":2514,"failed":0},{"name":"f64_bitwise.wast","passed":364,"failed":0},{"name":"f64_cmp.wast","passed":2407,"failed":0},{"name":"fac.wast","passed":8,"failed":0},{"name":"float_exprs.wast","passed":900,"failed":0},{"name":"float_literals.wast","passed":163,"failed":0},{"name":"float_memory.wast","passed":90,"failed":0},{"name":"float_misc.wast","passed":441,"failed":0},{"name":"forward.wast","passed":5,"failed":0},{"name":"func.wast","passed":172,"failed":0},{"name":"func_ptrs.wast","passed":36,"failed":0},{"name":"global.wast","passed":110,"failed":0},{"name":"i32.wast","passed":460,"failed":0},{"name":"i64.wast","passed":416,"failed":0},{"name":"if.wast","passed":241,"failed":0},{"name":"imports.wast","passed":183,"failed":0},{"name":"inline-module.wast","passed":1,"failed":0},{"name":"int_exprs.wast","passed":108,"failed":0},{"name":"int_literals.wast","passed":51,"failed":0},{"name":"labels.wast","passed":29,"failed":0},{"name":"left-to-right.wast","passed":96,"failed":0},{"name":"linking.wast","passed":132,"failed":0},{"name":"load.wast","passed":97,"failed":0},{"name":"local_get.wast","passed":36,"failed":0},{"name":"local_set.wast","passed":53,"failed":0},{"name":"local_tee.wast","passed":97,"failed":0},{"name":"loop.wast","passed":120,"failed":0},{"name":"memory.wast","passed":79,"failed":0},{"name":"memory_copy.wast","passed":4450,"failed":0},{"name":"memory_fill.wast","passed":100,"failed":0},{"name":"memory_grow.wast","passed":96,"failed":0},{"name":"memory_init.wast","passed":177,"failed":63},{"name":"memory_redundancy.wast","passed":8,"failed":0},{"name":"memory_size.wast","passed":42,"failed":0},{"name":"memory_trap.wast","passed":182,"failed":0},{"name":"names.wast","passed":486,"failed":0},{"name":"nop.wast","passed":88,"failed":0},{"name":"ref_func.wast","passed":9,"failed":8},{"name":"ref_is_null.wast","passed":4,"failed":12},{"name":"ref_null.wast","passed":1,"failed":2},{"name":"return.wast","passed":84,"failed":0},{"name":"select.wast","passed":148,"failed":0},{"name":"skip-stack-guard-page.wast","passed":11,"failed":0},{"name":"stack.wast","passed":7,"failed":0},{"name":"start.wast","passed":20,"failed":0},{"name":"store.wast","passed":68,"failed":0},{"name":"switch.wast","passed":28,"failed":0},{"name":"table-sub.wast","passed":2,"failed":0},{"name":"table.wast","passed":19,"failed":0},{"name":"table_copy.wast","passed":1613,"failed":115},{"name":"table_fill.wast","passed":23,"failed":22},{"name":"table_get.wast","passed":10,"failed":6},{"name":"table_grow.wast","passed":21,"failed":29},{"name":"table_init.wast","passed":720,"failed":60},{"name":"table_set.wast","passed":19,"failed":7},{"name":"table_size.wast","passed":8,"failed":31},{"name":"token.wast","passed":58,"failed":0},{"name":"traps.wast","passed":36,"failed":0},{"name":"type.wast","passed":3,"failed":0},{"name":"unreachable.wast","passed":64,"failed":0},{"name":"unreached-invalid.wast","passed":118,"failed":0},{"name":"unreached-valid.wast","passed":7,"failed":0},{"name":"unwind.wast","passed":50,"failed":0},{"name":"utf8-custom-section-id.wast","passed":176,"failed":0},{"name":"utf8-import-field.wast","passed":176,"failed":0},{"name":"utf8-import-module.wast","passed":176,"failed":0},{"name":"utf8-invalid-encoding.wast","passed":176,"failed":0}]
diff --git a/crates/types/src/instructions.rs b/crates/types/src/instructions.rs
index ba13979..9098812 100644
--- a/crates/types/src/instructions.rs
+++ b/crates/types/src/instructions.rs
@@ -1,4 +1,4 @@
-use crate::{ElemAddr, MemAddr};
+use crate::{DataAddr, ElemAddr, MemAddr};
 
 use super::{FuncAddr, GlobalAddr, LabelAddr, LocalAddr, TableAddr, TypeAddr, ValType};
 
@@ -266,4 +266,10 @@ pub enum Instruction {
     TableGrow(TableAddr),
     TableSize(TableAddr),
     TableFill(TableAddr),
+
+    // Bulk Memory Instructions
+    MemoryInit(MemAddr, DataAddr),
+    MemoryCopy(MemAddr, MemAddr),
+    MemoryFill(MemAddr),
+    DataDrop(DataAddr),
 }