From 4919d8c2f86f69f669a1afba2a28d5c344fe7197 Mon Sep 17 00:00:00 2001 From: Henry Gressmann Date: Mon, 29 Jan 2024 23:12:20 +0100 Subject: perf: improve benchmarks Signed-off-by: Henry Gressmann --- BENCHMARKS.md | 42 ++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 +- README.md | 6 +----- benches/README.md | 31 ------------------------------- benches/fibonacci.rs | 42 +++++++++++++++++++++--------------------- benches/selfhosted.rs | 18 +++++++++--------- benches/util/mod.rs | 30 ++++++++++++++++++++++++++++-- examples/README.md | 23 ----------------------- 8 files changed, 102 insertions(+), 92 deletions(-) create mode 100644 BENCHMARKS.md delete mode 100644 benches/README.md delete mode 100644 examples/README.md diff --git a/BENCHMARKS.md b/BENCHMARKS.md new file mode 100644 index 0000000..c86679c --- /dev/null +++ b/BENCHMARKS.md @@ -0,0 +1,42 @@ +# Benchmark results + +All benchmarks are run on a Ryzen 7 5800X, with 32GB of RAM, running Linux 6.6 with `intel_pstate=passive split_lock_detect=off mitigations=off`. + +## Results + +Coming soon. + +## WebAssembly Settings + +All WebAssembly files are compiled with the following settings: + +- `opt-level` is set to 3, `lto` is set to `thin`, `codegen-units` is set to 1. +- `reference-types`, `bulk-memory`, `mutable-globals` proposals are enabled. + +## Runtime Settings + +All runtimes are compiled with the following settings: + +- `unsafe` features are enabled +- `opt-level` is set to 3, `lto` is set to `thin`, `codegen-units` is set to 1. + +# Running benchmarks + +Benchmarks are run using [Criterion.rs](https://github.com/bheisler/criterion.rs). To run a benchmark, use the following command: + +```sh +$ cargo bench --bench +``` + +## Profiling + +To profile a benchmark, use the following command: + +```sh +$ cargo flamegraph --bench -- --bench +``` + +This will generate a flamegraph in `flamegraph.svg` and a `perf.data` file. +You can use [hotspot](https://github.com/KDAB/hotspot) to analyze the `perf.data` file. +Since a lot of functions are inlined, you probably want to remove the `#[inline]` attribute from the functions you care about. +Note that this will make the benchmark considerably slower, 2-10x slower in some cases. diff --git a/Cargo.toml b/Cargo.toml index 3bbbb35..fae5388 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,7 +43,7 @@ debug=true color-eyre="0.6" criterion={version="0.5", features=["html_reports"]} -tinywasm={path="crates/tinywasm"} +tinywasm={path="crates/tinywasm", features=["unsafe"]} wat={version="1.0"} wasmi={version="0.31", features=["std"]} wasmer={version="4.2", features=["cranelift", "singlepass"]} diff --git a/README.md b/README.md index 8a4aa93..17e966f 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ ## Why TinyWasm? - **Tiny** - Designed to be as small as possible without sacrificing too much performance or functionality. -- **Fast enough** - TinyWasm is reasonably fast, especially when compared to other interpreters. See [Performance](#performance) for more details. +- **Fast enough** - TinyWasm is reasonably fast, especially when compared to other interpreters. See [Benchmarks](./BENCHMARKS.md) for more details. - **Portable** - Runs on any platform llvm supports, including WebAssembly. Minimal external dependencies. ## Status @@ -69,10 +69,6 @@ $ tinywasm-cli --help With all these features disabled, TinyWasm only depends on `core`, `alloc` and `libm` and can be used in `no_std` environments. Since `libm` is not as performant as the compiler's math intrinsics, it is recommended to use the `std` feature if possible (at least [for now](https://github.com/rust-lang/rfcs/issues/2505)), especially on wasm32 targets. -## Performance - -> Benchmarks are coming soon. - ## License Licensed under either of [Apache License, Version 2.0](./LICENSE-APACHE) or [MIT license](./LICENSE-MIT) at your option. diff --git a/benches/README.md b/benches/README.md deleted file mode 100644 index 39ba321..0000000 --- a/benches/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Benchmark results - -All benchmarks are run on a Ryzen 7 5800X, with 32GB of RAM, running Linux 6.6 with `intel_pstate=passive split_lock_detect=off mitigations=off`. - -## Results - -Coming soon. - -## WebAssembly Settings - -All WebAssembly files are compiled with the following settings: - -- `opt-level` is set to 3, `lto` is set to `thin`, `codegen-units` is set to 1. -- `reference-types`, `bulk-memory`, `mutable-globals` proposals are enabled. - -## Runtime Settings - -All runtimes are compiled with the following settings: - -- `unsafe` features are enabled -- `opt-level` is set to 3, `lto` is set to `thin`, `codegen-units` is set to 1. - -## Benchmarking - -Benchmarks are run using [Criterion.rs](https://github.com/bheisler/criterion.rs) and can be found in the `benches` directory. - -## Running benchmarks - -```sh -$ cargo bench --bench -``` diff --git a/benches/fibonacci.rs b/benches/fibonacci.rs index 7c77ebf..ca83869 100644 --- a/benches/fibonacci.rs +++ b/benches/fibonacci.rs @@ -1,41 +1,41 @@ mod util; use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use tinywasm::types::TinyWasmModule; -use util::tinywasm_module; +use util::wasm_to_twasm; -fn run_tinywasm(module: TinyWasmModule, iterations: i32) { - use tinywasm::*; - let module = Module::from(module); - let mut store = Store::default(); - let imports = Imports::default(); - let instance = ModuleInstance::instantiate(&mut store, module, Some(imports)).expect("instantiate"); - let hello = instance.exported_func::(&store, "fibonacci").expect("exported_func"); +fn run_tinywasm(twasm: &[u8], iterations: i32, name: &str) { + let (mut store, instance) = util::tinywasm(twasm); + let hello = instance.exported_func::(&store, name).expect("exported_func"); hello.call(&mut store, iterations).expect("call"); } -fn run_wasmi(iterations: i32) { - use wasmi::*; - let engine = Engine::default(); - let module = wasmi::Module::new(&engine, FIBONACCI).expect("wasmi::Module::new"); - let mut store = Store::new(&engine, ()); - let linker = >::new(&engine); +fn run_wasmi(wasm: &[u8], iterations: i32, name: &str) { + let (module, mut store, linker) = util::wasmi(wasm); let instance = linker.instantiate(&mut store, &module).expect("instantiate").start(&mut store).expect("start"); - let hello = instance.get_typed_func::(&mut store, "fibonacci").expect("get_typed_func"); + let hello = instance.get_typed_func::(&mut store, name).expect("get_typed_func"); hello.call(&mut store, iterations).expect("call"); } const FIBONACCI: &[u8] = include_bytes!("../examples/rust/out/fibonacci.wasm"); fn criterion_benchmark(c: &mut Criterion) { - let module = tinywasm_module(FIBONACCI); + let twasm = wasm_to_twasm(FIBONACCI); - let mut group = c.benchmark_group("fibonacci"); - group.bench_function("tinywasm", |b| b.iter(|| run_tinywasm(module.clone(), black_box(60)))); - group.bench_function("wasmi", |b| b.iter(|| run_wasmi(black_box(60)))); + { + let mut group = c.benchmark_group("fibonacci"); + group.bench_function("tinywasm", |b| b.iter(|| run_tinywasm(&twasm, black_box(60), "fibonacci"))); + group.bench_function("wasmi", |b| b.iter(|| run_wasmi(&FIBONACCI, black_box(60), "fibonacci"))); + } + + { + let mut group = c.benchmark_group("fibonacci-recursive"); + group.measurement_time(std::time::Duration::from_secs(5)); + group.bench_function("tinywasm", |b| b.iter(|| run_tinywasm(&twasm, black_box(26), "fibonacci_recursive"))); + group.bench_function("wasmi", |b| b.iter(|| run_wasmi(&FIBONACCI, black_box(26), "fibonacci_recursive"))); + } } criterion_group!( name = benches; - config = Criterion::default().sample_size(50).measurement_time(std::time::Duration::from_secs(5)).significance_level(0.1); + config = Criterion::default().significance_level(0.1); targets = criterion_benchmark ); diff --git a/benches/selfhosted.rs b/benches/selfhosted.rs index b9df1af..4464579 100644 --- a/benches/selfhosted.rs +++ b/benches/selfhosted.rs @@ -1,11 +1,11 @@ mod util; use criterion::{criterion_group, criterion_main, Criterion}; -use tinywasm::types::TinyWasmModule; -use util::tinywasm_module; -fn run_tinywasm(module: TinyWasmModule) { +use crate::util::twasm_to_module; + +fn run_tinywasm(twasm: &[u8]) { use tinywasm::*; - let module = Module::from(module); + let module = twasm_to_module(twasm); let mut store = Store::default(); let mut imports = Imports::default(); imports.define("env", "printi32", Extern::typed_func(|_: FuncContext<'_>, _: i32| Ok(()))).expect("define"); @@ -14,10 +14,10 @@ fn run_tinywasm(module: TinyWasmModule) { hello.call(&mut store, ()).expect("call"); } -fn run_wasmi() { +fn run_wasmi(wasm: &[u8]) { use wasmi::*; let engine = Engine::default(); - let module = wasmi::Module::new(&engine, TINYWASM).expect("wasmi::Module::new"); + let module = wasmi::Module::new(&engine, wasm).expect("wasmi::Module::new"); let mut store = Store::new(&engine, ()); let mut linker = >::new(&engine); linker.define("env", "printi32", Func::wrap(&mut store, |_: Caller<'_, ()>, _: i32| {})).expect("define"); @@ -28,11 +28,11 @@ fn run_wasmi() { const TINYWASM: &[u8] = include_bytes!("../examples/rust/out/tinywasm.wasm"); fn criterion_benchmark(c: &mut Criterion) { - let module = tinywasm_module(TINYWASM); + let twasm = util::wasm_to_twasm(TINYWASM); let mut group = c.benchmark_group("selfhosted"); - group.bench_function("tinywasm", |b| b.iter(|| run_tinywasm(module.clone()))); - group.bench_function("wasmi", |b| b.iter(run_wasmi)); + group.bench_function("tinywasm", |b| b.iter(|| run_tinywasm(&twasm))); + group.bench_function("wasmi", |b| b.iter(|| run_wasmi(TINYWASM))); } criterion_group!( diff --git a/benches/util/mod.rs b/benches/util/mod.rs index 7baddb7..69510a5 100644 --- a/benches/util/mod.rs +++ b/benches/util/mod.rs @@ -1,6 +1,32 @@ +#![allow(dead_code)] + use tinywasm::{self, parser::Parser, types::TinyWasmModule}; -pub fn tinywasm_module(wasm: &[u8]) -> TinyWasmModule { +pub fn wasm_to_twasm(wasm: &[u8]) -> Vec { let parser = Parser::new(); - parser.parse_module_bytes(wasm).expect("parse_module_bytes") + let res = parser.parse_module_bytes(wasm).expect("parse_module_bytes"); + res.serialize_twasm().to_vec() +} + +#[inline] +pub fn twasm_to_module(twasm: &[u8]) -> tinywasm::Module { + unsafe { TinyWasmModule::from_twasm_unchecked(&twasm) }.into() +} + +pub fn tinywasm(twasm: &[u8]) -> (tinywasm::Store, tinywasm::ModuleInstance) { + use tinywasm::*; + let module = twasm_to_module(twasm); + let mut store = Store::default(); + let imports = Imports::default(); + let instance = ModuleInstance::instantiate(&mut store, module, Some(imports)).expect("instantiate"); + (store, instance) +} + +pub fn wasmi(wasm: &[u8]) -> (wasmi::Module, wasmi::Store<()>, wasmi::Linker<()>) { + use wasmi::*; + let engine = Engine::default(); + let module = wasmi::Module::new(&engine, wasm).expect("wasmi::Module::new"); + let store = Store::new(&engine, ()); + let linker = >::new(&engine); + (module, store, linker) } diff --git a/examples/README.md b/examples/README.md deleted file mode 100644 index 94f974b..0000000 --- a/examples/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Examples - -## Wasm-Rust - -These are examples using WebAssembly generated from Rust code. -To run these, you first need to build the Rust code, since the resulting wasm files are not included in the repository to keep it small. -This requires the `wasm32-unknown-unknown` target and `wasm-opt` to be installed (available via [Binaryen](https://github.com/WebAssembly/binaryen)). - -```bash -$ ./examples/rust/build.sh -``` - -Then you can run the examples: - -```bash -$ cargo run --example wasm-rust -``` - -Where `` is one of the following: - -- `hello`: A simple example that prints a number to the console. -- `tinywasm`: Runs `hello` using TinyWasm - inside of TinyWasm itself! -- `fibonacci`: Calculates the x-th Fibonacci number. -- cgit v1.3.1