perf: improve benchmarks

explodingcamera · explodingcamera · commit 4919d8c2f86f · 2024-01-29T23:12:20.000+01:00
Signed-off-by: Henry Gressmann &lt;mail@henrygressmann.de&gt;
diff --git a/BENCHMARKS.md b/BENCHMARKS.md
@@ -20,12 +20,23 @@ All runtimes are compiled with the following settings:
 - `unsafe` features are enabled
 - `opt-level` is set to 3, `lto` is set to `thin`, `codegen-units` is set to 1.
 
-## Benchmarking
+# Running benchmarks
 
-Benchmarks are run using [Criterion.rs](https://github.com/bheisler/criterion.rs) and can be found in the `benches` directory.
-
-## Running benchmarks
+Benchmarks are run using [Criterion.rs](https://github.com/bheisler/criterion.rs). To run a benchmark, use the following command:
 
 ```sh
 $ cargo bench --bench <name>
 ```
+
+## Profiling
+
+To profile a benchmark, use the following command:
+
+```sh
+$ cargo flamegraph --bench <name> -- --bench
+```
+
+This will generate a flamegraph in `flamegraph.svg` and a `perf.data` file.
+You can use [hotspot](https://github.com/KDAB/hotspot) to analyze the `perf.data` file.
+Since a lot of functions are inlined, you probably want to remove the `#[inline]` attribute from the functions you care about.
+Note that this will make the benchmark considerably slower, 2-10x slower in some cases.
diff --git a/Cargo.toml b/Cargo.toml
@@ -43,7 +43,7 @@ debug=true
 color-eyre="0.6"
 criterion={version="0.5", features=["html_reports"]}
 
-tinywasm={path="crates/tinywasm"}
+tinywasm={path="crates/tinywasm", features=["unsafe"]}
 wat={version="1.0"}
 wasmi={version="0.31", features=["std"]}
 wasmer={version="4.2", features=["cranelift", "singlepass"]}
diff --git a/README.md b/README.md
@@ -13,7 +13,7 @@
 ## Why TinyWasm?
 
 - **Tiny** - Designed to be as small as possible without sacrificing too much performance or functionality.
-- **Fast enough** - TinyWasm is reasonably fast, especially when compared to other interpreters. See [Performance](#performance) for more details.
+- **Fast enough** - TinyWasm is reasonably fast, especially when compared to other interpreters. See [Benchmarks](./BENCHMARKS.md) for more details.
 - **Portable** - Runs on any platform llvm supports, including WebAssembly. Minimal external dependencies.
 
 ## Status
@@ -69,10 +69,6 @@ $ tinywasm-cli --help
 With all these features disabled, TinyWasm only depends on `core`, `alloc` and `libm` and can be used in `no_std` environments.
 Since `libm` is not as performant as the compiler's math intrinsics, it is recommended to use the `std` feature if possible (at least [for now](https://github.com/rust-lang/rfcs/issues/2505)), especially on wasm32 targets.
 
-## Performance
-
-> Benchmarks are coming soon.
-
 ## License
 
 Licensed under either of [Apache License, Version 2.0](./LICENSE-APACHE) or [MIT license](./LICENSE-MIT) at your option.
diff --git a/benches/fibonacci.rs b/benches/fibonacci.rs
@@ -1,41 +1,41 @@
 mod util;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use tinywasm::types::TinyWasmModule;
-use util::tinywasm_module;
+use util::wasm_to_twasm;
 
-fn run_tinywasm(module: TinyWasmModule, iterations: i32) {
-    use tinywasm::*;
-    let module = Module::from(module);
-    let mut store = Store::default();
-    let imports = Imports::default();
-    let instance = ModuleInstance::instantiate(&mut store, module, Some(imports)).expect("instantiate");
-    let hello = instance.exported_func::<i32, i32>(&store, "fibonacci").expect("exported_func");
+fn run_tinywasm(twasm: &[u8], iterations: i32, name: &str) {
+    let (mut store, instance) = util::tinywasm(twasm);
+    let hello = instance.exported_func::<i32, i32>(&store, name).expect("exported_func");
     hello.call(&mut store, iterations).expect("call");
 }
 
-fn run_wasmi(iterations: i32) {
-    use wasmi::*;
-    let engine = Engine::default();
-    let module = wasmi::Module::new(&engine, FIBONACCI).expect("wasmi::Module::new");
-    let mut store = Store::new(&engine, ());
-    let linker = <Linker<()>>::new(&engine);
+fn run_wasmi(wasm: &[u8], iterations: i32, name: &str) {
+    let (module, mut store, linker) = util::wasmi(wasm);
     let instance = linker.instantiate(&mut store, &module).expect("instantiate").start(&mut store).expect("start");
-    let hello = instance.get_typed_func::<i32, i32>(&mut store, "fibonacci").expect("get_typed_func");
+    let hello = instance.get_typed_func::<i32, i32>(&mut store, name).expect("get_typed_func");
     hello.call(&mut store, iterations).expect("call");
 }
 
 const FIBONACCI: &[u8] = include_bytes!("../examples/rust/out/fibonacci.wasm");
 fn criterion_benchmark(c: &mut Criterion) {
-    let module = tinywasm_module(FIBONACCI);
+    let twasm = wasm_to_twasm(FIBONACCI);
 
-    let mut group = c.benchmark_group("fibonacci");
-    group.bench_function("tinywasm", |b| b.iter(|| run_tinywasm(module.clone(), black_box(60))));
-    group.bench_function("wasmi", |b| b.iter(|| run_wasmi(black_box(60))));
+    {
+        let mut group = c.benchmark_group("fibonacci");
+        group.bench_function("tinywasm", |b| b.iter(|| run_tinywasm(&twasm, black_box(60), "fibonacci")));
+        group.bench_function("wasmi", |b| b.iter(|| run_wasmi(&FIBONACCI, black_box(60), "fibonacci")));
+    }
+
+    {
+        let mut group = c.benchmark_group("fibonacci-recursive");
+        group.measurement_time(std::time::Duration::from_secs(5));
+        group.bench_function("tinywasm", |b| b.iter(|| run_tinywasm(&twasm, black_box(26), "fibonacci_recursive")));
+        group.bench_function("wasmi", |b| b.iter(|| run_wasmi(&FIBONACCI, black_box(26), "fibonacci_recursive")));
+    }
 }
 
 criterion_group!(
     name = benches;
-    config = Criterion::default().sample_size(50).measurement_time(std::time::Duration::from_secs(5)).significance_level(0.1);
+    config = Criterion::default().significance_level(0.1);
     targets = criterion_benchmark
 );
 
diff --git a/benches/selfhosted.rs b/benches/selfhosted.rs
@@ -1,11 +1,11 @@
 mod util;
 use criterion::{criterion_group, criterion_main, Criterion};
-use tinywasm::types::TinyWasmModule;
-use util::tinywasm_module;
 
-fn run_tinywasm(module: TinyWasmModule) {
+use crate::util::twasm_to_module;
+
+fn run_tinywasm(twasm: &[u8]) {
     use tinywasm::*;
-    let module = Module::from(module);
+    let module = twasm_to_module(twasm);
     let mut store = Store::default();
     let mut imports = Imports::default();
     imports.define("env", "printi32", Extern::typed_func(|_: FuncContext<'_>, _: i32| Ok(()))).expect("define");
@@ -14,10 +14,10 @@ fn run_tinywasm(module: TinyWasmModule) {
     hello.call(&mut store, ()).expect("call");
 }
 
-fn run_wasmi() {
+fn run_wasmi(wasm: &[u8]) {
     use wasmi::*;
     let engine = Engine::default();
-    let module = wasmi::Module::new(&engine, TINYWASM).expect("wasmi::Module::new");
+    let module = wasmi::Module::new(&engine, wasm).expect("wasmi::Module::new");
     let mut store = Store::new(&engine, ());
     let mut linker = <Linker<()>>::new(&engine);
     linker.define("env", "printi32", Func::wrap(&mut store, |_: Caller<'_, ()>, _: i32| {})).expect("define");
@@ -28,11 +28,11 @@ fn run_wasmi() {
 
 const TINYWASM: &[u8] = include_bytes!("../examples/rust/out/tinywasm.wasm");
 fn criterion_benchmark(c: &mut Criterion) {
-    let module = tinywasm_module(TINYWASM);
+    let twasm = util::wasm_to_twasm(TINYWASM);
 
     let mut group = c.benchmark_group("selfhosted");
-    group.bench_function("tinywasm", |b| b.iter(|| run_tinywasm(module.clone())));
-    group.bench_function("wasmi", |b| b.iter(run_wasmi));
+    group.bench_function("tinywasm", |b| b.iter(|| run_tinywasm(&twasm)));
+    group.bench_function("wasmi", |b| b.iter(|| run_wasmi(TINYWASM)));
 }
 
 criterion_group!(
diff --git a/benches/util/mod.rs b/benches/util/mod.rs
@@ -1,6 +1,32 @@
+#![allow(dead_code)]
+
 use tinywasm::{self, parser::Parser, types::TinyWasmModule};
 
-pub fn tinywasm_module(wasm: &[u8]) -> TinyWasmModule {
+pub fn wasm_to_twasm(wasm: &[u8]) -> Vec<u8> {
     let parser = Parser::new();
-    parser.parse_module_bytes(wasm).expect("parse_module_bytes")
+    let res = parser.parse_module_bytes(wasm).expect("parse_module_bytes");
+    res.serialize_twasm().to_vec()
+}
+
+#[inline]
+pub fn twasm_to_module(twasm: &[u8]) -> tinywasm::Module {
+    unsafe { TinyWasmModule::from_twasm_unchecked(&twasm) }.into()
+}
+
+pub fn tinywasm(twasm: &[u8]) -> (tinywasm::Store, tinywasm::ModuleInstance) {
+    use tinywasm::*;
+    let module = twasm_to_module(twasm);
+    let mut store = Store::default();
+    let imports = Imports::default();
+    let instance = ModuleInstance::instantiate(&mut store, module, Some(imports)).expect("instantiate");
+    (store, instance)
+}
+
+pub fn wasmi(wasm: &[u8]) -> (wasmi::Module, wasmi::Store<()>, wasmi::Linker<()>) {
+    use wasmi::*;
+    let engine = Engine::default();
+    let module = wasmi::Module::new(&engine, wasm).expect("wasmi::Module::new");
+    let store = Store::new(&engine, ());
+    let linker = <Linker<()>>::new(&engine);
+    (module, store, linker)
 }
diff --git a/examples/README.md b/examples/README.md
