Add benchmarks, fix quicksort/sieve, update bench.sh for version comparison

timfennis · claude · timfennis · commit fc51adbb16cc · 2026-03-20T13:20:46.000+01:00
Co-Authored-By: Claude Sonnet 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/bench.sh b/bench.sh
@@ -30,7 +30,7 @@ echo "Building release binary..."
 cargo build --release --manifest-path "$SCRIPT_DIR/Cargo.toml" -q
 
 echo ""
-echo "Validating the VM version can run '$FILE' without crashing..."
+echo "Validating versions can run '$FILE' without crashing..."
 
 validate() {
     local label="$1"
@@ -42,14 +42,52 @@ validate() {
     echo "  OK:   $label"
 }
 
-validate "ndc (PATH)"        ndc       run "$FILE"
+validate "ndc (PATH)"         ndc      run "$FILE"
 validate "local release (VM)" "$LOCAL" run "$FILE"
 
+# ndc2 and ndc1 are optional: older versions may not support all language features
+# ndc1 also has a different CLI (no 'run' subcommand)
+USE_NDC2=false
+if command -v ndc2 &>/dev/null; then
+    if ndc2 run "$FILE" &>/dev/null; then
+        echo "  OK:   ndc2 (v0.2.1)"
+        USE_NDC2=true
+    else
+        echo "  SKIP: ndc2 (v0.2.1) failed to run '$FILE', skipping"
+    fi
+else
+    echo "  SKIP: ndc2 not found, skipping"
+fi
+
+USE_NDC1=false
+if command -v ndc1 &>/dev/null; then
+    if ndc1 "$FILE" &>/dev/null; then
+        echo "  OK:   ndc1 (v0.1.0)"
+        USE_NDC1=true
+    else
+        echo "  SKIP: ndc1 (v0.1.0) failed to run '$FILE', skipping"
+    fi
+else
+    echo "  SKIP: ndc1 not found, skipping"
+fi
+
 echo ""
 echo "Running benchmark..."
 echo ""
 
+NDC2_ARGS=()
+if [[ "$USE_NDC2" == true ]]; then
+    NDC2_ARGS=(--command-name "ndc2 (v0.2.1)" "ndc2 run '$FILE'")
+fi
+
+NDC1_ARGS=()
+if [[ "$USE_NDC1" == true ]]; then
+    NDC1_ARGS=(--command-name "ndc1 (v0.1.0)" "ndc1 '$FILE'")
+fi
+
 hyperfine \
     --warmup 3 \
     --command-name "ndc (installed)" "ndc run '$FILE'" \
-    --command-name "local release (VM)" "'$LOCAL' run '$FILE'"
+    --command-name "local release (VM)" "'$LOCAL' run '$FILE'" \
+    "${NDC2_ARGS[@]}" \
+    "${NDC1_ARGS[@]}"
diff --git a/benches/Cargo.toml b/benches/Cargo.toml
@@ -15,3 +15,8 @@ rand_chacha.workspace = true
 name = "benchmark"
 path = "src/benchmark.rs"
 harness = false
+
+[[bench]]
+name = "output_sink"
+path = "src/output_sink.rs"
+harness = false
diff --git a/benches/programs/ackermann.ndc b/benches/programs/ackermann.ndc
@@ -0,0 +1,7 @@
+fn ackermann(m, n) {
+    if m == 0 { n + 1 }
+    else if n == 0 { ackermann(m - 1, 1) }
+    else { ackermann(m - 1, ackermann(m, n - 1)) }
+}
+
+print(ackermann(3, 7));
diff --git a/benches/programs/bigint.ndc b/benches/programs/bigint.ndc
@@ -0,0 +1,5 @@
+let result = 1;
+for i in 2..=5000 {
+    result *= i;
+}
+print(result.string.len, "digits");
diff --git a/benches/programs/closures.ndc b/benches/programs/closures.ndc
@@ -0,0 +1,3 @@
+let adders = [fn(x) => x + i for i in 0..10_000];
+let result = adders.map(fn(f) => f(0)).sum;
+print(result);
diff --git a/benches/programs/hof_pipeline.ndc b/benches/programs/hof_pipeline.ndc
@@ -0,0 +1,5 @@
+let result = list(1..=100_000)
+    .filter(fn(x) => x % 2 == 0)
+    .map(fn(x) => x * x)
+    .reduce(fn(acc, x) => acc + x);
+print(result);
diff --git a/benches/programs/map_ops.ndc b/benches/programs/map_ops.ndc
@@ -0,0 +1,11 @@
+let m = %{};
+for i in 1..=50_000 {
+    let key = (i * i) % 997;
+    if key in m {
+        m[key] += 1;
+    } else {
+        m[key] = 1;
+    }
+}
+let most_common = m.keys.max_by_key(fn(k) => m[k]);
+print("Unique keys:", m.keys.len, "Most common:", most_common, "count:", m[most_common]);
diff --git a/benches/programs/matrix_mul.ndc b/benches/programs/matrix_mul.ndc
@@ -0,0 +1,9 @@
+fn mat_mul(a, b, n) {
+    [[fold(0..n, 0, fn(acc, k) => acc + a[r][k] * b[k][c]) for c in 0..n] for r in 0..n]
+}
+
+let n = 50;
+let a = [[(r + c) % 7 for c in 0..n] for r in 0..n];
+let b = [[(r * c + 1) % 5 for c in 0..n] for r in 0..n];
+let result = mat_mul(a, b, n);
+print(result[0][0]);
diff --git a/benches/programs/print_heavy.ndc b/benches/programs/print_heavy.ndc
@@ -0,0 +1,3 @@
+for i in 0..10_000 {
+    print(i);
+}
diff --git a/benches/programs/quicksort.ndc b/benches/programs/quicksort.ndc
@@ -2,8 +2,9 @@ fn quicksort(arr) {
   if arr.len <= 1 { return arr };
   let pivot = arr[0];
   let left = arr.filter(fn(x) => x < pivot);
+  let mid = arr.filter(fn(x) => x == pivot);
   let right = arr.filter(fn(x) => x > pivot);
-  return quicksort(left) ++ [pivot] ++ quicksort(right);
+  return quicksort(left) ++ mid ++ quicksort(right);
 }
 
 print(quicksort([
diff --git a/benches/programs/sieve.ndc b/benches/programs/sieve.ndc
@@ -17,5 +17,5 @@ fn sieve(n) {
   return primes;
 }
 
-// Find lots of prime numbers
+// Compute primality for all numbers up to n
 sieve(200_000);
diff --git a/benches/src/output_sink.rs b/benches/src/output_sink.rs
@@ -0,0 +1,138 @@
+/// Microbenchmark comparing three candidate designs for the VM output sink.
+///
+/// - `enum`: static dispatch via a two-variant enum (current implementation)
+/// - `dyn`:  heap-allocated trait object (`Box<dyn OutputTrait>`)
+/// - `rc`:   `Rc<RefCell<Vec<u8>>>` shared-ownership buffer
+///
+/// All three implementations are defined locally so the benchmark does not
+/// depend on any particular version of the VM.  The goal is to measure the
+/// overhead of the *write path* in isolation from interpreter execution.
+use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
+use std::cell::RefCell;
+use std::hint::black_box;
+use std::io::Write;
+use std::rc::Rc;
+
+// ---------------------------------------------------------------------------
+// Implementation 1 – enum (current)
+// ---------------------------------------------------------------------------
+
+enum EnumSink {
+    Buffer(Vec<u8>),
+}
+
+impl Write for EnumSink {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        match self {
+            EnumSink::Buffer(v) => v.write(buf),
+        }
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Implementation 2 – trait object
+// ---------------------------------------------------------------------------
+
+trait OutputTrait: Write {
+    fn captured(&self) -> Option<&[u8]> {
+        None
+    }
+}
+
+struct DynBuffer(Vec<u8>);
+
+impl Write for DynBuffer {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        self.0.write(buf)
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+impl OutputTrait for DynBuffer {
+    fn captured(&self) -> Option<&[u8]> {
+        Some(&self.0)
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Implementation 3 – Rc<RefCell<Vec<u8>>>
+// ---------------------------------------------------------------------------
+
+struct RcSink(Rc<RefCell<Vec<u8>>>);
+
+impl Write for RcSink {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        self.0.borrow_mut().write(buf)
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Benchmark helpers
+// ---------------------------------------------------------------------------
+
+/// Write `n` copies of `payload` to `sink`.
+fn write_n(sink: &mut impl Write, payload: &[u8], n: usize) {
+    for _ in 0..n {
+        sink.write_all(black_box(payload)).unwrap();
+    }
+}
+
+fn bench_sinks(c: &mut Criterion) {
+    // Payloads of different sizes to expose cache / branch effects.
+    let payloads: &[(&str, &[u8])] = &[
+        ("4B", b"1234"),
+        ("16B", b"hello, world!!!!"),
+        ("64B", &[b'x'; 64]),
+        ("256B", &[b'x'; 256]),
+    ];
+    const WRITES: usize = 10_000;
+
+    let mut group = c.benchmark_group("output_sink");
+
+    for (label, payload) in payloads {
+        let bytes = (payload.len() * WRITES) as u64;
+        group.throughput(Throughput::Bytes(bytes));
+
+        group.bench_with_input(BenchmarkId::new("enum", label), payload, |b, payload| {
+            b.iter(|| {
+                let mut sink = EnumSink::Buffer(Vec::with_capacity(bytes as usize));
+                write_n(&mut sink, payload, WRITES);
+                sink
+            });
+        });
+
+        group.bench_with_input(BenchmarkId::new("dyn", label), payload, |b, payload| {
+            b.iter(|| {
+                let mut sink: Box<dyn OutputTrait> =
+                    Box::new(DynBuffer(Vec::with_capacity(bytes as usize)));
+                write_n(&mut sink, payload, WRITES);
+                sink
+            });
+        });
+
+        group.bench_with_input(BenchmarkId::new("rc", label), payload, |b, payload| {
+            b.iter(|| {
+                let buf = Rc::new(RefCell::new(Vec::with_capacity(bytes as usize)));
+                let mut sink = RcSink(Rc::clone(&buf));
+                write_n(&mut sink, payload, WRITES);
+                buf
+            });
+        });
+    }
+
+    group.finish();
+}
+
+criterion_group!(benches, bench_sinks);
+criterion_main!(benches);
diff --git a/ndc_interpreter/src/lib.rs b/ndc_interpreter/src/lib.rs
@@ -4,36 +4,12 @@ use ndc_lexer::{Lexer, TokenLocation};
 use ndc_parser::ExpressionLocation;
 use ndc_vm::compiler::Compiler;
 use ndc_vm::value::CompiledFunction;
-use ndc_vm::{NativeFunction, Value, Vm};
-use std::cell::RefCell;
-use std::io::Write;
+use ndc_vm::{NativeFunction, OutputSink, Value, Vm};
 use std::rc::Rc;
 
-#[derive(Clone)]
-enum OutputSink {
-    Stdout,
-    Buffer(Rc<RefCell<Vec<u8>>>),
-}
-
-impl Write for OutputSink {
-    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
-        match self {
-            OutputSink::Stdout => std::io::stdout().write(buf),
-            OutputSink::Buffer(vec) => vec.borrow_mut().write(buf),
-        }
-    }
-
-    fn flush(&mut self) -> std::io::Result<()> {
-        match self {
-            OutputSink::Stdout => std::io::stdout().flush(),
-            OutputSink::Buffer(vec) => vec.borrow_mut().flush(),
-        }
-    }
-}
-
 pub struct Interpreter {
     registry: FunctionRegistry<Rc<NativeFunction>>,
-    output: OutputSink,
+    capturing: bool,
     analyser: Analyser,
     /// Persistent REPL VM and the compiler checkpoint from the last run.
     /// `None` until the first `run_str` call; kept alive afterwards so that
@@ -45,20 +21,20 @@ impl Interpreter {
     /// Create an interpreter that writes output to stdout.
     #[must_use]
     pub fn new() -> Self {
-        Self::from_sink(OutputSink::Stdout)
+        Self::from_capturing(false)
     }
 
     /// Create an interpreter that captures output into an internal buffer,
     /// retrievable via [`get_output`].
     #[must_use]
     pub fn capturing() -> Self {
-        Self::from_sink(OutputSink::Buffer(Rc::new(RefCell::new(Vec::new()))))
+        Self::from_capturing(true)
     }
 
-    fn from_sink(output: OutputSink) -> Self {
+    fn from_capturing(capturing: bool) -> Self {
         Self {
             registry: FunctionRegistry::default(),
-            output,
+            capturing,
             analyser: Analyser::from_scope_tree(ScopeTree::from_global_scope(vec![])),
             repl_state: None,
         }
@@ -81,10 +57,9 @@ impl Interpreter {
 
     /// Returns the captured output, or `None` if this interpreter writes to stdout.
     pub fn get_output(&self) -> Option<Vec<u8>> {
-        match &self.output {
-            OutputSink::Stdout => None,
-            OutputSink::Buffer(buf) => Some(buf.borrow().clone()),
-        }
+        self.repl_state
+            .as_ref()
+            .and_then(|(vm, _)| vm.get_output().map(<[u8]>::to_vec))
     }
 
     pub fn analyse_str(
@@ -150,8 +125,13 @@ impl Interpreter {
 
         let result = match self.repl_state.take() {
             None => {
+                let output = if self.capturing {
+                    OutputSink::Buffer(Vec::new())
+                } else {
+                    OutputSink::Stdout
+                };
                 let (code, checkpoint) = Compiler::compile_resumable(expressions)?;
-                let mut vm = Vm::new(code, globals).with_output(Box::new(self.output.clone()));
+                let mut vm = Vm::new(code, globals).with_output(output);
                 #[cfg(feature = "vm-trace")]
                 {
                     vm = vm.with_source(input);
diff --git a/ndc_vm/src/vm.rs b/ndc_vm/src/vm.rs

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+let adders = [fn(x) => x + i for i in 0..10_000];`
	`2`	`+let result = adders.map(fn(f) => f(0)).sum;`
	`3`	`+print(result);`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+for i in 0..10_000 {`
	`2`	`+ print(i);`
	`3`	`+}`