Add ZynML runtime profiles, event bridge, and SIMD scaffolding

Author: darmie

.github/workflows/ci.yml

@@ -31,6 +31,23 @@ jobs:
       - name: Run clippy
         run: cargo clippy --workspace --all-targets
 
+  llvm-feature-check:
+    name: llvm feature check
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache cargo artifacts
+        uses: Swatinem/rust-cache@v2
+
+      - name: Check zynml with llvm-backend feature
+        run: cargo check -p zynml --features llvm-backend
+
   parse-conformance:
     name: parse conformance
     runs-on: ubuntu-latest
@@ -67,3 +84,5 @@ jobs:
         run: |
           RUST_MIN_STACK=134217728 cargo test -p zynml --test e2e_tests execution::test_execute_simple_expression -- --nocapture
           RUST_MIN_STACK=134217728 cargo test -p zynml --test e2e_tests execution::test_execute_prelude_only_example -- --nocapture
+          RUST_MIN_STACK=134217728 cargo test -p zynml --test e2e_tests runtime::test_runtime_profile_tiered_development -- --nocapture
+          RUST_MIN_STACK=134217728 cargo test -p zynml --test e2e_tests runtime::test_runtime_event_capture_render_and_stream -- --nocapture

crates/compiler/Cargo.toml

@@ -60,9 +60,14 @@ inkwell = { version = "0.7.1", features = ["llvm21-1"], optional = true }  # LLV
 
 [dev-dependencies]
 env_logger = "0.11"
+criterion = "0.5"
+
+[[bench]]
+name = "simd_vectorization"
+harness = false
 
 [features]
 default = ["cranelift-backend"]
 cranelift-backend = ["cranelift", "cranelift-codegen", "cranelift-frontend", "cranelift-jit", "cranelift-module", "cranelift-native"]
 llvm-backend = ["inkwell"]
-all-backends = ["cranelift-backend", "llvm-backend"]
+all-backends = ["cranelift-backend", "llvm-backend"]

crates/compiler/benches/simd_vectorization.rs

@@ -0,0 +1,44 @@
+use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
+
+fn scalar_add(dst: &mut [f32], a: &[f32], b: &[f32]) {
+    for i in 0..dst.len() {
+        dst[i] = a[i] + b[i];
+    }
+}
+
+fn chunked_add(dst: &mut [f32], a: &[f32], b: &[f32]) {
+    let mut i = 0;
+    while i + 4 <= dst.len() {
+        dst[i] = a[i] + b[i];
+        dst[i + 1] = a[i + 1] + b[i + 1];
+        dst[i + 2] = a[i + 2] + b[i + 2];
+        dst[i + 3] = a[i + 3] + b[i + 3];
+        i += 4;
+    }
+
+    while i < dst.len() {
+        dst[i] = a[i] + b[i];
+        i += 1;
+    }
+}
+
+fn bench_vector_add(c: &mut Criterion) {
+    let mut group = c.benchmark_group("vector-add");
+    for size in [1 << 10, 1 << 14, 1 << 18] {
+        let a = vec![1.0f32; size];
+        let b = vec![2.0f32; size];
+        let mut dst = vec![0.0f32; size];
+
+        group.bench_with_input(BenchmarkId::new("scalar", size), &size, |bencher, _| {
+            bencher.iter(|| scalar_add(black_box(&mut dst), black_box(&a), black_box(&b)));
+        });
+
+        group.bench_with_input(BenchmarkId::new("chunked", size), &size, |bencher, _| {
+            bencher.iter(|| chunked_add(black_box(&mut dst), black_box(&a), black_box(&b)));
+        });
+    }
+    group.finish();
+}
+
+criterion_group!(simd_benches, bench_vector_add);
+criterion_main!(simd_benches);

crates/compiler/src/cranelift_backend.rs

@@ -15,6 +15,7 @@ use cranelift_jit::{JITBuilder, JITModule};
 use cranelift_module::{DataDescription, FuncId, Linkage, Module};
 use log::{debug, error, info, warn};
 use std::collections::HashMap;
+use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, RwLock};
 
 use crate::effect_codegen::{
@@ -28,6 +29,18 @@ use crate::hir::{
 };
 use crate::{CompilerError, CompilerResult};
 
+static CRANELIFT_SKIPPED_FUNCTIONS: AtomicUsize = AtomicUsize::new(0);
+
+/// Number of function bodies skipped by Cranelift due to recoverable codegen errors.
+pub fn cranelift_skipped_function_count() -> usize {
+    CRANELIFT_SKIPPED_FUNCTIONS.load(Ordering::Relaxed)
+}
+
+/// Reset Cranelift skip diagnostics counter.
+pub fn reset_cranelift_skipped_function_count() {
+    CRANELIFT_SKIPPED_FUNCTIONS.store(0, Ordering::Relaxed);
+}
+
 /// Convert ZRTL TypeTag to Cranelift type
 fn type_tag_to_cranelift_type(tag: &crate::zrtl::TypeTag) -> types::Type {
     use crate::zrtl::{PrimitiveSize, TypeCategory};
@@ -372,6 +385,7 @@ impl CraneliftBackend {
             if !function.is_external {
                 // Skip functions that fail to compile (e.g., signature mismatches with ZRTL)
                 if let Err(e) = self.compile_function_body(*id, function, module) {
+                    CRANELIFT_SKIPPED_FUNCTIONS.fetch_add(1, Ordering::Relaxed);
                     eprintln!(
                         "[CRANELIFT WARN] Skipping function '{}': {:?}",
                         function.name, e
@@ -4061,10 +4075,16 @@ impl CraneliftBackend {
                 // Function pointers
                 Ok(self.module.target_config().pointer_type())
             }
-            HirType::Vector(elem_ty, _count) => {
-                // Vector types for SIMD - use the element type for now
-                self.translate_type(elem_ty)
-            }
+            HirType::Vector(elem_ty, count) => match (&**elem_ty, *count) {
+                (HirType::F32, 4) => Ok(types::F32X4),
+                (HirType::F64, 2) => Ok(types::F64X2),
+                (HirType::I32, 4) | (HirType::U32, 4) => Ok(types::I32X4),
+                (HirType::I64, 2) | (HirType::U64, 2) => Ok(types::I64X2),
+                _ => Err(CompilerError::CodeGen(format!(
+                    "unsupported SIMD vector lane shape in Cranelift backend: Vector({:?}, {})",
+                    elem_ty, count
+                ))),
+            },
             HirType::Union(_) => {
                 // Unions are treated as pointers to stack-allocated memory
                 Ok(self.module.target_config().pointer_type())

crates/compiler/src/lib.rs

@@ -69,6 +69,9 @@ pub use borrow_check::{
 };
 pub use cfg::{BasicBlock, CfgEdge, ControlFlowGraph};
 pub use const_eval::{ConstEvalContext, ConstEvaluator};
+pub use cranelift_backend::{
+    cranelift_skipped_function_count, reset_cranelift_skipped_function_count,
+};
 pub use effect_analysis::{
     analyze_effects, analyze_effects_with_call_graph, get_function_effect_summary,
     has_effect_errors, EffectAnalyzer, EffectCallGraph, EffectError, EffectErrorKind,
@@ -90,6 +93,8 @@ pub use hir::{HirBlock, HirFunction, HirId, HirInstruction, HirModule, HirValue}
 pub use hir_builder::HirBuilder; // HIR Builder API
 use log::debug;
 pub use lowering::{
+    lowering_skipped_function_count,
+    reset_lowering_skipped_function_count,
     AstLowering,
     BuiltinResolver,
     ChainedResolver,

crates/compiler/src/llvm_backend.rs

@@ -3291,6 +3291,18 @@ impl<'ctx> LLVMBackend<'ctx> {
                 // Return function pointer type
                 fn_type.ptr_type(AddressSpace::default()).into()
             }
+            Vector(elem_ty, count) => match (&**elem_ty, *count) {
+                (F32, 4) => self.context.f32_type().vec_type(4).into(),
+                (F64, 2) => self.context.f64_type().vec_type(2).into(),
+                (I32, 4) | (U32, 4) => self.context.i32_type().vec_type(4).into(),
+                (I64, 2) | (U64, 2) => self.context.i64_type().vec_type(2).into(),
+                _ => {
+                    return Err(CompilerError::CodeGen(format!(
+                        "unsupported SIMD vector lane shape in LLVM backend: Vector({:?}, {})",
+                        elem_ty, count
+                    )));
+                }
+            },
             _ => {
                 return Err(CompilerError::CodeGen(format!(
                     "Type translation not yet implemented: {:?}",

crates/compiler/src/lowering.rs

@@ -4,6 +4,7 @@
 //! ready for both Cranelift and LLVM backends.
 
 use std::hash::{Hash, Hasher};
+use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex};
 use zyntax_typed_ast::{
     AstArena, InternedString, Mutability, Type, TypeId, TypedDeclaration, TypedEffect,
@@ -26,6 +27,18 @@ use crate::ssa::{SsaBuilder, SsaForm};
 use crate::CompilerResult;
 use std::collections::HashMap;
 
+static LOWERING_SKIPPED_FUNCTIONS: AtomicUsize = AtomicUsize::new(0);
+
+/// Number of functions skipped by lowering due to recoverable errors.
+pub fn lowering_skipped_function_count() -> usize {
+    LOWERING_SKIPPED_FUNCTIONS.load(Ordering::Relaxed)
+}
+
+/// Reset lowering skip diagnostics counter.
+pub fn reset_lowering_skipped_function_count() {
+    LOWERING_SKIPPED_FUNCTIONS.store(0, Ordering::Relaxed);
+}
+
 /// Target data layout information for precise size and alignment calculations
 #[derive(Debug, Clone)]
 pub struct TargetData {
@@ -1650,6 +1663,7 @@ impl LoweringContext {
                 // but we shouldn't fail the entire compilation for unused code
                 if let Err(e) = self.lower_function(func) {
                     let func_name = func.name.resolve_global().unwrap_or_default();
+                    LOWERING_SKIPPED_FUNCTIONS.fetch_add(1, Ordering::Relaxed);
                     eprintln!("[LOWERING WARN] Skipping function '{}': {:?}", func_name, e);
                     // Remove from symbol table if it was registered
                     self.symbols.functions.remove(&func.name);

crates/compiler/src/ssa.rs

@@ -22,6 +22,8 @@ use zyntax_typed_ast::{
 /// Internal alias used when lowering `compute(...) { ... }` expressions.
 /// This must not collide with user code.
 const INTERNAL_COMPUTE_ALIAS: &str = "__internal_compute_dispatch";
+const INTERNAL_RENDER_EVENT_ALIAS: &str = "__internal_render_event";
+const INTERNAL_STREAM_EVENT_ALIAS: &str = "__internal_stream_event";
 
 /// SSA builder state
 pub struct SsaBuilder {
@@ -1957,6 +1959,20 @@ impl SsaBuilder {
                             .unwrap_or_default()
                     });
 
+                    if name_str == INTERNAL_RENDER_EVENT_ALIAS
+                        || name_str == INTERNAL_STREAM_EVENT_ALIAS
+                    {
+                        // Internal runtime event calls are lowered as side-effect markers.
+                        // Evaluate arguments for side effects, then erase the call from HIR.
+                        for arg in &call.positional_args {
+                            self.translate_expression(block_id, arg)?;
+                        }
+                        for named in &call.named_args {
+                            self.translate_expression(block_id, &named.value)?;
+                        }
+                        return Ok(self.create_undef(HirType::Void));
+                    }
+
                     // Check for enum constructors (Some, Ok, Err)
                     if name_str == "Some" || name_str == "Ok" || name_str == "Err" {
                         return self
@@ -2077,20 +2093,7 @@ impl SsaBuilder {
                 self.compute_yield_stack.push(Vec::new());
                 let mut current_block = block_id;
                 for stmt in &compute.body.statements {
-                    match &stmt.node {
-                        zyntax_typed_ast::typed_ast::TypedStatement::Let(_)
-                        | zyntax_typed_ast::typed_ast::TypedStatement::Expression(_)
-                        | zyntax_typed_ast::typed_ast::TypedStatement::Yield(_) => {
-                            current_block = self.process_statement(current_block, stmt)?;
-                        }
-                        _ => {
-                            self.compute_yield_stack.pop();
-                            return Err(crate::CompilerError::Analysis(
-                                "compute body fallback currently supports only let/expression/yield statements"
-                                    .to_string(),
-                            ));
-                        }
-                    }
+                    current_block = self.process_statement(current_block, stmt)?;
                 }
 
                 let yields = self.compute_yield_stack.pop().unwrap_or_default();

crates/zynml/Cargo.toml

@@ -35,3 +35,9 @@ env_logger = "0.11"
 
 [dev-dependencies]
 tempfile = "3.16"
+zyntax_compiler = { path = "../compiler" }
+
+[features]
+default = []
+llvm-backend = ["zyntax_embed/llvm-backend"]
+all-backends = ["llvm-backend"]

crates/zynml/ml.zyn

@@ -31,6 +31,8 @@
 @builtin {
     // Internal dispatch aliases (language-level constructs lowered to plugin calls)
     __internal_compute_dispatch: "$ZynML$compute",
+    __internal_render_event: "$ZynML$render_event",
+    __internal_stream_event: "$ZynML$stream_event",
 
     // === Tensor Operations (zrtl_tensor) ===
     tensor: "tensor_from_array",          // tensor([1.0, 2.0]) -> tensor_from_array([1.0, 2.0])
@@ -640,7 +642,7 @@ render_stmt = { stmt:render_with_options | stmt:render_simple }
 render_with_options = { "render" ~ value:expr ~ "{" ~ config_entries ~ "}" }
   -> TypedStatement::Expression {
       expr: TypedExpression::Call {
-          callee: Box::new(TypedExpression::Variable { name: intern("render") }),
+          callee: Box::new(TypedExpression::Variable { name: intern("__internal_render_event") }),
           args: [value],
           named_args: config_entries,
       }
@@ -650,7 +652,7 @@ render_with_options = { "render" ~ value:expr ~ "{" ~ config_entries ~ "}" }
 render_simple = { "render" ~ value:expr }
   -> TypedStatement::Expression {
       expr: TypedExpression::Call {
-          callee: Box::new(TypedExpression::Variable { name: intern("render") }),
+          callee: Box::new(TypedExpression::Variable { name: intern("__internal_render_event") }),
           args: [value],
       }
   }
@@ -659,12 +661,21 @@ render_simple = { "render" ~ value:expr }
 // Stream Statement
 // ============================================================================
 
+stream_source_call = { "stream" ~ source:identifier }
+  -> TypedExpression::Call {
+      callee: Box::new(TypedExpression::Variable { name: intern("stream") }),
+      args: [TypedExpression::Variable { name: intern(source) }],
+  }
+
+stream_chain_expr = { first:stream_source_call ~ rest:pipe_rest* }
+  -> fold_pipe(first, rest)
+
 // stream sensor_data |> window(100) |> sink(alert_system)
-stream_stmt = { "stream" ~ source:identifier ~ ("|>" ~ pipe_call)+ }
+stream_stmt = { chain:stream_chain_expr }
   -> TypedStatement::Expression {
       expr: TypedExpression::Call {
-          callee: Box::new(TypedExpression::Variable { name: intern("stream") }),
-          args: [TypedExpression::Variable { name: intern(source) }],
+          callee: Box::new(TypedExpression::Variable { name: intern("__internal_stream_event") }),
+          args: [chain],
       }
   }
 
@@ -1908,6 +1919,8 @@ statement = {
     break_stmt |
     continue_stmt |
     assign_stmt |
+    render_stmt |
+    stream_stmt |
     expr_stmt
 }