From 2fd885961092929339f3625cc4b7d141eee92976 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Tue, 7 Jul 2026 00:51:40 +0100 Subject: [PATCH 01/17] =?UTF-8?q?Complete=20image=E2=86=92mesh=20pipeline:?= =?UTF-8?q?=20DINOv3=20encoder,=20shape-SLAT=20stage,=20demo=20server?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finishes the stage-1 geometry pipeline end-to-end (image in, 3D mesh out, pure C++/ggml inference) following the depth-anything.cpp / privacy-filter.cpp process: layer-by-layer parity validation, sanitizer + fuzz builds, and a Go demo server with a browser mesh viewer. New pipeline stages (trellis2.cpp / trellis2.h): - DINOv3 ViT-L/16 image-conditioning encoder (axial-2D RoPE, LayerScale, exact-GELU MLP, affine-free final LN) — replaces the external .dinodata producer. Validated to rel-L2 ≤ 7e-7 across 40 taps. - PIL-exact fixed-point Lanczos preprocessing (alpha bbox crop, premultiply, 512 resize) — byte-exact vs Pillow. - Shape-SLAT flow DiT (1.3B sparse DiT over active voxels, 3D RoPE) and the FlexiDualGrid VAE decoder (sparse ConvNeXt U-Net, 3×3×3 submanifold conv as 27 gather+GEMM, learned subdivision 32³→512³). Decoder numerically exact through all 4 conv levels; forward rel-L2 2.9e-4. - flexible_dual_grid.h CPU mesher → triangle mesh (the real TRELLIS.2 geometry). Converters: convert_dino / convert_slat_flow / convert_shape_dec to GGUF (shape_slat_normalization baked into KV). scripts/{download_models,convert_all, refgen,demo}.sh. Validation: tests/parity.hpp + test_dino / test_preprocess / test_slat, ctest-registered with SKIP_RETURN_CODE 77 and model/slow labels; ref_common.py runs the PyTorch reference on stock CUDA/CPU with sparse ops monkeypatched to dense and TF32 disabled (true-fp32 golden). Existing ref scripts repathed off the missing trellis2-shiv layout. Fuzzing: libFuzzer harnesses (fuzz_image, fuzz_dinodata) under ASan/UBSan; found+fixed a .dinodata shape-product overflow (std::length_error → clean reject). Demo: flat C ABI (trellis2_capi) + Go server (purego dlopen, single inference mutex, job queue, progress) + self-contained WebGL2 viewer. Coarse (marching-cubes) or fine (dual-grid) path. CUDA via docker CDI; 3D-conv decoders pinned to CPU (no ggml CUDA CONV_3D kernel). Co-Authored-By: Claude Opus 4.8 --- .gitignore | 22 + CMakeLists.txt | 20 +- README.md | 88 +- convert_dino_to_gguf.py | 201 ++++ convert_shape_dec_to_gguf.py | 167 ++++ convert_slat_flow_to_gguf.py | 246 +++++ docker/Dockerfile.demo | 13 + docker/Dockerfile.ref | 25 + docs/PLAN.md | 81 ++ docs/VERIFICATION.md | 67 ++ examples/CMakeLists.txt | 5 + examples/dino_encode.cpp | 116 +++ examples/flexible_dual_grid.h | 153 +++ fuzz/CMakeLists.txt | 18 + fuzz/fuzz_dinodata.cpp | 57 ++ fuzz/fuzz_image.cpp | 33 + scripts/convert_all.sh | 25 + scripts/demo.sh | 27 + scripts/download_models.sh | 46 + scripts/dump_dino_reference.py | 127 +++ scripts/dump_slat_reference.py | 198 ++++ scripts/ref_common.py | 313 +++++++ scripts/refgen.sh | 26 + server/engine.go | 192 ++++ server/go.mod | 5 + server/go.sum | 2 + server/main.go | 286 ++++++ server/web/index.html | 428 +++++++++ tests/CMakeLists.txt | 61 ++ tests/parity.hpp | 92 ++ tests/ref_ss_dec.py | 12 +- tests/ref_ss_flow.py | 12 +- tests/ref_ss_sample.py | 14 +- tests/test_dino.cpp | 103 +++ tests/test_preprocess.cpp | 74 ++ tests/test_slat.cpp | 195 ++++ tests/test_ss_dec.cpp | 8 + tests/test_ss_flow_forward.cpp | 8 + tests/test_ss_sample.cpp | 8 + trellis2.cpp | 1590 +++++++++++++++++++++++++++++++- trellis2.h | 241 ++++- trellis2_capi.cpp | 301 ++++++ trellis2_capi.h | 97 ++ 43 files changed, 5760 insertions(+), 43 deletions(-) create mode 100644 convert_dino_to_gguf.py create mode 100644 convert_shape_dec_to_gguf.py create mode 100644 convert_slat_flow_to_gguf.py create mode 100644 docker/Dockerfile.demo create mode 100644 docker/Dockerfile.ref create mode 100644 docs/PLAN.md create mode 100644 docs/VERIFICATION.md create mode 100644 examples/dino_encode.cpp create mode 100644 examples/flexible_dual_grid.h create mode 100644 fuzz/CMakeLists.txt create mode 100644 fuzz/fuzz_dinodata.cpp create mode 100644 fuzz/fuzz_image.cpp create mode 100755 scripts/convert_all.sh create mode 100755 scripts/demo.sh create mode 100755 scripts/download_models.sh create mode 100644 scripts/dump_dino_reference.py create mode 100644 scripts/dump_slat_reference.py create mode 100644 scripts/ref_common.py create mode 100755 scripts/refgen.sh create mode 100644 server/engine.go create mode 100644 server/go.mod create mode 100644 server/go.sum create mode 100644 server/main.go create mode 100644 server/web/index.html create mode 100644 tests/parity.hpp create mode 100644 tests/test_dino.cpp create mode 100644 tests/test_preprocess.cpp create mode 100644 tests/test_slat.cpp create mode 100644 trellis2_capi.cpp create mode 100644 trellis2_capi.h diff --git a/.gitignore b/.gitignore index b600824..9ad0e98 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,25 @@ tests/ss_dec_ref.bin # clangd index .cache/ +corpus_*/ +crash-* + +# fuzzing +/corpus_*/ +crash-* +/fuzz/crashes-fixed/ + +# demo server build artifacts +/server/trellis2-server +/server/trellis2-server-linux +/server/vendor/ + +# python bytecode +__pycache__/ +*.pyc + +# model / reference artifacts +/models/ +/ggufs/ +/dumps/ +slow-unit-* diff --git a/CMakeLists.txt b/CMakeLists.txt index f1c48c5..8842e2b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,17 @@ endif() # ───────────────────────────────────────────────────────────────────────── option(TRELLIS2_USE_EXTERNAL_GGML "Consume external ggml via find_package() instead of the bundled submodule" OFF) +# libFuzzer harnesses (clang only). The sanitizer flags apply globally — set +# BEFORE any target so the whole tree (ggml, stb decode, preprocessing, +# loaders) is instrumented. Harness targets live in fuzz/ (added at the end). +option(TRELLIS2_FUZZ "Build libFuzzer harnesses (requires clang)" OFF) +set(TRELLIS2_FUZZ_SANITIZERS "address,undefined" CACHE STRING + "Sanitizers for the fuzz build") +if(TRELLIS2_FUZZ) + add_compile_options(-fsanitize=${TRELLIS2_FUZZ_SANITIZERS} -fsanitize=fuzzer-no-link -fno-omit-frame-pointer -g -O1) + add_link_options(-fsanitize=${TRELLIS2_FUZZ_SANITIZERS}) +endif() + # ggml options — enable Metal on Apple (bundled-ggml path only) option(TRELLIS2_METAL "Enable Metal backend" ON) if(APPLE AND TRELLIS2_METAL AND NOT TRELLIS2_USE_EXTERNAL_GGML) @@ -35,11 +46,13 @@ else() set(TRELLIS2_GGML_TARGET ggml) endif() -add_library(trellis2 trellis2.cpp trellis2.h) +add_library(trellis2 trellis2.cpp trellis2.h trellis2_capi.cpp trellis2_capi.h) target_include_directories(trellis2 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/stb ) +# trellis2_capi.cpp reuses the example isosurface extractor for the demo path. +target_include_directories(trellis2 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/examples) if(TRELLIS2_USE_EXTERNAL_GGML AND DEFINED GGML_INCLUDE_DIR) target_include_directories(trellis2 PUBLIC ${GGML_INCLUDE_DIR}) @@ -62,5 +75,10 @@ endif() option(TRELLIS2_BUILD_TESTS "Build test executables" OFF) if(TRELLIS2_BUILD_TESTS) + enable_testing() add_subdirectory(tests) endif() + +if(TRELLIS2_FUZZ) + add_subdirectory(fuzz) +endif() diff --git a/README.md b/README.md index cbc7f64..6744be3 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,75 @@ # trellis2.cpp A C++/[ggml](https://github.com/ggml-org/ggml) implementation of the -**TRELLIS.2** image-to-3D pipeline (stage 1 first: sparse-structure flow). +**TRELLIS.2** image-to-3D geometry pipeline: an image goes in, a 3D mesh comes +out, with all inference in C++/ggml (no PyTorch at runtime). No PBR textures +yet — geometry only. Modeled structurally after [sam3.cpp](https://github.com/rms80/sam3.cpp): single-file library (`trellis2.h` / `trellis2.cpp`), bundled ggml as a submodule (Metal on by default on Apple), DLL-export decoration, and a -CMake build with example executables. +CMake build with example executables. A flat C ABI (`trellis2_capi.h`) drives +a Go demo server with a browser mesh viewer. -## Status +## Quick start (demo) -Early scaffolding. Implemented so far: +```sh +git submodule update --init --depth 1 # ggml +scripts/download_models.sh # HF checkpoints -> models/ (~7 GB) +docker build -f docker/Dockerfile.ref -t trellis2-ref docker # convert weights / gen refs +docker build -f docker/Dockerfile.demo -t trellis2-demo docker # CUDA runtime + Go +# convert every checkpoint to GGUF (f16 for the demo, f32 for validation) +docker run --rm -v "$PWD":/work -w /work trellis2-ref bash scripts/convert_all.sh +# build the CUDA shared lib + Go server, then run +docker run --rm -v "$PWD":/work -w /work trellis2-demo bash -c ' + cmake -B build-cuda-shared -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=ON \ + -DCMAKE_CUDA_ARCHITECTURES=120 -DBUILD_SHARED_LIBS=ON && cmake --build build-cuda-shared -j + cd server && go build -o trellis2-server-linux .' +docker run --rm --device nvidia.com/gpu=all -v "$PWD":/work -w /work/server -p 8742:8742 \ + trellis2-demo ./trellis2-server-linux -lib /work/build-cuda-shared/libtrellis2.so -ggufs /work/ggufs +# open http://localhost:8742 , drop an image (transparent background works best) +``` + +Drop the shape-SLAT models (`-coarse`, or omit `slat_flow`/`shape_dec` GGUFs) +to fall back to the fast 64³ marching-cubes preview. On a 16 GB RTX 50-series +the full fine path (1.3B SS-flow + 1.3B shape-SLAT + sparse U-Net decoder) +runs image→mesh in ~110 s and yields a ~1M-vertex 512³ dual-grid mesh. + +## Pipeline + +``` +image (RGBA) + → preprocess alpha bbox crop, premultiply, PIL-exact Lanczos-512 [C++, byte-exact] + → DINOv3 ViT-L/16 [1, 1029, 1024] conditioning tokens [C++/ggml] + → SS-flow DiT 1.3B dense DiT, 12-step CFG flow-Euler → z_s [C++/ggml] + → SS decoder dense 3D-conv → 64³ occupancy → 32³ voxel scaffold [C++/ggml] + → shape-SLAT DiT 1.3B sparse DiT over active voxels, 12-step CFG [C++/ggml] + → shape VAE decoder sparse ConvNeXt U-Net, 16× up → 512³ dual-grid fields [C++/ggml] + → flexible dual grid → triangle mesh [C++] +``` -- **`.dinodata` loader** — reads the DINOv3 conditioning tensor that the - sparse-structure flow DiT consumes as cross-attention K/V. This is the full - DINOv3 ViT-L/16 token sequence (`[1, 1029, 1024]` = 1 CLS + 4 register + - 1024 patch) of the **last transformer layer** with an **affine-free - LayerNorm** (NOT HF's `last_hidden_state`). `neg_cond = zeros_like(cond)`, - so it is never stored. Files are produced by `trellis2-shiv/dump_dinodata.py`. +Every stage is validated tap-by-tap against the PyTorch reference — see +[docs/VERIFICATION.md](docs/VERIFICATION.md). Highlights: preprocessing is +byte-exact, the DINOv3 encoder matches to rel-L2 ≤ 7e-7 across 40 taps, and the +sparse U-Net decoder is numerically exact through all four conv levels. + +## Components + +- **Image preprocessing + DINOv3 encoder** — `trellis2_preprocess_rgba()` + reproduces `pipeline.preprocess_image` (the has-alpha path) with a + PIL-compatible fixed-point Lanczos-3 resampler (byte-exact vs Pillow). + `trellis2_dino_encode()` runs the full DINOv3 ViT-L/16 (axial-2D RoPE, + LayerScale, exact-GELU MLP) and applies the affine-free final LayerNorm the + flow models expect — the `[1, 1029, 1024]` conditioning that used to come + from an external `dump_dinodata.py`. `dino_encode` chains them: + + ```sh + ./build/examples/dino_encode ggufs/dino_f16.gguf image.png cond.dinodata + ``` + +- **`.dinodata` loader** — `trellis2_load_dinodata()` still reads/writes the + precomputed conditioning tensor (1 CLS + 4 register + 1024 patch, last layer, + affine-free LN; `neg_cond = zeros_like(cond)`), for testing and CLI chaining. - **SS-flow DiT weights** — `convert_ss_flow_to_gguf.py` converts the stage-1 `ss_flow_img_dit_1_3B_64_bf16` checkpoint to GGUF; `trellis2_ss_flow_load()` @@ -58,10 +110,10 @@ Early scaffolding. Implemented so far: # -> logits [1,64,64,64], occupied(>0) grid (the coarse voxel scaffold) ``` -- **Occupancy → mesh** — `ss_mesh` decodes a z_s latent and exports the +- **Occupancy → coarse mesh** — `ss_mesh` decodes a z_s latent and exports the `{logit = 0}` isosurface as a watertight OBJ via a self-contained marching cubes (`examples/marching_cubes.h`, the tetrahedral / Freudenthal variant — no - 256-row table, provably manifold). Chain it after sampling to *see* the result: + 256-row table, provably manifold). This is the fast preview path: ```sh ./build/examples/ss_sample ss_flow_dit_f16.gguf /path/img.dinodata z_s.latent @@ -69,6 +121,18 @@ Early scaffolding. Implemented so far: # -> watertight shape.obj in the centered unit cube; open in any 3D viewer ``` +- **Shape-SLAT flow + decoder (fine geometry)** — `trellis2_slat_flow_sample()` + runs the sparse 1.3B DiT over the active voxels of the 32³ scaffold (same + block structure as the SS-flow DiT, 3D RoPE over each voxel's coords), + denormalized with `shape_slat_normalization` baked into the GGUF. + `trellis2_shape_dec_decode()` runs `FlexiDualGridVaeDecoder` — a sparse + ConvNeXt U-Net whose 3×3×3 submanifold convolutions are expressed as 27 + gather+GEMM steps, with each level's learned subdivision growing the active + set (32³ → 512³, 16×). `examples/flexible_dual_grid.h` turns the 7-channel + per-voxel output (dual-vertex offset, per-axis intersection flags, quad split + weight) into the triangle mesh. This is the real TRELLIS.2 geometry, driven + end-to-end by the demo server. + ## Validate the forward pass ```sh diff --git a/convert_dino_to_gguf.py b/convert_dino_to_gguf.py new file mode 100644 index 0000000..d5a102f --- /dev/null +++ b/convert_dino_to_gguf.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +""" +Convert the DINOv3 ViT-L/16 image-conditioning encoder (HF format, +facebook/dinov3-vitl16-pretrain-lvd1689m or a mirror) to a GGUF file for +trellis2.cpp. + +TRELLIS.2 uses this model (transformers DINOv3ViTModel) to turn the +preprocessed 512x512 input image into the [1, 1029, 1024] conditioning tensor +consumed by the flow models: 1 CLS + 4 register + 32x32 patch tokens, last +transformer layer output passed through an affine-free layer norm (the model's +own final `norm` layer is NOT applied). + +Architecture (config.json): hidden 1024, 24 layers, 16 heads, MLP 4096 (exact +GELU, not gated), patch 16, axial 2D RoPE over patch-center coords with +theta=100, LayerScale on both residual branches, q/v/o biases but no k bias. + +Self-contained like the other converters (safetensors + numpy). Tensors keep +their HF state-dict names; hyperparameters travel as `trellis2.dino.*` KV. + +Usage: + python convert_dino_to_gguf.py --model models/dinov3-vitl16/model.safetensors \ + --output ggufs/dino_f32.gguf --ftype 0 + +ftype: 0 = f32 (validation), 1 = f16 (matrices f16; norms/biases/1-D f32). +""" + +import argparse +import json +import os +import struct +import sys + +import numpy as np + +# ── GGUF / GGML constants (must match the bundled ggml) ────────────────────── +GGUF_MAGIC = b"GGUF" +GGUF_VERSION = 3 +GGUF_ALIGNMENT = 32 + +GGML_TYPE_F32 = 0 +GGML_TYPE_F16 = 1 + +GGUF_VT_UINT32 = 4 +GGUF_VT_FLOAT32 = 6 +GGUF_VT_BOOL = 7 +GGUF_VT_STRING = 8 + +ARCH = "trellis2-dino" +KV_PREFIX = "trellis2.dino." + + +def _gguf_str(s: str) -> bytes: + b = s.encode("utf-8") + return struct.pack(" bytes: + return _gguf_str(key) + struct.pack(" int: + return (n + a - 1) // a * a + + +def choose_type(name, shape, ftype: int) -> int: + if ftype == 0: + return GGML_TYPE_F32 + # matrices f16; 1-D (biases, norms, layerscale) and token embeddings f32 + if len(shape) < 2: + return GGML_TYPE_F32 + if "cls_token" in name or "register_tokens" in name or "mask_token" in name: + return GGML_TYPE_F32 + return GGML_TYPE_F16 + + +def to_bytes(arr_f32: np.ndarray, ggml_type: int) -> bytes: + if ggml_type == GGML_TYPE_F32: + return arr_f32.astype("