From 25558730f592796d10874adc1d5af2d850540214 Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 11:51:27 +0000 Subject: [PATCH 1/2] Add tiled-storage user docs and performance benchmark Add the README 'Tiled storage' and 'Building and testing' sections covering the TiledTree usage snippet, the fresh-only/rejection contract, LONG_TESTS, and links to the guide and design doc. Add doc/tiles-guide.md (a how-to for flushing, compaction, rollback, proofs, and the lower-level building blocks) and doc/tiles-illustrated.md (a worked, diagram-led walkthrough of the tiled tree lifecycle). No new storage semantics: these are documentation-only additions over the tiled-storage API completed in earlier branches. Add time_tiles, benchmarking append, flush, in-memory and tile-served inclusion/consistency proofs, and compaction against a plain in-memory Tree baseline; wire it into the existing LONG_TESTS group alongside tiles_level2. Enable the documentation workflow on pull requests (building, but not publishing, the site) so doc changes get functional CI coverage. Finalize the design doc's testing-strategy and build/backwards- compatibility-impact sections, completing the document that has grown incrementally across the previous branches. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build-docs.yml | 5 +- README.md | 63 +++ doc/design/tlog-tiles.md | 36 ++ doc/tiles-guide.md | 325 +++++++++++++ doc/tiles-illustrated.md | 764 +++++++++++++++++++++++++++++++ test/CMakeLists.txt | 2 + test/time_tiles.cpp | 192 ++++++++ 7 files changed, 1386 insertions(+), 1 deletion(-) create mode 100644 doc/tiles-guide.md create mode 100644 doc/tiles-illustrated.md create mode 100644 test/time_tiles.cpp diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index e3d617c..d6e7c4f 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -4,10 +4,11 @@ on: push: branches: - main + pull_request: workflow_dispatch: concurrency: - group: pages + group: pages-${{ github.ref }} cancel-in-progress: false permissions: read-all @@ -24,6 +25,7 @@ jobs: - name: Setup Pages id: pages uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 + if: github.event_name != 'pull_request' - name: Install dependencies run: | @@ -42,6 +44,7 @@ jobs: uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: doc/build + if: github.event_name != 'pull_request' deploy: name: Deploy diff --git a/README.md b/README.md index 904c442..efadb1e 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,69 @@ unusual features like flushing, retracting, and tree segment serialisation. assert(path->verify(root)); +## Tiled storage (tlog-tiles) + +The companion header `merklecpp_tiles.h` adds optional, header-only support for +persisting a tree as [tlog-tiles](https://c2sp.org/tlog-tiles) tile files +*progressively* (optionally dropping already-tiled leaves from memory) and for +retrieving inclusion and consistency proofs from those tiles, from the in-memory +tree, or from a combination of the two. The hashing is unchanged: tiles and tile-derived proofs are templated on +the tree's existing hash function, so a tile-derived inclusion proof is +byte-identical to one from `merkle::Tree::path()` and verifies with the same +`merkle::Path::verify()`. + + #include + + merkle::tiles::TiledTree::Config cfg; + cfg.prefix = "/var/log/mylog"; // tile files live here + cfg.retention_margin = 1024; // keep the most recent leaves in memory + cfg.compact_on_flush = true; // opt in to dropping already-tiled leaves + + merkle::tiles::TiledTree log(cfg); + for (const auto& leaf_hash : batch) + log.append(leaf_hash); + + // Write newly-complete tiles. With compaction enabled + // this also drops from memory the leaves already covered by a full tile; + // otherwise the tree keeps every leaf and you can call log.compact() later. + log.flush(); + + // Proofs are served from tiles + the resident tree, even for flushed leaves. + auto inclusion = log.inclusion_proof(/*index=*/0, log.size()); + assert(inclusion->verify(log.root())); + + auto consistency = log.consistency_proof(/*m=*/100, /*n=*/log.size()); + +`TiledTree` creates a new tiled tree: the configured directory may exist, but +its `tile` subdirectory must be absent or empty. It deliberately rejects +existing tile data because tile files alone do not identify or restore the +tree that produced them. Applications with externally persisted tree state can +use the lower-level `TileStore` and `TileWriter` APIs to resume a store. + +See the [tiled storage guide](doc/tiles-guide.md) for a how-to covering +flushing, compaction, rollback, proofs, and the lower-level building blocks, +and [doc/design/tlog-tiles.md](doc/design/tlog-tiles.md) for the full design, +file/directory layout, and the proof algorithms. + + +## Building and testing + +Enable the test suite with CMake's `TESTS` option: + + cmake -S . -B build -DTESTS=ON + cmake --build build + ctest --test-dir build + +Some tile coverage is intentionally long-running. `LONG_TESTS` is off by +default for local builds; turn it on when you want the full tile stress suite, +including level-2 tile coverage and tile proof timing: + + cmake -S . -B build -DTESTS=ON -DLONG_TESTS=ON + +The repository CI enables `LONG_TESTS` so pull requests continue to exercise the +full tiled-storage matrix. + + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index bafc628..3caa26d 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -710,6 +710,30 @@ one-method core addition; README/docs updates. --- +## 11. Testing strategy + +The existing library is the **oracle** — this is how we guarantee "proofs remain +compatible": + +- **Equality, not just verification.** Assert `*inclusion_proof(i,size) == + *tree.path(i)` using the existing `PathT::operator==`, and likewise against + `past_path`. This is stronger than `verify()` and pins byte-compatibility. +- **Root agreement.** `ProofEngineT::root(size)` == `tree.root()` / + `tree.past_root(size-1)` for many random sizes. +- **Spec vectors.** Exact full-tile counts and on-disk file sets for sizes 256 + and 70 000; index-encoding strings (incl. `x001/x234/067`). +- **Flush/compaction.** Cross-check proofs for flushed indices against a twin + tree that was never flushed (same inserts) — roots must match and proofs must + verify. +- **Consistency proofs.** Standard RFC 6962 `verify_consistency` plus the + reconciliation-of-`past_root`s check; randomized `m < n`. +- **Templating.** Run a subset under `Tree`, `Tree384`, `Tree512` to confirm + hash-size independence. +- **Style.** Mirror existing standalone-`main()` tests (`test/util.h`, + `make_hashes`, `get_timeout`) and register via `add_merklecpp_test`. + +--- + ## 12. Risks, edge cases, open questions - **External interop (by design, no).** With the default combiner the tiles are @@ -748,3 +772,15 @@ one-method core addition; README/docs updates. --- +## 13. Build & backwards-compatibility impact + +- **Additive only.** New header + tests; existing `merklecpp.h` behaviour and + API are unchanged. Default builds (no tiles) are unaffected. +- **No new mandatory dependencies.** Roll-ups/proofs use the existing + `HASH_FUNCTION` (two-hash combiner), so `sha256_compress` suffices; OpenSSL + stays optional. +- **Optional core change** is a single read-only accessor that performs no + hashing and does not alter any existing code path. +- **CMake.** Add a `tiles` test group guarded like the others; no changes to the + `merklecpp` INTERFACE target are required for consumers that don't use tiles. +``` diff --git a/doc/tiles-guide.md b/doc/tiles-guide.md new file mode 100644 index 0000000..396dbaa --- /dev/null +++ b/doc/tiles-guide.md @@ -0,0 +1,325 @@ +# Tiled storage and proofs — a guide to `merklecpp_tiles.h` + +`merklecpp_tiles.h` is an optional, header-only companion to `merklecpp.h`. It +lets you persist a Merkle tree as a set of immutable **tile files** on disk and +serve **inclusion** and **consistency** proofs from those tiles, from the +in-memory tree, or from a combination of the two — so proofs stay available even +after old entries are dropped from memory. + +It builds on the [tlog-tiles](https://c2sp.org/tlog-tiles) file/directory layout +but is **not** trying to be wire-compatible with external tlog-tiles clients. See +[`design/tlog-tiles.md`](design/tlog-tiles.md) for the design and internals; this +page is a practical how-to. + +## Contents + +- [Requirements and a note on hashing](#requirements-and-a-note-on-hashing) +- [Thread safety](#thread-safety) +- [Quick start: `TiledTree`](#quick-start-tiledtree) +- [Flushing and compaction](#flushing-and-compaction) +- [Rollback](#rollback) +- [Proofs](#proofs) +- [Lower-level building blocks](#lower-level-building-blocks) +- [Entry bundles (optional)](#entry-bundles-optional) +- [On-disk layout](#on-disk-layout) + +## Requirements and a note on hashing + +- C++17 (the header uses `` and small platform-specific file-sync + calls for durable tile writes). +- Include the companion header; it pulls in `merklecpp.h` for you: + + ```cpp + #include + ``` + +- Everything lives in `namespace merkle::tiles` and is templated on the same + `` as your tree. The default aliases + (`merkle::tiles::TiledTree`, `TileStore`, `TileWriter`, `ProofEngine`, …) use + the **same** SHA-256 as `merkle::Tree`, so a tile-derived inclusion proof is + byte-identical to one from `merkle::Tree::path()` and verifies with the usual + `merkle::Path::verify()`. +- You insert **leaf hashes**, not raw entries — exactly like `merkle::Tree`. + Deriving a leaf hash from an entry (e.g. `leaf = H(entry)`) is your + application's job. The tile hash values are whatever your `HASH_FUNCTION` + produces; they are not RFC 6962 unless you instantiate your tree with an + RFC 6962 hash function (not required, and not the goal here). + +## Thread safety + +The tiled-storage API provides no internal synchronization. Treat every +`TileStore`, `TileWriter`, `TileHashSource`, `ProofEngine`, `TiledTree`, and +`EntryBundleWriter` instance as single-threaded. + +If an object or store prefix is shared between threads, the caller must +serialize every operation. This includes methods declared `const`: proof +generation updates the `TileHashSource` LRU cache. The library deliberately +does not add locks or otherwise coordinate concurrent readers and writers. + +## Quick start: `TiledTree` + +`TiledTree` is the high-level wrapper: append leaf hashes, flush them to disk +(which writes tiles), and ask for proofs. + +```cpp +#include + +merkle::tiles::TiledTree::Config cfg; +cfg.prefix = "/var/log/mylog"; // directory for tile files + +merkle::tiles::TiledTree log(cfg); + +// Append leaf hashes (compute these from your entries however you like). +for (const merkle::Hash& leaf : batch) + log.append(leaf); + +// Persist newly-complete tiles to disk. +log.flush(); + +merkle::Hash root = log.root(); // current Merkle root +uint64_t n = log.size(); // number of leaves + +// Inclusion proof for leaf 0 in the tree of `n` leaves. +auto inclusion = log.inclusion_proof(/*index=*/0, /*size=*/n); +assert(inclusion->verify(root)); + +// Consistency proof that size 100 is a prefix of size n. +auto consistency = log.consistency_proof(/*m=*/100, /*n=*/n); +``` + +`TiledTree` can be move-constructed, but it cannot be copied or assigned. Move +construction keeps its writer bound to the destination tree's tile store. + +`TiledTree` always creates a new tiled tree. The configured directory may +already exist, but its `tile` subdirectory must be absent or empty. Construction +throws rather than adopting existing tiles because those files do not identify +the tree that produced them or contain enough state to restore its size and +root. If your application persists and validates that state separately, use +the lower-level `TileStore` and `TileWriter` APIs; `TileWriter` intentionally +resumes existing full tiles and therefore trusts the caller to supply the same +tree and hash function. A fresh writer scans the requested range in order, +stopping at the first missing or malformed file, so an interior hole is +rewritten rather than hidden by later files. + +`flush()` is incremental: each call writes only the full tiles that became +complete since the previous call. Full tiles are immutable: written once after +all 256 entries are final and never rewritten. The remaining frontier stays in +memory until it crosses the next full-tile boundary. + +Tile files are written through unique temporary files, synced, then published +with an atomic replace. On POSIX systems, each newly created directory is made +durable by syncing its parent, and the destination directory is synced after +the rename. Before reusing a visible file, a writer also re-confirms its +directory chain and destination directory. This makes a retry repeat a failed +directory sync even when the rename or directory creation is already visible. +A wrong-size file at a tile path is not a published tile and is rewritten. + +## Flushing and compaction + +By default `flush()` only *writes* tiles; it keeps every leaf resident in +memory. Dropping already-tiled leaves from memory ("compaction") is **opt-in**, +because once you drop them you can only prove them from the tiles. + +```cpp +merkle::tiles::TiledTree::Config cfg; +cfg.prefix = "/var/log/mylog"; +cfg.compact_on_flush = true; // drop tiled leaves after each flush +cfg.retention_margin = 4096; // ...but keep the most recent 4096 resident + +merkle::tiles::TiledTree log(cfg); +``` + +- `compact_on_flush` (default `false`): when set, `flush()` calls + `compact()` for you. +- `compact()` can also be called explicitly at any time. It drops from memory + only leaves already covered by a **durably written full tile**, keeping at + least `retention_margin` recent leaves resident. It also retains the final + tiled leaf so rollback to exactly `immutable_size()` remains representable. + It returns the new minimum (smallest still-resident) leaf index. +- Proofs for dropped leaves are still produced — they are served from the tiles + and transparently combined with the resident frontier. + +`flushed_size()` is the boundary completed successfully at every required tile +level, and it is the only boundary used for proof reads and compaction. +`immutable_size()` is the rollback boundary. A flush seals that boundary before +it starts writing, because an error can occur after a full tile becomes visible. +If a flush throws, `immutable_size()` may advance while `flushed_size()` does +not. Keep the same tree contents, correct the I/O failure, and retry `flush()`; +finalized tiles are reused rather than rewritten. + +```cpp +log.compact(); // free memory now +uint64_t resident_from = log.tree_ref().min_index(); +``` + +## Rollback + +Tiles are immutable, so you may only roll back entries beyond the boundary +returned by `immutable_size()`. `retract_to` enforces this: + +```cpp +log.retract_to(index); // keep leaves [0, index], drop the rest +``` + +- Allowed when the resulting size is `>= immutable_size()`. +- Throws otherwise, because a flush may already have published an immutable + full tile for that range. +- The exact `immutable_size()` boundary remains available after compaction; + compaction retains the final tiled leaf needed by the in-memory tree. +- After a successful flush, `immutable_size() == flushed_size()`. After an + interrupted flush, `immutable_size()` may be larger until the same tree state + is flushed successfully. +- `retract_to` mirrors `merkle::Tree::retract_to`: `index` is the new *last* + leaf, so the resulting size is `index + 1`. + +> **Warning:** Treat `tree_ref()` as an inspection escape hatch unless you also +> maintain every tiled-tree invariant yourself. Direct retraction bypasses the +> guard, can make `flushed_size()` and `immutable_size()` exceed `size()`, and +> can make `flushed_size()` regress. Use `TiledTree::retract_to` instead. + +`store_ref()` is similarly unsafe for mutation. A later flush trusts any +correctly sized tile written through it without checking that the hashes match +the in-memory tree. A mismatched tile can silently invalidate proofs after +compaction. + +## Proofs + +Both proof types come from `TiledTree` (or, at a lower level, from +`ProofEngine`). They are produced with your tree's hash function, so they match +what `merkle::Tree` would produce. Requests outside the current tree (e.g. a +size greater than `size()`, or an out-of-range index) throw `std::runtime_error` +rather than returning an incorrect proof. + +### Inclusion proofs + +```cpp +// Prove leaf `index` in a tree of `size` leaves. +std::shared_ptr p = log.inclusion_proof(index, size); +bool ok = p->verify(root_at_size); +``` + +`size` is the tree size you are proving against: + +- `size == log.size()` ⇒ equivalent to `merkle::Tree::path(index)`; verify + against `log.root()`. +- a past `size` ⇒ equivalent to `merkle::Tree::past_path(index, size - 1)`; + verify against the root at that size (e.g. a past root you are + auditing). + +`size` may even exceed `flushed_size()`: the recent, not-yet-tiled frontier is +taken from the resident tree while the older part comes from tiles. + +### Consistency proofs + +```cpp +std::vector proof = log.consistency_proof(m, n); // m <= n +bool ok = merkle::tiles::ProofEngine::verify_consistency( + m, n, old_root /* root at size m */, new_root /* root at size n */, proof); +``` + +`verify_consistency` is a static helper, so you can verify on a client that only +has the two roots and the proof. + +The arguments are tree **sizes** (leaf counts). If you have leaf **indices** +instead, use the variant that maps index `i` to the tree of size `i + 1` (the +"last leaf" convention, matching `past_path`/`retract_to`): + +```cpp +// Equivalent to consistency_proof(i + 1, j + 1). +auto proof = log.consistency_proof_from_indices(i, j); // i <= j +``` + +Both `TiledTree` and the lower-level `ProofEngine` provide +`consistency_proof_from_indices`. + +## Lower-level building blocks + +If you manage your own tree/storage you can use the pieces directly instead of +`TiledTree`. + +### Writing tiles from your own tree + +```cpp +merkle::Tree tree; +for (auto& leaf : batch) tree.insert(leaf); + +merkle::tiles::TileStore store("/var/log/mylog"); +merkle::tiles::TileWriter writer(store); + +// Write all newly-complete full tiles; keep the remaining frontier in memory. +auto stats = writer.write_up_to( + tree.num_leaves(), + [&](uint64_t i) -> const merkle::Hash& { return tree.leaf(i); }); +// stats.full_written +``` + +`TileWriter` keeps an in-memory next-file cursor. A new writer reconstructs it +by checking the contiguous prefix only up to the number of full files relevant +to the requested tree size. Existing files are re-confirmed as durably +published before reuse; malformed files and holes are rewritten. + +### Reading tiles and computing proofs + +A `HashSource` resolves the root of a complete subtree; pick where it reads from: + +- `TileHashSource(store, available_size)` — from full tile files; resolves the + full-tile-covered prefix only (the frontier needs a memory source). +- `MemoryHashSource(tree)` — from a resident `merkle::Tree`. +- `CombinedHashSource(primary, secondary)` — try `primary` first, then + `secondary` (e.g. memory then tiles). + +```cpp +merkle::tiles::TileHashSource src(store, /*available_size=*/tree.num_leaves()); +merkle::tiles::ProofEngine engine(src); + +merkle::Hash root = engine.root(size); +auto inclusion = engine.inclusion_proof(index, size); +auto consistency = engine.consistency_proof(m, n); +``` + +A tile-only source can resolve proofs whose subtrees all lie within the +full-tile-covered prefix (`available_size` is rounded down to a whole number of +tiles). For the live frontier, combine it with a `MemoryHashSource` — which is +exactly what `TiledTree` does for you. + +`TiledTree` simply wires a `CombinedHashSource(MemoryHashSource, TileHashSource)` +into a `ProofEngine` for you. It creates these sources for each proof call, so +its tile cache is per-call. A long-lived lower-level `TileHashSource` retains +its cache across calls. + +## Entry bundles (optional) + +If you also want to store the raw log entries (tlog-tiles "entry bundles"), use +`EntryBundleWriter`. Bundles are level-0 only and application-owned — merklecpp +stores leaf hashes; you supply the raw bytes and decide how an entry maps to its +leaf hash. Only full bundles (256 entries) are written; the incomplete tail +stays with your application until it completes a bundle. + +```cpp +merkle::tiles::EntryBundleWriter bundles(store); +bundles.write_up_to(num_entries, + [&](uint64_t i) -> std::vector { return raw_entry_bytes(i); }); + +// Read a full bundle back (256 entries). +std::vector> e = store.read_entry_bundle(/*index=*/0); +``` + +Entries are encoded as big-endian `uint16` length-prefixed byte strings. + +## On-disk layout + +Under the configured `prefix`: + +``` +/ + tile/0/000, tile/0/001 … # level-0 tiles (leaf hashes), 256 hashes each + tile/1/… # higher levels (roll-ups of full tiles below) + tile/entries/… # optional raw entry bundles +``` + +Tile indices use the tlog-tiles path encoding: zero-padded 3-digit groups with +all but the last prefixed by `x` (e.g. index `1234067` -> `x001/x234/067`). Every +tile is full (256-wide), final, and immutable. Entries beyond the last full-tile +boundary remain in memory. See +[`design/tlog-tiles.md`](design/tlog-tiles.md) for the full specification of the +geometry and proof algorithms. diff --git a/doc/tiles-illustrated.md b/doc/tiles-illustrated.md new file mode 100644 index 0000000..30bf65f --- /dev/null +++ b/doc/tiles-illustrated.md @@ -0,0 +1,764 @@ +# Tiled Merkle trees: an illustrated walkthrough + +This page builds a visual model of how an append-only Merkle tree moves from +memory into immutable tile files, and how proofs continue to work across both +places. + +> [!IMPORTANT] +> **Every example on this page uses an imaginary tile width of 32 entries +> purely to keep the diagrams readable.** +> +> The merklecpp implementation remains fixed at `TILE_WIDTH = 256` and +> `TILE_HEIGHT = 8`. A width of 32 is not a configuration option, and this page +> does not propose changing the code, file format, defaults, or examples +> elsewhere. Unless a section explicitly says "illustrative", use 256. + +## The scaled-down model + +A production tile contains 256 entries and spans 8 binary tree levels because +`256 = 2^8`. This page scales that geometry down to 32 entries and 5 levels +because `32 = 2^5`. + +| Property | This page only | Production merklecpp | +|---|---:|---:| +| Tile width | 32 entries | 256 entries | +| Tree levels spanned by one tile | 5 | 8 | +| Leaves covered by one full level-0 tile | 32 | 256 | +| Leaves covered by one level-1 entry | 32 | 256 | +| Leaves covered by one full level-1 tile | 1,024 | 65,536 | + +The scaling changes only the numbers in the drawings. The rules are the same: + +1. Only full tiles are written. +2. A level-0 tile contains leaf hashes. +3. A higher-level tile contains roots of complete tiles from the level below. +4. The incomplete right-hand frontier remains in memory. +5. Published tiles are immutable. +6. Proofs can resolve subtree roots from memory, tiles, or both. + +### Notation + +- `h7` is the hash of leaf 7. +- `R[a, b)` is the Merkle root of the half-open leaf range `[a, b)`. +- `tile/L/NNN` is tile index `NNN` at tile level `L`. +- "Resident" means the in-memory tree can still expand that range to answer + proof requests. +- "Compacted" means the in-memory tree retains enough summary hashes to keep + its root correct, but no longer retains all detail below that range. + +### Colors used below + +```mermaid +flowchart TB + T["Tile-backed hash or range"]:::tile + M["Resident in-memory hash or range"]:::memory + B["Available from both tiles and memory"]:::both + S["Compacted in-memory summary"]:::summary + X["Leaf being proved"]:::target + PT["Hash emitted in a proof
blue outline: from tiles"]:::proofTile + PM["Hash emitted in a proof
green outline: from memory"]:::proofMemory + + T ~~~ M + M ~~~ B + B ~~~ S + S ~~~ X + X ~~~ PT + PT ~~~ PM + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef summary fill:#e5e7eb,stroke:#6b7280,color:#111827 + classDef target fill:#fed7aa,stroke:#ea580c,stroke-width:3px,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 + classDef proofMemory fill:#fef3c7,stroke:#16a34a,stroke-width:3px,color:#111827 +``` + +## What `flush()` and `compact()` each do + +Appending, flushing, and compacting are separate operations: + +```mermaid +flowchart TB + A["append(h)
Add a leaf hash to the in-memory tree"]:::memory + B["A complete 32-entry range now exists
but no file is written automatically"]:::memory + C["flush()
Write every newly complete full tile"]:::tile + D["The same range exists on disk and in memory
(the default after flush)"]:::both + E["compact()
Optionally discard old resident detail"]:::summary + F["Old detail is served from tiles;
the incomplete frontier stays in memory"]:::both + + A --> B + B --> C + C --> D + D --> E + E --> F + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef summary fill:#e5e7eb,stroke:#6b7280,color:#111827 +``` + +`flush()` does not compact by default. Setting `compact_on_flush = true` makes +the final two steps happen in one call, but the durability rule is unchanged: +compaction happens only after all required tile writes succeed. + +If a tile write fails, `immutable_size()` may advance past `flushed_size()` +because a published tile cannot be rolled back. Keep the same tree contents and +retry the flush. See +[Flushing and compaction](tiles-guide.md#flushing-and-compaction) for the full +interrupted-write contract. + +## What is inside a tile file? + +In the illustrative model, `tile/0/000` is the concatenation of 32 leaf hashes: + +```mermaid +flowchart TB + F["tile/0/000
32 serialized hashes"]:::tile + A["entries 0..7
h0 ... h7"]:::tile + B["entries 8..15
h8 ... h15"]:::tile + C["entries 16..23
h16 ... h23"]:::tile + D["entries 24..31
h24 ... h31"]:::tile + R["R[0, 32)
reconstructed by hashing the entries"]:::computed + N["Internal binary-tree nodes are reconstructed;
they are not separately stored in the file"]:::note + + F -->|first bytes| A + A -->|followed by| B + B -->|followed by| C + C -->|followed by| D + D --> R + R --> N + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef note fill:#f9fafb,stroke:#9ca3af,color:#374151 +``` + +At level 1, each entry is already the root of 32 leaves: + +```mermaid +flowchart TB + L1["tile/1/000
32 serialized subtree roots"]:::tile + L0A["entry 0
root(tile/0/000) = R[0, 32)"]:::tile + L0B["entry 1
root(tile/0/001) = R[32, 64)"]:::tile + L0C["entries 2..30
..."]:::tile + L0Z["entry 31
root(tile/0/031) = R[992, 1024)"]:::tile + ROOT["R[0, 1024)
reconstructed from tile/1/000"]:::computed + + L1 --> L0A + L0A --> L0B + L0B --> L0C + L0C --> L0Z + L0Z --> ROOT + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +The production version of the second diagram needs 256 level-0 tile roots, so +its first full level-1 tile appears at 65,536 leaves rather than 1,024. + +## On-disk file layout + +After an illustrative 1,030-leaf tree is flushed, the full-tile boundary is +1,024: + +```text +prefix/ + tile/ + 0/ + 000 # h0 ... h31 + 001 # h32 ... h63 + ... + 031 # h992 ... h1023 + 1/ + 000 # R[0,32), R[32,64), ... R[992,1024) +``` + +Leaves `[1024, 1030)` do not appear in a tile file because they do not complete +another 32-entry tile. They remain in memory. + +```mermaid +flowchart TB + N["n = 1,030 leaves"]:::computed + C["covered = floor(1,030 / 32) * 32 = 1,024"]:::computed + L0["32 full level-0 files
tile/0/000 through tile/0/031"]:::tile + L1["1 full level-1 file
tile/1/000"]:::tile + M["6-leaf frontier
[1024, 1030) in memory"]:::memory + + N --> C + C --> L0 + L0 --> L1 + L1 --> M + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +The optional `tile/entries/` bundles are omitted here. They store raw +application entries, not Merkle tree nodes, and do not change proof generation. + +## Tree growth, one snapshot at a time + +The next snapshots assume `retention_margin = 0`. Where compaction is shown, +merklecpp still retains the final tiled leaf as a boundary leaf. This is why +the "both" range below is one leaf wide. + +### Snapshot A: 20 leaves + +No full 32-entry tile exists: + +```mermaid +flowchart TB + N["n = 20"]:::computed + C["full-tile boundary = 0"]:::computed + M["Memory only
[0, 20)"]:::memory + D["Disk
no tile files"]:::empty + + N --> C + C --> M + M --> D + + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef empty fill:#f9fafb,stroke:#9ca3af,color:#374151 +``` + +Calling `flush()` at this point writes nothing. Every root and proof is served +from the in-memory tree. + +### Snapshot B: 40 leaves, before the first flush + +The first 32 leaves form a complete tile, but tile creation is explicit: + +```mermaid +flowchart TB + N["n = 40"]:::computed + M0["[0, 32)
complete and eligible, still memory only"]:::memory + M1["[32, 40)
incomplete frontier, memory only"]:::memory + D["Disk
still empty until flush()"]:::empty + + N --> M0 + M0 --> M1 + M1 --> D + + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef empty fill:#f9fafb,stroke:#9ca3af,color:#374151 +``` + +### Snapshot B: 40 leaves, after `flush()` + +The default `flush()` writes the full prefix but does not remove it from +memory: + +```mermaid +flowchart TB + F["flush() succeeds
flushed_size() = 32"]:::computed + B["[0, 32)
tile/0/000 + resident memory"]:::both + M["[32, 40)
resident memory only"]:::memory + + F --> B + B --> M + + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +At this point a proof may be answered entirely from memory even though a tile +copy exists. + +### Snapshot B: 40 leaves, after compaction + +With zero retention, compaction drops old leaf detail while preserving leaf 31 +as the rollback boundary: + +```mermaid +flowchart TB + R["R[0, 40)
current in-memory root"]:::computed + P["R[0, 32)
prefix represented by compacted summaries"]:::summary + C["[0, 31)
not leaf-addressable in memory"]:::summary + B["h31
retained boundary leaf"]:::both + M["R[32, 40)
fully resident frontier"]:::memory + T["tile/0/000
proof detail for [0, 32)"]:::tile + + R --> P + P --> C + P --> B + R --> M + P -.->|subtree and leaf detail| T + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef summary fill:#e5e7eb,stroke:#6b7280,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +There are now three logical ownership ranges: + +| Leaf range | Proof detail available from | +|---|---| +| `[0, 31)` | tiles only | +| `[31, 32)` | tiles and memory | +| `[32, 40)` | memory only | + +The compacted in-memory summaries still contribute to `root()`. "Tiles only" +means that a request for a leaf or complete subtree in that range must use the +tile source; it does not mean the in-memory root forgot the prefix hash. + +### Snapshot C: grow from 40 to 72 leaves + +Assume the tree was flushed and compacted at size 40, then 32 more leaves were +appended. + +Before the second flush: + +```mermaid +flowchart TB + N["n = 72
flushed_size() is still 32"]:::computed + T["[0, 31)
tiles only"]:::tile + B["[31, 32)
boundary leaf in both"]:::both + E["[32, 64)
complete and eligible, but still memory only"]:::memory + F["[64, 72)
incomplete memory frontier"]:::memory + + N --> T + T --> B + B --> E + E --> F + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +After the second flush and compaction: + +```mermaid +flowchart TB + F["flush() writes tile/0/001
flushed_size() = 64"]:::computed + T0["tile/0/000 covers [0, 32)"]:::tile + T1["tile/0/001 covers [32, 64)"]:::tile + C["[0, 63)
tiles only after compaction"]:::tile + B["[63, 64)
new boundary leaf in both"]:::both + M["[64, 72)
memory-only frontier"]:::memory + + F --> T0 + T0 --> T1 + T1 --> C + C --> B + B --> M + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +### Snapshot D: 1,030 leaves + +This is the first snapshot with a full illustrative level-1 tile: + +```mermaid +flowchart TB + N["n = 1,030"]:::computed + T0["Level 0
32 files cover [0, 1024)"]:::tile + T1["Level 1
tile/1/000 contains their 32 roots"]:::tile + C["After compaction
[0, 1023) uses tiles for proof detail"]:::tile + B["h1023
boundary leaf in both"]:::both + M["[1024, 1030)
memory-only frontier"]:::memory + + N --> T0 + T0 --> T1 + T1 --> C + C --> B + B --> M + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +### Snapshot summary + +This table assumes each snapshot has just completed a successful flush and +compaction with zero retention: + +| Tree size | `flushed_size()` | Files written | Tiles only | Tiles + memory | Memory only | +|---:|---:|---|---|---|---| +| 20 | 0 | none | none | none | `[0, 20)` | +| 40 | 32 | `tile/0/000` | `[0, 31)` | `[31, 32)` | `[32, 40)` | +| 72 | 64 | `tile/0/000..001` | `[0, 63)` | `[63, 64)` | `[64, 72)` | +| 1,030 | 1,024 | 32 level-0 tiles + `tile/1/000` | `[0, 1023)` | `[1023, 1024)` | `[1024, 1030)` | + +Again, multiply the tile geometry back to 256 for production. In particular, +the production level-1 example starts at 65,536 leaves, not 1,024. + +## How a proof finds a subtree root + +`TiledTree` gives `ProofEngine` a combined source. It tries the resident tree +first because that avoids I/O, then falls back to tiles: + +```mermaid +flowchart TB + Q["ProofEngine requests R[a, b)"]:::computed + L{"Is the range one leaf?"}:::decision + A{"Otherwise, is its width a power of two
and is the range aligned to that width?"}:::decision + P{"Is this complete subtree
fully resident in memory?"}:::decision + M["Return the in-memory hash"]:::memory + T{"Can the tile source resolve it
inside flushed_size()?"}:::decision + D["Read the appropriate tile entries
and roll them up"]:::tile + S["Split the range into smaller subtrees
and resolve each one"]:::computed + E["Fail if no source can resolve a required leaf"]:::error + + Q --> L + L -->|yes| P + L -->|no| A + A -->|yes| P + A -->|no| S + P -->|yes| M + P -->|no| T + T -->|yes| D + T -->|no, and range has multiple leaves| S + T -->|no, and range is one leaf| E + S -->|smaller range| Q + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef decision fill:#fff7ed,stroke:#c2410c,color:#111827 + classDef error fill:#fee2e2,stroke:#dc2626,color:#111827 +``` + +For example, after compacting the 40-leaf tree: + +- `R[0, 32)` is not fully resident, so memory declines it and tiles return it. +- `R[32, 36)` is resident, so memory returns it without touching disk. +- `R[24, 40)` crosses the boundary and is not one complete aligned subtree. + The proof engine splits it into resolvable pieces. + +## Inclusion proof 1: entirely from one tile + +Consider a proof against tree size 32 after `tile/0/000` has been written and +the old leaves have been compacted. This may be the current size or a historical +prefix of a larger tree. We want to prove leaf 5. + +Every required hash is reconstructed from `tile/0/000`: + +```mermaid +flowchart TB + R032["R[0, 32)"]:::tile + R016["R[0, 16)"]:::tile + P1632["R[16, 32)
proof"]:::proofTile + R08["R[0, 8)"]:::tile + P816["R[8, 16)
proof"]:::proofTile + P04["R[0, 4)
proof"]:::proofTile + R48["R[4, 8)"]:::tile + R46["R[4, 6)"]:::tile + P68["R[6, 8)
proof"]:::proofTile + P4["h4
proof"]:::proofTile + X5["h5
target leaf"]:::target + + R032 --> R016 + R032 --> P1632 + R016 --> R08 + R016 --> P816 + R08 --> P04 + R08 --> R48 + R48 --> R46 + R48 --> P68 + R46 --> P4 + R46 --> X5 + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef target fill:#fed7aa,stroke:#ea580c,stroke-width:3px,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 +``` + +The proof payload is ordered from the leaf toward the root: + +| Order | Proof hash | Position relative to the running hash | Source | +|---:|---|---|---| +| 1 | `h4` | left | `tile/0/000` | +| 2 | `R[6, 8)` | right | `tile/0/000` | +| 3 | `R[0, 4)` | left | `tile/0/000` | +| 4 | `R[8, 16)` | right | `tile/0/000` | +| 5 | `R[16, 32)` | right | `tile/0/000` | + +The internal roots in this table are computed on demand from the tile's leaf +hashes. They are not additional files. + +Verification starts with `h5`, combines the five proof hashes in order, and +arrives at `R[0, 32)`. + +## Inclusion proof 2: tiles and memory together + +Return to the compacted 40-leaf tree and prove leaf 36 against the current root +`R[0, 40)`. + +The target and its nearby siblings are in the resident frontier. The old +32-leaf prefix is supplied as one tile-backed subtree root: + +```mermaid +flowchart TB + R040["R[0, 40)"]:::computed + P032["R[0, 32)
proof from tile"]:::proofTile + R3240["R[32, 40)
resident frontier"]:::memory + P3236["R[32, 36)
proof from memory"]:::proofMemory + R3640["R[36, 40)"]:::memory + R3638["R[36, 38)"]:::memory + P3840["R[38, 40)
proof from memory"]:::proofMemory + X36["h36
target leaf"]:::target + P37["h37
proof from memory"]:::proofMemory + + R040 --> P032 + R040 --> R3240 + R3240 --> P3236 + R3240 --> R3640 + R3640 --> R3638 + R3640 --> P3840 + R3638 --> X36 + R3638 --> P37 + + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef target fill:#fed7aa,stroke:#ea580c,stroke-width:3px,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 + classDef proofMemory fill:#fef3c7,stroke:#16a34a,stroke-width:3px,color:#111827 +``` + +The mixed proof payload is: + +| Order | Proof hash | Position | Source | +|---:|---|---|---| +| 1 | `h37` | right | memory | +| 2 | `R[38, 40)` | right | memory | +| 3 | `R[32, 36)` | left | memory | +| 4 | `R[0, 32)` | left | `tile/0/000` | + +The caller sees one ordinary `merkle::Path`. Source selection is internal; the +proof format does not mark some hashes as "tile" and others as "memory". + +Proving an old leaf in the current tree is mixed in the opposite direction. +For example, a proof for leaf 5 at size 40 gets its target and lower siblings +from `tile/0/000`, then gets the final sibling `R[32, 40)` from memory. + +## Consistency proofs: the idea + +An inclusion proof answers: + +> Is this leaf part of this tree root? + +A consistency proof answers: + +> Can the tree with `m` leaves be extended, without changing its first `m` +> leaves, to produce the tree with `n` leaves? + +The verifier already knows: + +- `m` and the old root `R[0, m)`; +- `n` and the new root `R[0, n)`. + +The proof supplies enough complete subtree roots to reconstruct both roots +through a shared history. + +The producer recursively follows the part of the new tree that contains the +old boundary and emits the sibling subtree at each split: + +```mermaid +flowchart TB + A["Start with [0, n) and old size m"]:::computed + B["Split at the largest power of two
smaller than the current range"]:::computed + C{"Which side contains
the old boundary?"}:::decision + D["Recurse into that side"]:::computed + E["Emit the other side's root
as a proof hash"]:::proof + F{"Reached exactly
the old boundary?"}:::decision + G["Return proof hashes
from deepest to highest"]:::proof + + A --> B + B --> C + C --> D + D --> E + E --> F + F -->|no| B + F -->|yes| G + + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef decision fill:#fff7ed,stroke:#c2410c,color:#111827 + classDef proof fill:#fef3c7,stroke:#b45309,color:#111827 +``` + +Each emitted range is resolved through the same memory-first, tile-second +source used by inclusion proofs. + +## Consistency proof 1: a perfect old tree + +First prove that the 32-leaf tree is a prefix of the 40-leaf tree: + +```cpp +auto proof = log.consistency_proof(32, 40); +``` + +Because 32 is a power of two, the old root is already one complete left +subtree. The proof needs only the new right-hand range: + +```mermaid +flowchart TB + OLD["Known old root
R[0, 32)"]:::tile + EXT["proof[0]
R[32, 40) from memory"]:::proofMemory + JOIN["H(R[0, 32), R[32, 40))"]:::computed + NEW["Expected new root
R[0, 40)"]:::result + + OLD --> JOIN + EXT --> JOIN + JOIN --> NEW + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef proofMemory fill:#fef3c7,stroke:#16a34a,stroke-width:3px,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef result fill:#dcfce7,stroke:#15803d,stroke-width:3px,color:#111827 +``` + +The old root may have been calculated from `tile/0/000`; the extension is +resident in memory. Verification combines the known old root with the single +proof hash and compares the result with the known new root. + +## Consistency proof 2: a non-perfect old tree + +Now prove that the 20-leaf tree is a prefix of the 40-leaf tree: + +```cpp +auto proof = log.consistency_proof(20, 40); +``` + +Size 20 is not a power of two, so the old root does not line up with a single +node in the 40-leaf tree. The proof decomposes the relevant ranges: + +```mermaid +flowchart TB + R040["R[0, 40)"]:::computed + R032["R[0, 32)"]:::tile + P3240["P4 = R[32, 40)
memory"]:::proofMemory + P016["P3 = R[0, 16)
tile"]:::proofTile + R1632["R[16, 32)"]:::tile + R1624["R[16, 24)"]:::tile + P2432["P2 = R[24, 32)
tile"]:::proofTile + P1620["P0 = R[16, 20)
tile seed"]:::proofTile + P2024["P1 = R[20, 24)
tile"]:::proofTile + + R040 --> R032 + R040 --> P3240 + R032 --> P016 + R032 --> R1632 + R1632 --> R1624 + R1632 --> P2432 + R1624 --> P1620 + R1624 --> P2024 + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 + classDef proofMemory fill:#fef3c7,stroke:#16a34a,stroke-width:3px,color:#111827 +``` + +The proof vector contains hashes only; the range labels are shown here to make +the algorithm visible. Given `m = 20` and `n = 40`, the verifier derives where +each hash belongs. + +| Order | Illustrative range | Source | Why it is needed | +|---:|---|---|---| +| `P0` | `R[16, 20)` | tile | Seed shared by old and new reconstructions | +| `P1` | `R[20, 24)` | tile | Extend only the new reconstruction | +| `P2` | `R[24, 32)` | tile | Extend only the new reconstruction | +| `P3` | `R[0, 16)` | tile | Complete both the old and new left sides | +| `P4` | `R[32, 40)` | memory | Extend the new reconstruction to size 40 | + +Verification evolves two accumulators: + +The verifier uses the bit structure of `m` and `n` to decide which accumulator +each proof hash updates. Intuitively, `P0` seeds a subtree shared by both +histories and `P3` completes that shared old-tree boundary. `P1`, `P2`, and +`P4` cover leaves at or beyond the old size, so they extend only the new +accumulator. + +```mermaid +flowchart TB + S["Seed both accumulators with P0
old = new = R[16, 20)"]:::proofTile + A["Combine P1 on the right
new = R[16, 24)"]:::tile + B["Combine P2 on the right
new = R[16, 32)"]:::tile + C["Combine P3 on the left
old = R[0, 20)
new = R[0, 32)"]:::tile + D["Combine P4 on the right
new = R[0, 40)"]:::memory + V["Compare both reconstructed roots
with the caller's old and new roots"]:::result + + S --> A + A --> B + B --> C + C --> D + D --> V + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 + classDef result fill:#dcfce7,stroke:#15803d,stroke-width:3px,color:#111827 +``` + +This example is mixed in a useful way: + +- The old 20-leaf state can be reconstructed from the first tile even though + the live in-memory tree has compacted those leaves. +- The newly appended range `[32, 40)` comes from memory. +- The proof is still an ordinary vector of hashes, independent of where each + hash was found. + +## The complete mental model + +```mermaid +flowchart TB + A["Append leaf hashes"]:::memory + B["In-memory left-balanced Merkle tree"]:::memory + C["A 32-entry range becomes complete
(256 entries in production)"]:::both + D["flush() publishes immutable full tiles"]:::tile + E["compact() optionally drops old resident detail"]:::summary + F["MemoryHashSource
serves whatever remains resident"]:::memory + G["TileHashSource
serves the flushed prefix"]:::tile + H["CombinedHashSource
tries memory, then tiles"]:::both + I["ProofEngine"]:::computed + J["Current or historical root"]:::result + K["Inclusion proof"]:::result + L["Consistency proof"]:::result + + A --> B + B --> C + C --> D + D -.->|optional| E + D --> F + D --> G + E --> F + F --> H + G --> H + H --> I + I --> J + I --> K + I --> L + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef summary fill:#e5e7eb,stroke:#6b7280,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef result fill:#dcfce7,stroke:#15803d,stroke-width:3px,color:#111827 +``` + +The important boundary is always the last successfully flushed full tile: + +- Below it, immutable tiles can preserve proof detail after compaction. +- Above it, the incomplete frontier must remain resident in memory. +- A proof may resolve several component subtrees from either side of the + boundary, but the caller receives one normal proof. +- None of these rules depends on the illustrative width of 32. Production uses + the same model with 256-entry tiles. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 155a4e1..de022ae 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -41,8 +41,10 @@ endif() if(LONG_TESTS) add_merklecpp_test(tiles_level2 tiles_level2.cpp) + add_merklecpp_test(time_tiles time_tiles.cpp) set_tests_properties( ${MERKLECPP_TEST_PREFIX}tiles_level2 + ${MERKLECPP_TEST_PREFIX}time_tiles PROPERTIES TIMEOUT 900 ) endif() diff --git a/test/time_tiles.cpp b/test/time_tiles.cpp new file mode 100644 index 0000000..dce7d8d --- /dev/null +++ b/test/time_tiles.cpp @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TiledTree; + +static double secs_since( + const std::chrono::high_resolution_clock::time_point& start) +{ + const auto stop = std::chrono::high_resolution_clock::now(); + return (double)std::chrono::duration_cast( + stop - start) + .count() / + 1e9; +} + +// Uniform random in [0, bound). +static uint64_t rand_below(uint64_t bound) +{ + if (bound == 0) + { + return 0; + } + return (uint64_t)((std::rand() / (RAND_MAX + 1.0)) * (double)bound); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + +#ifndef NDEBUG + const uint64_t num_leaves = 50000; + const uint64_t num_proofs = 100; +#else + const uint64_t num_leaves = 1000000; + const uint64_t num_proofs = 10000; +#endif + + const fs::path dir = fs::temp_directory_path() / + ("merklecpp_time_tiles_" + std::to_string((unsigned long long)seed)); + + // Accumulator that consumes each proof, so the work is not optimised away. + volatile uint64_t sink = 0; + + std::cout << std::fixed << std::setprecision(3); + + try + { + const auto hashes = make_hashes(num_leaves); + + TiledTree::Config cfg; + cfg.prefix = dir; + TiledTree log(cfg); + + // 1. Append: grow the in-memory tree. + auto t = std::chrono::high_resolution_clock::now(); + for (uint64_t i = 0; i < num_leaves; i++) + { + log.append(hashes[i]); + } + log.root(); + const double append_s = secs_since(t); + std::cout << "append : " << num_leaves << " leaves in " + << append_s << " sec (" + << (uint64_t)((double)num_leaves / append_s) << " leaves/sec)\n"; + + // 2. Flush: write newly-complete full tiles to disk. + t = std::chrono::high_resolution_clock::now(); + const auto stats = log.flush(); + const double flush_s = secs_since(t); + std::cout << "flush (to disk) : " << stats.full_written + << " full tiles in " << flush_s << " sec (" + << (uint64_t)((double)stats.full_written / flush_s) + << " tiles/sec)\n"; + + const uint64_t n = log.size(); + + // 3. Inclusion proofs while everything is still resident (memory path). + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += log.inclusion_proof(rand_below(n), n)->size(); + } + const double inc_mem_s = secs_since(t); + std::cout << "inclusion (memory) : " << num_proofs << " proofs in " + << inc_mem_s << " sec (" + << (uint64_t)((double)num_proofs / inc_mem_s) << " proofs/sec)\n"; + + // 4. Consistency proofs while resident (memory path). + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += log.consistency_proof(1 + rand_below(n - 1), n).size(); + } + const double con_mem_s = secs_since(t); + std::cout << "consistency(memory): " << num_proofs << " proofs in " + << con_mem_s << " sec (" + << (uint64_t)((double)num_proofs / con_mem_s) << " proofs/sec)\n"; + + // 5. Compact: drop tiled leaves from memory. + t = std::chrono::high_resolution_clock::now(); + const uint64_t min_idx = log.compact(); + const double compact_s = secs_since(t); + std::cout << "compact : dropped " << min_idx + << " leaves from memory in " << compact_s << " sec\n"; + + // 6. Inclusion proofs for evicted leaves: served from the on-disk tiles. + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += log.inclusion_proof(rand_below(min_idx), n)->size(); + } + const double inc_tile_s = secs_since(t); + std::cout << "inclusion (tiles) : " << num_proofs << " proofs in " + << inc_tile_s << " sec (" + << (uint64_t)((double)num_proofs / inc_tile_s) + << " proofs/sec)\n"; + + // 7. Consistency proofs spanning the tiled (evicted) past. + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += log.consistency_proof(1 + rand_below(min_idx - 1), n).size(); + } + const double con_tile_s = secs_since(t); + std::cout << "consistency(tiles) : " << num_proofs << " proofs in " + << con_tile_s << " sec (" + << (uint64_t)((double)num_proofs / con_tile_s) + << " proofs/sec)\n"; + + // 8. Baseline: identical inclusion proofs from a plain in-memory Tree. + merkle::Tree ref; + for (uint64_t i = 0; i < num_leaves; i++) + { + ref.insert(hashes[i]); + } + ref.root(); + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += ref.path(rand_below(n))->size(); + } + const double inc_ref_s = secs_since(t); + std::cout << "inclusion (Tree) : " << num_proofs << " proofs in " + << inc_ref_s << " sec (" + << (uint64_t)((double)num_proofs / inc_ref_s) << " proofs/sec)\n"; + + // Sanity: a tile-served proof still verifies against the reference root. + if (!log.inclusion_proof(0, n)->verify(ref.root())) + { + throw std::runtime_error("benchmark proof failed to verify"); + } + + std::cout << "time_tiles: OK (checksum " << sink << ")\n"; + + std::error_code ec; + fs::remove_all(dir, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + + return 0; +} From 55dd47290ed647eaef6a8ade1a2d5d1f07417fdb Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 11:56:50 +0000 Subject: [PATCH 2/2] Fix tiled-storage design fence Remove the unmatched trailing code fence so the completed design document renders correctly. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/design/tlog-tiles.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 3caa26d..2d87ff2 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -783,4 +783,3 @@ compatible": hashing and does not alter any existing code path. - **CMake.** Add a `tiles` test group guarded like the others; no changes to the `merklecpp` INTERFACE target are required for consumers that don't use tiles. -```