diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b12b1bf..3f3a2e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,9 +51,9 @@ jobs: working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} run: | if [ "$RUNNER_OS" == "Linux" ]; then - cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=ON -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DLONG_TESTS=ON -DOPENSSL=ON -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON else - cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON + cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DLONG_TESTS=ON fi - name: Build @@ -64,7 +64,9 @@ jobs: - name: Clang-Tidy Header if: matrix.os == 'ubuntu-latest' shell: bash - run: clang-tidy merklecpp.h -p build/${{ matrix.build_type }} --warnings-as-errors='*' + run: | + clang-tidy merklecpp.h -p build/${{ matrix.build_type }} --warnings-as-errors='*' + clang-tidy merklecpp_tiles.h -p build/${{ matrix.build_type }} --warnings-as-errors='*' - name: Test working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index e3ecf9f..0bf9149 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,7 +43,7 @@ jobs: - name: Configure merklecpp working-directory: ${{github.workspace}}/build - run: cmake -DCMAKE_BUILD_TYPE=Debug -DTESTS=ON $GITHUB_WORKSPACE + run: cmake -DCMAKE_BUILD_TYPE=Debug -DTESTS=ON -DLONG_TESTS=ON $GITHUB_WORKSPACE - name: Build merklecpp working-directory: ${{github.workspace}}/build diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e35781..55123ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,7 @@ option(EVERCRYPT "enable comparison with EverCrypt Merkle trees" OFF) option(OPENSSL "enable OpenSSL" OFF) option(TRACE "enable debug traces" OFF) option(CLANG_TIDY "enable clang-tidy checks during build" OFF) +option(LONG_TESTS "enable long-running tests" OFF) if(CLANG_TIDY) find_program(CLANG_TIDY_PROGRAM clang-tidy) diff --git a/Doxyfile b/Doxyfile index 33229bb..d045b46 100644 --- a/Doxyfile +++ b/Doxyfile @@ -790,7 +790,8 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = merklecpp.h +INPUT = merklecpp.h \ + merklecpp_tiles.h # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md new file mode 100644 index 0000000..bafc628 --- /dev/null +++ b/doc/design/tlog-tiles.md @@ -0,0 +1,750 @@ +# Design: Tiled storage and tile-backed proofs for merklecpp + +Status: Proposal / Draft +Audience: merklecpp maintainers and contributors +Scope: Extend the header-only library to (a) persist the Merkle tree as +[tlog-tiles](https://c2sp.org/tlog-tiles) tile files progressively (on +compaction), and (b) serve **inclusion** and **consistency** proofs from those +tiles, from the in-memory tree, or from a combination of the two. + +--- + +## 1. Goals and non-goals + +### Goals + +1. Write the tree to disk as a set of immutable, cacheable **tile files** using + the [tlog-tiles](https://c2sp.org/tlog-tiles) file/directory layout and tile + geometry (256-wide tiles, 8 tree levels per tile, path encoding). Only + **full, balanced** tiles are produced. The remaining frontier stays in the + in-memory tree until it crosses the next full-tile boundary. +2. Produce tiles **progressively**: each batch/flush writes only the newly + completed tiles, and the write integrates with the existing `flush_to()` + "compaction" so that flushed (evicted) subtrees are durably persisted before + their nodes are freed from memory. +3. Provide an API to retrieve, **after compaction**, an **inclusion path** and a + **consistency path**: + - from the in-memory tree alone (existing behaviour), + - from full tiles alone, for the tiled (full-tile-covered) prefix, or + - from a combination of full tiles (old, flushed part) and the in-memory + tree (recent frontier) — the normal case. +4. Keep everything **header-only** and templated, matching the existing + `TreeT` style. + +### Full-tile-only policy + +Partial tiles are out of scope: merklecpp never emits or reads them. During a +flush, a tile becomes eligible only after all 256 entries are complete and +final. Entries beyond that boundary remain in memory. Once published, a full +tile is immutable and is never rewritten. + +### Thread-safety policy + +The tiled-storage API provides no internal synchronization. Every object is +single-threaded unless its caller serializes all access. This requirement also +applies to separate objects that share a store prefix and to methods declared +`const`, because proof reads update the `TileHashSourceT` LRU cache. Locking and +reader/writer coordination are deliberately left to the application. + +### Hard constraints (from the request) + +- **Hashing logic is unchanged.** No modification to `HASH_FUNCTION`, to node + hashing, or to leaf handling. +- **Proofs remain compatible with the existing library.** A proof produced from + tiles MUST be identical to (and verify against the same root as) the proof the + existing library would produce via `Tree::path()` / `Tree::past_path()` / + `Tree::root()` / `Tree::past_root()`. + +### Non-goals + +- Byte-level interoperability with *external* RFC 6962 / Go-ecosystem tlog + clients. See [§2.3](#23-compatibility-statement): such interop is only + possible if the consumer instantiates `TreeT` with an RFC 6962 combiner, which + is **out of scope** here and not required by this design. +- An HTTP server. We define the on-disk layout and the read/write/proof APIs; + serving the static files is an application concern. + +--- + +## 2. Background + +### 2.1 tlog-tiles, the parts we adopt + +A tiled log exposes the Merkle tree as a set of static resources: + +- **Tiles** at `/tile//`, `application/octet-stream`. + - `` = level, decimal `0..63`, no leading zeros. + - `` = tile index within the level, encoded as zero-padded 3-digit path + elements where **all but the last element are prefixed with `x`**. Example: + `1234067` → `x001/x234/067`. +- A **full tile** is exactly **256 hashes** wide (`256 * HASH_SIZE` bytes). The + spec fixes `HASH_SIZE = 32` (SHA-256); here it is the template parameter. +- Level 0 hashes are **leaf hashes**. At level `L ≥ 1`, each hash is the Merkle + Tree Hash of a *full* tile at level `L-1`. A tile spans **8 tree levels**: its + 256 entries are the leaves of a height-8 perfect subtree, and intra-tile + internal nodes are reconstructed by hashing entries. +- The `n`-th tile at level `l` contains, for `i = 0..255`: + `MTH(D[(n*256+i)*256^l : (n*256+i+1)*256^l])`. + Its **start index** is `n*256^(l+1)`, its **end index** `(n+1)*256^(l+1)`. +- The frontier beyond the last full tile is held in the in-memory tree, so the + on-disk tile set is always a full-tile prefix. +- **Entry bundles** at `/tile/entries/`: big-endian `uint16` + length-prefixed raw entries, 256 per full bundle. (See + [section 6.8](#68-entry-bundles-optional).) +- **Pruning**: a log keeps a *minimum index*; tiles/bundles whose end index is + `≤ minimum index` may be denied. This maps cleanly onto merklecpp's + `flush_to()` / `min_index()`. + +### 2.2 The merklecpp model + +- `TreeT` is a **left-balanced binary Merkle tree**. + Its shape is identical to RFC 6962: a node's left child is the largest perfect + subtree of `2^k` leaves with `2^k < n`. A *full* node of height `h` covers + exactly `2^(h-1)` aligned leaves, and its hash is the root of that perfect + subtree (verified from `walk_to`, `is_full`, `update_sizes`). +- Internal nodes are combined with `HASH_FUNCTION(left, right, out)` over exactly + two child hashes (`merklecpp.h` node hashing). There is **no** domain + separation; the combiner is the only cryptographic operation. +- `flush_to(index)` is the existing **compaction** primitive: it conflates the + left part of the tree into single hash-only nodes and drops `num_flushed` + leaves from memory, raising `min_index()` to `index`. After a flush, paths for + leaves `< index` can no longer be produced from memory. +- Existing proof APIs we must stay compatible with: + - `root()` / `past_root(i)` — current / historical root. + - `path(i)` — inclusion path for leaf `i` in the current tree. + - `past_path(i, as_of)` — inclusion path for leaf `i` as of size `as_of+1`. + - `PathT::verify(root)` / `PathT::root()` — verification via `HASH_FUNCTION`. + - merklecpp has **no** classic two-size consistency proof today; this design + adds one (built from the same combiner, so it reconciles `past_root`s). + +### 2.3 Compatibility statement + +The tile geometry, path encoding, and the inclusion / consistency proof +**algorithms** are exactly those of tlog-tiles / RFC 6962. The implementation +emits only immutable full tiles. +The **hash values stored in tiles** are produced by the tree's existing +`HASH_FUNCTION`. Because + +1. merklecpp's tree shape equals the RFC 6962 left-balanced shape, and +2. tile entries and proof building blocks are *perfect-subtree roots* combined + with the **same** `HASH_FUNCTION` the tree already uses, + +a tile entry at level `L`, index `g` equals the in-memory full node of height +`8L+1` over leaves `[g·256^L, (g+1)·256^L)`, and a proof assembled from tiles is +**byte-identical** to the one `Tree::path()` / `Tree::past_path()` would emit. +Tile-derived proofs therefore verify with the unchanged `PathT::verify()`. + +> External RFC 6962 interop would additionally require RFC 6962 domain +> separation (`SHA256(0x00‖entry)` leaves, `SHA256(0x01‖l‖r)` nodes). merklecpp +> already lets a consumer pass such a `HASH_FUNCTION`; doing so is **optional and +> out of scope**. This design never assumes it. + +--- + +## 3. Tile ↔ merklecpp mapping (the math) + +Let `TILE_HEIGHT = 8` and `TILE_WIDTH = 256 = 2^8`. All combiners below are the +tree's `HASH_FUNCTION`; `MTH` denotes the Merkle Tree Hash computed with it. + +- **Tile entry = perfect subtree root.** Entry `i` of tile `(L, N)` (global + entry index `g = N·256 + i`) is `MTH(D[g·256^L : (g+1)·256^L])`, i.e. the root + of a perfect subtree of `2^(8L)` leaves — the in-memory full node of height + `8L + 1`. +- **Level roll-up.** Because a full level-`L` entry is the root of 256 full + level-`(L-1)` entries, a level-`L` tile can be computed from level-`(L-1)` + tiles alone: `tile(L,N)[i] = perfect_root( tile(L-1, N·256+i)[0..255] )`. So + the **write path needs no tree internals above level 0** — only leaf hashes + plus roll-ups. +- **Intra-tile internal nodes.** A tile's 256 entries are the leaves of a + height-8 perfect subtree. The root of `2^r` (`0 ≤ r ≤ 8`) consecutive, + aligned entries is `perfect_root` of that run. Hence any RFC 6962 subtree root + at tree level `k = 8L + r` is obtained from **one** level-`L` tile by hashing + `2^r` consecutive entries (a single entry when `r = 0`). +- **Full-tile prefix.** Only complete level-`L` tiles are written: + `full_tiles_L = floor( floor(s / 256^L) / 256 )`, together covering the leaf + prefix `covered = floor(s / 256) * 256`. The incomplete frontier `[covered, s)` + is not tiled; it is served from the in-memory tree. When no higher-level full + tile contains a subtree, it is resolved by **descending to the highest + available full tile and rolling up** (ultimately from full level-0 tiles). + +This yields a single read primitive that both proof types are built on: + +``` +subtree_root(level k, index j) = MTH(D[j·2^k : (j+1)·2^k]) # within `covered` + L = k / 8 ; r = k % 8 + first = j << r ; n = first / 256 ; off = first % 256 + if a full level-L tile n exists (first + 2^r within full_tiles_L · 256): + return perfect_root( tile(L, n)[off : off + 2^r] ) # single entry if r==0 + else: # higher-level tile unavailable: descend + return HASH_FUNCTION( subtree_root(k-1, 2j), subtree_root(k-1, 2j+1) ) +``` + +and a range primitive for non-perfect (right-frontier) subtrees: + +``` +mth_range(a, b): // MTH(D[a:b]); a aligned to 2^ceil(log2(b-a)) + w = b - a + if w == 2^k and source can resolve subtree_root(k, a>>k): return it + k = largest power of two < w // RFC 6962 split + return HASH_FUNCTION( mth_range(a, a+k), mth_range(a+k, b) ) +``` + +`mth_range` falls back to splitting when a perfect subtree is not directly +resolvable from the chosen source; the recursion always bottoms out at resolvable +pieces (see the [combined-source invariant](#7-progressive-production--compaction)). + +--- + +## 4. File and directory layout + +Rooted at a configurable `prefix` directory on local disk: + +``` +/ + tile/ + 0/ # level 0 (leaf hashes), full tiles only + 000 001 ... 255 # full tiles (8192 B each for HASH_SIZE=32) + x001/ + 000 ... 255 + 1/ # level 1 (roll-ups of level-0 full tiles) + 000 ... + .../ + entries/ # optional raw entry bundles (full only) + 000 001 ... +``` + +### Index encoding (`encode_tile_index`) + +``` +encode_tile_index(N): + parts = [] + do { parts.push_front(printf("%03d", N % 1000)); N /= 1000 } while (N > 0) + for each part except the last: prepend 'x' + return join(parts, "/") +``` + +Examples: `5 → "005"`, `255 → "255"`, `1000 → "x001/000"`, +`1234067 → "x001/x234/067"`. + +Resource paths: + +- Full tile: `tile//` (the only tiles this + implementation writes) +- Entry bundle: `tile/entries/` (full bundles only) + +Tile byte format: the 256 entries concatenated, each `HASH_SIZE` raw bytes +(`HashT::bytes`); a full tile is `256 * HASH_SIZE` bytes. + +--- + +## 5. Architecture overview + +``` + append(leaf hash) flush() + caller ───────────────▶ TiledTreeT ───────────────▶ TileStoreT (disk I/O) + │ owns Tree ▲ + │ │ read tiles + inclusion/consistency proof requests │ + ▼ │ + ProofEngine ──▶ HashSource ◀────────┘ + ├─ MemoryHashSource (in-memory Tree) + ├─ TileHashSource (TileStore + size) + └─ CombinedHashSource +``` + +New, all in a companion header `merklecpp_tiles.h` (includes `merklecpp.h`), +namespace `merkle::tiles`, every type templated on `` +with default aliases mirroring the bottom of `merklecpp.h`: + +| Component | Responsibility | +|----------------------|-----------------------------------------------------------------------| +| `TileCoord` | pure index math + path encoding (no I/O) | +| `TileStoreT` | read/write tile files on a local filesystem | +| `TileWriterT` | compute & persist newly-complete full tiles (write path) | +| `HashSource` impls | resolve `subtree_root` from tiles, memory, or both | +| `ProofEngineT` | `mth_range`, `inclusion_proof`, `consistency_proof`, verifiers | +| `TiledTreeT` | convenience wrapper: `append` / `flush` / `prove*` / compaction | + +The **only** (optional) change to the core header is a small, read-only, +**non-hashing** accessor (`subtree_root`) used by `MemoryHashSource`; see +[§6.2](#62-optional-core-accessor-non-hashing). + +--- + +## 6. Public API design + +### 6.1 Header, namespace, aliases + +```cpp +// merklecpp_tiles.h +#include "merklecpp.h" +#include +#include + +namespace merkle { +namespace tiles { + +static constexpr uint16_t TILE_HEIGHT = 8; +static constexpr uint16_t TILE_WIDTH = 256; // 2^TILE_HEIGHT + +template &, + const HashT&, + HashT&)> +class TileStoreT { /* ... */ }; +// ... TileWriterT, ProofEngineT, TiledTreeT ... + +} // namespace tiles + +// Convenience aliases (mirror merklecpp.h) +using TileStore = tiles::TileStoreT<32, sha256_compress>; +using TiledTree = tiles::TiledTreeT<32, sha256_compress>; +// + 384/512 variants +} // namespace merkle +``` + +Note the roll-up/proof combiner is the **same** `HASH_FUNCTION` the tree uses, +and it only ever combines two `HASH_SIZE`-byte hashes — so the default +`sha256_compress` single-block path is sufficient and **no OpenSSL dependency is +introduced**. + +### 6.2 Optional core accessor (non-hashing) + +To let proofs be served partly from the resident tree, add one small read-only +method to `TreeT`. It performs **no hashing changes**: it navigates to an +existing node and returns its already-computed hash. + +```cpp +// In TreeT (merklecpp.h). Returns true and sets `out` to MTH(D[j< read_tile(const TileRef&) const; // 256 entries + void write_tile(const TileRef&, const std::vector&); // synced atomic replace +}; +``` + +- Writes use a unique temporary file, sync the file contents, publish with an + atomic replace, and on POSIX sync every newly created directory link plus the + destination directory after the rename. Before reusing a visible file, the + writer re-confirms the directory chain and destination directory, so a retry + repeats a failed sync even if directory creation or rename is already + visible. A wrong-size tile is not treated as a durable full tile and is + rewritten by the writer. +- Full tiles are written once and never rewritten (immutability), so every file + under `tile//` is write-once. +- A small in-process cache of recently read tiles avoids repeated I/O during + proof generation. +- `TileStoreT` does not synchronize access. Callers must serialize all + operations on a store and across stores that share a prefix. + +### 6.4 Write path — `TileWriterT` (progressive, on compaction) + +```cpp +class TileWriterT { +public: + explicit TileWriterT(TileStoreT& store); + + // Persist all newly-complete full tiles (all levels 0..63). Incremental: + // only writes tiles not already present. Higher levels are always rolled up, + // since proof generation relies on them. + // `leaf_at` supplies level-0 leaf hashes for [0, size) (e.g. Tree::leaf). + void write_up_to(uint64_t size, + const std::function& leaf_at); +}; +``` + +Algorithm (`write_up_to`): + +``` +for L = 0, 1, 2, ... while entries_L = size >> (8*L) is > 0: + full_L = entries_L / 256 + for N in [first_unwritten_full(L) .. full_L): # new full tiles only + entries = [ entry(L, N*256 + i) for i in 0..255 ] + store.write_tile({L, N}, entries) + # rightmost incomplete entries are not written; they stay in memory until + # they complete a full tile + +entry(L, g): + if L == 0: return leaf_at(g) + else: return perfect_root( store.read_tile({L-1, g}) ) # 256→1 +``` + +- Level 0 reads leaf hashes via the supplied `leaf_at` (e.g. `Tree::leaf(i)`), + which exist **before** flushing. Higher levels roll up from the level-0 tiles + just written — independent of tree internals and robust to prior flushes. +- `first_unwritten_full(L)` is cached in each writer. A fresh writer reconstructs + it with a bounded, ordered scan of the contiguous prefix up to `full_L`. + Scanning in order is required because malformed or externally created files + can make raw file presence non-monotonic. + +Compaction integration (the "on compaction" story): + +```cpp +// In TiledTreeT::flush(): seal, write durable full tiles, THEN free memory. +uint64_t covered = (size / TILE_WIDTH) * TILE_WIDTH; // full level-0 coverage +immutable_size = max(immutable_size, covered); // before any write +writer.write_up_to(size, [&](uint64_t i) -> const Hash& { return tree.leaf(i); }); +flushed_size = covered; // after all levels succeed +// compact() retains the last covered leaf (when any) and the entire frontier. +``` + +If `write_up_to` throws, `immutable_size` remains advanced because a full tile +may already be visible, while `flushed_size` remains at its previous successful +boundary. The in-memory tree is not compacted. The caller fixes the I/O error +and retries with the same tree state; existing finalized tiles are reused. + +### 6.5 `HashSource` — tiles, memory, or both + +```cpp +struct HashSource { // concept (duck-typed or virtual) + // MTH(D[index< inclusion_proof(uint64_t index, uint64_t size) const; + + // RFC 6962 consistency proof that size `m` is a prefix of size `n` (m<=n). + std::vector consistency_proof(uint64_t m, uint64_t n) const; + std::vector consistency_proof_from_indices( + uint64_t first_index, uint64_t second_index) const; + + // Verifier (consistency is new to merklecpp; inclusion reuses PathT::verify). + static bool verify_consistency(uint64_t m, uint64_t n, + const Hash& old_root, const Hash& new_root, + const std::vector& proof); +}; +``` + +Inclusion (top-down; element order/`direction` chosen to match `Tree::path`): + +``` +elements = [] # leaf→root order via push_front +lo = 0, hi = size, idx = index +while hi - lo > 1: + k = largest_pow2_lt(hi - lo) # split at lo+k + if idx - lo < k: # target in left ⇒ sibling on the RIGHT + sib = mth_range(lo+k, hi); dir = PATH_RIGHT; hi = lo + k + else: # target in right ⇒ sibling on the LEFT + sib = mth_range(lo, lo+k); dir = PATH_LEFT; lo = lo + k + elements.push_front({sib, dir}) +leaf = src.leaf(index) +return Path(leaf, index, elements, max_index = size - 1) +``` + +Consistency (RFC 6962 `SUBPROOF`): + +``` +consistency_proof(m, n): # 0 < m <= n + if m == n: return [] + subproof(m, lo=0, hi=n, complete=true) + +subproof(m, lo, hi, complete): + if m == hi - lo: + if not complete: proof.push_back(mth_range(lo, hi)) + return + k = largest_pow2_lt(hi - lo) + if m <= k: + subproof(m, lo, lo+k, complete) + proof.push_back(mth_range(lo+k, hi)) + else: + subproof(m-k, lo+k, hi, false) + proof.push_back(mth_range(lo, lo+k)) +``` + +Because every emitted hash is an `mth_range` computed with `HASH_FUNCTION`, the +consistency proof reconciles `Tree::past_root(m-1)` with +`Tree::past_root(n-1)` — i.e. it is consistent with the existing library. + +### 6.7 `TiledTreeT` — convenience wrapper + +```cpp +class TiledTreeT { +public: + struct Config { + std::filesystem::path prefix; + uint64_t retention_margin = 0; // keep this many recent leaves resident + bool compact_on_flush = false; // opt in to dropping tiled leaves + }; + explicit TiledTreeT(Config); + + void append(const Hash& leaf_hash); // tree.insert + uint64_t size() const; // tree.num_leaves + Hash root(); // tree.root + uint64_t flushed_size() const; // successful tile boundary + uint64_t immutable_size() const; // rollback boundary + + // Write newly-complete full tiles. Compaction (dropping already-tiled + // leaves from memory) happens only if compact_on_flush. + Stats flush(); + + // Drop from memory the leaves already covered by a full tile (opt-in); the + // un-tiled frontier is always retained, and proofs for dropped leaves remain + // available from the tiles. + uint64_t compact(); + + // Roll back only beyond immutable_size(). + void retract_to(size_t index); + + // Proofs over tiles ∪ resident tree (works for flushed indices). + std::shared_ptr inclusion_proof(uint64_t index, uint64_t size); + std::vector consistency_proof(uint64_t m, uint64_t n); + std::vector consistency_proof_from_indices(uint64_t i, uint64_t j); + + Tree& tree_ref(); // mutable escape hatch + Store& store_ref(); // mutable escape hatch +}; +``` + +`TiledTreeT` performs no internal locking. The caller must serialize every +operation on a shared instance, including proof calls. + +Callers with their own storage can construct a `TileWriterT` and call +`write_up_to`, then build a `ProofEngineT` on a `CombinedHashSource`, so the +wrapper is optional sugar. + +### 6.8 Entry bundles (optional) + +merklecpp never sees raw entries (callers insert pre-computed leaf hashes), so +entry bundles are an **application-owned** add-on, included for completeness: + +- `tile/entries/` stores big-endian `uint16` length-prefixed entries, + 256 per full bundle (full bundles only). +- The application is responsible for the leaf-hash derivation it uses (e.g. + `leaf_hash = H(entry)`); merklecpp stores whatever leaf hash is inserted. +- An `EntryBundleWriterT` mirrors `TileWriterT`: it writes full bundles on + 256-entry boundaries only; the incomplete tail stays with the application. + Marked optional/secondary. + +--- + +## 7. Progressive production & compaction + +The pairing of tile writing with `flush_to` gives two central correctness +invariants: + +> **Compaction invariant.** Retain the final leaf of the last fully successful +> flush: `flushed_size == 0 || min_index() < flushed_size`. +> +> **Immutability invariant.** Never roll back below a full-tile boundary that a +> flush may have published: `size >= immutable_size`. + +`TiledTree` is fresh-only: its configured directory may exist, but the `tile` +subdirectory must be absent or empty. Tile files do not carry the size, root, +hash identity, or ownership information needed to reopen a tree safely, so the +wrapper rejects an existing tile namespace rather than adopting it. The +lower-level `TileWriter` supports resume for applications that persist and +validate the matching tree state themselves. + +Per flush: + +1. `append(...)` new leaf hashes; compute `root()`. +2. Compute `covered = floor(size / 256) * 256` and advance `immutable_size` to + `covered` before any write can publish a full tile. +3. `write_up_to(size, leaf_at)` - persist newly-complete **full** tiles at all + levels (incremental). +4. After every level succeeds, set `flushed_size = covered`. +5. *(optional)* `compact()` computes an aligned retention target, capped below + nonzero `covered`, then calls `flush_to(target)`. This reclaims memory only + when `compact_on_flush` is set (or `compact()` is called explicitly), while + retaining the final tiled leaf and the entire un-tiled frontier. By default + nothing is dropped and the tree stays whole. + +If step 3 fails, steps 4 and 5 do not run. `immutable_size` stays advanced to +prevent stale-tile rollback, while `flushed_size` stays at the last complete +all-level write so proofs and compaction do not trust an incomplete flush. + +Given the invariant, every leaf and every perfect subtree is resolvable: + +- leaf `i < covered` ⇒ in a full level-0 tile; leaf `i ≥ min_index` ⇒ resident. + Since `min_index ≤ covered`, **every** leaf is in tiles ∪ memory. The frontier + `[covered, size)` is always resident, because `compact()` never flushes past + `covered`. +- `mth_range` resolves a perfect subtree directly when it lies wholly in tiles + (`end ≤ covered`) or wholly in memory (`start ≥ min_index`); otherwise it + splits and recurses, terminating at resolvable pieces (leaves at worst). A + subtree within `covered` whose level has no completed full tile is resolved by + descending to the highest available full tile. + +Hence inclusion and consistency proofs are always producible after compaction, +from full tiles alone (for the tiled prefix), from memory alone (when nothing +relevant was flushed), or from the combination — satisfying the request. + +Cost per flush is `O(new full tiles)`; higher-level tiles are cheap roll-ups of +256 child hashes. Proof generation is `O(log(size))` `mth_range` calls, each at +most a few tile reads plus a `≤ 256`-leaf roll-up, with repeated tile reads +served from the per-source cache in the common case. + +--- + +## 8. Pruning / minimum index + +tlog-tiles pruning maps directly onto merklecpp: + +- The log's *minimum index* is `tree.min_index()` (== `num_flushed`). +- "Deny tiles/bundles whose end index ≤ minimum index" is implemented by the + serving layer consulting `min_index()`; on-disk tiles may be retained + (recommended by the spec) so historical proofs remain producible. +- The unpruned default is `min_index() == 0` (no `flush_to`). + +`flush_to` is the mechanism; *retention policy* (when/whether to prune) is left +to the application, exactly as the spec leaves it to log ecosystems. + +--- + +## 9. Worked examples (used as test vectors) + +**Size 256:** exactly one full level-0 tile (`tile/0/000`). No level-1 tile (its +single entry is the un-tiled frontier root, kept in memory). + +**Size 70 000:** 273 full level-0 tiles (`covered = 69 888`) + one full level-1 +tile (`tile/1/000`, rolling up level-0 tiles 0..255) = **274 full tiles**. The +incomplete frontier at every level is not written; the remaining 112 leaves and +the higher-level roll-ups stay in the in-memory tree. +These exact counts are asserted in tests. + +**Index encoding:** `1234067 → x001/x234/067`; `1000 → x001/000`; `255 → 255`. + +--- + +## 10. Implementation plan + +Each phase is independently testable; phases 1–4 require **no** core changes. + +**Phase 0 — Scaffolding.** Add `merklecpp_tiles.h` (includes `merklecpp.h`), +`merkle::tiles` namespace, geometry constants, default aliases. Wire a new test +group in `test/CMakeLists.txt` following `add_merklecpp_test`. + +**Phase 1 — Coordinates & store.** `TileCoord`/`encode_index`, `TileRef`, +`TileStoreT` (path building, atomic `write_tile`, `read_tile`). *Tests:* +index-encoding vectors; tile byte round-trip. + +**Phase 2 — Write path.** `TileWriterT::write_up_to` (level-0 from `leaf_at`, +roll-ups, incremental cursor; full tiles only — the frontier is never tiled). +*Tests:* sizes 256 & 70 000 produce exactly the expected full-tile set; +full-tile immutability (re-running writes nothing new). + +**Phase 3 — Hash sources & proof engine.** `TileHashSource`, `mth_range`, +`ProofEngineT::root/inclusion_proof/consistency_proof/verify_consistency`. +*Tests:* `root(size)` from tiles == `tree.root()`; `inclusion_proof(i,size)` +**equals** `tree.path(i)` (operator==) and verifies; `inclusion_proof(i, m)` +equals `tree.past_path(i, m-1)`; consistency proof reconciles +`tree.past_root(m-1)`/`tree.past_root(n-1)`. + +**Phase 4 — Combination & wrapper.** Optional core `subtree_root` accessor; +`MemoryHashSource`, `CombinedHashSource`, `TiledTreeT` (append/flush/prove +with `flush_to`). *Tests:* build N leaves, flush, then prove +inclusion for a **flushed** index and a **resident** index against a non-flushed +reference tree's root; consistency across a flush boundary. + +**Phase 5 — Optional entry bundles** (`BundleWriter`) and docs (README "Usage" +snippet, link this design doc, add `merklecpp_tiles.h` to Doxygen inputs). + +Deliverables: `merklecpp_tiles.h`; `test/tiles_*.cpp`; CMake wiring; optional +one-method core addition; README/docs updates. + +--- + +## 12. Risks, edge cases, open questions + +- **External interop (by design, no).** With the default combiner the tiles are + *not* byte-compatible with RFC 6962 tooling. Documented in + [§2.3](#23-compatibility-statement); opt-in via an RFC 6962 `HASH_FUNCTION` is + the consumer's choice and out of scope. +- **Filesystem dependency.** Tile I/O needs ``/``; isolated + in the companion header so the core stays dependency-free. +- **Immutable full tiles.** A tile is emitted only after all of its entries are + final, and every emitted tile is write-once. A stand-alone tile reader cannot + serve the frontier; that is the in-memory tree's job (or the application must + keep it elsewhere). +- **`flush_to` alignment.** Compaction normally flushes to a 256-multiple + derived from retention. When that target equals `flushed_size`, it stops one + leaf earlier so `TreeT` can still retract to exactly that size. This one-leaf + overlap is enforced inside `TiledTreeT::compact`. +- **Rollback vs. immutable tiles.** Tiles are write-once, so rolling the tree + back (`retract_to`) over a range that a flush may have published would leave + stale, never-rewritten tiles. `TiledTreeT::retract_to` therefore throws if the + resulting size is below `immutable_size()`. A failed flush may advance + `immutable_size()` without advancing `flushed_size()`; retry with the same + tree state. Retracting the underlying tree directly via `tree_ref()` bypasses + this guard, can make the size boundaries inconsistent or non-monotonic, and + must be avoided. Files written through `store_ref()` are trusted without + checking that they match the tree and can invalidate proofs after compaction. +- **No internal synchronization.** Every tiled-storage object and shared store + prefix requires external serialization. This includes `const` proof reads, + which update the tile cache. +- **Very large indices.** Index math uses `uint64_t`; encoding handles + multi-group indices. Level bound `<= 63` per spec (8 suffices for `2^64`). + Resume scans are bounded by the requested tree size and cannot follow sparse + files beyond that range. +- **Open question — `subtree_root` in core vs. `past_path`-derived memory + source.** Recommend the tiny non-hashing accessor; falls back to zero-core- + change if maintainers prefer. Either keeps hashing untouched. + +--- + diff --git a/merklecpp.h b/merklecpp.h index 1170b1c..803df88 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -1316,6 +1317,82 @@ namespace merkle leaf_node(index)->hash, index, std::move(path), as_of); } + /// @brief Extracts the root hash of a complete subtree resident in memory + /// @param level The height of the subtree (it spans 2**level leaves) + /// @param index The index of the subtree at that height + /// @param out Set to the subtree root hash on success + /// @return Whether the subtree is a complete (balanced) subtree fully + /// resident in memory + /// @note This is read-only and does not change the hashing of the tree: it + /// returns an existing node hash (computing it on demand exactly as root() + /// and path() do). It returns false if any leaf of the subtree has been + /// flushed, if the subtree extends past the last leaf, or if the node at + /// that position is not a full subtree. The subtree spans leaf indices + /// [index << level, (index + 1) << level). + bool subtree_root(uint8_t level, size_t index, Hash& out) + { + const size_t leaves = num_leaves(); + if (leaves == 0 || level >= std::numeric_limits::digits) + { + return false; + } + if (index > (std::numeric_limits::max() >> level)) + { + return false; + } + + const size_t lo = index << level; + const size_t count = (size_t)1 << level; + + if (lo < min_index() || count > leaves || lo > leaves - count) + { + return false; + } + + if (level == 0) + { + out = leaf(lo); + return true; + } + + compute_root(); + + const uint8_t target_height = level + 1; + if (!_root || _root->height < target_height) + { + return false; + } + + Node* cur = _root; + size_t it = lo << (sizeof(lo) * 8 - _root->height + 1); + for (uint8_t height = _root->height; height > target_height;) + { + const bool go_right = ((it >> (8 * sizeof(it) - 1)) & 0x01) != 0U; + if (cur->height == height) + { + Node* next = go_right ? cur->right : cur->left; + if (!next) + { + return false; // conflated/flushed: not resident + } + cur = next; + } + it <<= 1; + height--; + } + + if (cur->height != target_height || !cur->is_full()) + { + return false; + } + if (cur->dirty) + { + hash(cur); + } + out = cur->hash; + return true; + } + /// @brief Serialises the tree /// @param bytes The vector of bytes to serialise to void serialise(std::vector& bytes) diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h new file mode 100644 index 0000000..1e3ce90 --- /dev/null +++ b/merklecpp_tiles.h @@ -0,0 +1,1904 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "merklecpp.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include +#else +# include +# include +#endif + +// Tiled storage for merklecpp trees, following the full-tile geometry and path +// encoding of the C2SP tlog-tiles layout (https://c2sp.org/tlog-tiles). Only +// complete, immutable tiles are stored. Their hash values are produced by the +// tree's existing HASH_FUNCTION, so tile-derived proofs are byte-identical to +// those produced by merkle::TreeT (see doc/design/tlog-tiles.md). +// +// Thread safety: types in this header do not synchronize access internally. +// Callers must serialize all operations on shared objects and store prefixes, +// including const proof operations that update the tile cache. + +namespace merkle // NOLINT(modernize-concat-nested-namespaces) +{ + namespace tiles + { + /// @brief Number of tree levels spanned by a single tile. + static constexpr uint16_t TILE_HEIGHT = 8; + + /// @brief Number of hashes in a full tile (2**TILE_HEIGHT). + static constexpr uint16_t TILE_WIDTH = 256; + + namespace detail + { + template + uint64_t contiguous_prefix_length(uint64_t limit, const Present& present) + { + uint64_t length = 0; + while (length < limit && present(length)) + { + length++; + } + return length; + } + } + + /// @brief Encodes a tile index as tlog-tiles path elements. + /// @param n The tile index + /// @return The index as zero-padded, '/'-separated 3-digit groups, where + /// all but the last group are prefixed with 'x'. For example, 1234067 is + /// encoded as "x001/x234/067" and 5 as "005". + static inline std::string encode_tile_index(uint64_t n) + { + std::vector parts; + char buf[8]; + do + { + std::snprintf(buf, sizeof(buf), "%03u", (unsigned)(n % 1000)); + parts.emplace_back(buf); + n /= 1000; + } while (n > 0); + + std::string r; + for (size_t i = 0; i < parts.size(); i++) + { + const std::string& part = parts[parts.size() - 1 - i]; + if (i != 0) + { + r += "/"; + } + if (i + 1 < parts.size()) + { + r += "x"; + } + r += part; + } + return r; + } + + /// @brief Identifies a single (full) tile within a tiled log. + /// @note Only full, TILE_WIDTH-wide tiles are produced and consumed; the + /// incomplete frontier is never tiled (see doc/design/tlog-tiles.md). + struct TileRef + { + /// @brief The level of the tile (0 == leaf hashes). + uint8_t level = 0; + + /// @brief The index of the tile within its level. + uint64_t index = 0; + }; + + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TileWriterT; + + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class EntryBundleWriterT; + + /// @brief Reads and writes tlog-tiles tile files on a local filesystem. + /// @tparam HASH_SIZE Size of each hash in bytes + /// @tparam HASH_FUNCTION The tree's node hash function (carried for use by + /// later components; tile I/O itself does not hash). + /// @warning No internal synchronization is provided. Callers must serialize + /// access to a store and to all stores that share its prefix. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TileStoreT + { + friend class TileWriterT; + friend class EntryBundleWriterT; + + public: + /// @brief The type of hashes stored in tiles. + using Hash = HashT; + + /// @brief Constructs a tile store rooted at @p prefix. + /// @param prefix The directory under which the tiles live. + explicit TileStoreT(std::filesystem::path prefix) : + prefix(std::move(prefix)) + {} + + /// @brief The root directory of the store. + [[nodiscard]] const std::filesystem::path& root() const + { + return prefix; + } + + /// @brief Encodes a tile index (see encode_tile_index). + static std::string encode_index(uint64_t n) + { + return encode_tile_index(n); + } + + /// @brief The filesystem path of a tile. + [[nodiscard]] std::filesystem::path tile_path(const TileRef& ref) const + { + return prefix / + ("tile/" + std::to_string((unsigned)ref.level) + "/" + + encode_tile_index(ref.index)); + } + + /// @brief The filesystem path of an entry bundle. + [[nodiscard]] std::filesystem::path entries_path(uint64_t index) const + { + return prefix / ("tile/entries/" + encode_tile_index(index)); + } + + /// @brief Whether a full tile exists on disk. + [[nodiscard]] bool has_full_tile(uint8_t level, uint64_t index) const + { + std::error_code ec; + const auto path = tile_path(TileRef{level, index}); + if (!std::filesystem::is_regular_file(path, ec) || ec) + { + return false; + } + return std::filesystem::file_size(path, ec) == + (uintmax_t)TILE_WIDTH * HASH_SIZE && + !ec; + } + + /// @brief Writes a tile to disk atomically. + /// @param ref The tile to write + /// @param hashes The tile's hashes (exactly TILE_WIDTH of them) + void write_tile(const TileRef& ref, const std::vector& hashes) + { + if (hashes.size() != TILE_WIDTH) + { + throw std::runtime_error("tile width mismatch"); + } + + std::vector bytes; + bytes.reserve(hashes.size() * HASH_SIZE); + for (const auto& h : hashes) + { + h.serialise(bytes); + } + + write_file_atomically(tile_path(ref), bytes); + } + + /// @brief Reads a tile from disk. + /// @param ref The tile to read + /// @return The tile's hashes (TILE_WIDTH of them) + [[nodiscard]] std::vector read_tile(const TileRef& ref) const + { + std::vector bytes = read_file(tile_path(ref)); + const size_t expected = (size_t)TILE_WIDTH * HASH_SIZE; + if (bytes.size() != expected) + { + throw std::runtime_error("unexpected tile size"); + } + + std::vector hashes; + hashes.reserve(TILE_WIDTH); + size_t position = 0; + for (uint16_t i = 0; i < TILE_WIDTH; i++) + { + hashes.emplace_back(bytes, position); + } + return hashes; + } + + /// @brief Whether a full entry bundle exists on disk. + [[nodiscard]] bool has_entry_bundle(uint64_t index) const + { + std::error_code ec; + const auto path = entries_path(index); + if (!std::filesystem::is_regular_file(path, ec) || ec) + { + return false; + } + try + { + (void)decode_entries(read_file(path), TILE_WIDTH); + return true; + } + catch (const std::exception&) + { + return false; + } + } + + /// @brief Writes a full entry bundle to disk atomically. + /// @param index The bundle index + /// @param entries The raw log entries (exactly TILE_WIDTH of them) + /// @note Entries are stored in the tlog-tiles entry-bundle format: a + /// sequence of big-endian uint16 length-prefixed byte strings. + void write_entry_bundle( + uint64_t index, const std::vector>& entries) + { + if (entries.size() != TILE_WIDTH) + { + throw std::runtime_error("entry bundle width mismatch"); + } + write_file_atomically(entries_path(index), encode_entries(entries)); + } + + /// @brief Reads a full entry bundle from disk. + /// @param index The bundle index + /// @return The raw log entries (TILE_WIDTH of them) + [[nodiscard]] std::vector> read_entry_bundle( + uint64_t index) const + { + return decode_entries(read_file(entries_path(index)), TILE_WIDTH); + } + + /// @brief Encodes log entries into the tlog-tiles entry-bundle format. + static std::vector encode_entries( + const std::vector>& entries) + { + std::vector bytes; + for (const auto& e : entries) + { + if (e.size() > 0xFFFF) + { + throw std::runtime_error( + "entry too large for uint16 length prefix"); + } + bytes.push_back((uint8_t)((e.size() >> 8) & 0xFF)); + bytes.push_back((uint8_t)(e.size() & 0xFF)); + bytes.insert(bytes.end(), e.begin(), e.end()); + } + return bytes; + } + + /// @brief Decodes @p count entries from the entry-bundle format. + static std::vector> decode_entries( + const std::vector& bytes, size_t count) + { + std::vector> out; + out.reserve(count); + size_t pos = 0; + for (size_t i = 0; i < count; i++) + { + if (bytes.size() - pos < 2) + { + throw std::runtime_error("truncated entry bundle"); + } + const auto len = + (uint16_t)(((uint16_t)bytes[pos] << 8) | bytes[pos + 1]); + pos += 2; + if (len > bytes.size() - pos) + { + throw std::runtime_error("truncated entry bundle"); + } + out.emplace_back( + bytes.begin() + static_cast(pos), + bytes.begin() + static_cast(pos + len)); + pos += len; + } + if (pos != bytes.size()) + { + throw std::runtime_error("trailing bytes in entry bundle"); + } + return out; + } + + protected: + using DirectorySync = std::function; + + TileStoreT(std::filesystem::path prefix, DirectorySync directory_sync) : + prefix(std::move(prefix)), directory_sync(std::move(directory_sync)) + {} + + /// @brief The root directory of the store. + std::filesystem::path prefix; + + DirectorySync directory_sync; + std::set durable_directory_entries; + std::set durable_directory_contents; + + [[nodiscard]] bool confirm_full_tile(uint8_t level, uint64_t index) + { + const auto path = tile_path(TileRef{level, index}); + if (!has_full_tile(level, index)) + { + return false; + } + confirm_file_durable(path); + return true; + } + + [[nodiscard]] bool confirm_entry_bundle(uint64_t index) + { + const auto path = entries_path(index); + if (!has_entry_bundle(index)) + { + return false; + } + confirm_file_durable(path); + return true; + } + + void begin_write_attempt() + { + durable_directory_contents.clear(); + } + + /// @brief Reads an entire file into a byte vector. + static std::vector read_file(const std::filesystem::path& path) + { + std::ifstream f(path, std::ios::binary); + if (!f.good()) + { + throw std::runtime_error("cannot open file: " + path.string()); + } + return { + std::istreambuf_iterator(f), std::istreambuf_iterator()}; + } + + /// @brief Writes a file atomically via a synced temporary file. + /// @note Uses unique temp names, cleans them up on errors, and syncs the + /// file before publishing it with an atomic replace. POSIX builds also + /// confirm each directory entry in the path and sync the destination + /// directory after rename. Existing files are only reused after these + /// syncs are re-confirmed by the current write attempt. + void write_file_atomically( + const std::filesystem::path& path, const std::vector& bytes) + { + create_directories_durably(path.parent_path()); + + const std::filesystem::path tmp = temp_path(path); + TempFileGuard guard(tmp); + write_and_sync_file(tmp, bytes); + const auto parent = directory_or_dot(path.parent_path()); + durable_directory_contents.erase(parent); + replace_file(tmp, path); + sync_directory_contents(parent); + guard.dismiss(); + } + + static std::filesystem::path directory_or_dot( + const std::filesystem::path& directory) + { + return directory.empty() ? std::filesystem::path(".") : directory; + } + + void create_directories_durably( + const std::filesystem::path& requested_directory) + { + const auto directory = directory_or_dot(requested_directory); + if (durable_directory_entries.count(directory) != 0) + { + return; + } + + auto parent = directory_or_dot(directory.parent_path()); + if (parent != directory) + { + create_directories_durably(parent); + } + + std::error_code ec; + const bool exists = std::filesystem::exists(directory, ec); + if (ec) + { + throw std::runtime_error( + "cannot inspect directory " + directory.string() + ": " + + ec.message()); + } + if (exists) + { + const bool is_directory = + std::filesystem::is_directory(directory, ec); + if (ec) + { + throw std::runtime_error( + "cannot inspect directory " + directory.string() + ": " + + ec.message()); + } + if (!is_directory) + { + throw std::runtime_error( + "cannot create directory " + directory.string() + + ": path exists and is not a directory"); + } + } + else + { + const bool created = std::filesystem::create_directory(directory, ec); + if (ec) + { + throw std::runtime_error( + "cannot create directory " + directory.string() + ": " + + ec.message()); + } + if (!created && !std::filesystem::is_directory(directory, ec)) + { + throw std::runtime_error( + "cannot create directory " + directory.string() + + (ec ? ": " + ec.message() : "")); + } + } + + if (parent != directory) + { + sync_directory_for_durability(parent); + durable_directory_contents.insert(parent); + } + durable_directory_entries.insert(directory); + } + + void confirm_file_durable(const std::filesystem::path& path) + { + const auto parent = directory_or_dot(path.parent_path()); + create_directories_durably(parent); + sync_directory_contents(parent); + } + + void sync_directory_contents(const std::filesystem::path& directory) + { + const auto path = directory_or_dot(directory); + if (durable_directory_contents.count(path) != 0) + { + return; + } + sync_directory_for_durability(path); + durable_directory_contents.insert(path); + } + + class TempFileGuard + { + public: + explicit TempFileGuard(std::filesystem::path path) : + path(std::move(path)) + {} + + ~TempFileGuard() + { + if (active) + { + std::error_code ec; + std::filesystem::remove(path, ec); + } + } + + void dismiss() + { + active = false; + } + + private: + std::filesystem::path path; + bool active = true; + }; + + static std::filesystem::path temp_path(const std::filesystem::path& path) + { + static std::atomic counter{0}; + const auto stamp = + std::chrono::steady_clock::now().time_since_epoch().count(); + std::filesystem::path tmp = path; + tmp += ".tmp." + std::to_string(process_id()) + "." + + std::to_string((uint64_t)stamp) + "." + + std::to_string(counter.fetch_add(1, std::memory_order_relaxed)); + return tmp; + } + + static uint64_t process_id() + { +#ifdef _WIN32 + return (uint64_t)GetCurrentProcessId(); +#else + return (uint64_t)::getpid(); +#endif + } + + static std::string system_error_message(const std::string& what) + { +#ifdef _WIN32 + return what + ": error " + std::to_string(GetLastError()); +#else + return what + ": " + std::strerror(errno); +#endif + } + + static void require_write_progress( + size_t written, const std::filesystem::path& path) + { + if (written == 0) + { + throw std::runtime_error("short write: " + path.string()); + } + } + + static void write_and_sync_file( + const std::filesystem::path& path, const std::vector& bytes) + { +#ifdef _WIN32 + HANDLE handle = CreateFileW( + path.wstring().c_str(), + GENERIC_WRITE, + 0, + nullptr, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (handle == INVALID_HANDLE_VALUE) + { + throw std::runtime_error("cannot open file: " + path.string()); + } + bool close_handle = true; + try + { + size_t written = 0; + while (written < bytes.size()) + { + const auto remaining = bytes.size() - written; + const auto chunk = (DWORD)std::min( + remaining, (size_t)std::numeric_limits::max()); + DWORD done = 0; + if (!WriteFile( + handle, bytes.data() + written, chunk, &done, nullptr)) + { + throw std::runtime_error( + system_error_message("error writing file " + path.string())); + } + require_write_progress((size_t)done, path); + written += done; + } + if (!FlushFileBuffers(handle)) + { + throw std::runtime_error( + system_error_message("error syncing file " + path.string())); + } + close_handle = false; + if (!CloseHandle(handle)) + { + throw std::runtime_error( + system_error_message("error closing file " + path.string())); + } + } + catch (...) + { + if (close_handle) + { + CloseHandle(handle); + } + throw; + } +#else + int flags = O_WRONLY | O_CREAT | O_TRUNC; +# ifdef O_CLOEXEC + flags |= O_CLOEXEC; +# endif + int fd = ::open(path.c_str(), flags, 0666); + if (fd < 0) + { + throw std::runtime_error( + system_error_message("cannot open file " + path.string())); + } + try + { + size_t written = 0; + while (written < bytes.size()) + { + const ssize_t done = + ::write(fd, bytes.data() + written, bytes.size() - written); + if (done < 0) + { + if (errno == EINTR) + { + continue; + } + throw std::runtime_error( + system_error_message("error writing file " + path.string())); + } + require_write_progress((size_t)done, path); + written += (size_t)done; + } + if (::fsync(fd) != 0) + { + throw std::runtime_error( + system_error_message("error syncing file " + path.string())); + } + if (::close(fd) != 0) + { + fd = -1; + throw std::runtime_error( + system_error_message("error closing file " + path.string())); + } + fd = -1; + } + catch (...) + { + if (fd >= 0) + { + ::close(fd); + } + throw; + } +#endif + } + + static void replace_file( + const std::filesystem::path& tmp, const std::filesystem::path& path) + { +#ifdef _WIN32 + if (!MoveFileExW( + tmp.wstring().c_str(), + path.wstring().c_str(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) + { + throw std::runtime_error(system_error_message( + "cannot rename temp file " + tmp.string() + " to " + + path.string())); + } +#else + std::error_code ec; + std::filesystem::rename(tmp, path, ec); + if (ec) + { + throw std::runtime_error( + "cannot rename temp file " + tmp.string() + " to " + path.string() + + ": " + ec.message()); + } +#endif + } + + void sync_directory_for_durability(const std::filesystem::path& path) + { + if (directory_sync) + { + directory_sync(path); + return; + } + sync_directory_on_disk(path); + } + + static void sync_directory_on_disk(const std::filesystem::path& path) + { +#ifndef _WIN32 + int flags = O_RDONLY; +# ifdef O_DIRECTORY + flags |= O_DIRECTORY; +# endif +# ifdef O_CLOEXEC + flags |= O_CLOEXEC; +# endif + const int fd = ::open(path.c_str(), flags); + if (fd < 0) + { + throw std::runtime_error( + system_error_message("cannot open directory " + path.string())); + } + if (::fsync(fd) != 0) + { + const std::string message = + system_error_message("error syncing directory " + path.string()); + ::close(fd); + throw std::runtime_error(message); + } + if (::close(fd) != 0) + { + throw std::runtime_error( + system_error_message("error closing directory " + path.string())); + } +#else + (void)path; +#endif + } + }; + + /// @brief Computes the Merkle Tree Hash of a perfect (balanced) subtree. + /// @param leaves The subtree's leaves; the count MUST be a power of two. + /// @return The subtree root, computed with the tree's HASH_FUNCTION. + /// @note This is exactly a merkle::TreeT full-node hash, which is why tile + /// entries (such roots) are immutable: an unbalanced subtree would still + /// change as leaves are added and must therefore never be tiled. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + inline HashT perfect_root( + const std::vector>& leaves) + { + if (leaves.empty()) + { + throw std::runtime_error("perfect_root requires at least one leaf"); + } + if ((leaves.size() & (leaves.size() - 1)) != 0) + { + throw std::runtime_error( + "perfect_root requires a power-of-two number of leaves"); + } + + std::vector> level = leaves; + while (level.size() > 1) + { + std::vector> next; + next.reserve(level.size() / 2); + for (size_t i = 0; i + 1 < level.size(); i += 2) + { + HashT h; + HASH_FUNCTION(level[i], level[i + 1], h); + next.push_back(h); + } + level.swap(next); + } + return level.front(); + } + + /// @brief Computes and persists tlog-tiles tiles for a growing tree. + /// @tparam HASH_SIZE Size of each hash in bytes + /// @tparam HASH_FUNCTION The tree's node hash function + /// @note Only balanced subtrees are tiled: a level-L entry is the root of a + /// complete 2**(8L)-leaf subtree. Only full tiles (256 such entries) are + /// written; they are therefore immutable and written exactly once. Entries + /// beyond the last full-tile boundary remain in memory until a later flush + /// completes the next tile. + /// @warning No internal synchronization is provided. Callers must serialize + /// access to a writer and its store. + /// @warning A writer trusts existing full tiles as output from the same + /// tree and hash function. Callers resuming a store must establish that + /// ownership and restore the matching tree state. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TileWriterT + { + public: + /// @brief The type of hashes stored in tiles. + using Hash = HashT; + + /// @brief The associated tile store type. + using Store = TileStoreT; + + /// @brief Supplies the level-0 leaf hash for a given leaf index. + using LeafFn = std::function; + + /// @brief Counts of work performed by a write_up_to call. + struct Stats + { + /// @brief Number of full tiles written. + uint64_t full_written = 0; + }; + + /// @brief Constructs a writer over @p store. + explicit TileWriterT(Store& store) : store(store) {} + + /// @brief Writes all newly-complete full tiles for a tree of @p size + /// leaves. + /// @param size The current tree size + /// @param leaf_at Returns the level-0 leaf hash for a leaf index in + /// [0, size); only ever queried for leaves of complete subtrees. + /// @return Counts of tiles written + /// @note Incremental: full tiles already on disk are immutable and are + /// never rewritten once validated and confirmed durable. Malformed files + /// are replaced. Entries that do not complete a tile are not written. + /// Tiles are always rolled up at every level (0..63), so the on-disk set + /// always contains the higher-level roll-ups that proof generation relies + /// on. + Stats write_up_to(uint64_t size, const LeafFn& leaf_at) + { + Stats stats; + store.begin_write_attempt(); + + // tlog-tiles defines levels 0..63; the loop stops early once a level + // has no complete entries (see the entries == 0 break below). + for (uint8_t level = 0; level <= 63; level++) + { + // Number of complete (balanced) level-L entries available; this + // deliberately excludes the incomplete frontier subtree. + const uint64_t entries = entries_at_level(size, level); + if (entries == 0) + { + break; + } + ensure_level(level); + + const uint64_t full_tiles = entries / TILE_WIDTH; + + if (cursor_inited[level] == 0) + { + next_full[level] = full_prefix_length(level, full_tiles); + cursor_inited[level] = 1; + } + + for (uint64_t n = next_full[level]; n < full_tiles; n++) + { + if (store.confirm_full_tile(level, n)) + { + continue; // immutable: never rewrite an existing full tile + } + store.write_tile( + TileRef{level, n}, + collect(level, n * TILE_WIDTH, TILE_WIDTH, leaf_at)); + stats.full_written++; + } + if (full_tiles > next_full[level]) + { + next_full[level] = full_tiles; + } + } + + return stats; + } + + protected: + /// @brief The tile store written to. + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + Store& store; + + /// @brief Per-level index of the next full tile to write. + std::vector next_full; + + /// @brief Per-level flag indicating next_full has been initialised. + std::vector cursor_inited; + + /// @brief Number of complete level-@p level entries for a tree of @p + /// size. + static uint64_t entries_at_level(uint64_t size, uint8_t level) + { + const unsigned shift = 8U * (unsigned)level; + return shift >= 64 ? 0 : (size >> shift); + } + + /// @brief Ensures per-level bookkeeping vectors cover @p level. + void ensure_level(uint8_t level) + { + const size_t needed = (size_t)level + 1; + if (next_full.size() < needed) + { + next_full.resize(needed, 0); + cursor_inited.resize(needed, 0); + } + } + + /// @brief Length of the confirmed contiguous prefix, bounded by @p limit. + [[nodiscard]] uint64_t full_prefix_length(uint8_t level, uint64_t limit) + { + return detail::contiguous_prefix_length(limit, [&](uint64_t index) { + return store.confirm_full_tile(level, index); + }); + } + + /// @brief Collects @p count consecutive level-@p level entries, each the + /// root of a complete (balanced) subtree. + std::vector collect( + uint8_t level, + uint64_t first_entry, + uint64_t count, + const LeafFn& leaf_at) + { + std::vector out; + out.reserve(count); + for (uint64_t i = 0; i < count; i++) + { + const uint64_t g = first_entry + i; + if (level == 0) + { + out.push_back(leaf_at(g)); + } + else + { + // Roll up the complete child full tile (256 complete entries). + out.push_back( + perfect_root( + store.read_tile(TileRef{(uint8_t)(level - 1), g}))); + } + } + return out; + } + }; + + /// @brief Abstract source of Merkle subtree roots for proof generation. + /// @note Implementations resolve the root of a complete (balanced) subtree + /// from tiles, from an in-memory tree, or from a combination of the two. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + struct HashSourceT + { + /// @brief The type of hashes resolved. + using Hash = HashT; + + virtual ~HashSourceT() = default; + + /// @brief Resolves MTH(D[index << level : (index + 1) << level]). + /// @param level The subtree height (the subtree spans 2**level leaves) + /// @param index The subtree index at that height + /// @param out Set to the subtree root on success + /// @return Whether the complete, balanced subtree could be resolved + virtual bool subtree_root( + uint8_t level, uint64_t index, Hash& out) const = 0; + + /// @brief Resolves the level-0 leaf hash at @p index. + virtual bool leaf(uint64_t index, Hash& out) const + { + return subtree_root(0, index, out); + } + }; + + /// @brief Resolves subtree roots from tlog-tiles tile files. + /// @note @p available_size is rounded down to a whole number of full tiles: + /// only complete, durably-written full tiles are read. A complete subtree + /// within that full-tile prefix is resolvable; anything reaching into the + /// incomplete frontier yields false so that a proof builder can fall back + /// to another source (e.g. an in-memory tree). + /// @warning No internal synchronization is provided. Even const operations + /// update the internal LRU cache, so callers must serialize all access to a + /// shared source. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TileHashSourceT : public HashSourceT + { + public: + using Hash = HashT; + using Store = TileStoreT; + + /// @brief Constructs a source over @p store for trees up to + /// @p available_size leaves. @p available_size is rounded down to a whole + /// number of full tiles, since only full tiles are durable. + TileHashSourceT(const Store& store, uint64_t available_size) : + store(store), available_size((available_size / TILE_WIDTH) * TILE_WIDTH) + { + tile_cache.reserve(TILE_CACHE_SIZE); + } + + bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override + { + // The subtree covers leaves [index << level, (index + 1) << level). It + // is resolvable only when it lies entirely within the full-tile-covered + // prefix; the incomplete frontier is served from another source. + if (level >= 64 || index >= (available_size >> level)) + { + return false; + } + resolve(level, index, out); + return true; + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Store& store; + uint64_t available_size; // full-tile prefix length (a multiple of WIDTH) + + /// @brief Combines the @p span entries at @p off of @p tile into a root. + static Hash roll_up( + const std::vector& tile, uint64_t off, uint64_t span) + { + if (span == 1) + { + return tile.at(off); + } + return perfect_root(std::vector( + tile.begin() + (std::ptrdiff_t)off, + tile.begin() + (std::ptrdiff_t)(off + span))); + } + + /// @brief Resolves a complete subtree known to lie within the full-tile + /// prefix, reading the highest-level full tile that holds it (and rolling + /// up); descends to lower full tiles when a higher-level full tile has not + /// completed. Terminates because full level-0 tiles always cover the + /// prefix. + void resolve(uint8_t level, uint64_t index, Hash& out) const + { + if (level <= TILE_HEIGHT) + { + // Spans 2**level <= TILE_WIDTH leaves: held by one level-0 tile. + const uint64_t span = (uint64_t)1 << level; + const uint64_t start = index << level; + const std::vector tile = + read_tile(TileRef{0, start / TILE_WIDTH}); + out = roll_up(tile, start % TILE_WIDTH, span); + return; + } + + const uint8_t L = level / TILE_HEIGHT; + const uint8_t r = level % TILE_HEIGHT; + const uint64_t first = index << r; // first level-L entry + const uint64_t n = first / TILE_WIDTH; // level-L tile index + const unsigned full_shift = 8U * ((unsigned)L + 1U); + const uint64_t full_tiles = + full_shift >= 64 ? 0 : (available_size >> full_shift); + + if (n < full_tiles) + { + // One full level-L tile holds all 2**r entries of this subtree. + const std::vector tile = read_tile(TileRef{L, n}); + out = roll_up(tile, first % TILE_WIDTH, (uint64_t)1 << r); + return; + } + + // No full level-L tile here: split into two level-(level-1) subtrees. + Hash lo; + Hash hi; + resolve((uint8_t)(level - 1), index * 2, lo); + resolve((uint8_t)(level - 1), index * 2 + 1, hi); + HASH_FUNCTION(lo, hi, out); + } + + struct TileCacheEntry + { + TileRef ref; + std::vector hashes; + }; + + static constexpr size_t TILE_CACHE_SIZE = 64; + mutable std::vector tile_cache; + + std::vector read_tile(const TileRef& ref) const + { + for (auto it = tile_cache.begin(); it != tile_cache.end(); it++) + { + if (it->ref.level == ref.level && it->ref.index == ref.index) + { + TileCacheEntry entry = std::move(*it); + tile_cache.erase(it); + std::vector hashes = entry.hashes; + tile_cache.push_back(std::move(entry)); + return hashes; + } + } + + if (tile_cache.size() >= TILE_CACHE_SIZE) + { + tile_cache.erase(tile_cache.begin()); + } + tile_cache.push_back(TileCacheEntry{ref, store.read_tile(ref)}); + return tile_cache.back().hashes; + } + }; + + /// @brief Builds and verifies inclusion and consistency proofs. + /// @note Proofs are assembled from a HashSource using the tree's + /// HASH_FUNCTION, so an inclusion proof is byte-identical to the one + /// produced by merkle::TreeT::path()/past_path() and verifies with + /// PathT::verify(). + /// @warning Thread safety is inherited from the supplied HashSource. Callers + /// must serialize operations when the source is shared. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class ProofEngineT + { + public: + using Hash = HashT; + using Path = PathT; + using Source = HashSourceT; + + explicit ProofEngineT(const Source& source) : source(source) {} + + /// @brief The Merkle root of a tree of @p size leaves. + Hash root(uint64_t size) const + { + if (size == 0) + { + throw std::runtime_error("empty tree has no root"); + } + Hash out; + if (!mth_range(0, size, out)) + { + throw std::runtime_error("unresolved subtree while computing root"); + } + return out; + } + + /// @brief Inclusion proof for leaf @p index in a tree of @p size leaves. + /// @note Equivalent to TreeT::path(index) when size == num_leaves(), and + /// to TreeT::past_path(index, size - 1) otherwise. + std::shared_ptr inclusion_proof(uint64_t index, uint64_t size) const + { + if (index >= size) + { + throw std::runtime_error("leaf index out of bounds"); + } + + std::list elements; // leaf -> root order + uint64_t lo = 0; + uint64_t hi = size; + while (hi - lo > 1) + { + const uint64_t k = largest_pow2_lt(hi - lo); + typename Path::Element e; + if (index - lo < k) + { + if (!mth_range(lo + k, hi, e.hash)) + { + throw std::runtime_error("unresolved subtree in inclusion proof"); + } + e.direction = Path::PATH_RIGHT; + hi = lo + k; + } + else + { + if (!mth_range(lo, lo + k, e.hash)) + { + throw std::runtime_error("unresolved subtree in inclusion proof"); + } + e.direction = Path::PATH_LEFT; + lo = lo + k; + } + elements.push_front(std::move(e)); + } + + Hash leaf; + if (!source.leaf(index, leaf)) + { + throw std::runtime_error("unresolved leaf in inclusion proof"); + } + return std::make_shared( + leaf, index, std::move(elements), size - 1); + } + + /// @brief Consistency proof that a tree of @p m leaves is a prefix of a + /// tree of @p n leaves (RFC 6962). + std::vector consistency_proof(uint64_t m, uint64_t n) const + { + if (m == 0 || m > n) + { + throw std::runtime_error("invalid consistency proof sizes"); + } + std::vector proof; + if (m == n) + { + return proof; + } + subproof(m, 0, n, true, proof); + return proof; + } + + /// @brief Consistency proof between the trees whose last leaves are at + /// indices @p first_index and @p second_index (first_index <= + /// second_index). + /// @note Equivalent to consistency_proof(first_index + 1, + /// second_index + 1): it proves the tree of the first first_index + 1 + /// leaves is a prefix of the tree of the first second_index + 1 leaves. + std::vector consistency_proof_from_indices( + uint64_t first_index, uint64_t second_index) const + { + return consistency_proof(first_index + 1, second_index + 1); + } + + /// @brief Verifies an RFC 6962 consistency proof reconciling the roots of + /// trees of @p m and @p n leaves. + static bool verify_consistency( + uint64_t m, + uint64_t n, + const Hash& first_hash, + const Hash& second_hash, + std::vector proof) + { + if (m > n) + { + return false; + } + if (m == n) + { + return proof.empty() && first_hash == second_hash; + } + if (m == 0) + { + return proof.empty(); + } + + if (is_pow2(m)) + { + proof.insert(proof.begin(), first_hash); + } + if (proof.empty()) + { + return false; + } + + uint64_t fn = m - 1; + uint64_t sn = n - 1; + while ((fn & 1) != 0) + { + fn >>= 1; + sn >>= 1; + } + + Hash fr = proof[0]; + Hash sr = proof[0]; + for (size_t i = 1; i < proof.size(); i++) + { + if (sn == 0) + { + return false; + } + const Hash& c = proof[i]; + if ((fn & 1) != 0 || fn == sn) + { + HASH_FUNCTION(c, fr, fr); + HASH_FUNCTION(c, sr, sr); + if ((fn & 1) == 0) + { + while ((fn & 1) == 0 && fn != 0) + { + fn >>= 1; + sn >>= 1; + } + } + } + else + { + HASH_FUNCTION(sr, c, sr); + } + fn >>= 1; + sn >>= 1; + } + + return fr == first_hash && sr == second_hash && sn == 0; + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Source& source; + + static bool is_pow2(uint64_t n) + { + return n != 0 && (n & (n - 1)) == 0; + } + + static uint64_t largest_pow2_lt(uint64_t n) + { + uint64_t k = 1; + while (k <= (n - 1) / 2) + { + k <<= 1; + } + return k; + } + + static uint8_t log2_exact(uint64_t n) + { + uint8_t r = 0; + while (n > 1) + { + n >>= 1; + r++; + } + return r; + } + + /// @brief MTH(D[a:b]) via the source; falls back to splitting when a + /// perfect subtree cannot be resolved directly. + bool mth_range(uint64_t a, uint64_t b, Hash& out) const + { + const uint64_t w = b - a; + if (w == 0) + { + return false; + } + if (w == 1) + { + return source.leaf(a, out); + } + if (is_pow2(w) && (a % w == 0)) + { + if (source.subtree_root(log2_exact(w), a / w, out)) + { + return true; + } + } + const uint64_t k = largest_pow2_lt(w); + Hash left; + Hash right; + if (!mth_range(a, a + k, left) || !mth_range(a + k, b, right)) + { + return false; + } + HASH_FUNCTION(left, right, out); + return true; + } + + void subproof( + uint64_t m, + uint64_t lo, + uint64_t hi, + bool complete, + std::vector& proof) const + { + if (m == hi - lo) + { + if (!complete) + { + Hash h; + if (!mth_range(lo, hi, h)) + { + throw std::runtime_error( + "unresolved subtree in consistency proof"); + } + proof.push_back(h); + } + return; + } + const uint64_t k = largest_pow2_lt(hi - lo); + Hash h; + if (m <= k) + { + subproof(m, lo, lo + k, complete, proof); + if (!mth_range(lo + k, hi, h)) + { + throw std::runtime_error("unresolved subtree in consistency proof"); + } + } + else + { + subproof(m - k, lo + k, hi, false, proof); + if (!mth_range(lo, lo + k, h)) + { + throw std::runtime_error("unresolved subtree in consistency proof"); + } + } + proof.push_back(h); + } + }; + + /// @brief Resolves subtree roots from an in-memory merkle::TreeT. + /// @note Resolves only complete subtrees that are fully resident (not + /// flushed), returning false otherwise so that a builder can fall back to + /// another source. Performs no hashing changes (see TreeT::subtree_root). + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class MemoryHashSourceT : public HashSourceT + { + public: + using Hash = HashT; + using Tree = TreeT; + + explicit MemoryHashSourceT(Tree& tree) : tree(tree) {} + + bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override + { + return tree.subtree_root(level, (size_t)index, out); + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + Tree& tree; + }; + + /// @brief Resolves subtree roots from a primary source, falling back to a + /// secondary source. + /// @note Used to combine an in-memory tree (primary: no I/O, serves the + /// resident frontier) with tile files (secondary: serve the flushed past). + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class CombinedHashSourceT : public HashSourceT + { + public: + using Hash = HashT; + using Source = HashSourceT; + + CombinedHashSourceT(const Source& primary, const Source& secondary) : + primary(primary), secondary(secondary) + {} + + bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override + { + return primary.subtree_root(level, index, out) || + secondary.subtree_root(level, index, out); + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Source& primary; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Source& secondary; + }; + + /// @brief A merkle tree backed by tlog-tiles storage. + /// @note Appends grow an in-memory tree; flush() durably writes only full + /// (balanced) tiles, so the incomplete frontier is never tiled and stays + /// resident in memory. Compaction (dropping from memory the leaves already + /// covered by a full tile) is optional: enable it per flush with + /// Config::compact_on_flush, or call compact() explicitly; it never drops + /// the un-tiled frontier. Proofs are served from the combination of the + /// resident tree (frontier) and the full tiles (compacted past). + /// @note TiledTree creates a new tiled tree and cannot reopen one from tile + /// files alone. Construction rejects a non-empty tile namespace because + /// the files do not identify their tree or record enough state to restore + /// it. Use TileWriter directly only when the caller owns and restores that + /// state. + /// @warning No internal synchronization is provided. Callers must serialize + /// all access to a shared tree, including proof operations. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TiledTreeT + { + public: + using Hash = HashT; + using Tree = TreeT; + using Path = PathT; + using Store = TileStoreT; + using Writer = TileWriterT; + using Stats = typename Writer::Stats; + + /// @brief Configuration for a tiled tree. + struct Config + { + /// @brief Root directory for a new tiled tree. + /// @note The directory itself may exist, but its tile subdirectory must + /// be absent or empty. + std::filesystem::path prefix; + + /// @brief Number of most-recent leaves to keep resident when + /// compacting (i.e. never dropped from memory). + /// @note Compaction may retain one additional tiled boundary leaf so + /// rollback to exactly immutable_size() remains possible. + uint64_t retention_margin = 0; + + /// @brief If set, flush() compacts after writing tiles, dropping + /// from memory the leaves already covered by a full tile. Off by + /// default: tiles are written but the tree keeps every leaf resident. + bool compact_on_flush = false; + }; + + explicit TiledTreeT(Config config) : + config(std::move(config)), store(this->config.prefix), writer(store) + { + require_empty_tile_namespace(); + } + + TiledTreeT(const TiledTreeT&) = delete; + TiledTreeT& operator=(const TiledTreeT&) = delete; + + /// @brief Moves a tiled tree, rebinding its writer to the moved store. + TiledTreeT(TiledTreeT&& other) noexcept : + config(std::move(other.config)), + store(std::move(other.store)), + writer(store), + tree(std::move(other.tree)), + tiles_size(std::exchange(other.tiles_size, 0)), + sealed_size(std::exchange(other.sealed_size, 0)) + {} + + TiledTreeT& operator=(TiledTreeT&&) = delete; + + /// @brief Appends a leaf hash. + void append(const Hash& leaf_hash) + { + tree.insert(leaf_hash); + } + + /// @brief The number of leaves (including flushed ones). + [[nodiscard]] uint64_t size() const + { + return tree.num_leaves(); + } + + /// @brief The current Merkle root. + Hash root() + { + return tree.root(); + } + + /// @brief The number of leaves covered by the last fully successful flush. + /// @note This is always a multiple of TILE_WIDTH. It advances only after + /// every required tile level has been written successfully, and controls + /// proof reads and compaction. + [[nodiscard]] uint64_t flushed_size() const + { + return tiles_size; + } + + /// @brief The rollback seal for ranges a flush may have published. + /// @note A flush seals its full-tile boundary before writing. If the write + /// fails, this may exceed flushed_size(); keep the same tree contents and + /// retry the flush. + [[nodiscard]] uint64_t immutable_size() const + { + return sealed_size; + } + + /// @brief Access to the underlying tree. + /// @warning Mutating the tree directly bypasses tiled-tree bookkeeping. + /// In particular, direct retraction can make flushed_size() and + /// immutable_size() exceed size() and can make flushed_size() regress. + /// Use TiledTreeT operations whenever they are available. + Tree& tree_ref() + { + return tree; + } + + /// @brief Access to the underlying tile store. + /// @warning Files written or changed through this reference are trusted + /// by later flushes without checking that their hashes match this tree. + /// Mismatched files can silently invalidate proofs after compaction. + Store& store_ref() + { + return store; + } + + /// @brief Writes newly-complete full tiles to disk; compacts only if + /// Config::compact_on_flush is set. + /// @return Counts of the full tiles written by this flush + /// @note The full-tile boundary is made immutable before any tile write. + /// Only after every required tile level succeeds does flushed_size() + /// advance to that boundary. On failure, immutable_size() may advance + /// while flushed_size() does not; the tree remains resident and the flush + /// can be retried without rewriting finalized tiles. + Stats flush() + { + Stats stats; + const uint64_t n = tree.num_leaves(); + if (n == 0) + { + return stats; + } + + const uint64_t covered = (n / TILE_WIDTH) * TILE_WIDTH; + if (covered > sealed_size) + { + sealed_size = covered; + } + + stats = writer.write_up_to(n, [this](uint64_t i) -> const Hash& { + return tree.leaf((size_t)i); + }); + tiles_size = covered; + + if (config.compact_on_flush) + { + compact(); + } + return stats; + } + + /// @brief Drops old leaves covered by durably-written full tiles, keeping + /// retention_margin recent leaves and one tiled boundary leaf. + /// @return The new minimum (smallest still-resident) leaf index + /// @note Only leaves covered by a full tile are dropped, so the un-tiled + /// frontier is always retained in memory and inclusion/consistency proofs + /// remain available (the past from tiles, the frontier from memory). The + /// leaf at flushed_size() - 1 also remains resident so retract_to() can + /// represent a tree whose size is exactly immutable_size(). Has no effect + /// until tiling has produced full tiles. + uint64_t compact() + { + const uint64_t covered = (tiles_size / TILE_WIDTH) * TILE_WIDTH; + uint64_t target = covered > config.retention_margin ? + covered - config.retention_margin : + 0; + target = (target / TILE_WIDTH) * TILE_WIDTH; + // TreeT cannot retract below min_index(). Keep the final tiled leaf + // resident so rollback to a size of exactly immutable_size() remains + // representable after compaction. + if (covered > 0 && target == covered) + { + target--; + } + if (target > tree.min_index()) + { + tree.flush_to((size_t)target); + } + return tree.min_index(); + } + + /// @brief Rolls the tree back so that @p index becomes the last leaf, + /// removing all leaves after it (same semantics as TreeT::retract_to). + /// @note Only full tiles are immutable: this throws if the resulting size + /// would be smaller than immutable_size(). A failed flush may advance + /// immutable_size() without advancing flushed_size(). + void retract_to(size_t index) + { + if ((uint64_t)index + 1 < sealed_size) + { + throw std::runtime_error( + "TiledTree::retract_to: cannot roll back entries sealed for " + "immutable tiles (resulting size < immutable size)"); + } + tree.retract_to(index); + } + + /// @brief Inclusion proof for @p index in a tree of @p proof_size leaves. + /// @note Served from tiles (flushed past) combined with the resident tree + /// (recent frontier); @p proof_size may exceed flushed_size(). + std::shared_ptr inclusion_proof(uint64_t index, uint64_t proof_size) + { + return with_engine([&](const auto& engine) { + return engine.inclusion_proof(index, proof_size); + }); + } + + /// @brief Consistency proof between tree sizes @p m and @p n. + std::vector consistency_proof(uint64_t m, uint64_t n) + { + return with_engine( + [&](const auto& engine) { return engine.consistency_proof(m, n); }); + } + + /// @brief Consistency proof between the trees whose last leaves are at + /// indices @p first_index and @p second_index (first_index <= + /// second_index). + /// @note Equivalent to consistency_proof(first_index + 1, + /// second_index + 1). + std::vector consistency_proof_from_indices( + uint64_t first_index, uint64_t second_index) + { + return consistency_proof(first_index + 1, second_index + 1); + } + + protected: + Config config; + Store store; + Writer writer; + Tree tree; + uint64_t tiles_size = 0; + uint64_t sealed_size = 0; + + void require_empty_tile_namespace() const + { + const auto tile_root = store.root() / "tile"; + std::error_code ec; + const bool exists = std::filesystem::exists(tile_root, ec); + if (ec) + { + throw std::runtime_error( + "TiledTree: cannot inspect tile namespace " + tile_root.string() + + ": " + ec.message()); + } + if (!exists) + { + return; + } + + const bool is_directory = std::filesystem::is_directory(tile_root, ec); + if (ec) + { + throw std::runtime_error( + "TiledTree: cannot inspect tile namespace " + tile_root.string() + + ": " + ec.message()); + } + const bool is_empty = + is_directory && std::filesystem::is_empty(tile_root, ec); + if (ec) + { + throw std::runtime_error( + "TiledTree: cannot inspect tile namespace " + tile_root.string() + + ": " + ec.message()); + } + if (!is_empty) + { + throw std::runtime_error( + "TiledTree: tile namespace is not empty; reopening an existing " + "tiled tree is not supported"); + } + } + + /// @brief Builds a proof engine over the combined resident-tree + /// (frontier) and full-tile (flushed past) source, and invokes @p fn with + /// it. + /// @note The sources and engine are stack-local; @p fn must consume the + /// engine before returning (proofs are returned by value, holding hash + /// copies, so the result outlives the engine). + template + auto with_engine(Fn fn) + { + MemoryHashSourceT mem(tree); + TileHashSourceT tile_src(store, tiles_size); + CombinedHashSourceT combined(mem, tile_src); + ProofEngineT engine(combined); + return fn(engine); + } + }; + + /// @brief Writes tlog-tiles entry bundles (raw log entries) for a growing + /// log. + /// @note Entry bundles are level-0 only and application-owned: merklecpp + /// stores leaf hashes, while the raw entries (and the leaf-hash derivation + /// linking each entry to its level-0 tile hash) are the application's + /// responsibility. Only full bundles (TILE_WIDTH entries) are written; they + /// are immutable and written exactly once. The incomplete tail stays with + /// the application until it grows into a full bundle, mirroring the + /// un-tiled Merkle frontier. + /// @warning No internal synchronization is provided. Callers must serialize + /// access to a writer and its store. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class EntryBundleWriterT + { + public: + using Store = TileStoreT; + + /// @brief Supplies the raw bytes of the log entry at a given index. + using EntryFn = std::function(uint64_t)>; + + /// @brief Counts of work performed by a write_up_to call. + struct Stats + { + /// @brief Number of full bundles written. + uint64_t full_written = 0; + }; + + explicit EntryBundleWriterT(Store& store) : store(store) {} + + /// @brief Writes all newly-complete full bundles for a log of @p size + /// entries. + /// @param size The current number of entries + /// @param entry_at Returns the raw bytes of the entry at an index in + /// [0, size); only ever queried for entries of complete bundles. + /// @return Counts of bundles written + /// @note Incremental: full bundles already on disk are immutable and are + /// never rewritten once validated and confirmed durable. Malformed files + /// are replaced. The incomplete tail is never bundled. + Stats write_up_to(uint64_t size, const EntryFn& entry_at) + { + Stats stats; + store.begin_write_attempt(); + const uint64_t full = size / TILE_WIDTH; + + if (!cursor_inited) + { + next_full = full_prefix_length(full); + cursor_inited = true; + } + + for (uint64_t n = next_full; n < full; n++) + { + if (store.confirm_entry_bundle(n)) + { + continue; // immutable: never rewrite an existing full bundle + } + store.write_entry_bundle( + n, collect(n * TILE_WIDTH, TILE_WIDTH, entry_at)); + stats.full_written++; + } + if (full > next_full) + { + next_full = full; + } + return stats; + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + Store& store; + uint64_t next_full = 0; + bool cursor_inited = false; + + std::vector> collect( + uint64_t first, uint64_t count, const EntryFn& entry_at) + { + std::vector> out; + out.reserve(count); + for (uint64_t i = 0; i < count; i++) + { + out.push_back(entry_at(first + i)); + } + return out; + } + + [[nodiscard]] uint64_t full_prefix_length(uint64_t limit) + { + return detail::contiguous_prefix_length(limit, [&](uint64_t index) { + return store.confirm_entry_bundle(index); + }); + } + }; + + /// @brief Default tile store (SHA256, default hash function). + using TileStore = TileStoreT<32, sha256_compress>; + + /// @brief Default tile writer (SHA256, default hash function). + using TileWriter = TileWriterT<32, sha256_compress>; + + /// @brief Default abstract hash source (SHA256, default hash function). + using HashSource = HashSourceT<32, sha256_compress>; + + /// @brief Default tile-backed hash source (SHA256, default hash function). + using TileHashSource = TileHashSourceT<32, sha256_compress>; + + /// @brief Default proof engine (SHA256, default hash function). + using ProofEngine = ProofEngineT<32, sha256_compress>; + + /// @brief Default in-memory hash source (SHA256, default hash function). + using MemoryHashSource = MemoryHashSourceT<32, sha256_compress>; + + /// @brief Default combined hash source (SHA256, default hash function). + using CombinedHashSource = CombinedHashSourceT<32, sha256_compress>; + + /// @brief Default tiled tree (SHA256, default hash function). + using TiledTree = TiledTreeT<32, sha256_compress>; + + /// @brief Default entry-bundle writer (SHA256, default hash function). + using EntryBundleWriter = EntryBundleWriterT<32, sha256_compress>; + +#ifdef HAVE_OPENSSL + /// @brief SHA384 tile store. + using TileStore384 = TileStoreT<48, sha384_openssl>; + + /// @brief SHA512 tile store. + using TileStore512 = TileStoreT<64, sha512_openssl>; + + /// @brief SHA384 tile writer. + using TileWriter384 = TileWriterT<48, sha384_openssl>; + + /// @brief SHA512 tile writer. + using TileWriter512 = TileWriterT<64, sha512_openssl>; + + /// @brief SHA384 hash source, tile-backed source and proof engine. + using HashSource384 = HashSourceT<48, sha384_openssl>; + using TileHashSource384 = TileHashSourceT<48, sha384_openssl>; + using ProofEngine384 = ProofEngineT<48, sha384_openssl>; + + /// @brief SHA512 hash source, tile-backed source and proof engine. + using HashSource512 = HashSourceT<64, sha512_openssl>; + using TileHashSource512 = TileHashSourceT<64, sha512_openssl>; + using ProofEngine512 = ProofEngineT<64, sha512_openssl>; + + /// @brief SHA384 memory/combined sources and tiled tree. + using MemoryHashSource384 = MemoryHashSourceT<48, sha384_openssl>; + using CombinedHashSource384 = CombinedHashSourceT<48, sha384_openssl>; + using TiledTree384 = TiledTreeT<48, sha384_openssl>; + + /// @brief SHA512 memory/combined sources and tiled tree. + using MemoryHashSource512 = MemoryHashSourceT<64, sha512_openssl>; + using CombinedHashSource512 = CombinedHashSourceT<64, sha512_openssl>; + using TiledTree512 = TiledTreeT<64, sha512_openssl>; + + /// @brief SHA384/512 entry-bundle writers. + using EntryBundleWriter384 = EntryBundleWriterT<48, sha384_openssl>; + using EntryBundleWriter512 = EntryBundleWriterT<64, sha512_openssl>; +#endif + } +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e90a87c..155a4e1 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -29,6 +29,23 @@ add_merklecpp_test(serialisation serialisation.cpp) add_merklecpp_test(partial_serialisation partial_serialisation.cpp) add_merklecpp_test(serialise_to_file serialise_to_file.cpp) add_merklecpp_test(coverage coverage.cpp) +add_merklecpp_test(tiles_store tiles_store.cpp) +add_merklecpp_test(tiles_writer tiles_writer.cpp) +add_merklecpp_test(tiles_proofs tiles_proofs.cpp) +add_merklecpp_test(tiles_tree tiles_tree.cpp) +add_merklecpp_test(tiles_entries tiles_entries.cpp) + +if(OPENSSL) + add_merklecpp_test(tiles_hashes tiles_hashes.cpp) +endif() + +if(LONG_TESTS) + add_merklecpp_test(tiles_level2 tiles_level2.cpp) + set_tests_properties( + ${MERKLECPP_TEST_PREFIX}tiles_level2 + PROPERTIES TIMEOUT 900 + ) +endif() if(TARGET evercrypt.host) add_merklecpp_test(compare_evercrypt compare_evercrypt.cpp) diff --git a/test/tiles_entries.cpp b/test/tiles_entries.cpp new file mode 100644 index 0000000..e48ae37 --- /dev/null +++ b/test/tiles_entries.cpp @@ -0,0 +1,225 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::EntryBundleWriter; +using merkle::tiles::TileRef; +using merkle::tiles::TileStore; +using merkle::tiles::TileWriter; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +static void overwrite_file(const fs::path& p, const std::vector& bytes) +{ + std::ofstream f(p, std::ios::binary | std::ios::trunc); + f.write( + reinterpret_cast(bytes.data()), (std::streamsize)bytes.size()); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + + const fs::path base = fs::temp_directory_path() / + ("merklecpp_tiles_entries_" + std::to_string((unsigned long long)seed) + + "_" + std::to_string(std::rand())); + + try + { + // 1. Encode/decode round-trip with variable-length entries. + { + const std::vector> entries = { + {}, {0x01}, {0x02, 0x03, 0x04}, std::vector(300, 0xAB)}; + const auto enc = TileStore::encode_entries(entries); + // 4 length prefixes (8 bytes) + 0 + 1 + 3 + 300 payload bytes. + expect(enc.size() == 8 + 0 + 1 + 3 + 300, "encoded size"); + const auto dec = TileStore::decode_entries(enc, entries.size()); + expect(dec == entries, "encode/decode round-trip"); + + auto trailing = enc; + trailing.push_back(0xFF); + bool trailing_threw = false; + try + { + (void)TileStore::decode_entries(trailing, entries.size()); + } + catch (const std::exception&) + { + trailing_threw = true; + } + expect(trailing_threw, "decode rejects trailing bytes"); + } + + const auto hashes = make_hashes(70000); + const auto entry_at = [&](uint64_t i) -> std::vector { + return hashes[i]; // HashT -> vector + }; + + // 2. size 256: a single full bundle, no tail. + { + TileStore store(base / "e256"); + EntryBundleWriter writer(store); + const auto s = writer.write_up_to(256, entry_at); + expect(s.full_written == 1, "256 counts"); + expect(store.has_entry_bundle(0), "256 full bundle"); + expect(!store.has_entry_bundle(1), "256 no second bundle"); + + const auto b0 = store.read_entry_bundle(0); + expect(b0.size() == 256, "256 bundle width"); + for (size_t i = 0; i < 256; i++) + { + expect(Hash(b0[i]) == hashes[i], "256 entry round-trip"); + } + } + + // 3. size 70000: 273 full bundles (the 112-entry tail is not bundled), + // plus tile linkage. + { + TileStore store(base / "e70k"); + EntryBundleWriter writer(store); + const auto s = writer.write_up_to(70000, entry_at); + expect(s.full_written == 273, "70000 counts"); + expect(store.has_entry_bundle(0), "70000 bundle 0"); + expect(store.has_entry_bundle(272), "70000 bundle 272"); + expect(!store.has_entry_bundle(273), "70000 no bundle 273"); + + // Level-0 tile entries (leaf hashes) correspond to the bundle entries + // under the identity leaf-hash used here. + TileWriter tw(store); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + tw.write_up_to(70000, leaf_at); + const auto tile0 = store.read_tile(TileRef{0, 0}); + const auto bundle0 = store.read_entry_bundle(0); + for (size_t i = 0; i < 256; i++) + { + expect(Hash(bundle0[i]) == tile0[i], "bundle<->tile linkage"); + } + } + + // 4. Incremental writes: full bundles are immutable, the tail is not + // bundled. + { + TileStore store(base / "einc"); + EntryBundleWriter writer(store); + expect(writer.write_up_to(256, entry_at).full_written == 1, "inc s1"); + const auto s2 = writer.write_up_to(256, entry_at); + expect(s2.full_written == 0, "inc immutable rerun"); + overwrite_file(store.entries_path(0), {0x00, 0x05, 0x01}); + expect(!store.has_entry_bundle(0), "inc corrupt bundle not durable"); + EntryBundleWriter resumed(store); + expect( + resumed.write_up_to(256, entry_at).full_written == 1, + "inc rewrite corrupt"); + expect(store.has_entry_bundle(0), "inc rewritten bundle durable"); + const auto s3 = writer.write_up_to(600, entry_at); + expect(s3.full_written == 1, "inc s3 full"); + expect(store.has_entry_bundle(1), "inc bundle 1"); + expect(!store.has_entry_bundle(2), "inc no tail bundle"); + } + + // 5. Resume repairs an interior malformed bundle even when later files are + // valid. + { + const fs::path dir = base / "eholes"; + { + TileStore store(dir); + EntryBundleWriter writer(store); + expect( + writer.write_up_to(2048, entry_at).full_written == 8, + "holes initial bundles"); + overwrite_file(store.entries_path(3), {0x00, 0x05, 0x01}); + expect(!store.has_entry_bundle(3), "holes interior bundle malformed"); + expect(store.has_entry_bundle(7), "holes later bundle remains valid"); + } + + TileStore store(dir); + EntryBundleWriter writer(store); + expect( + writer.write_up_to(2048, entry_at).full_written == 1, + "holes rewrites interior bundle"); + std::vector> expected; + expected.reserve(merkle::tiles::TILE_WIDTH); + constexpr uint64_t hole_begin = (uint64_t)merkle::tiles::TILE_WIDTH * 3; + constexpr uint64_t hole_end = (uint64_t)merkle::tiles::TILE_WIDTH * 4; + for (uint64_t i = hole_begin; i < hole_end; i++) + { + expected.push_back(entry_at(i)); + } + expect( + store.read_entry_bundle(3) == expected, + "holes repaired bundle contents"); + } + + // 6. Recovery examines only bundle indices relevant to the requested size. + { + const fs::path dir = base / "esparse"; + const std::vector> bundle( + merkle::tiles::TILE_WIDTH, {0x42}); + { + TileStore store(dir); + store.write_entry_bundle(0, bundle); + for (uint64_t index = 1;; index <<= 1) + { + store.write_entry_bundle(index, bundle); + if (index == (uint64_t{1} << 63)) + { + break; + } + } + } + + TileStore store(dir); + EntryBundleWriter writer(store); + expect( + writer.write_up_to(merkle::tiles::TILE_WIDTH, entry_at).full_written == + 0, + "sparse bounded recovery"); + expect( + store.has_entry_bundle(0), "sparse requested bundle remains valid"); + } + + std::cout << "tiles_entries: OK" << '\n'; + + std::error_code ec; + fs::remove_all(base, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + + return 0; +} diff --git a/test/tiles_hashes.cpp b/test/tiles_hashes.cpp new file mode 100644 index 0000000..4ac86a5 --- /dev/null +++ b/test/tiles_hashes.cpp @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +template < + size_t HASH_SIZE, + typename Tree, + typename TiledTree, + typename ProofEngine> +static void exercise_tiled_hash(const fs::path& prefix, const std::string& name) +{ + const auto hashes = make_hashesT(300); + Tree reference; + typename TiledTree::Config config; + config.prefix = prefix; + config.compact_on_flush = true; + TiledTree tree(config); + + for (const auto& hash : hashes) + { + reference.insert(hash); + tree.append(hash); + } + + expect(tree.flush().full_written == 1, name + " full tile written"); + expect(tree.flushed_size() == 256, name + " flushed size"); + expect(tree.root() == reference.root(), name + " root"); + expect( + fs::file_size(prefix / "tile" / "0" / "000") == + (uintmax_t)merkle::tiles::TILE_WIDTH * HASH_SIZE, + name + " tile byte size"); + + const auto inclusion = tree.inclusion_proof(0, tree.size()); + expect(inclusion->verify(reference.root()), name + " inclusion proof"); + + const auto consistency = tree.consistency_proof(256, tree.size()); + expect( + ProofEngine::verify_consistency( + 256, + tree.size(), + *reference.past_root(255), + reference.root(), + consistency), + name + " consistency proof"); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + + const fs::path base = fs::temp_directory_path() / + ("merklecpp_tiles_hashes_" + std::to_string((unsigned long long)seed) + + "_" + std::to_string(std::rand())); + + try + { + exercise_tiled_hash< + 48, + merkle::Tree384, + merkle::tiles::TiledTree384, + merkle::tiles::ProofEngine384>(base / "sha384", "SHA384"); + exercise_tiled_hash< + 64, + merkle::Tree512, + merkle::tiles::TiledTree512, + merkle::tiles::ProofEngine512>(base / "sha512", "SHA512"); + + std::cout << "tiles_hashes: OK" << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + } + catch (const std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + + return 0; +} diff --git a/test/tiles_level2.cpp b/test/tiles_level2.cpp new file mode 100644 index 0000000..e1ea541 --- /dev/null +++ b/test/tiles_level2.cpp @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// End-to-end coverage of the level-2 tile paths. A full level-2 tile requires +// 256^3 == 16,777,216 leaves, so this is a deliberately large test: it writes +// the tiles for that many leaves (~65k tile files) from a deterministic leaf +// source -- no in-memory tree is built -- and cross-checks the writer's +// level-by-level roll-up against TileHashSourceT::resolve (which reads the +// level-2 tile) and against the underlying leaves. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TILE_WIDTH; +using merkle::tiles::TileHashSource; +using merkle::tiles::TileRef; +using merkle::tiles::TileStore; +using merkle::tiles::TileWriter; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +// Roll up a perfect (power-of-two) set of hashes with the default combiner. +static Hash rollup(const std::vector& hashes) +{ + return merkle::tiles::perfect_root<32, merkle::sha256_compress>(hashes); +} + +int main() +{ + const fs::path dir = fs::temp_directory_path() / + ("merklecpp_tiles_level2_" + + std::to_string((unsigned long long)std::time(nullptr))); + + try + { + // 256^3 leaves == exactly one full level-2 tile. + const uint64_t n = + (uint64_t)TILE_WIDTH * (uint64_t)TILE_WIDTH * (uint64_t)TILE_WIDTH; + + TileStore store(dir); + TileWriter writer(store); + + // Deterministic leaf hash derived from the index (low 8 bytes); avoids + // materialising a 16.7M-entry vector. + Hash leaf; + const auto leaf_at = [&](uint64_t i) -> const Hash& { + leaf.zero(); + for (int b = 0; b < 8; b++) + { + leaf.bytes[b] = (uint8_t)(i >> (8 * b)); + } + return leaf; + }; + + const auto stats = writer.write_up_to(n, leaf_at); + + // 65536 full L0 tiles + 256 full L1 tiles + 1 full L2 tile. + expect( + stats.full_written == (uint64_t)TILE_WIDTH * TILE_WIDTH + TILE_WIDTH + 1, + "level2: tile counts"); + expect(store.has_full_tile(2, 0), "level2: L2 tile present"); + expect(!store.has_full_tile(2, 1), "level2: no second L2 tile"); + expect(!store.has_full_tile(3, 0), "level2: no L3 tile"); + + const auto l2 = store.read_tile(TileRef{2, 0}); + expect(l2.size() == TILE_WIDTH, "level2: L2 tile width"); + + const TileHashSource src(store, n); + + // Each level-2 entry j is the root of level-1 tile j, which rolls up + // level-0 tiles, which are the leaves verbatim. Cross-check the writer's + // roll-up, resolve's level-2 read, and the leaf chain on a sample of + // indices. + for (const uint64_t j : + {(uint64_t)0, (uint64_t)1, (uint64_t)200, (uint64_t)255}) + { + const auto l1j = store.read_tile(TileRef{1, j}); + expect(l2[j] == rollup(l1j), "level2: L2[j] == rollup(L1 tile j)"); + + // resolve reads the level-2 tile for the 2^16-leaf subtree at index j. + Hash r16; + expect( + src.subtree_root(16, j, r16), "level2: subtree_root(16,j) resolves"); + expect(r16 == l2[j], "level2: resolve(16,j) == L2[j]"); + + // Anchor to leaves: L1 tile j entry 0 == root of L0 tile (j*256), whose + // first entry is leaf (j * 65536). + const auto l0 = store.read_tile(TileRef{0, j * TILE_WIDTH}); + expect(l1j[0] == rollup(l0), "level2: L1[j][0] == rollup(L0 tile)"); + expect( + l0[0] == leaf_at(j * (uint64_t)TILE_WIDTH * TILE_WIDTH), + "level2: L0 entry == leaf"); + } + + // Intra-tile roll-up: the 2^17-leaf subtree at 0 hashes L2[0] and L2[1]. + Hash r17; + expect(src.subtree_root(17, 0, r17), "level2: subtree_root(17,0) resolves"); + expect( + r17 == rollup({l2[0], l2[1]}), "level2: resolve(17,0) == H(L2[0],L2[1])"); + + std::cout << "tiles_level2: OK" << '\n'; + + std::error_code ec; + fs::remove_all(dir, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + + return 0; +} diff --git a/test/tiles_proofs.cpp b/test/tiles_proofs.cpp new file mode 100644 index 0000000..1697dbd --- /dev/null +++ b/test/tiles_proofs.cpp @@ -0,0 +1,383 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::CombinedHashSource; +using merkle::tiles::MemoryHashSource; +using merkle::tiles::ProofEngine; +using merkle::tiles::TileHashSource; +using merkle::tiles::TileStore; +using merkle::tiles::TileWriter; + +class ProofEngineProbe : public ProofEngine +{ +public: + using ProofEngine::largest_pow2_lt; +}; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +// Validates the TreeT::subtree_root accessor via memory-only proofs: they must +// match the library exactly. +static void check_memory_source(uint64_t n, const std::vector& hashes) +{ + const std::string at = " @n=" + std::to_string(n); + + merkle::Tree tree; + for (uint64_t i = 0; i < n; i++) + { + tree.insert(hashes[i]); + } + const Hash root = tree.root(); + + const MemoryHashSource source(tree); + const ProofEngine engine(source); + + expect(engine.root(n) == root, "mem root" + at); + + std::vector indices; + if (n <= 16) + { + for (uint64_t i = 0; i < n; i++) + { + indices.push_back(i); + } + } + else + { + for (const uint64_t i : {(uint64_t)0, (uint64_t)1, n / 2, n - 1}) + { + indices.push_back(i); + } + } + + for (const uint64_t i : indices) + { + const auto p = engine.inclusion_proof(i, n); + expect( + *p == *tree.path(i), "mem inclusion==path i=" + std::to_string(i) + at); + expect(p->verify(root), "mem inclusion verify i=" + std::to_string(i) + at); + } + + std::vector> pairs; + if (n <= 16) + { + for (uint64_t m = 1; m < n; m++) + { + for (uint64_t k = m + 1; k <= n; k++) + { + pairs.emplace_back(m, k); + } + } + } + else + { + pairs = {{1, n}, {n / 2, n}, {n - 1, n}}; + } + + for (const auto& pr : pairs) + { + const uint64_t m = pr.first; + const uint64_t k = pr.second; + const Hash rm = engine.root(m); + const Hash rk = engine.root(k); + expect(rm == *tree.past_root(m - 1), "mem past_root m" + at); + + const auto pp = engine.inclusion_proof(m / 2, m); + expect( + *pp == *tree.past_path(m / 2, m - 1), "mem inclusion(m)==past_path" + at); + + const auto cp = engine.consistency_proof(m, k); + expect( + ProofEngine::verify_consistency(m, k, rm, rk, cp), + "mem consistency" + at); + } +} + +// Exercises tile-derived proofs for a tree of `n` leaves against the existing +// library (which acts as the oracle: proofs must be byte-identical). Full tiles +// serve the covered prefix and an in-memory tree serves the un-tiled frontier, +// exactly as TiledTree combines them. +static void check_size( + const fs::path& dir, uint64_t n, const std::vector& hashes) +{ + const std::string at = " @n=" + std::to_string(n); + + TileStore store(dir); + TileWriter writer(store); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + writer.write_up_to(n, leaf_at); + + // Oracle: a full, never-flushed tree with the same leaves. + merkle::Tree tree; + for (uint64_t i = 0; i < n; i++) + { + tree.insert(hashes[i]); + } + const Hash root = tree.root(); + + // Production-shaped source: full tiles serve the covered prefix, an in-memory + // tree serves the un-tiled frontier. Drop the tiled past from the frontier + // tree so proofs over it are genuinely served from the tiles. merklecpp keeps + // at least one resident leaf, so never flush the whole tree. + const uint64_t covered = (n / 256) * 256; // 256 == TILE_WIDTH + merkle::Tree frontier; + for (uint64_t i = 0; i < n; i++) + { + frontier.insert(hashes[i]); + } + uint64_t drop_to = covered; + if (n > 0 && drop_to >= n) + { + drop_to = n - 1; + } + if (drop_to > 0) + { + frontier.flush_to((size_t)drop_to); + } + const MemoryHashSource mem(frontier); + const TileHashSource tiles(store, covered); + const CombinedHashSource source(mem, tiles); + const ProofEngine engine(source); + + // Root recomputed from tiles equals the library root. + expect(engine.root(n) == root, "root" + at); + + // Indices to probe: all of them for small trees, else a spread. + std::vector indices; + if (n <= 16) + { + for (uint64_t i = 0; i < n; i++) + { + indices.push_back(i); + } + } + else + { + for (const uint64_t i : + {(uint64_t)0, (uint64_t)1, n / 3, n / 2, n - 2, n - 1}) + { + indices.push_back(i); + } + } + + // Inclusion proofs are identical to TreeT::path and verify. + for (const uint64_t i : indices) + { + if (i >= n) + { + continue; + } + const auto p = engine.inclusion_proof(i, n); + expect(*p == *tree.path(i), "inclusion==path i=" + std::to_string(i) + at); + expect(p->verify(root), "inclusion verify i=" + std::to_string(i) + at); + } + + // Consistency pairs: exhaustive for small trees, else a fixed spread. + std::vector> pairs; + if (n <= 16) + { + for (uint64_t m = 1; m < n; m++) + { + for (uint64_t k = m + 1; k <= n; k++) + { + pairs.emplace_back(m, k); + } + } + } + else + { + pairs = {{1, n}, {n / 2, n}, {n - 1, n}, {1, 2}}; + // Tile-boundary crossings. + if (n > 256) + { + pairs.emplace_back(256, n); + pairs.emplace_back(257, n); + } + } + + for (const auto& pr : pairs) + { + const uint64_t m = pr.first; + const uint64_t k = pr.second; + if (m == 0 || m >= k || k > n) + { + continue; + } + + const Hash rm = engine.root(m); + const Hash rk = engine.root(k); + expect(rm == *tree.past_root(m - 1), "past_root m" + at); + expect(rk == *tree.past_root(k - 1), "past_root k" + at); + + // Past inclusion proof matches TreeT::past_path. + const uint64_t i = m / 2; + const auto pp = engine.inclusion_proof(i, m); + expect( + *pp == *tree.past_path(i, m - 1), + "inclusion(m)==past_path i=" + std::to_string(i) + at); + expect(pp->verify(rm), "inclusion(m) verify" + at); + + // Consistency proof reconciles the two roots. + const auto cp = engine.consistency_proof(m, k); + expect( + ProofEngine::verify_consistency(m, k, rm, rk, cp), + "consistency " + std::to_string(m) + "->" + std::to_string(k) + at); + + // The index-based variant is consistency_proof(i+1, j+1). + expect( + engine.consistency_proof_from_indices(m - 1, k - 1) == cp, + "consistency index variant" + at); + + // Tampering with a proof element or a root is rejected. + auto bad = cp; + bad[0].bytes[0] ^= 0xFFU; + expect( + !ProofEngine::verify_consistency(m, k, rm, rk, bad), + "consistency tamper rejected" + at); + + Hash wrong = rk; + wrong.bytes[0] ^= 0xFFU; + expect( + !ProofEngine::verify_consistency(m, k, rm, wrong, cp), + "consistency wrong root rejected" + at); + } + + std::error_code ec; + fs::remove_all(dir, ec); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + + const fs::path base = fs::temp_directory_path() / + ("merklecpp_tiles_proofs_" + std::to_string((unsigned long long)seed) + + "_" + std::to_string(std::rand())); + + try + { + const auto hashes = make_hashes(300000); + + // ---- Memory-only proofs (exercises TreeT::subtree_root). + for (const uint64_t n : + {(uint64_t)1, + (uint64_t)2, + (uint64_t)3, + (uint64_t)5, + (uint64_t)8, + (uint64_t)13, + (uint64_t)16, + (uint64_t)256, + (uint64_t)257, + (uint64_t)1000}) + { + check_memory_source(n, hashes); + } + std::cout << "memory source: OK" << '\n'; + + // ---- Hostile arithmetic inputs are rejected without UB or overflow + // loops. + { + merkle::Tree tree; + tree.insert(hashes[0]); + Hash out; + expect(!tree.subtree_root(64, 0, out), "subtree_root rejects level 64"); + expect(!tree.subtree_root(100, 0, out), "subtree_root rejects level 100"); + expect( + !tree.subtree_root(1, std::numeric_limits::max(), out), + "subtree_root rejects overflowing index"); + + expect(ProofEngineProbe::largest_pow2_lt(2) == 1, "pow2_lt 2"); + expect( + ProofEngineProbe::largest_pow2_lt((uint64_t)1 << 63) == + ((uint64_t)1 << 62), + "pow2_lt 2^63"); + expect( + ProofEngineProbe::largest_pow2_lt(((uint64_t)1 << 63) + 1) == + ((uint64_t)1 << 63), + "pow2_lt 2^63+1"); + expect( + ProofEngineProbe::largest_pow2_lt( + std::numeric_limits::max()) == ((uint64_t)1 << 63), + "pow2_lt uint64 max"); + std::cout << "hostile arithmetic inputs: OK" << '\n'; + } + + for (const uint64_t n : + {(uint64_t)1, + (uint64_t)2, + (uint64_t)3, + (uint64_t)4, + (uint64_t)5, + (uint64_t)7, + (uint64_t)8, + (uint64_t)13, + (uint64_t)16, + (uint64_t)255, + (uint64_t)256, + (uint64_t)257, + (uint64_t)1000}) + { + check_size(base / ("n" + std::to_string(n)), n, hashes); + } + std::cout << "small/medium sizes: OK" << '\n'; + + // Large trees. 65536 == 256 full L0 tiles == one full L1 tile (exact L1 + // boundary); 65537 is one past it; 70000 exercises a full L1 tile plus an + // in-memory frontier; 300000 forces proofs over height->=16 subtrees, so + // TileHashSource::resolve descends through level-2 logic (full_shift = 24) + // before reaching the level-1 tiles -- the only coverage of the L>=2 path. + for (const uint64_t n : + {(uint64_t)65536, (uint64_t)65537, (uint64_t)70000, (uint64_t)300000}) + { + check_size(base / ("big" + std::to_string(n)), n, hashes); + std::cout << "size " << n << ": OK" << '\n'; + } + + std::cout << "tiles_proofs: OK" << '\n'; + + std::error_code ec; + fs::remove_all(base, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + + return 0; +} diff --git a/test/tiles_store.cpp b/test/tiles_store.cpp new file mode 100644 index 0000000..6d45d01 --- /dev/null +++ b/test/tiles_store.cpp @@ -0,0 +1,451 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TileRef; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +static void expect_eq( + const std::string& got, const std::string& expected, const std::string& what) +{ + if (got != expected) + { + throw std::runtime_error( + what + ": got '" + got + "', expected '" + expected + "'"); + } +} + +static std::string rel(const merkle::tiles::TileStore& store, const fs::path& p) +{ + return p.lexically_relative(store.root()).generic_string(); +} + +static bool any_tmp_files(const fs::path& dir) +{ + if (!fs::exists(dir)) + { + return false; + } + return std::any_of( + fs::recursive_directory_iterator(dir), + fs::recursive_directory_iterator(), + [](const auto& entry) { + return entry.path().filename().string().find(".tmp") != std::string::npos; + }); +} + +class TileStoreProbe : public merkle::tiles::TileStore +{ +public: + using Store = merkle::tiles::TileStore; + + static void check_write_progress(size_t written) + { + Store::require_write_progress(written, "test"); + } +}; + +struct SyncFault +{ + fs::path fail_path; + size_t failures_remaining = 0; + size_t matching_calls_before_failure = 0; + std::vector calls; + + void sync(const fs::path& path) + { + const auto normalised = path.lexically_normal(); + calls.push_back(normalised); + if (failures_remaining > 0 && normalised == fail_path.lexically_normal()) + { + if (matching_calls_before_failure > 0) + { + matching_calls_before_failure--; + return; + } + failures_remaining--; + throw std::runtime_error("injected directory sync failure"); + } + } + + [[nodiscard]] size_t call_count(const fs::path& path) const + { + const auto normalised = path.lexically_normal(); + return (size_t)std::count(calls.begin(), calls.end(), normalised); + } +}; + +class FaultInjectingTileStore : public merkle::tiles::TileStore +{ +public: + using Store = merkle::tiles::TileStore; + + FaultInjectingTileStore( + fs::path prefix, const std::shared_ptr& fault) : + Store( + std::move(prefix), [fault](const fs::path& path) { fault->sync(path); }) + {} +}; + +// Overwrites a file with exactly the given bytes (to simulate corruption). +static void overwrite_file(const fs::path& p, const std::vector& bytes) +{ + std::ofstream f(p, std::ios::binary | std::ios::trunc); + f.write( + reinterpret_cast(bytes.data()), (std::streamsize)bytes.size()); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + + const fs::path dir = fs::temp_directory_path() / + ("merklecpp_tiles_" + std::to_string((unsigned long long)seed) + "_" + + std::to_string(std::rand())); + + try + { + // 1. Tile index encoding vectors (tlog-tiles examples). + expect_eq(merkle::tiles::encode_tile_index(0), "000", "encode 0"); + expect_eq(merkle::tiles::encode_tile_index(5), "005", "encode 5"); + expect_eq(merkle::tiles::encode_tile_index(255), "255", "encode 255"); + expect_eq(merkle::tiles::encode_tile_index(999), "999", "encode 999"); + expect_eq( + merkle::tiles::encode_tile_index(1000), "x001/000", "encode 1000"); + expect_eq( + merkle::tiles::encode_tile_index(1234067), + "x001/x234/067", + "encode 1234067"); + + merkle::tiles::TileStore store(dir); + const size_t hsz = Hash().size(); + const auto full = make_hashes(merkle::tiles::TILE_WIDTH); + + // 2. Resource path layout (full tiles and bundles only). + expect_eq( + rel(store, store.tile_path(TileRef{0, 0})), + "tile/0/000", + "tile_path L0 N0"); + expect_eq( + rel(store, store.tile_path(TileRef{1, 1234067})), + "tile/1/x001/x234/067", + "tile_path L1 big index"); + expect_eq( + rel(store, store.entries_path(5)), "tile/entries/005", "entries full"); + + // 2b. Production writes sync each directory link and the destination + // directory in order. + { + const fs::path prefix = dir / "durable"; + const auto fault = std::make_shared(); + FaultInjectingTileStore durable_store(prefix, fault); + durable_store.write_tile(TileRef{1, 0}, full); + const std::vector expected = { + dir.parent_path(), dir, prefix, prefix / "tile", prefix / "tile" / "1"}; + expect( + fault->calls.size() >= expected.size() && + std::equal( + expected.begin(), + expected.end(), + fault->calls.end() - (std::ptrdiff_t)expected.size()), + "directory parents and destination synced in order"); + } + + // 2c. A successful write must make progress. This shared guard prevents + // the Windows WriteFile loop from spinning if it reports zero bytes. + { + bool zero_threw = false; + try + { + TileStoreProbe::check_write_progress(0); + } + catch (const std::exception&) + { + zero_threw = true; + } + expect(zero_threw, "zero-byte write rejected"); + TileStoreProbe::check_write_progress(1); + } + + // 2d. A failed destination-directory sync after rename is retried before + // another already-open writer trusts the visible tile. + { + const fs::path prefix = dir / "retry_destination"; + const fs::path destination = prefix / "tile" / "0"; + const auto fault = std::make_shared(); + fault->fail_path = destination; + constexpr uint64_t growing_size = (uint64_t)merkle::tiles::TILE_WIDTH * 2; + const auto growing = make_hashes((size_t)growing_size); + const auto leaf_at = [&](uint64_t i) -> const Hash& { + return growing[i]; + }; + + FaultInjectingTileStore retry_store(prefix, fault); + merkle::tiles::TileWriter retry_writer(retry_store); + expect( + retry_writer.write_up_to(merkle::tiles::TILE_WIDTH, leaf_at) + .full_written == 1, + "first writer publishes initial tile"); + + fault->matching_calls_before_failure = 1; + fault->failures_remaining = 1; + { + FaultInjectingTileStore failed_store(prefix, fault); + merkle::tiles::TileWriter writer(failed_store); + bool sync_threw = false; + try + { + (void)writer.write_up_to(growing_size, leaf_at); + } + catch (const std::exception&) + { + sync_threw = true; + } + expect(sync_threw, "destination sync failure propagated"); + expect( + failed_store.has_full_tile(0, 1), + "renamed tile remains visible after sync failure"); + } + + expect( + retry_writer.write_up_to(growing_size, leaf_at).full_written == 0, + "retry certifies visible tile without rewriting"); + expect( + fault->call_count(destination) == 4, + "destination directory sync retried"); + const std::vector second( + growing.begin() + (std::ptrdiff_t)merkle::tiles::TILE_WIDTH, + growing.end()); + expect( + retry_store.read_tile(TileRef{0, 1}) == second, + "retried tile remains intact"); + } + + // 2e. A failed ancestor-parent sync is retried even though the directory + // created before the failure remains visible. + { + const fs::path prefix = dir / "retry_ancestor"; + const auto fault = std::make_shared(); + fault->fail_path = prefix; + fault->failures_remaining = 1; + + { + FaultInjectingTileStore failed_store(prefix, fault); + merkle::tiles::TileWriter writer(failed_store); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return full[i]; }; + bool sync_threw = false; + try + { + (void)writer.write_up_to(merkle::tiles::TILE_WIDTH, leaf_at); + } + catch (const std::exception&) + { + sync_threw = true; + } + expect(sync_threw, "ancestor sync failure propagated"); + expect( + fs::is_directory(prefix / "tile"), + "directory remains visible after parent sync failure"); + expect( + !failed_store.has_full_tile(0, 0), + "tile not published before ancestor sync succeeds"); + } + + FaultInjectingTileStore retry_store(prefix, fault); + merkle::tiles::TileWriter retry_writer(retry_store); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return full[i]; }; + expect( + retry_writer.write_up_to(merkle::tiles::TILE_WIDTH, leaf_at) + .full_written == 1, + "retry writes tile after certifying existing ancestor"); + expect(fault->call_count(prefix) == 2, "ancestor parent sync retried"); + expect(retry_store.has_full_tile(0, 0), "retry publishes full tile"); + } + + // 3a. Full tile byte round-trip. + const TileRef full_ref{0, 0}; + store.write_tile(full_ref, full); + expect(store.has_full_tile(0, 0), "has_full_tile after write"); + expect(!store.has_full_tile(0, 5), "missing full tile"); + expect( + fs::file_size(store.tile_path(full_ref)) == + (uintmax_t)merkle::tiles::TILE_WIDTH * hsz, + "full tile file size"); + const auto full_rt = store.read_tile(full_ref); + expect(full_rt.size() == full.size(), "full tile width round-trip"); + for (size_t i = 0; i < full.size(); i++) + { + expect(full_rt[i] == full[i], "full tile hash round-trip"); + } + + // 3b. Wrong-width writes are rejected (only 256-wide tiles are valid). + const std::vector three(full.begin(), full.begin() + 3); + bool threw = false; + try + { + store.write_tile(TileRef{0, 2}, three); // 3 hashes for a full tile + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "width mismatch rejected"); + + // 3c. Corrupt / truncated files are rejected on read (integrity check), so + // a torn write can never be served as a valid tile or bundle. + { + const auto expect_throws = + [](const std::function& fn, const std::string& what) { + bool t = false; + try + { + fn(); + } + catch (const std::exception&) + { + t = true; + } + expect(t, what); + }; + + // Truncated tile: fewer bytes than a full tile. + overwrite_file(store.tile_path(full_ref), std::vector(hsz, 0)); + expect(!store.has_full_tile(0, 0), "truncated tile is not durable"); + expect_throws( + [&] { (void)store.read_tile(full_ref); }, "truncated tile rejected"); + store.write_tile(full_ref, full); + expect(store.has_full_tile(0, 0), "truncated tile rewritten"); + expect(store.read_tile(full_ref) == full, "rewritten tile round-trip"); + + // Oversized tile: more bytes than a full tile. + overwrite_file( + store.tile_path(full_ref), + std::vector((merkle::tiles::TILE_WIDTH + 1) * hsz, 0)); + expect(!store.has_full_tile(0, 0), "oversized tile is not durable"); + expect_throws( + [&] { (void)store.read_tile(full_ref); }, "oversized tile rejected"); + + // A valid full entry bundle whose file is then cut short. + std::vector> entries(merkle::tiles::TILE_WIDTH); + for (size_t i = 0; i < entries.size(); i++) + { + entries[i] = {(uint8_t)i, 0x7F}; + } + store.write_entry_bundle(0, entries); + expect(store.has_entry_bundle(0), "bundle durable before truncation"); + expect( + store.read_entry_bundle(0).size() == merkle::tiles::TILE_WIDTH, + "bundle valid before truncation"); + // Claims a 5-byte entry but supplies only one trailing byte. + overwrite_file(store.entries_path(0), {0x00, 0x05, 0x01}); + expect(!store.has_entry_bundle(0), "truncated bundle is not durable"); + expect_throws( + [&] { (void)store.read_entry_bundle(0); }, + "truncated entry bundle rejected"); + + // A syntactically complete bundle with trailing bytes is not durable. + auto encoded = merkle::tiles::TileStore::encode_entries(entries); + encoded.push_back(0xFF); + overwrite_file(store.entries_path(0), encoded); + expect(!store.has_entry_bundle(0), "trailing bundle is not durable"); + expect_throws( + [&] { (void)store.read_entry_bundle(0); }, + "trailing entry bundle rejected"); + + // decode_entries directly: a length prefix that overruns the buffer. + expect_throws( + [] { + (void)merkle::tiles::TileStore::decode_entries({0xFF, 0xFF, 0x00}, 1); + }, + "decode_entries oversized length rejected"); + expect_throws( + [] { + (void)merkle::tiles::TileStore::decode_entries({0x00, 0x00, 0xFF}, 1); + }, + "decode_entries trailing bytes rejected"); + } + +// Windows file replacement semantics can reject racing same-path replaces even +// when the final tile remains valid, so keep this stress check POSIX-only. +#ifndef _WIN32 + // 3d. Concurrent same-tile writes use unique temp files and leave no + // temporary files behind after success. Each thread owns its store object; + // sharing one object requires caller-provided synchronization. + { + const TileRef concurrent_ref{0, 42}; + std::atomic ok{true}; + std::vector threads; + for (size_t i = 0; i < 8; i++) + { + threads.emplace_back([&] { + try + { + merkle::tiles::TileStore thread_store(dir); + thread_store.write_tile(concurrent_ref, full); + } + catch (...) + { + ok = false; + } + }); + } + for (auto& thread : threads) + { + thread.join(); + } + expect(ok, "concurrent writes succeeded"); + expect(store.read_tile(concurrent_ref) == full, "concurrent tile valid"); + expect(!any_tmp_files(dir), "no temp files left after writes"); + } +#endif + + std::cout << "tiles_store: OK" << '\n'; + + std::error_code ec; + fs::remove_all(dir, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + + return 0; +} diff --git a/test/tiles_tree.cpp b/test/tiles_tree.cpp new file mode 100644 index 0000000..bed6c87 --- /dev/null +++ b/test/tiles_tree.cpp @@ -0,0 +1,667 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::ProofEngine; +using merkle::tiles::TiledTree; + +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_copy_assignable_v); +static_assert(std::is_nothrow_move_constructible_v); +static_assert(!std::is_move_assignable_v); + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + + const fs::path base = fs::temp_directory_path() / + ("merklecpp_tiles_tree_" + std::to_string((unsigned long long)seed) + "_" + + std::to_string(std::rand())); + + try + { + const auto hashes = make_hashes(1500); + + // ---- Part 0: the empty tree. flush()/compact() are no-ops and there is no + // root. + { + TiledTree::Config ecfg; + ecfg.prefix = base / "tt_empty"; + TiledTree ett(ecfg); + expect(ett.size() == 0, "empty: size 0"); + expect(ett.flush().full_written == 0, "empty: flush writes nothing"); + expect(ett.flushed_size() == 0, "empty: flushed size 0"); + expect(ett.immutable_size() == 0, "empty: immutable size 0"); + expect(ett.compact() == 0, "empty: compact no-op"); + bool ethrew = false; + try + { + (void)ett.root(); + } + catch (const std::exception&) + { + ethrew = true; + } + expect(ethrew, "empty: root throws"); + std::cout << "empty tree: OK" << '\n'; + } + + // ---- Part 0b: moving a tree rebinds its writer to the destination store. + { + TiledTree::Config move_cfg; + move_cfg.prefix = base / "tt_move_expected"; + TiledTree source(move_cfg); + for (uint64_t i = 0; i < 300; i++) + { + source.append(hashes[i]); + } + source.flush(); + + const fs::path work = base / "tt_move_cwd"; + fs::create_directories(work); + const fs::path previous_cwd = fs::current_path(); + fs::current_path(work); + try + { + TiledTree moved(std::move(source)); + expect(moved.flushed_size() == 256, "move: flushed size retained"); + expect(moved.immutable_size() == 256, "move: immutable size retained"); + expect( + moved.store_ref().root() == move_cfg.prefix, + "move: destination store retained"); + for (uint64_t i = 300; i < 512; i++) + { + moved.append(hashes[i]); + } + expect(moved.flush().full_written == 1, "move: next tile written"); + expect( + fs::is_regular_file(move_cfg.prefix / "tile/0/001"), + "move: tile written to configured store"); + expect( + !fs::exists(work / "tile/0/001"), + "move: no tile written relative to current directory"); + } + catch (...) + { + fs::current_path(previous_cwd); + throw; + } + fs::current_path(previous_cwd); + std::cout << "move construction: OK" << '\n'; + } + + // ---- Part 0c: a TiledTree is fresh-only. It accepts an existing root + // directory, but rejects a tile namespace that may belong to another + // tree rather than silently adopting its immutable files. + { + TiledTree::Config existing_cfg; + existing_cfg.prefix = base / "tt_existing"; + fs::create_directories(existing_cfg.prefix / "tile"); + { + TiledTree first(existing_cfg); + for (uint64_t i = 0; i < 256; i++) + { + first.append(hashes[i]); + } + expect( + first.flush().full_written == 1, + "existing prefix: first tree writes tile"); + } + + bool construction_threw = false; + try + { + const TiledTree second(existing_cfg); + } + catch (const std::exception&) + { + construction_threw = true; + } + expect( + construction_threw, + "existing prefix: second tree rejects existing tiles"); + std::cout << "existing tile namespace: OK" << '\n'; + } + + // ---- Part 2: TiledTree flush, proofs over tiles + memory. + const uint64_t n1 = 1000; // first flush size + const uint64_t N = 1500; // final size + + // Reference: a plain (never flushed) tree with the same leaves. + merkle::Tree ref; + for (uint64_t i = 0; i < N; i++) + { + ref.insert(hashes[i]); + } + const Hash ref_root = ref.root(); + + // Default mode: flush() writes tiles but drops nothing from memory; + // an explicit compact() drops only the leaves already covered by a tile. + { + TiledTree::Config dcfg; + dcfg.prefix = base / "tt_default"; + TiledTree dtt(dcfg); + for (uint64_t i = 0; i < n1; i++) + { + dtt.append(hashes[i]); + } + dtt.flush(); + expect(dtt.tree_ref().min_index() == 0, "default: nothing dropped"); + dtt.compact(); + expect( + dtt.tree_ref().min_index() == 767, "compact() keeps boundary leaf"); + } + + TiledTree::Config cfg; + cfg.prefix = base / "tt"; + cfg.retention_margin = 0; + cfg.compact_on_flush = true; + TiledTree tt(cfg); + + for (uint64_t i = 0; i < n1; i++) + { + tt.append(hashes[i]); + } + tt.flush(); // flushes full tiles; flushed_size = covered = 768 + expect(tt.flushed_size() == 768, "flushed size"); + expect(tt.tree_ref().min_index() == 767, "compacted with boundary overlap"); + + for (uint64_t i = n1; i < N; i++) + { + tt.append(hashes[i]); + } + expect(tt.size() == N, "size after appends"); + expect(tt.root() == ref_root, "tiled root == reference root"); + + // Indices that are: flushed (tiles only), in the flushed-but-resident + // overlap, and on the un-flushed resident frontier. + for (const uint64_t i : + {(uint64_t)0, + (uint64_t)767, + (uint64_t)800, + (uint64_t)999, + (uint64_t)1000, + (uint64_t)1499}) + { + const auto p = tt.inclusion_proof(i, N); + expect( + *p == *ref.path(i), "combined inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(ref_root), + "combined inclusion verify i=" + std::to_string(i)); + } + + // The resident tree alone cannot prove a flushed index. + bool threw = false; + try + { + (void)tt.tree_ref().path(0); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "memory-only path throws for flushed index"); + + // Consistency across the flush boundary: flushed size -> current size, + // and a flushed-era size -> current size. + { + const auto cp = tt.consistency_proof(n1, N); + expect( + ProofEngine::verify_consistency( + n1, N, *ref.past_root(n1 - 1), ref_root, cp), + "consistency n1->N across flush"); + + const uint64_t m = 500; // flushed era + const auto cp2 = tt.consistency_proof(m, N); + expect( + ProofEngine::verify_consistency( + m, N, *ref.past_root(m - 1), ref_root, cp2), + "consistency m->N across flush"); + + // Index-based variant: consistency_proof_from_indices(i, j) == + // consistency_proof(i + 1, j + 1), spanning tiles and the live tree. + const auto cpi = tt.consistency_proof_from_indices(m - 1, N - 1); + expect(cpi == cp2, "consistency index variant across flush"); + expect( + ProofEngine::verify_consistency( + m, N, *ref.past_root(m - 1), ref_root, cpi), + "consistency index variant verifies"); + } + + // A second flush writes further tiles; proofs for now-flushed indices still + // work (this also confirms the writer never reads a flushed leaf). + tt.flush(); // flushes full tiles; flushed_size = covered = 1280 + expect(tt.flushed_size() == 1280, "second flushed size"); + expect(tt.tree_ref().min_index() == 1279, "second boundary leaf retained"); + + for (const uint64_t i : + {(uint64_t)0, + (uint64_t)1000, + (uint64_t)1279, + (uint64_t)1280, + (uint64_t)1499}) + { + const auto p = tt.inclusion_proof(i, N); + expect( + *p == *ref.path(i), "post-2nd inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(ref_root), + "post-2nd inclusion verify i=" + std::to_string(i)); + } + + std::cout << "tiled tree (flush + combination): OK" << '\n'; + + // ---- Part 2b: compaction when the size is an exact multiple of TILE_WIDTH + // (here 512). flush_to cannot drain the whole tree, so compact() keeps + // one resident leaf rather than throwing; proofs still resolve. + { + TiledTree::Config mcfg; + mcfg.prefix = base / "tt_multiple"; + mcfg.compact_on_flush = true; + TiledTree mtt(mcfg); + for (uint64_t i = 0; i < 512; i++) + { + mtt.append(hashes[i]); + } + mtt.flush(); // covered == size == 512; compaction must not throw + expect(mtt.flushed_size() == 512, "multiple: flushed size 512"); + expect(mtt.tree_ref().min_index() == 511, "multiple: one leaf retained"); + + merkle::Tree mref; + for (uint64_t i = 0; i < 512; i++) + { + mref.insert(hashes[i]); + } + const Hash mroot = mref.root(); + expect(mtt.root() == mroot, "multiple: root matches reference"); + for (const uint64_t i : {(uint64_t)0, (uint64_t)256, (uint64_t)511}) + { + const auto p = mtt.inclusion_proof(i, 512); + expect(*p == *mref.path(i), "multiple: inclusion==ref"); + expect(p->verify(mroot), "multiple: inclusion verify"); + } + std::cout << "tiled tree (exact-multiple compaction): OK" << '\n'; + } + + // ---- Part 2c: compaction with a non-zero retention_margin keeps the most + // recent leaves resident while flushed indices are still served from + // tiles. The immutable prefix remains the full-tile prefix, regardless + // of the margin. + { + TiledTree::Config rcfg; + rcfg.prefix = base / "tt_margin"; + rcfg.retention_margin = 300; + rcfg.compact_on_flush = true; + TiledTree rtt(rcfg); + for (uint64_t i = 0; i < n1; i++) // n1 == 1000 + { + rtt.append(hashes[i]); + } + rtt.flush(); // covered = 768; target = floor((768 - 300) / 256) * 256 = + // 256 + expect(rtt.flushed_size() == 768, "margin: flushed size 768"); + expect( + rtt.tree_ref().min_index() == 256, + "margin: retained >= 300 recent leaves"); + + for (uint64_t i = n1; i < N; i++) + { + rtt.append(hashes[i]); + } + expect(rtt.root() == ref_root, "margin: root matches reference"); + + // Flushed-only (0, 255), flushed-but-resident overlap (256, 767), and the + // un-flushed frontier (1000, 1499). + for (const uint64_t i : + {(uint64_t)0, + (uint64_t)255, + (uint64_t)256, + (uint64_t)767, + (uint64_t)1000, + (uint64_t)1499}) + { + const auto p = rtt.inclusion_proof(i, N); + expect( + *p == *ref.path(i), "margin inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(ref_root), + "margin inclusion verify i=" + std::to_string(i)); + } + + bool threw = false; + try + { + rtt.retract_to(700); // size 701 < immutable_size 768 + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "margin: retract below flushed prefix throws"); + + std::cout << "tiled tree (retention margin): OK" << '\n'; + } + + // ---- Part 3: rollback. Tiles are immutable, so only un-tiled + // (post-flush) entries may be rolled back. + { + // 3a. Before any flush nothing is tiled, so rollback is + // unrestricted. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_pre"; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 50; i++) + { + rb.append(hashes[i]); + } + rb.retract_to(29); // tiles_size == 0 -> allowed + expect(rb.size() == 30, "rb pre-flush retract allowed"); + } + + // 3b. After a flush, the un-tiled frontier can be rolled back and + // the + // tiled region stays consistent and provable; committed entries + // can't. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb"; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 300; i++) + { + rb.append(hashes[i]); + } + rb.flush(); // flushed_size = covered = 256 + for (uint64_t i = 300; i < 400; i++) + { + rb.append(hashes[i]); + } + rb.retract_to(349); // keep [0,349] + expect(rb.size() == 350, "rb retracted to 350"); + for (uint64_t i = 350; i < 400; i++) + { + rb.append(hashes[1000 + i]); // re-append DIFFERENT leaves + } + rb.flush(); // flushed_size = covered = 256 (tile [0,256) already + // written) + + // Reference tree of the exact post-rollback state. + merkle::Tree exp_tree; + for (uint64_t i = 0; i < 350; i++) + { + exp_tree.insert(hashes[i]); + } + for (uint64_t i = 350; i < 400; i++) + { + exp_tree.insert(hashes[1000 + i]); + } + const Hash exp_root = exp_tree.root(); + expect(rb.root() == exp_root, "rb root matches reference"); + + // Proofs for a tiled index and a frontier index match the reference. + for (const uint64_t i : {(uint64_t)100, (uint64_t)299, (uint64_t)399}) + { + const auto p = rb.inclusion_proof(i, 400); + expect( + *p == *exp_tree.path(i), + "rb inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(exp_root), "rb inclusion verify i=" + std::to_string(i)); + } + const auto cp = rb.consistency_proof(300, 400); + expect( + ProofEngine::verify_consistency( + 300, 400, *exp_tree.past_root(299), exp_root, cp), + "rb consistency 300->400"); + + // Only the immutable full-tile prefix [0,256) is protected: rolling + // back into it is refused, while rolling back within the un-tiled + // frontier (>= immutable_size()) is allowed. + expect(rb.flushed_size() == 256, "rb flushed to full-tile prefix"); + + bool threw = false; + try + { + rb.retract_to(100); // size 101 < 256 + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "rb retract below tiled prefix throws"); + threw = false; + try + { + rb.retract_to(254); // size 255 < 256 + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "rb retract just below tiled prefix throws"); + rb.retract_to(255); // size 256 == immutable_size(): drops only frontier + expect(rb.size() == 256, "rb retract to tiled prefix allowed"); + } + + // 3c. Compaction keeps the final tiled leaf resident, so rollback to a + // tree size of exactly immutable_size() remains representable. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_exact_boundary"; + cfg.compact_on_flush = true; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 1000; i++) + { + rb.append(hashes[i]); + } + rb.flush(); + expect(rb.flushed_size() == 768, "rb boundary flushed size"); + expect(rb.immutable_size() == 768, "rb boundary immutable size"); + expect( + rb.tree_ref().min_index() == 767, + "rb boundary final tiled leaf retained"); + + rb.retract_to(767); + expect(rb.size() == 768, "rb exact immutable boundary allowed"); + + merkle::Tree expected; + for (uint64_t i = 0; i < 768; i++) + { + expected.insert(hashes[i]); + } + const Hash expected_root = expected.root(); + expect( + rb.root() == expected_root, "rb boundary root matches reference"); + const auto proof = rb.inclusion_proof(0, 768); + expect( + *proof == *expected.path(0), + "rb boundary inclusion matches reference"); + expect(proof->verify(expected_root), "rb boundary inclusion verifies"); + } + + // 3d. Rollback interacts correctly with compaction (flushed + tiled + // past). + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_compact"; + cfg.compact_on_flush = true; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 1000; i++) + { + rb.append(hashes[i]); + } + rb.flush(); // flushed_size = covered = 768; retain boundary leaf 767 + expect( + rb.tree_ref().min_index() == 767, + "rb compact boundary leaf retained"); + for (uint64_t i = 1000; i < 1200; i++) + { + rb.append(hashes[i]); + } + // Frontier rollback (>= immutable_size()) is allowed. + rb.retract_to(1099); + expect(rb.size() == 1100, "rb compact frontier retract ok"); + + merkle::Tree exp_tree; + for (uint64_t i = 0; i < 1100; i++) + { + exp_tree.insert(hashes[i]); + } + const Hash exp_root = exp_tree.root(); + const auto p = rb.inclusion_proof(0, 1100); // leaf 0 is flushed + tiled + expect(*p == *exp_tree.path(0), "rb compact flushed inclusion==ref"); + expect(p->verify(exp_root), "rb compact flushed inclusion verify"); + + bool threw = false; + try + { + rb.retract_to(500); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "rb compact retract below tiled throws"); + } + + // 3e. A failed flush may publish an immutable full tile before a later + // write fails. The attempted full-tile boundary is sealed against + // rollback, while flushed_size advances only after a successful retry. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_interrupted"; + TiledTree interrupted(cfg); + for (uint64_t i = 0; i < 512; i++) + { + interrupted.append(hashes[i]); + } + + const fs::path blocker = cfg.prefix / "tile/0/001"; + fs::create_directories(blocker); + bool flush_threw = false; + try + { + interrupted.flush(); + } + catch (const std::exception&) + { + flush_threw = true; + } + + expect(flush_threw, "interrupted flush throws"); + expect( + fs::is_regular_file(cfg.prefix / "tile/0/000"), + "interrupted flush published first full tile"); + expect( + interrupted.flushed_size() == 0, + "interrupted flush does not advance flushed size"); + expect( + interrupted.immutable_size() == 512, + "interrupted flush seals attempted boundary"); + expect( + interrupted.compact() == 0, + "interrupted flush does not permit compaction"); + + TiledTree recovered(std::move(interrupted)); + expect( + recovered.immutable_size() == 512, + "interrupted flush seal survives move"); + + bool retract_threw = false; + try + { + recovered.retract_to(0); + } + catch (const std::exception&) + { + retract_threw = true; + } + expect( + retract_threw, + "interrupted flush rejects rollback across attempted tiles"); + expect(recovered.size() == 512, "interrupted rollback changes nothing"); + + fs::remove(blocker); + expect( + recovered.flush().full_written == 1, + "interrupted flush retry writes only missing tile"); + expect( + recovered.flushed_size() == 512, + "interrupted flush retry advances flushed size"); + expect( + recovered.immutable_size() == 512, + "interrupted flush retry preserves immutable size"); + recovered.compact(); + + merkle::Tree expected; + for (uint64_t i = 0; i < 512; i++) + { + expected.insert(hashes[i]); + } + const Hash expected_root = expected.root(); + expect( + recovered.root() == expected_root, + "interrupted flush root matches reference"); + for (const uint64_t i : + {(uint64_t)0, (uint64_t)255, (uint64_t)256, (uint64_t)511}) + { + const auto proof = recovered.inclusion_proof(i, 512); + expect( + *proof == *expected.path(i), + "interrupted flush inclusion matches reference"); + expect( + proof->verify(expected_root), + "interrupted flush inclusion verifies"); + } + } + } + std::cout << "rollback: OK" << '\n'; + + std::cout << "tiles_tree: OK" << '\n'; + + std::error_code ec; + fs::remove_all(base, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + + return 0; +} diff --git a/test/tiles_writer.cpp b/test/tiles_writer.cpp new file mode 100644 index 0000000..05e847f --- /dev/null +++ b/test/tiles_writer.cpp @@ -0,0 +1,337 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TileRef; +using merkle::tiles::TileStore; +using merkle::tiles::TileWriter; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +static size_t tile_file_count(const TileStore& store) +{ + const fs::path tiles = store.root() / "tile"; + if (!fs::exists(tiles)) + { + return 0; + } + const fs::recursive_directory_iterator it(tiles); + return (size_t)std::count_if( + begin(it), end(it), [](const fs::directory_entry& e) { + return e.is_regular_file(); + }); +} + +// Roll up a full level-0 tile and compare with a level-1 tile entry. +static Hash rollup(const std::vector& leaves) +{ + return merkle::tiles::perfect_root<32, merkle::sha256_compress>(leaves); +} + +static void overwrite_file( + const fs::path& path, const std::vector& bytes) +{ + std::ofstream file(path, std::ios::binary | std::ios::trunc); + file.write( + reinterpret_cast(bytes.data()), (std::streamsize)bytes.size()); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + + const fs::path base = fs::temp_directory_path() / + ("merklecpp_tiles_writer_" + std::to_string((unsigned long long)seed) + + "_" + std::to_string(std::rand())); + + try + { + // ---- A. size 256: exactly one full L0 tile and no L1 tile (the L1 root is + // not yet a full tile, so it stays in memory). The level-1 entry it + // would hold equals the real merkle::Tree root. + { + const auto hashes = make_hashes(256); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + + TileStore store(base / "a"); + TileWriter writer(store); + const auto s = writer.write_up_to(256, leaf_at); + + expect(s.full_written == 1, "A full_written"); + + expect(store.has_full_tile(0, 0), "A L0 full tile"); + expect( + fs::file_size(store.tile_path(TileRef{0, 0})) == 256U * Hash().size(), + "A L0 full tile size"); + expect(!store.has_full_tile(1, 0), "A no L1 full tile"); + expect(tile_file_count(store) == 1, "A exact tile file count"); + + // Level-0 tile is the leaf hashes verbatim. + const auto l0 = store.read_tile(TileRef{0, 0}); + for (size_t i = 0; i < hashes.size(); i++) + { + expect(l0[i] == hashes[i], "A L0 entry == leaf"); + } + + // The (un-tiled) level-1 entry == root of the equivalent merkle::Tree. + merkle::Tree tree; + for (const auto& h : hashes) + { + tree.insert(h); + } + expect(rollup(l0) == tree.root(), "A rollup(L0) == tree root"); + + std::cout << "A (size 256): OK" << '\n'; + } + + // ---- B. size 70000: full tiles only (273 full L0 + 1 full L1 = 274); the + // incomplete frontier at every level is left untiled. + { + const auto hashes = make_hashes(70000); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + + TileStore store(base / "b"); + TileWriter writer(store); + const auto s = writer.write_up_to(70000, leaf_at); + + expect(s.full_written == 274, "B full_written"); + + expect(store.has_full_tile(0, 0), "B L0 tile 0"); + expect(store.has_full_tile(0, 272), "B L0 tile 272"); + expect(!store.has_full_tile(0, 273), "B no L0 tile 273"); + + expect(store.has_full_tile(1, 0), "B L1 tile 0"); + expect(!store.has_full_tile(1, 1), "B no L1 tile 1"); + + expect(!fs::exists(store.root() / "tile" / "2"), "B no level 2"); + expect(!fs::exists(store.root() / "tile" / "3"), "B no level 3"); + expect(tile_file_count(store) == 274, "B exact tile file count"); + + // Higher-level entries are roll-ups of the complete child tiles. + const auto l1 = store.read_tile(TileRef{1, 0}); + expect(l1.size() == 256, "B L1 full width"); + expect(l1[0] == rollup(store.read_tile(TileRef{0, 0})), "B L1[0]"); + expect(l1[255] == rollup(store.read_tile(TileRef{0, 255})), "B L1[255]"); + + std::cout << "B (size 70000): OK" << '\n'; + } + + // ---- C. incremental writes preserve immutability and idempotency. + { + const auto hashes = make_hashes(1024); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + + TileStore store(base / "c"); + TileWriter writer(store); + + const auto s1 = writer.write_up_to(256, leaf_at); + expect(s1.full_written == 1, "C s1 full"); + + // Re-running at the same size writes nothing (full tiles are immutable). + const auto s2 = writer.write_up_to(256, leaf_at); + expect(s2.full_written == 0, "C s2 full immutable"); + + // Grow to 600: one new full L0 tile (covering 512), nothing else. + const auto s3 = writer.write_up_to(600, leaf_at); + expect(s3.full_written == 1, "C s3 full"); + + expect(store.has_full_tile(0, 0), "C L0 tile 0"); + expect(store.has_full_tile(0, 1), "C L0 tile 1"); + expect(!store.has_full_tile(0, 2), "C no L0 tile 2"); + expect(!store.has_full_tile(1, 0), "C no L1 full tile"); + expect(tile_file_count(store) == 2, "C exact tile file count"); + + std::cout << "C (incremental): OK" << '\n'; + } + + // ---- D. crossing into a full level-1 tile: the roll-up appears as a full + // tile and prior full tiles are never rewritten. + { + const auto hashes = make_hashes(65536); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + + TileStore store(base / "d"); + TileWriter writer(store); + + // One leaf short of a full L1 tile: 255 full L0 tiles, no L1 yet. + const auto s1 = writer.write_up_to(65535, leaf_at); + expect(s1.full_written == 255, "D s1 255 full L0"); + expect(!store.has_full_tile(1, 0), "D no L1 before completion"); + expect(tile_file_count(store) == 255, "D initial tile file count"); + + // Completing the 256th L0 tile yields one new L0 tile and one full L1. + const auto s2 = writer.write_up_to(65536, leaf_at); + expect(s2.full_written == 2, "D s2 new L0 + new L1"); + expect(store.has_full_tile(0, 255), "D L0 tile 255"); + expect(store.has_full_tile(1, 0), "D L1 tile 0"); + expect(!store.has_full_tile(1, 1), "D no L1 tile 1"); + expect(!store.has_full_tile(2, 0), "D no L2 tile"); + expect(tile_file_count(store) == 257, "D final tile file count"); + + const auto l1 = store.read_tile(TileRef{1, 0}); + expect(l1[0] == rollup(store.read_tile(TileRef{0, 0})), "D L1[0]"); + + // Re-running writes nothing (everything already full and immutable). + const auto s3 = writer.write_up_to(65536, leaf_at); + expect(s3.full_written == 0, "D s3 immutable rerun"); + + std::cout << "D (full L1 tile): OK" << '\n'; + } + + // ---- E. resume: a fresh writer over an existing store rebuilds its cursor + // from disk via full_prefix_length, never rewriting full tiles and + // writing only the newly-complete ones. + { + const auto hashes = make_hashes(70000); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + + const fs::path dir = base / "e"; + { + TileStore store(dir); + TileWriter writer(store); + expect( + writer.write_up_to(600, leaf_at).full_written == 2, + "E first writer 2 L0 tiles"); // 600 / 256 == 2 full L0 tiles + } + + // A brand-new store + writer over the same directory: its cursor must + // resume from what is already on disk. + TileStore store(dir); + TileWriter writer(store); + expect( + writer.write_up_to(600, leaf_at).full_written == 0, + "E resume rewrites nothing"); + + // Extending to 70000 writes only the missing tiles: 273 - 2 == 271 new L0 + // plus 1 new L1 (which rolls up the L0 tiles the first writer wrote). + expect( + writer.write_up_to(70000, leaf_at).full_written == 272, + "E resume writes only new tiles"); + expect(store.has_full_tile(0, 0), "E L0 tile 0 still present"); + expect(store.has_full_tile(0, 272), "E L0 tile 272"); + expect(store.has_full_tile(1, 0), "E L1 tile 0"); + expect( + store.read_tile(TileRef{1, 0})[0] == + rollup(store.read_tile(TileRef{0, 0})), + "E L1[0] rolled up from resumed L0"); + expect(tile_file_count(store) == 274, "E exact tile file count"); + + // A third fresh writer confirms full idempotence after a resume. + TileStore store3(dir); + TileWriter writer3(store3); + expect( + writer3.write_up_to(70000, leaf_at).full_written == 0, + "E second resume idempotent"); + + std::cout << "E (writer resume): OK" << '\n'; + } + + // ---- F. Resume discovers the first hole rather than trusting later valid + // files as proof that the prefix is contiguous. + { + const auto hashes = make_hashes(2048); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + const fs::path dir = base / "f"; + + { + TileStore store(dir); + TileWriter writer(store); + expect( + writer.write_up_to(2048, leaf_at).full_written == 8, + "F initial tiles"); + overwrite_file(store.tile_path(TileRef{0, 3}), {0}); + expect(!store.has_full_tile(0, 3), "F interior tile corrupt"); + expect(store.has_full_tile(0, 7), "F later tile remains valid"); + } + + TileStore store(dir); + TileWriter writer(store); + expect( + writer.write_up_to(2048, leaf_at).full_written == 1, + "F rewrites interior hole"); + const std::vector expected( + hashes.begin() + (std::ptrdiff_t)merkle::tiles::TILE_WIDTH * 3, + hashes.begin() + (std::ptrdiff_t)merkle::tiles::TILE_WIDTH * 4); + expect( + store.read_tile(TileRef{0, 3}) == expected, "F repaired tile contents"); + expect(tile_file_count(store) == 8, "F exact tile file count"); + + std::cout << "F (interior recovery): OK" << '\n'; + } + + // ---- G. Recovery is bounded by the requested tree size, so sparse files + // at geometrically increasing indices cannot overflow its search. + { + const fs::path dir = base / "g"; + const auto hashes = make_hashes(merkle::tiles::TILE_WIDTH); + { + TileStore store(dir); + store.write_tile(TileRef{0, 0}, hashes); + for (uint64_t index = 1;; index <<= 1) + { + store.write_tile(TileRef{0, index}, hashes); + if (index == (uint64_t{1} << 63)) + { + break; + } + } + } + + TileStore store(dir); + TileWriter writer(store); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + expect( + writer.write_up_to(merkle::tiles::TILE_WIDTH, leaf_at).full_written == + 0, + "G bounded sparse recovery"); + expect(store.has_full_tile(0, 0), "G requested tile remains valid"); + + std::cout << "G (bounded sparse recovery): OK" << '\n'; + } + + std::cout << "tiles_writer: OK" << '\n'; + + std::error_code ec; + fs::remove_all(base, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + + return 0; +}