diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index e3d617c..d6e7c4f 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -4,10 +4,11 @@ on: push: branches: - main + pull_request: workflow_dispatch: concurrency: - group: pages + group: pages-${{ github.ref }} cancel-in-progress: false permissions: read-all @@ -24,6 +25,7 @@ jobs: - name: Setup Pages id: pages uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 + if: github.event_name != 'pull_request' - name: Install dependencies run: | @@ -42,6 +44,7 @@ jobs: uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: doc/build + if: github.event_name != 'pull_request' deploy: name: Deploy diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b12b1bf..3f3a2e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,9 +51,9 @@ jobs: working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} run: | if [ "$RUNNER_OS" == "Linux" ]; then - cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=ON -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DLONG_TESTS=ON -DOPENSSL=ON -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON else - cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON + cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DLONG_TESTS=ON fi - name: Build @@ -64,7 +64,9 @@ jobs: - name: Clang-Tidy Header if: matrix.os == 'ubuntu-latest' shell: bash - run: clang-tidy merklecpp.h -p build/${{ matrix.build_type }} --warnings-as-errors='*' + run: | + clang-tidy merklecpp.h -p build/${{ matrix.build_type }} --warnings-as-errors='*' + clang-tidy merklecpp_tiles.h -p build/${{ matrix.build_type }} --warnings-as-errors='*' - name: Test working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index e3ecf9f..0bf9149 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,7 +43,7 @@ jobs: - name: Configure merklecpp working-directory: ${{github.workspace}}/build - run: cmake -DCMAKE_BUILD_TYPE=Debug -DTESTS=ON $GITHUB_WORKSPACE + run: cmake -DCMAKE_BUILD_TYPE=Debug -DTESTS=ON -DLONG_TESTS=ON $GITHUB_WORKSPACE - name: Build merklecpp working-directory: ${{github.workspace}}/build diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e35781..78122cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,9 +1,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -project(merklecpp LANGUAGES CXX C ASM) - cmake_minimum_required(VERSION 3.11) + +project(merklecpp LANGUAGES CXX C ASM) set(CMAKE_CXX_STANDARD 17) set(MERKLECPP_DIR ${CMAKE_CURRENT_SOURCE_DIR}) @@ -14,6 +14,7 @@ option(EVERCRYPT "enable comparison with EverCrypt Merkle trees" OFF) option(OPENSSL "enable OpenSSL" OFF) option(TRACE "enable debug traces" OFF) option(CLANG_TIDY "enable clang-tidy checks during build" OFF) +option(LONG_TESTS "enable long-running tests" OFF) if(CLANG_TIDY) find_program(CLANG_TIDY_PROGRAM clang-tidy) diff --git a/Doxyfile b/Doxyfile index 33229bb..d045b46 100644 --- a/Doxyfile +++ b/Doxyfile @@ -790,7 +790,8 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = merklecpp.h +INPUT = merklecpp.h \ + merklecpp_tiles.h # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/README.md b/README.md index 904c442..efadb1e 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,69 @@ unusual features like flushing, retracting, and tree segment serialisation. assert(path->verify(root)); +## Tiled storage (tlog-tiles) + +The companion header `merklecpp_tiles.h` adds optional, header-only support for +persisting a tree as [tlog-tiles](https://c2sp.org/tlog-tiles) tile files +*progressively* (optionally dropping already-tiled leaves from memory) and for +retrieving inclusion and consistency proofs from those tiles, from the in-memory +tree, or from a combination of the two. The hashing is unchanged: tiles and tile-derived proofs are templated on +the tree's existing hash function, so a tile-derived inclusion proof is +byte-identical to one from `merkle::Tree::path()` and verifies with the same +`merkle::Path::verify()`. + + #include + + merkle::tiles::TiledTree::Config cfg; + cfg.prefix = "/var/log/mylog"; // tile files live here + cfg.retention_margin = 1024; // keep the most recent leaves in memory + cfg.compact_on_flush = true; // opt in to dropping already-tiled leaves + + merkle::tiles::TiledTree log(cfg); + for (const auto& leaf_hash : batch) + log.append(leaf_hash); + + // Write newly-complete tiles. With compaction enabled + // this also drops from memory the leaves already covered by a full tile; + // otherwise the tree keeps every leaf and you can call log.compact() later. + log.flush(); + + // Proofs are served from tiles + the resident tree, even for flushed leaves. + auto inclusion = log.inclusion_proof(/*index=*/0, log.size()); + assert(inclusion->verify(log.root())); + + auto consistency = log.consistency_proof(/*m=*/100, /*n=*/log.size()); + +`TiledTree` creates a new tiled tree: the configured directory may exist, but +its `tile` subdirectory must be absent or empty. It deliberately rejects +existing tile data because tile files alone do not identify or restore the +tree that produced them. Applications with externally persisted tree state can +use the lower-level `TileStore` and `TileWriter` APIs to resume a store. + +See the [tiled storage guide](doc/tiles-guide.md) for a how-to covering +flushing, compaction, rollback, proofs, and the lower-level building blocks, +and [doc/design/tlog-tiles.md](doc/design/tlog-tiles.md) for the full design, +file/directory layout, and the proof algorithms. + + +## Building and testing + +Enable the test suite with CMake's `TESTS` option: + + cmake -S . -B build -DTESTS=ON + cmake --build build + ctest --test-dir build + +Some tile coverage is intentionally long-running. `LONG_TESTS` is off by +default for local builds; turn it on when you want the full tile stress suite, +including level-2 tile coverage and tile proof timing: + + cmake -S . -B build -DTESTS=ON -DLONG_TESTS=ON + +The repository CI enables `LONG_TESTS` so pull requests continue to exercise the +full tiled-storage matrix. + + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/audit.md b/audit.md new file mode 100644 index 0000000..8e8ff1d --- /dev/null +++ b/audit.md @@ -0,0 +1,344 @@ +# Audit: tiled storage & tile-backed proofs (PR #48) + +**Scope.** All PR #48 changes relative to `main`: the new header +`merklecpp_tiles.h`, the `TreeT::subtree_root` addition in `merklecpp.h`, the +`test/tiles_*.cpp` / `test/time_tiles.cpp` suite, and +`doc/design/tlog-tiles.md` + `doc/tiles-guide.md`. + +**Method.** Four independent review passes: a manual read plus three specialist +reviewers. Two reviewers built and ran the tile suite under GCC **and** Clang +with **ASan + UBSan**, and one cross-checked **113,462** generated proofs against +the `merkle::TreeT` oracle across every tile boundary. Where a finding is marked +*reproduced*, it was triggered in a running build, not merely read. + +**Headline.** The core proof math is **sound**: inclusion proofs are +byte-identical to `TreeT::path()/past_path()`, consistency proofs round-trip +through `verify_consistency`, and both were checked exhaustively for small trees +and by a large oracle sweep. Follow-up review found additional durability retry +and cursor-recovery defects outside the proof math; these are recorded below +and fixed. + +> **Update (follow-up commit).** Acting on this audit: the `write_higher_levels` +> option has been **removed** entirely — higher-level roll-up tiles are now +> always written — which makes **C1 unreachable** (no way to disable the tiles +> `resolve` relies on) and **T3 obsolete** (no flag to test). The test gaps +> **T1, T2, T4, T5, T6** are now **implemented**: corrupt/truncated-file reads +> (`tiles_store`), level-2 tiles end-to-end via a 256³-leaf test (`tiles_level2`) +> plus level-2 `resolve` descend coverage in the `tiles_proofs` oracle sweep +> (sizes 65536/65537/300000), writer reopen/resume (`tiles_writer` §E), +> non-zero `retention_margin` compaction (`tiles_tree` Part 2c), and the empty +> tree (`tiles_tree` Part 0). +> +> **Update (hardening commit).** The remaining implementation findings are now +> addressed: **C2/D2** by synced unique-temp writes, durable creation of every +> directory ancestor on POSIX, and size-validated tile presence; **C3/C4** by +> overflow-safe public arithmetic guards; **C5** by unique temp names and +> cleanup; and **P1/D1** by a per-source tile read cache plus corrected docs. +> +> **Update (fleet-review follow-up).** **C6** now re-confirms directory entries +> and destination directories before reusing visible files, so failed syncs are +> retried. **C7** replaces non-monotonic binary/exponential cursor discovery +> with a bounded contiguous scan for tiles and entry bundles. **C8** documents +> the concrete `tree_ref()` and `store_ref()` mutation hazards. Production-path +> fault injection, interior-corruption, sparse-index, SHA-384/SHA-512, timeout, +> and pull-request documentation checks cover the fixes. + +--- + +## Findings summary + +| # | Severity | Area | Summary | +|---|----------|------|---------| +| C1 | Medium | Correctness (config) | **Fixed:** the `write_higher_levels` option was removed, so required roll-up tiles are always written | +| C2 | Medium | Durability | **Fixed:** `write_file_atomically` syncs unique temp files, atomically replaces, durably creates POSIX directory ancestors, and rewrites bad-size tiles | +| C3 | Low | Robustness (public API) | **Fixed:** `TreeT::subtree_root` validates `level`/`index` before shifting and uses overflow-safe range checks | +| C4 | Low | Robustness (hostile input) | **Fixed:** `largest_pow2_lt` uses an overflow-safe loop condition for `size > 2^63` | +| C5 | Low | Concurrency / cleanup | **Fixed:** temp file names are unique per process/time/counter and cleaned up on error | +| C6 | High | Durability retry | **Fixed:** visible files and directory chains are re-synced before writer reuse | +| C7 | Medium | Cursor recovery | **Fixed:** recovery scans the bounded contiguous prefix and repairs interior holes | +| C8 | Low | Public escape hatches | **Fixed:** `tree_ref()` and `store_ref()` warn about bookkeeping and proof corruption | +| P1 | Low–Med | Performance | **Fixed:** `TileHashSource` keeps a small decoded tile cache for proof generation | +| T1 | Medium | Tests | **Fixed:** corrupt/truncated tile and entry-bundle read tests added | +| T2 | Medium | Tests | **Fixed:** level-2 coverage added via `tiles_level2` and large proof sweeps | +| T3 | Medium | Tests | **Obsolete:** the `write_higher_levels` flag was removed with C1 | +| T4 | Low–Med | Tests | **Fixed:** reopen/resume coverage added for `full_prefix_length` cursor recovery | +| T5 | Low | Tests | **Fixed:** non-zero `retention_margin` compaction and guarded rollback are covered | +| T6 | Low | Tests | **Fixed:** empty tree and `65537` boundary coverage added | +| T7 | Medium | Tests | **Fixed:** production-path directory-sync failures, interior corruption, and sparse indices are covered | +| T8 | Low | Tests | **Fixed:** SHA-384/SHA-512 tiled runtime coverage and CTest timeouts were added | +| D1 | Medium | Docs | **Fixed:** design doc now matches the per-source tile cache | +| D2 | Low | Docs | **Fixed:** durability wording now describes synced temp-file atomic replace and bad-size tile rewrite | +| D3 | Low | Docs | **Fixed:** stale API/cursor text was corrected and documentation now builds on pull requests | + +--- + +## Correctness & robustness + +### C1 — `TileHashSourceT::resolve` assumes higher-level tiles exist (Medium) + +`merklecpp_tiles.h:636–642`. + +> **Status:** Fixed by removing the public `write_higher_levels` option. Higher +> level roll-up tiles are now always written, so the arithmetic existence +> invariant used by `resolve` is maintained. + +```cpp +const uint64_t full_tiles = full_shift >= 64 ? 0 : (available_size >> full_shift); +if (n < full_tiles) +{ + const std::vector tile = store.read_tile(TileRef{L, n}); // unchecked read + ... +} +``` + +`resolve` infers the presence of a level-`L` (`L ≥ 1`) tile purely from +`available_size`. That inference is valid only with the default +`write_higher_levels = true` (`merklecpp_tiles.h:373`). The option is public and +reachable through `TiledTreeT::Config::writer.write_higher_levels`, and the design +doc (`§6.4`) does not state it is incompatible with proof generation. With it set +to `false`, only level-0 tiles are written, but `resolve` still believes level-≥1 +tiles exist and calls `read_tile`, which **throws** rather than returning `false`, +so the `||` fallback in `CombinedHashSourceT::subtree_root` never engages. + +*Reproduced:* 70,000-leaf tree, `write_higher_levels=false`, +`TileHashSource(store, 69888)`, `engine.root(69888)` → +`cannot open file: .../tile/1/000`. Triggers for any subtree of height ≥ 9 once +the tiled prefix reaches 65,536 leaves. + +*Default path is safe:* `available_size >> (8·(L+1))` was shown (algebraically and +by the oracle sweep) to equal exactly the number of level-`L` tiles on disk when +higher levels are written, so the default never reads a missing tile. + +**Fix.** Gate the read on real presence and fall through to the existing split +path, which already reconstructs the same root from level-0 tiles: +```cpp +if (n < full_tiles && store.has_full_tile(L, n)) { ... } +``` +This makes proofs correct (just slower) regardless of the option. Alternatively, +remove/guard the option or document the incompatibility. + +### C2 — `write_file_atomically` is atomic but not durable; the store can wedge after a crash (Medium) + +> **Status:** Fixed by the hardening and fleet-review follow-up commits. +> `write_file_atomically` writes a unique synced temp file, publishes it with +> atomic replace, syncs every directory link and the destination directory on +> POSIX, and removes the temp file on error. Before reuse, an existing file's +> directory chain and destination directory are re-confirmed. Exact file +> validation means a malformed tile or bundle is rewritten. + +The original implementation used stream flush plus rename and treated file +existence as the idempotence signal. A crash could therefore leave a present but +wrong-size tile that future writes skipped and future reads rejected. The +current implementation syncs the temp file before publishing, durably creates +each POSIX directory ancestor, syncs the destination directory after +publishing, and re-confirms those syncs before an existing file is reused. +`has_full_tile` requires the exact full tile size, so wrong-size tiles are +rewritten. + +### C3 — `TreeT::subtree_root` does not validate its arguments (Low) + +> **Status:** Fixed by the hardening commit. The method now rejects unsupported +> levels and overflowing indices before shifting, and uses an overflow-safe +> `lo/count/leaves` containment check. + +`merklecpp.h:1333–1336`. + +```cpp +const size_t lo = index << level; // line 1333 +const size_t count = (size_t)1 << level; // line 1334 -- run BEFORE any guard +if (num_leaves() == 0 || lo < min_index() || lo + count > num_leaves()) ... +``` + +`subtree_root` is a public method (used by `MemoryHashSourceT`). It shifts by the +caller-supplied `uint8_t level` *before* any bound check: + +* **`level ≥ 64` → UB.** `index << level` / `1 << level` are shifts past the type + width. *Reproduced under UBSan:* `subtree_root(64, 0, out)` → + `shift exponent 64 is too large`. +* **`lo + count` overflow → silent wrong `true`.** With `level=1` and + `index ≈ 2^63`, `lo + count` wraps to `0`, the upper-bound guard passes, and the + call returns `true` with `out` set to an unrelated node hash (ASan-clean — + memory-safe, but a wrong positive). + +No in-repo caller hits either case (internal `level = log2_exact(w) ≤ 63` or `0`); +this is strictly out-of-contract input to a public API. + +> Related robustness note (no behavioural bug today): unlike the canonical +> `walk_to`, which guards its identical shift with `if (_root->height > 1)` +> (`merklecpp.h:1068`), `subtree_root`'s shift at `:1356` is left unguarded and is +> only safe because the `level==0` early-return plus the `_root->height ≥ +> target_height ≥ 2` guard keep the exponent in `[1,63]`. Verified safe under +> UBSan, but it is a fragile invariant; an explicit guard/assert would harden it. + +**Fix.** First lines of the method: +```cpp +if (level >= sizeof(size_t) * 8) return false; +if (num_leaves() == 0 || index > (num_leaves() >> level)) return false; // overflow-safe +``` + +### C4 — `largest_pow2_lt` hangs for `size > 2^63` (Low) + +> **Status:** Fixed by the hardening commit. The loop condition is now based on +> `(n - 1) / 2`, so it cannot shift past `2^63` while searching for the largest +> power of two below `n`. + +`merklecpp_tiles.h:846–854`. + +```cpp +uint64_t k = 1; +while ((k << 1) < n) k <<= 1; // k reaches 2^63, k<<1 overflows to 0, loop never ends +``` + +Reachable from the public `root(size)`, `inclusion_proof(index, size)` and +`consistency_proof(m, n)` (via `mth_range`, *before* any leaf resolution), so an +oversized `size` argument hangs the call. *Reproduced:* `n = 2^63+1`, `2^63+5`, +`2^64−1` never terminate. Not reachable with a real tree (>9.2×10¹⁸ leaves); a +hostile/buggy size only. + +**Fix.** `while (k <= (n - 1) / 2) k <<= 1;` (or stop at `k == (1ull << 63)`). + +### C5 — fixed `*.tmp` name: concurrent collision + leak on error (Low) + +> **Status:** Fixed by the hardening commit. Temp names now include process, +> timestamp, and an atomic counter, and a guard removes the temp path unless the +> atomic replace succeeds. + +`merklecpp_tiles.h:282` (`tmp += ".tmp"`). + +A single fixed temp name per target means two writers persisting the *same* tile +race on the same `*.tmp`; the second `rename` can fail with `ENOENT` and throw, and +there is a window where one writer renames the other's partially-written temp into +place (corruption-safe only because tile content is deterministic). On a write or +`rename` error the `*.tmp` is also left behind. + +**Fix.** Unique temp suffix (pid + counter/random) and `remove` it on the error +path. + +### C6 - failed directory syncs can be skipped on retry (High) + +> **Status:** Fixed. Directory-entry and directory-content sync state is only +> cached after a successful sync. A writer re-confirms this state before +> trusting any visible existing tile or entry bundle, including through a fresh +> `TileStore` instance. + +The earlier write-once check used valid file presence as its retry signal. If a +rename succeeded and the following directory sync failed, a retry could skip +the visible file and advance `flushed_size()` even though a crash could still +lose the directory entry. Similarly, a directory created before its parent +sync failed remained visible and was not synced again. + +Production-path fault injection now covers both interruption points and verifies +that retry repeats the failed sync before a tile is reused or published. + +### C7 - prefix recovery assumes filesystem presence is monotonic (Medium) + +> **Status:** Fixed. Fresh writers linearly inspect the contiguous prefix only +> up to the full-file count relevant to the requested size. + +Binary search cannot locate the first missing or malformed file when a later +file is valid. Exponential search can also overflow when crafted sparse files +exist at powers of two. The bounded ordered scan repairs the first hole, checks +later files individually in the write loop, and never examines irrelevant high +indices. Tiles and entry bundles have matching regressions. + +### C8 - mutable escape-hatch consequences were understated (Low) + +> **Status:** Fixed. API comments and the guide now describe the concrete +> invariants callers assume responsibility for. + +Direct retraction through `tree_ref()` can make `flushed_size()` and +`immutable_size()` exceed `size()` and can make `flushed_size()` regress. +Correctly sized but unrelated files written through `store_ref()` are trusted +by a later flush and can invalidate proofs after compaction. + +--- + +## Performance + +### P1 — no tile read cache (Low–Medium) + +> **Status:** Fixed for proof generation by the hardening commit. `TileHashSource` +> now keeps a small decoded tile cache and serves repeated reads from memory. + +The original implementation opened, read, and deserialized a full 256-hash tile +for every `resolve` call, and the design doc described a cache that had not yet +been implemented. `TileHashSource` now keeps a small decoded tile cache keyed by +`{level, index}` and reuses hot tiles during proof generation. + +--- + +## Test coverage gaps + +Empirical validation is otherwise strong: `tiles_proofs.cpp` checks generated +proofs against `TreeT::path/past_path/past_root`, round-trips consistency proofs +through `verify_consistency`, exercises tamper/wrong-root rejection, and is +**exhaustive** over `(m,k)` consistency pairs for `n ≤ 16`. The gaps: + +The historical and follow-up gaps are now covered: corrupt and wrong-size +tile/bundle files, interior holes, sparse high indices, production-path +directory-sync failures, level-2 tiles, SHA-384/SHA-512 tiled trees, +reopen/resume, non-zero retention margin, guarded rollback below the flushed +prefix, empty trees, and `65537` / large-boundary proof cases. Long tests have +CTest timeouts. The `write_higher_levels=false` test became obsolete when that +option was removed. + +--- + +## Documentation + +* **D1 (Medium)** — Fixed: `doc/design/tlog-tiles.md` now describes the + implemented per-source tile cache. +* **D2 (Low)** — Fixed: `doc/design/tlog-tiles.md` and the code comment now + describe synced unique-temp writes, durable directory creation, atomic + replace, and bad-size tile rewrite. +* **D3 (Low)** - Fixed: the design uses the implemented `tree_ref()` / + `store_ref()` and lower-level `TileWriterT` APIs, describes bounded + in-memory cursor recovery and cache lifetime, and is built for pull requests. + +--- + +## Examined and found correct (no action) + +Recorded so the breadth of the review is auditable: + +* **Shift safety** — every shift in `merklecpp_tiles.h` uses 64-bit operands or is + guarded; the `subtree_root` `:1356` shift is provably in `[1,63]` (UBSan-clean). + The only shift defect is C4. +* **Recursion / termination** — `resolve` decrements `level` to a `≤ 8` base + (depth ≤ ~55); `mth_range`/`subproof` strictly shrink their range; + `inclusion_proof` is iterative. No stack risk at 2⁴⁰ leaves. +* **Inclusion proofs** — sibling ranges, `PATH_LEFT/RIGHT` directions, leaf→root + ordering, and `max_index = size − 1` match `PathT::verify`; byte-identical to + `TreeT::path/past_path` across exhaustive index sweeps over the 256/512/768/ + 1024/2048 boundaries. +* **Consistency proofs** — `subproof` matches RFC 6962 SUBPROOF (pow2 and non-pow2 + `m`); `verify_consistency` matches the standard CT verifier (`m==n`, `m==0`, + `m>n`, `is_pow2(m)` prepend, the `fn/sn` reduction, final `sn==0`). Round-trips + for all `m 2³⁰ leaves). `subtree_root` calls `is_full()`, + but the same expression is already on the core insertion path, so this change + neither introduces nor newly reaches it. Flagged for awareness only. + +--- + +## Recommended priority + +All implementation, test, and documentation findings from this audit have now +been addressed. diff --git a/doc/conf.py b/doc/conf.py index 5a7ef0e..6d84c31 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -12,17 +12,18 @@ # import os import sys -sys.path.insert(0, os.path.abspath('.')) + +sys.path.insert(0, os.path.abspath(".")) # -- Project information ----------------------------------------------------- -project = 'merklecpp' -copyright = '2021, Microsoft' -author = 'Christoph M. Wintersteiger' +project = "merklecpp" +copyright = "2021, Microsoft" +author = "Christoph M. Wintersteiger" # The full version, including alpha/beta/rc tags -release = '1.0.0' +release = "1.0.0" # -- General configuration --------------------------------------------------- @@ -31,12 +32,12 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - "breathe", - "sphinx.ext.githubpages", + "breathe", + "sphinx.ext.githubpages", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -49,17 +50,15 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'pydata_sphinx_theme' +html_theme = "pydata_sphinx_theme" -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# Add custom static paths here when the documentation needs local assets. +html_static_path = [] # Setup the breathe extension breathe_projects = {"merklecpp": "xml"} breathe_default_project = "merklecpp" -breathe_domain_by_extension = {"h" : "cpp"} +breathe_domain_by_extension = {"h": "cpp"} def setup(self): @@ -70,4 +69,4 @@ def setup(self): breathe_projects["merklecpp"] = str(srcdir / breathe_projects["merklecpp"]) if not os.environ.get("SKIP_DOXYGEN"): - subprocess.run(["doxygen"], cwd=srcdir / "..", check=True) \ No newline at end of file + subprocess.run(["doxygen"], cwd=srcdir / "..", check=True) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md new file mode 100644 index 0000000..3caa26d --- /dev/null +++ b/doc/design/tlog-tiles.md @@ -0,0 +1,786 @@ +# Design: Tiled storage and tile-backed proofs for merklecpp + +Status: Proposal / Draft +Audience: merklecpp maintainers and contributors +Scope: Extend the header-only library to (a) persist the Merkle tree as +[tlog-tiles](https://c2sp.org/tlog-tiles) tile files progressively (on +compaction), and (b) serve **inclusion** and **consistency** proofs from those +tiles, from the in-memory tree, or from a combination of the two. + +--- + +## 1. Goals and non-goals + +### Goals + +1. Write the tree to disk as a set of immutable, cacheable **tile files** using + the [tlog-tiles](https://c2sp.org/tlog-tiles) file/directory layout and tile + geometry (256-wide tiles, 8 tree levels per tile, path encoding). Only + **full, balanced** tiles are produced. The remaining frontier stays in the + in-memory tree until it crosses the next full-tile boundary. +2. Produce tiles **progressively**: each batch/flush writes only the newly + completed tiles, and the write integrates with the existing `flush_to()` + "compaction" so that flushed (evicted) subtrees are durably persisted before + their nodes are freed from memory. +3. Provide an API to retrieve, **after compaction**, an **inclusion path** and a + **consistency path**: + - from the in-memory tree alone (existing behaviour), + - from full tiles alone, for the tiled (full-tile-covered) prefix, or + - from a combination of full tiles (old, flushed part) and the in-memory + tree (recent frontier) — the normal case. +4. Keep everything **header-only** and templated, matching the existing + `TreeT` style. + +### Full-tile-only policy + +Partial tiles are out of scope: merklecpp never emits or reads them. During a +flush, a tile becomes eligible only after all 256 entries are complete and +final. Entries beyond that boundary remain in memory. Once published, a full +tile is immutable and is never rewritten. + +### Thread-safety policy + +The tiled-storage API provides no internal synchronization. Every object is +single-threaded unless its caller serializes all access. This requirement also +applies to separate objects that share a store prefix and to methods declared +`const`, because proof reads update the `TileHashSourceT` LRU cache. Locking and +reader/writer coordination are deliberately left to the application. + +### Hard constraints (from the request) + +- **Hashing logic is unchanged.** No modification to `HASH_FUNCTION`, to node + hashing, or to leaf handling. +- **Proofs remain compatible with the existing library.** A proof produced from + tiles MUST be identical to (and verify against the same root as) the proof the + existing library would produce via `Tree::path()` / `Tree::past_path()` / + `Tree::root()` / `Tree::past_root()`. + +### Non-goals + +- Byte-level interoperability with *external* RFC 6962 / Go-ecosystem tlog + clients. See [§2.3](#23-compatibility-statement): such interop is only + possible if the consumer instantiates `TreeT` with an RFC 6962 combiner, which + is **out of scope** here and not required by this design. +- An HTTP server. We define the on-disk layout and the read/write/proof APIs; + serving the static files is an application concern. + +--- + +## 2. Background + +### 2.1 tlog-tiles, the parts we adopt + +A tiled log exposes the Merkle tree as a set of static resources: + +- **Tiles** at `/tile//`, `application/octet-stream`. + - `` = level, decimal `0..63`, no leading zeros. + - `` = tile index within the level, encoded as zero-padded 3-digit path + elements where **all but the last element are prefixed with `x`**. Example: + `1234067` → `x001/x234/067`. +- A **full tile** is exactly **256 hashes** wide (`256 * HASH_SIZE` bytes). The + spec fixes `HASH_SIZE = 32` (SHA-256); here it is the template parameter. +- Level 0 hashes are **leaf hashes**. At level `L ≥ 1`, each hash is the Merkle + Tree Hash of a *full* tile at level `L-1`. A tile spans **8 tree levels**: its + 256 entries are the leaves of a height-8 perfect subtree, and intra-tile + internal nodes are reconstructed by hashing entries. +- The `n`-th tile at level `l` contains, for `i = 0..255`: + `MTH(D[(n*256+i)*256^l : (n*256+i+1)*256^l])`. + Its **start index** is `n*256^(l+1)`, its **end index** `(n+1)*256^(l+1)`. +- The frontier beyond the last full tile is held in the in-memory tree, so the + on-disk tile set is always a full-tile prefix. +- **Entry bundles** at `/tile/entries/`: big-endian `uint16` + length-prefixed raw entries, 256 per full bundle. (See + [section 6.8](#68-entry-bundles-optional).) +- **Pruning**: a log keeps a *minimum index*; tiles/bundles whose end index is + `≤ minimum index` may be denied. This maps cleanly onto merklecpp's + `flush_to()` / `min_index()`. + +### 2.2 The merklecpp model + +- `TreeT` is a **left-balanced binary Merkle tree**. + Its shape is identical to RFC 6962: a node's left child is the largest perfect + subtree of `2^k` leaves with `2^k < n`. A *full* node of height `h` covers + exactly `2^(h-1)` aligned leaves, and its hash is the root of that perfect + subtree (verified from `walk_to`, `is_full`, `update_sizes`). +- Internal nodes are combined with `HASH_FUNCTION(left, right, out)` over exactly + two child hashes (`merklecpp.h` node hashing). There is **no** domain + separation; the combiner is the only cryptographic operation. +- `flush_to(index)` is the existing **compaction** primitive: it conflates the + left part of the tree into single hash-only nodes and drops `num_flushed` + leaves from memory, raising `min_index()` to `index`. After a flush, paths for + leaves `< index` can no longer be produced from memory. +- Existing proof APIs we must stay compatible with: + - `root()` / `past_root(i)` — current / historical root. + - `path(i)` — inclusion path for leaf `i` in the current tree. + - `past_path(i, as_of)` — inclusion path for leaf `i` as of size `as_of+1`. + - `PathT::verify(root)` / `PathT::root()` — verification via `HASH_FUNCTION`. + - merklecpp has **no** classic two-size consistency proof today; this design + adds one (built from the same combiner, so it reconciles `past_root`s). + +### 2.3 Compatibility statement + +The tile geometry, path encoding, and the inclusion / consistency proof +**algorithms** are exactly those of tlog-tiles / RFC 6962. The implementation +emits only immutable full tiles. +The **hash values stored in tiles** are produced by the tree's existing +`HASH_FUNCTION`. Because + +1. merklecpp's tree shape equals the RFC 6962 left-balanced shape, and +2. tile entries and proof building blocks are *perfect-subtree roots* combined + with the **same** `HASH_FUNCTION` the tree already uses, + +a tile entry at level `L`, index `g` equals the in-memory full node of height +`8L+1` over leaves `[g·256^L, (g+1)·256^L)`, and a proof assembled from tiles is +**byte-identical** to the one `Tree::path()` / `Tree::past_path()` would emit. +Tile-derived proofs therefore verify with the unchanged `PathT::verify()`. + +> External RFC 6962 interop would additionally require RFC 6962 domain +> separation (`SHA256(0x00‖entry)` leaves, `SHA256(0x01‖l‖r)` nodes). merklecpp +> already lets a consumer pass such a `HASH_FUNCTION`; doing so is **optional and +> out of scope**. This design never assumes it. + +--- + +## 3. Tile ↔ merklecpp mapping (the math) + +Let `TILE_HEIGHT = 8` and `TILE_WIDTH = 256 = 2^8`. All combiners below are the +tree's `HASH_FUNCTION`; `MTH` denotes the Merkle Tree Hash computed with it. + +- **Tile entry = perfect subtree root.** Entry `i` of tile `(L, N)` (global + entry index `g = N·256 + i`) is `MTH(D[g·256^L : (g+1)·256^L])`, i.e. the root + of a perfect subtree of `2^(8L)` leaves — the in-memory full node of height + `8L + 1`. +- **Level roll-up.** Because a full level-`L` entry is the root of 256 full + level-`(L-1)` entries, a level-`L` tile can be computed from level-`(L-1)` + tiles alone: `tile(L,N)[i] = perfect_root( tile(L-1, N·256+i)[0..255] )`. So + the **write path needs no tree internals above level 0** — only leaf hashes + plus roll-ups. +- **Intra-tile internal nodes.** A tile's 256 entries are the leaves of a + height-8 perfect subtree. The root of `2^r` (`0 ≤ r ≤ 8`) consecutive, + aligned entries is `perfect_root` of that run. Hence any RFC 6962 subtree root + at tree level `k = 8L + r` is obtained from **one** level-`L` tile by hashing + `2^r` consecutive entries (a single entry when `r = 0`). +- **Full-tile prefix.** Only complete level-`L` tiles are written: + `full_tiles_L = floor( floor(s / 256^L) / 256 )`, together covering the leaf + prefix `covered = floor(s / 256) * 256`. The incomplete frontier `[covered, s)` + is not tiled; it is served from the in-memory tree. When no higher-level full + tile contains a subtree, it is resolved by **descending to the highest + available full tile and rolling up** (ultimately from full level-0 tiles). + +This yields a single read primitive that both proof types are built on: + +``` +subtree_root(level k, index j) = MTH(D[j·2^k : (j+1)·2^k]) # within `covered` + L = k / 8 ; r = k % 8 + first = j << r ; n = first / 256 ; off = first % 256 + if a full level-L tile n exists (first + 2^r within full_tiles_L · 256): + return perfect_root( tile(L, n)[off : off + 2^r] ) # single entry if r==0 + else: # higher-level tile unavailable: descend + return HASH_FUNCTION( subtree_root(k-1, 2j), subtree_root(k-1, 2j+1) ) +``` + +and a range primitive for non-perfect (right-frontier) subtrees: + +``` +mth_range(a, b): // MTH(D[a:b]); a aligned to 2^ceil(log2(b-a)) + w = b - a + if w == 2^k and source can resolve subtree_root(k, a>>k): return it + k = largest power of two < w // RFC 6962 split + return HASH_FUNCTION( mth_range(a, a+k), mth_range(a+k, b) ) +``` + +`mth_range` falls back to splitting when a perfect subtree is not directly +resolvable from the chosen source; the recursion always bottoms out at resolvable +pieces (see the [combined-source invariant](#7-progressive-production--compaction)). + +--- + +## 4. File and directory layout + +Rooted at a configurable `prefix` directory on local disk: + +``` +/ + tile/ + 0/ # level 0 (leaf hashes), full tiles only + 000 001 ... 255 # full tiles (8192 B each for HASH_SIZE=32) + x001/ + 000 ... 255 + 1/ # level 1 (roll-ups of level-0 full tiles) + 000 ... + .../ + entries/ # optional raw entry bundles (full only) + 000 001 ... +``` + +### Index encoding (`encode_tile_index`) + +``` +encode_tile_index(N): + parts = [] + do { parts.push_front(printf("%03d", N % 1000)); N /= 1000 } while (N > 0) + for each part except the last: prepend 'x' + return join(parts, "/") +``` + +Examples: `5 → "005"`, `255 → "255"`, `1000 → "x001/000"`, +`1234067 → "x001/x234/067"`. + +Resource paths: + +- Full tile: `tile//` (the only tiles this + implementation writes) +- Entry bundle: `tile/entries/` (full bundles only) + +Tile byte format: the 256 entries concatenated, each `HASH_SIZE` raw bytes +(`HashT::bytes`); a full tile is `256 * HASH_SIZE` bytes. + +--- + +## 5. Architecture overview + +``` + append(leaf hash) flush() + caller ───────────────▶ TiledTreeT ───────────────▶ TileStoreT (disk I/O) + │ owns Tree ▲ + │ │ read tiles + inclusion/consistency proof requests │ + ▼ │ + ProofEngine ──▶ HashSource ◀────────┘ + ├─ MemoryHashSource (in-memory Tree) + ├─ TileHashSource (TileStore + size) + └─ CombinedHashSource +``` + +New, all in a companion header `merklecpp_tiles.h` (includes `merklecpp.h`), +namespace `merkle::tiles`, every type templated on `` +with default aliases mirroring the bottom of `merklecpp.h`: + +| Component | Responsibility | +|----------------------|-----------------------------------------------------------------------| +| `TileCoord` | pure index math + path encoding (no I/O) | +| `TileStoreT` | read/write tile files on a local filesystem | +| `TileWriterT` | compute & persist newly-complete full tiles (write path) | +| `HashSource` impls | resolve `subtree_root` from tiles, memory, or both | +| `ProofEngineT` | `mth_range`, `inclusion_proof`, `consistency_proof`, verifiers | +| `TiledTreeT` | convenience wrapper: `append` / `flush` / `prove*` / compaction | + +The **only** (optional) change to the core header is a small, read-only, +**non-hashing** accessor (`subtree_root`) used by `MemoryHashSource`; see +[§6.2](#62-optional-core-accessor-non-hashing). + +--- + +## 6. Public API design + +### 6.1 Header, namespace, aliases + +```cpp +// merklecpp_tiles.h +#include "merklecpp.h" +#include +#include + +namespace merkle { +namespace tiles { + +static constexpr uint16_t TILE_HEIGHT = 8; +static constexpr uint16_t TILE_WIDTH = 256; // 2^TILE_HEIGHT + +template &, + const HashT&, + HashT&)> +class TileStoreT { /* ... */ }; +// ... TileWriterT, ProofEngineT, TiledTreeT ... + +} // namespace tiles + +// Convenience aliases (mirror merklecpp.h) +using TileStore = tiles::TileStoreT<32, sha256_compress>; +using TiledTree = tiles::TiledTreeT<32, sha256_compress>; +// + 384/512 variants +} // namespace merkle +``` + +Note the roll-up/proof combiner is the **same** `HASH_FUNCTION` the tree uses, +and it only ever combines two `HASH_SIZE`-byte hashes — so the default +`sha256_compress` single-block path is sufficient and **no OpenSSL dependency is +introduced**. + +### 6.2 Optional core accessor (non-hashing) + +To let proofs be served partly from the resident tree, add one small read-only +method to `TreeT`. It performs **no hashing changes**: it navigates to an +existing node and returns its already-computed hash. + +```cpp +// In TreeT (merklecpp.h). Returns true and sets `out` to MTH(D[j< read_tile(const TileRef&) const; // 256 entries + void write_tile(const TileRef&, const std::vector&); // synced atomic replace +}; +``` + +- Writes use a unique temporary file, sync the file contents, publish with an + atomic replace, and on POSIX sync every newly created directory link plus the + destination directory after the rename. Before reusing a visible file, the + writer re-confirms the directory chain and destination directory, so a retry + repeats a failed sync even if directory creation or rename is already + visible. A wrong-size tile is not treated as a durable full tile and is + rewritten by the writer. +- Full tiles are written once and never rewritten (immutability), so every file + under `tile//` is write-once. +- A small in-process cache of recently read tiles avoids repeated I/O during + proof generation. +- `TileStoreT` does not synchronize access. Callers must serialize all + operations on a store and across stores that share a prefix. + +### 6.4 Write path — `TileWriterT` (progressive, on compaction) + +```cpp +class TileWriterT { +public: + explicit TileWriterT(TileStoreT& store); + + // Persist all newly-complete full tiles (all levels 0..63). Incremental: + // only writes tiles not already present. Higher levels are always rolled up, + // since proof generation relies on them. + // `leaf_at` supplies level-0 leaf hashes for [0, size) (e.g. Tree::leaf). + void write_up_to(uint64_t size, + const std::function& leaf_at); +}; +``` + +Algorithm (`write_up_to`): + +``` +for L = 0, 1, 2, ... while entries_L = size >> (8*L) is > 0: + full_L = entries_L / 256 + for N in [first_unwritten_full(L) .. full_L): # new full tiles only + entries = [ entry(L, N*256 + i) for i in 0..255 ] + store.write_tile({L, N}, entries) + # rightmost incomplete entries are not written; they stay in memory until + # they complete a full tile + +entry(L, g): + if L == 0: return leaf_at(g) + else: return perfect_root( store.read_tile({L-1, g}) ) # 256→1 +``` + +- Level 0 reads leaf hashes via the supplied `leaf_at` (e.g. `Tree::leaf(i)`), + which exist **before** flushing. Higher levels roll up from the level-0 tiles + just written — independent of tree internals and robust to prior flushes. +- `first_unwritten_full(L)` is cached in each writer. A fresh writer reconstructs + it with a bounded, ordered scan of the contiguous prefix up to `full_L`. + Scanning in order is required because malformed or externally created files + can make raw file presence non-monotonic. + +Compaction integration (the "on compaction" story): + +```cpp +// In TiledTreeT::flush(): seal, write durable full tiles, THEN free memory. +uint64_t covered = (size / TILE_WIDTH) * TILE_WIDTH; // full level-0 coverage +immutable_size = max(immutable_size, covered); // before any write +writer.write_up_to(size, [&](uint64_t i) -> const Hash& { return tree.leaf(i); }); +flushed_size = covered; // after all levels succeed +// compact() retains the last covered leaf (when any) and the entire frontier. +``` + +If `write_up_to` throws, `immutable_size` remains advanced because a full tile +may already be visible, while `flushed_size` remains at its previous successful +boundary. The in-memory tree is not compacted. The caller fixes the I/O error +and retries with the same tree state; existing finalized tiles are reused. + +### 6.5 `HashSource` — tiles, memory, or both + +```cpp +struct HashSource { // concept (duck-typed or virtual) + // MTH(D[index< inclusion_proof(uint64_t index, uint64_t size) const; + + // RFC 6962 consistency proof that size `m` is a prefix of size `n` (m<=n). + std::vector consistency_proof(uint64_t m, uint64_t n) const; + std::vector consistency_proof_from_indices( + uint64_t first_index, uint64_t second_index) const; + + // Verifier (consistency is new to merklecpp; inclusion reuses PathT::verify). + static bool verify_consistency(uint64_t m, uint64_t n, + const Hash& old_root, const Hash& new_root, + const std::vector& proof); +}; +``` + +Inclusion (top-down; element order/`direction` chosen to match `Tree::path`): + +``` +elements = [] # leaf→root order via push_front +lo = 0, hi = size, idx = index +while hi - lo > 1: + k = largest_pow2_lt(hi - lo) # split at lo+k + if idx - lo < k: # target in left ⇒ sibling on the RIGHT + sib = mth_range(lo+k, hi); dir = PATH_RIGHT; hi = lo + k + else: # target in right ⇒ sibling on the LEFT + sib = mth_range(lo, lo+k); dir = PATH_LEFT; lo = lo + k + elements.push_front({sib, dir}) +leaf = src.leaf(index) +return Path(leaf, index, elements, max_index = size - 1) +``` + +Consistency (RFC 6962 `SUBPROOF`): + +``` +consistency_proof(m, n): # 0 < m <= n + if m == n: return [] + subproof(m, lo=0, hi=n, complete=true) + +subproof(m, lo, hi, complete): + if m == hi - lo: + if not complete: proof.push_back(mth_range(lo, hi)) + return + k = largest_pow2_lt(hi - lo) + if m <= k: + subproof(m, lo, lo+k, complete) + proof.push_back(mth_range(lo+k, hi)) + else: + subproof(m-k, lo+k, hi, false) + proof.push_back(mth_range(lo, lo+k)) +``` + +Because every emitted hash is an `mth_range` computed with `HASH_FUNCTION`, the +consistency proof reconciles `Tree::past_root(m-1)` with +`Tree::past_root(n-1)` — i.e. it is consistent with the existing library. + +### 6.7 `TiledTreeT` — convenience wrapper + +```cpp +class TiledTreeT { +public: + struct Config { + std::filesystem::path prefix; + uint64_t retention_margin = 0; // keep this many recent leaves resident + bool compact_on_flush = false; // opt in to dropping tiled leaves + }; + explicit TiledTreeT(Config); + + void append(const Hash& leaf_hash); // tree.insert + uint64_t size() const; // tree.num_leaves + Hash root(); // tree.root + uint64_t flushed_size() const; // successful tile boundary + uint64_t immutable_size() const; // rollback boundary + + // Write newly-complete full tiles. Compaction (dropping already-tiled + // leaves from memory) happens only if compact_on_flush. + Stats flush(); + + // Drop from memory the leaves already covered by a full tile (opt-in); the + // un-tiled frontier is always retained, and proofs for dropped leaves remain + // available from the tiles. + uint64_t compact(); + + // Roll back only beyond immutable_size(). + void retract_to(size_t index); + + // Proofs over tiles ∪ resident tree (works for flushed indices). + std::shared_ptr inclusion_proof(uint64_t index, uint64_t size); + std::vector consistency_proof(uint64_t m, uint64_t n); + std::vector consistency_proof_from_indices(uint64_t i, uint64_t j); + + Tree& tree_ref(); // mutable escape hatch + Store& store_ref(); // mutable escape hatch +}; +``` + +`TiledTreeT` performs no internal locking. The caller must serialize every +operation on a shared instance, including proof calls. + +Callers with their own storage can construct a `TileWriterT` and call +`write_up_to`, then build a `ProofEngineT` on a `CombinedHashSource`, so the +wrapper is optional sugar. + +### 6.8 Entry bundles (optional) + +merklecpp never sees raw entries (callers insert pre-computed leaf hashes), so +entry bundles are an **application-owned** add-on, included for completeness: + +- `tile/entries/` stores big-endian `uint16` length-prefixed entries, + 256 per full bundle (full bundles only). +- The application is responsible for the leaf-hash derivation it uses (e.g. + `leaf_hash = H(entry)`); merklecpp stores whatever leaf hash is inserted. +- An `EntryBundleWriterT` mirrors `TileWriterT`: it writes full bundles on + 256-entry boundaries only; the incomplete tail stays with the application. + Marked optional/secondary. + +--- + +## 7. Progressive production & compaction + +The pairing of tile writing with `flush_to` gives two central correctness +invariants: + +> **Compaction invariant.** Retain the final leaf of the last fully successful +> flush: `flushed_size == 0 || min_index() < flushed_size`. +> +> **Immutability invariant.** Never roll back below a full-tile boundary that a +> flush may have published: `size >= immutable_size`. + +`TiledTree` is fresh-only: its configured directory may exist, but the `tile` +subdirectory must be absent or empty. Tile files do not carry the size, root, +hash identity, or ownership information needed to reopen a tree safely, so the +wrapper rejects an existing tile namespace rather than adopting it. The +lower-level `TileWriter` supports resume for applications that persist and +validate the matching tree state themselves. + +Per flush: + +1. `append(...)` new leaf hashes; compute `root()`. +2. Compute `covered = floor(size / 256) * 256` and advance `immutable_size` to + `covered` before any write can publish a full tile. +3. `write_up_to(size, leaf_at)` - persist newly-complete **full** tiles at all + levels (incremental). +4. After every level succeeds, set `flushed_size = covered`. +5. *(optional)* `compact()` computes an aligned retention target, capped below + nonzero `covered`, then calls `flush_to(target)`. This reclaims memory only + when `compact_on_flush` is set (or `compact()` is called explicitly), while + retaining the final tiled leaf and the entire un-tiled frontier. By default + nothing is dropped and the tree stays whole. + +If step 3 fails, steps 4 and 5 do not run. `immutable_size` stays advanced to +prevent stale-tile rollback, while `flushed_size` stays at the last complete +all-level write so proofs and compaction do not trust an incomplete flush. + +Given the invariant, every leaf and every perfect subtree is resolvable: + +- leaf `i < covered` ⇒ in a full level-0 tile; leaf `i ≥ min_index` ⇒ resident. + Since `min_index ≤ covered`, **every** leaf is in tiles ∪ memory. The frontier + `[covered, size)` is always resident, because `compact()` never flushes past + `covered`. +- `mth_range` resolves a perfect subtree directly when it lies wholly in tiles + (`end ≤ covered`) or wholly in memory (`start ≥ min_index`); otherwise it + splits and recurses, terminating at resolvable pieces (leaves at worst). A + subtree within `covered` whose level has no completed full tile is resolved by + descending to the highest available full tile. + +Hence inclusion and consistency proofs are always producible after compaction, +from full tiles alone (for the tiled prefix), from memory alone (when nothing +relevant was flushed), or from the combination — satisfying the request. + +Cost per flush is `O(new full tiles)`; higher-level tiles are cheap roll-ups of +256 child hashes. Proof generation is `O(log(size))` `mth_range` calls, each at +most a few tile reads plus a `≤ 256`-leaf roll-up, with repeated tile reads +served from the per-source cache in the common case. + +--- + +## 8. Pruning / minimum index + +tlog-tiles pruning maps directly onto merklecpp: + +- The log's *minimum index* is `tree.min_index()` (== `num_flushed`). +- "Deny tiles/bundles whose end index ≤ minimum index" is implemented by the + serving layer consulting `min_index()`; on-disk tiles may be retained + (recommended by the spec) so historical proofs remain producible. +- The unpruned default is `min_index() == 0` (no `flush_to`). + +`flush_to` is the mechanism; *retention policy* (when/whether to prune) is left +to the application, exactly as the spec leaves it to log ecosystems. + +--- + +## 9. Worked examples (used as test vectors) + +**Size 256:** exactly one full level-0 tile (`tile/0/000`). No level-1 tile (its +single entry is the un-tiled frontier root, kept in memory). + +**Size 70 000:** 273 full level-0 tiles (`covered = 69 888`) + one full level-1 +tile (`tile/1/000`, rolling up level-0 tiles 0..255) = **274 full tiles**. The +incomplete frontier at every level is not written; the remaining 112 leaves and +the higher-level roll-ups stay in the in-memory tree. +These exact counts are asserted in tests. + +**Index encoding:** `1234067 → x001/x234/067`; `1000 → x001/000`; `255 → 255`. + +--- + +## 10. Implementation plan + +Each phase is independently testable; phases 1–4 require **no** core changes. + +**Phase 0 — Scaffolding.** Add `merklecpp_tiles.h` (includes `merklecpp.h`), +`merkle::tiles` namespace, geometry constants, default aliases. Wire a new test +group in `test/CMakeLists.txt` following `add_merklecpp_test`. + +**Phase 1 — Coordinates & store.** `TileCoord`/`encode_index`, `TileRef`, +`TileStoreT` (path building, atomic `write_tile`, `read_tile`). *Tests:* +index-encoding vectors; tile byte round-trip. + +**Phase 2 — Write path.** `TileWriterT::write_up_to` (level-0 from `leaf_at`, +roll-ups, incremental cursor; full tiles only — the frontier is never tiled). +*Tests:* sizes 256 & 70 000 produce exactly the expected full-tile set; +full-tile immutability (re-running writes nothing new). + +**Phase 3 — Hash sources & proof engine.** `TileHashSource`, `mth_range`, +`ProofEngineT::root/inclusion_proof/consistency_proof/verify_consistency`. +*Tests:* `root(size)` from tiles == `tree.root()`; `inclusion_proof(i,size)` +**equals** `tree.path(i)` (operator==) and verifies; `inclusion_proof(i, m)` +equals `tree.past_path(i, m-1)`; consistency proof reconciles +`tree.past_root(m-1)`/`tree.past_root(n-1)`. + +**Phase 4 — Combination & wrapper.** Optional core `subtree_root` accessor; +`MemoryHashSource`, `CombinedHashSource`, `TiledTreeT` (append/flush/prove +with `flush_to`). *Tests:* build N leaves, flush, then prove +inclusion for a **flushed** index and a **resident** index against a non-flushed +reference tree's root; consistency across a flush boundary. + +**Phase 5 — Optional entry bundles** (`BundleWriter`) and docs (README "Usage" +snippet, link this design doc, add `merklecpp_tiles.h` to Doxygen inputs). + +Deliverables: `merklecpp_tiles.h`; `test/tiles_*.cpp`; CMake wiring; optional +one-method core addition; README/docs updates. + +--- + +## 11. Testing strategy + +The existing library is the **oracle** — this is how we guarantee "proofs remain +compatible": + +- **Equality, not just verification.** Assert `*inclusion_proof(i,size) == + *tree.path(i)` using the existing `PathT::operator==`, and likewise against + `past_path`. This is stronger than `verify()` and pins byte-compatibility. +- **Root agreement.** `ProofEngineT::root(size)` == `tree.root()` / + `tree.past_root(size-1)` for many random sizes. +- **Spec vectors.** Exact full-tile counts and on-disk file sets for sizes 256 + and 70 000; index-encoding strings (incl. `x001/x234/067`). +- **Flush/compaction.** Cross-check proofs for flushed indices against a twin + tree that was never flushed (same inserts) — roots must match and proofs must + verify. +- **Consistency proofs.** Standard RFC 6962 `verify_consistency` plus the + reconciliation-of-`past_root`s check; randomized `m < n`. +- **Templating.** Run a subset under `Tree`, `Tree384`, `Tree512` to confirm + hash-size independence. +- **Style.** Mirror existing standalone-`main()` tests (`test/util.h`, + `make_hashes`, `get_timeout`) and register via `add_merklecpp_test`. + +--- + +## 12. Risks, edge cases, open questions + +- **External interop (by design, no).** With the default combiner the tiles are + *not* byte-compatible with RFC 6962 tooling. Documented in + [§2.3](#23-compatibility-statement); opt-in via an RFC 6962 `HASH_FUNCTION` is + the consumer's choice and out of scope. +- **Filesystem dependency.** Tile I/O needs ``/``; isolated + in the companion header so the core stays dependency-free. +- **Immutable full tiles.** A tile is emitted only after all of its entries are + final, and every emitted tile is write-once. A stand-alone tile reader cannot + serve the frontier; that is the in-memory tree's job (or the application must + keep it elsewhere). +- **`flush_to` alignment.** Compaction normally flushes to a 256-multiple + derived from retention. When that target equals `flushed_size`, it stops one + leaf earlier so `TreeT` can still retract to exactly that size. This one-leaf + overlap is enforced inside `TiledTreeT::compact`. +- **Rollback vs. immutable tiles.** Tiles are write-once, so rolling the tree + back (`retract_to`) over a range that a flush may have published would leave + stale, never-rewritten tiles. `TiledTreeT::retract_to` therefore throws if the + resulting size is below `immutable_size()`. A failed flush may advance + `immutable_size()` without advancing `flushed_size()`; retry with the same + tree state. Retracting the underlying tree directly via `tree_ref()` bypasses + this guard, can make the size boundaries inconsistent or non-monotonic, and + must be avoided. Files written through `store_ref()` are trusted without + checking that they match the tree and can invalidate proofs after compaction. +- **No internal synchronization.** Every tiled-storage object and shared store + prefix requires external serialization. This includes `const` proof reads, + which update the tile cache. +- **Very large indices.** Index math uses `uint64_t`; encoding handles + multi-group indices. Level bound `<= 63` per spec (8 suffices for `2^64`). + Resume scans are bounded by the requested tree size and cannot follow sparse + files beyond that range. +- **Open question — `subtree_root` in core vs. `past_path`-derived memory + source.** Recommend the tiny non-hashing accessor; falls back to zero-core- + change if maintainers prefer. Either keeps hashing untouched. + +--- + +## 13. Build & backwards-compatibility impact + +- **Additive only.** New header + tests; existing `merklecpp.h` behaviour and + API are unchanged. Default builds (no tiles) are unaffected. +- **No new mandatory dependencies.** Roll-ups/proofs use the existing + `HASH_FUNCTION` (two-hash combiner), so `sha256_compress` suffices; OpenSSL + stays optional. +- **Optional core change** is a single read-only accessor that performs no + hashing and does not alter any existing code path. +- **CMake.** Add a `tiles` test group guarded like the others; no changes to the + `merklecpp` INTERFACE target are required for consumers that don't use tiles. +``` diff --git a/doc/tiles-guide.md b/doc/tiles-guide.md new file mode 100644 index 0000000..396dbaa --- /dev/null +++ b/doc/tiles-guide.md @@ -0,0 +1,325 @@ +# Tiled storage and proofs — a guide to `merklecpp_tiles.h` + +`merklecpp_tiles.h` is an optional, header-only companion to `merklecpp.h`. It +lets you persist a Merkle tree as a set of immutable **tile files** on disk and +serve **inclusion** and **consistency** proofs from those tiles, from the +in-memory tree, or from a combination of the two — so proofs stay available even +after old entries are dropped from memory. + +It builds on the [tlog-tiles](https://c2sp.org/tlog-tiles) file/directory layout +but is **not** trying to be wire-compatible with external tlog-tiles clients. See +[`design/tlog-tiles.md`](design/tlog-tiles.md) for the design and internals; this +page is a practical how-to. + +## Contents + +- [Requirements and a note on hashing](#requirements-and-a-note-on-hashing) +- [Thread safety](#thread-safety) +- [Quick start: `TiledTree`](#quick-start-tiledtree) +- [Flushing and compaction](#flushing-and-compaction) +- [Rollback](#rollback) +- [Proofs](#proofs) +- [Lower-level building blocks](#lower-level-building-blocks) +- [Entry bundles (optional)](#entry-bundles-optional) +- [On-disk layout](#on-disk-layout) + +## Requirements and a note on hashing + +- C++17 (the header uses `` and small platform-specific file-sync + calls for durable tile writes). +- Include the companion header; it pulls in `merklecpp.h` for you: + + ```cpp + #include + ``` + +- Everything lives in `namespace merkle::tiles` and is templated on the same + `` as your tree. The default aliases + (`merkle::tiles::TiledTree`, `TileStore`, `TileWriter`, `ProofEngine`, …) use + the **same** SHA-256 as `merkle::Tree`, so a tile-derived inclusion proof is + byte-identical to one from `merkle::Tree::path()` and verifies with the usual + `merkle::Path::verify()`. +- You insert **leaf hashes**, not raw entries — exactly like `merkle::Tree`. + Deriving a leaf hash from an entry (e.g. `leaf = H(entry)`) is your + application's job. The tile hash values are whatever your `HASH_FUNCTION` + produces; they are not RFC 6962 unless you instantiate your tree with an + RFC 6962 hash function (not required, and not the goal here). + +## Thread safety + +The tiled-storage API provides no internal synchronization. Treat every +`TileStore`, `TileWriter`, `TileHashSource`, `ProofEngine`, `TiledTree`, and +`EntryBundleWriter` instance as single-threaded. + +If an object or store prefix is shared between threads, the caller must +serialize every operation. This includes methods declared `const`: proof +generation updates the `TileHashSource` LRU cache. The library deliberately +does not add locks or otherwise coordinate concurrent readers and writers. + +## Quick start: `TiledTree` + +`TiledTree` is the high-level wrapper: append leaf hashes, flush them to disk +(which writes tiles), and ask for proofs. + +```cpp +#include + +merkle::tiles::TiledTree::Config cfg; +cfg.prefix = "/var/log/mylog"; // directory for tile files + +merkle::tiles::TiledTree log(cfg); + +// Append leaf hashes (compute these from your entries however you like). +for (const merkle::Hash& leaf : batch) + log.append(leaf); + +// Persist newly-complete tiles to disk. +log.flush(); + +merkle::Hash root = log.root(); // current Merkle root +uint64_t n = log.size(); // number of leaves + +// Inclusion proof for leaf 0 in the tree of `n` leaves. +auto inclusion = log.inclusion_proof(/*index=*/0, /*size=*/n); +assert(inclusion->verify(root)); + +// Consistency proof that size 100 is a prefix of size n. +auto consistency = log.consistency_proof(/*m=*/100, /*n=*/n); +``` + +`TiledTree` can be move-constructed, but it cannot be copied or assigned. Move +construction keeps its writer bound to the destination tree's tile store. + +`TiledTree` always creates a new tiled tree. The configured directory may +already exist, but its `tile` subdirectory must be absent or empty. Construction +throws rather than adopting existing tiles because those files do not identify +the tree that produced them or contain enough state to restore its size and +root. If your application persists and validates that state separately, use +the lower-level `TileStore` and `TileWriter` APIs; `TileWriter` intentionally +resumes existing full tiles and therefore trusts the caller to supply the same +tree and hash function. A fresh writer scans the requested range in order, +stopping at the first missing or malformed file, so an interior hole is +rewritten rather than hidden by later files. + +`flush()` is incremental: each call writes only the full tiles that became +complete since the previous call. Full tiles are immutable: written once after +all 256 entries are final and never rewritten. The remaining frontier stays in +memory until it crosses the next full-tile boundary. + +Tile files are written through unique temporary files, synced, then published +with an atomic replace. On POSIX systems, each newly created directory is made +durable by syncing its parent, and the destination directory is synced after +the rename. Before reusing a visible file, a writer also re-confirms its +directory chain and destination directory. This makes a retry repeat a failed +directory sync even when the rename or directory creation is already visible. +A wrong-size file at a tile path is not a published tile and is rewritten. + +## Flushing and compaction + +By default `flush()` only *writes* tiles; it keeps every leaf resident in +memory. Dropping already-tiled leaves from memory ("compaction") is **opt-in**, +because once you drop them you can only prove them from the tiles. + +```cpp +merkle::tiles::TiledTree::Config cfg; +cfg.prefix = "/var/log/mylog"; +cfg.compact_on_flush = true; // drop tiled leaves after each flush +cfg.retention_margin = 4096; // ...but keep the most recent 4096 resident + +merkle::tiles::TiledTree log(cfg); +``` + +- `compact_on_flush` (default `false`): when set, `flush()` calls + `compact()` for you. +- `compact()` can also be called explicitly at any time. It drops from memory + only leaves already covered by a **durably written full tile**, keeping at + least `retention_margin` recent leaves resident. It also retains the final + tiled leaf so rollback to exactly `immutable_size()` remains representable. + It returns the new minimum (smallest still-resident) leaf index. +- Proofs for dropped leaves are still produced — they are served from the tiles + and transparently combined with the resident frontier. + +`flushed_size()` is the boundary completed successfully at every required tile +level, and it is the only boundary used for proof reads and compaction. +`immutable_size()` is the rollback boundary. A flush seals that boundary before +it starts writing, because an error can occur after a full tile becomes visible. +If a flush throws, `immutable_size()` may advance while `flushed_size()` does +not. Keep the same tree contents, correct the I/O failure, and retry `flush()`; +finalized tiles are reused rather than rewritten. + +```cpp +log.compact(); // free memory now +uint64_t resident_from = log.tree_ref().min_index(); +``` + +## Rollback + +Tiles are immutable, so you may only roll back entries beyond the boundary +returned by `immutable_size()`. `retract_to` enforces this: + +```cpp +log.retract_to(index); // keep leaves [0, index], drop the rest +``` + +- Allowed when the resulting size is `>= immutable_size()`. +- Throws otherwise, because a flush may already have published an immutable + full tile for that range. +- The exact `immutable_size()` boundary remains available after compaction; + compaction retains the final tiled leaf needed by the in-memory tree. +- After a successful flush, `immutable_size() == flushed_size()`. After an + interrupted flush, `immutable_size()` may be larger until the same tree state + is flushed successfully. +- `retract_to` mirrors `merkle::Tree::retract_to`: `index` is the new *last* + leaf, so the resulting size is `index + 1`. + +> **Warning:** Treat `tree_ref()` as an inspection escape hatch unless you also +> maintain every tiled-tree invariant yourself. Direct retraction bypasses the +> guard, can make `flushed_size()` and `immutable_size()` exceed `size()`, and +> can make `flushed_size()` regress. Use `TiledTree::retract_to` instead. + +`store_ref()` is similarly unsafe for mutation. A later flush trusts any +correctly sized tile written through it without checking that the hashes match +the in-memory tree. A mismatched tile can silently invalidate proofs after +compaction. + +## Proofs + +Both proof types come from `TiledTree` (or, at a lower level, from +`ProofEngine`). They are produced with your tree's hash function, so they match +what `merkle::Tree` would produce. Requests outside the current tree (e.g. a +size greater than `size()`, or an out-of-range index) throw `std::runtime_error` +rather than returning an incorrect proof. + +### Inclusion proofs + +```cpp +// Prove leaf `index` in a tree of `size` leaves. +std::shared_ptr p = log.inclusion_proof(index, size); +bool ok = p->verify(root_at_size); +``` + +`size` is the tree size you are proving against: + +- `size == log.size()` ⇒ equivalent to `merkle::Tree::path(index)`; verify + against `log.root()`. +- a past `size` ⇒ equivalent to `merkle::Tree::past_path(index, size - 1)`; + verify against the root at that size (e.g. a past root you are + auditing). + +`size` may even exceed `flushed_size()`: the recent, not-yet-tiled frontier is +taken from the resident tree while the older part comes from tiles. + +### Consistency proofs + +```cpp +std::vector proof = log.consistency_proof(m, n); // m <= n +bool ok = merkle::tiles::ProofEngine::verify_consistency( + m, n, old_root /* root at size m */, new_root /* root at size n */, proof); +``` + +`verify_consistency` is a static helper, so you can verify on a client that only +has the two roots and the proof. + +The arguments are tree **sizes** (leaf counts). If you have leaf **indices** +instead, use the variant that maps index `i` to the tree of size `i + 1` (the +"last leaf" convention, matching `past_path`/`retract_to`): + +```cpp +// Equivalent to consistency_proof(i + 1, j + 1). +auto proof = log.consistency_proof_from_indices(i, j); // i <= j +``` + +Both `TiledTree` and the lower-level `ProofEngine` provide +`consistency_proof_from_indices`. + +## Lower-level building blocks + +If you manage your own tree/storage you can use the pieces directly instead of +`TiledTree`. + +### Writing tiles from your own tree + +```cpp +merkle::Tree tree; +for (auto& leaf : batch) tree.insert(leaf); + +merkle::tiles::TileStore store("/var/log/mylog"); +merkle::tiles::TileWriter writer(store); + +// Write all newly-complete full tiles; keep the remaining frontier in memory. +auto stats = writer.write_up_to( + tree.num_leaves(), + [&](uint64_t i) -> const merkle::Hash& { return tree.leaf(i); }); +// stats.full_written +``` + +`TileWriter` keeps an in-memory next-file cursor. A new writer reconstructs it +by checking the contiguous prefix only up to the number of full files relevant +to the requested tree size. Existing files are re-confirmed as durably +published before reuse; malformed files and holes are rewritten. + +### Reading tiles and computing proofs + +A `HashSource` resolves the root of a complete subtree; pick where it reads from: + +- `TileHashSource(store, available_size)` — from full tile files; resolves the + full-tile-covered prefix only (the frontier needs a memory source). +- `MemoryHashSource(tree)` — from a resident `merkle::Tree`. +- `CombinedHashSource(primary, secondary)` — try `primary` first, then + `secondary` (e.g. memory then tiles). + +```cpp +merkle::tiles::TileHashSource src(store, /*available_size=*/tree.num_leaves()); +merkle::tiles::ProofEngine engine(src); + +merkle::Hash root = engine.root(size); +auto inclusion = engine.inclusion_proof(index, size); +auto consistency = engine.consistency_proof(m, n); +``` + +A tile-only source can resolve proofs whose subtrees all lie within the +full-tile-covered prefix (`available_size` is rounded down to a whole number of +tiles). For the live frontier, combine it with a `MemoryHashSource` — which is +exactly what `TiledTree` does for you. + +`TiledTree` simply wires a `CombinedHashSource(MemoryHashSource, TileHashSource)` +into a `ProofEngine` for you. It creates these sources for each proof call, so +its tile cache is per-call. A long-lived lower-level `TileHashSource` retains +its cache across calls. + +## Entry bundles (optional) + +If you also want to store the raw log entries (tlog-tiles "entry bundles"), use +`EntryBundleWriter`. Bundles are level-0 only and application-owned — merklecpp +stores leaf hashes; you supply the raw bytes and decide how an entry maps to its +leaf hash. Only full bundles (256 entries) are written; the incomplete tail +stays with your application until it completes a bundle. + +```cpp +merkle::tiles::EntryBundleWriter bundles(store); +bundles.write_up_to(num_entries, + [&](uint64_t i) -> std::vector { return raw_entry_bytes(i); }); + +// Read a full bundle back (256 entries). +std::vector> e = store.read_entry_bundle(/*index=*/0); +``` + +Entries are encoded as big-endian `uint16` length-prefixed byte strings. + +## On-disk layout + +Under the configured `prefix`: + +``` +/ + tile/0/000, tile/0/001 … # level-0 tiles (leaf hashes), 256 hashes each + tile/1/… # higher levels (roll-ups of full tiles below) + tile/entries/… # optional raw entry bundles +``` + +Tile indices use the tlog-tiles path encoding: zero-padded 3-digit groups with +all but the last prefixed by `x` (e.g. index `1234067` -> `x001/x234/067`). Every +tile is full (256-wide), final, and immutable. Entries beyond the last full-tile +boundary remain in memory. See +[`design/tlog-tiles.md`](design/tlog-tiles.md) for the full specification of the +geometry and proof algorithms. diff --git a/doc/tiles-illustrated.md b/doc/tiles-illustrated.md new file mode 100644 index 0000000..30bf65f --- /dev/null +++ b/doc/tiles-illustrated.md @@ -0,0 +1,764 @@ +# Tiled Merkle trees: an illustrated walkthrough + +This page builds a visual model of how an append-only Merkle tree moves from +memory into immutable tile files, and how proofs continue to work across both +places. + +> [!IMPORTANT] +> **Every example on this page uses an imaginary tile width of 32 entries +> purely to keep the diagrams readable.** +> +> The merklecpp implementation remains fixed at `TILE_WIDTH = 256` and +> `TILE_HEIGHT = 8`. A width of 32 is not a configuration option, and this page +> does not propose changing the code, file format, defaults, or examples +> elsewhere. Unless a section explicitly says "illustrative", use 256. + +## The scaled-down model + +A production tile contains 256 entries and spans 8 binary tree levels because +`256 = 2^8`. This page scales that geometry down to 32 entries and 5 levels +because `32 = 2^5`. + +| Property | This page only | Production merklecpp | +|---|---:|---:| +| Tile width | 32 entries | 256 entries | +| Tree levels spanned by one tile | 5 | 8 | +| Leaves covered by one full level-0 tile | 32 | 256 | +| Leaves covered by one level-1 entry | 32 | 256 | +| Leaves covered by one full level-1 tile | 1,024 | 65,536 | + +The scaling changes only the numbers in the drawings. The rules are the same: + +1. Only full tiles are written. +2. A level-0 tile contains leaf hashes. +3. A higher-level tile contains roots of complete tiles from the level below. +4. The incomplete right-hand frontier remains in memory. +5. Published tiles are immutable. +6. Proofs can resolve subtree roots from memory, tiles, or both. + +### Notation + +- `h7` is the hash of leaf 7. +- `R[a, b)` is the Merkle root of the half-open leaf range `[a, b)`. +- `tile/L/NNN` is tile index `NNN` at tile level `L`. +- "Resident" means the in-memory tree can still expand that range to answer + proof requests. +- "Compacted" means the in-memory tree retains enough summary hashes to keep + its root correct, but no longer retains all detail below that range. + +### Colors used below + +```mermaid +flowchart TB + T["Tile-backed hash or range"]:::tile + M["Resident in-memory hash or range"]:::memory + B["Available from both tiles and memory"]:::both + S["Compacted in-memory summary"]:::summary + X["Leaf being proved"]:::target + PT["Hash emitted in a proof
blue outline: from tiles"]:::proofTile + PM["Hash emitted in a proof
green outline: from memory"]:::proofMemory + + T ~~~ M + M ~~~ B + B ~~~ S + S ~~~ X + X ~~~ PT + PT ~~~ PM + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef summary fill:#e5e7eb,stroke:#6b7280,color:#111827 + classDef target fill:#fed7aa,stroke:#ea580c,stroke-width:3px,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 + classDef proofMemory fill:#fef3c7,stroke:#16a34a,stroke-width:3px,color:#111827 +``` + +## What `flush()` and `compact()` each do + +Appending, flushing, and compacting are separate operations: + +```mermaid +flowchart TB + A["append(h)
Add a leaf hash to the in-memory tree"]:::memory + B["A complete 32-entry range now exists
but no file is written automatically"]:::memory + C["flush()
Write every newly complete full tile"]:::tile + D["The same range exists on disk and in memory
(the default after flush)"]:::both + E["compact()
Optionally discard old resident detail"]:::summary + F["Old detail is served from tiles;
the incomplete frontier stays in memory"]:::both + + A --> B + B --> C + C --> D + D --> E + E --> F + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef summary fill:#e5e7eb,stroke:#6b7280,color:#111827 +``` + +`flush()` does not compact by default. Setting `compact_on_flush = true` makes +the final two steps happen in one call, but the durability rule is unchanged: +compaction happens only after all required tile writes succeed. + +If a tile write fails, `immutable_size()` may advance past `flushed_size()` +because a published tile cannot be rolled back. Keep the same tree contents and +retry the flush. See +[Flushing and compaction](tiles-guide.md#flushing-and-compaction) for the full +interrupted-write contract. + +## What is inside a tile file? + +In the illustrative model, `tile/0/000` is the concatenation of 32 leaf hashes: + +```mermaid +flowchart TB + F["tile/0/000
32 serialized hashes"]:::tile + A["entries 0..7
h0 ... h7"]:::tile + B["entries 8..15
h8 ... h15"]:::tile + C["entries 16..23
h16 ... h23"]:::tile + D["entries 24..31
h24 ... h31"]:::tile + R["R[0, 32)
reconstructed by hashing the entries"]:::computed + N["Internal binary-tree nodes are reconstructed;
they are not separately stored in the file"]:::note + + F -->|first bytes| A + A -->|followed by| B + B -->|followed by| C + C -->|followed by| D + D --> R + R --> N + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef note fill:#f9fafb,stroke:#9ca3af,color:#374151 +``` + +At level 1, each entry is already the root of 32 leaves: + +```mermaid +flowchart TB + L1["tile/1/000
32 serialized subtree roots"]:::tile + L0A["entry 0
root(tile/0/000) = R[0, 32)"]:::tile + L0B["entry 1
root(tile/0/001) = R[32, 64)"]:::tile + L0C["entries 2..30
..."]:::tile + L0Z["entry 31
root(tile/0/031) = R[992, 1024)"]:::tile + ROOT["R[0, 1024)
reconstructed from tile/1/000"]:::computed + + L1 --> L0A + L0A --> L0B + L0B --> L0C + L0C --> L0Z + L0Z --> ROOT + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +The production version of the second diagram needs 256 level-0 tile roots, so +its first full level-1 tile appears at 65,536 leaves rather than 1,024. + +## On-disk file layout + +After an illustrative 1,030-leaf tree is flushed, the full-tile boundary is +1,024: + +```text +prefix/ + tile/ + 0/ + 000 # h0 ... h31 + 001 # h32 ... h63 + ... + 031 # h992 ... h1023 + 1/ + 000 # R[0,32), R[32,64), ... R[992,1024) +``` + +Leaves `[1024, 1030)` do not appear in a tile file because they do not complete +another 32-entry tile. They remain in memory. + +```mermaid +flowchart TB + N["n = 1,030 leaves"]:::computed + C["covered = floor(1,030 / 32) * 32 = 1,024"]:::computed + L0["32 full level-0 files
tile/0/000 through tile/0/031"]:::tile + L1["1 full level-1 file
tile/1/000"]:::tile + M["6-leaf frontier
[1024, 1030) in memory"]:::memory + + N --> C + C --> L0 + L0 --> L1 + L1 --> M + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +The optional `tile/entries/` bundles are omitted here. They store raw +application entries, not Merkle tree nodes, and do not change proof generation. + +## Tree growth, one snapshot at a time + +The next snapshots assume `retention_margin = 0`. Where compaction is shown, +merklecpp still retains the final tiled leaf as a boundary leaf. This is why +the "both" range below is one leaf wide. + +### Snapshot A: 20 leaves + +No full 32-entry tile exists: + +```mermaid +flowchart TB + N["n = 20"]:::computed + C["full-tile boundary = 0"]:::computed + M["Memory only
[0, 20)"]:::memory + D["Disk
no tile files"]:::empty + + N --> C + C --> M + M --> D + + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef empty fill:#f9fafb,stroke:#9ca3af,color:#374151 +``` + +Calling `flush()` at this point writes nothing. Every root and proof is served +from the in-memory tree. + +### Snapshot B: 40 leaves, before the first flush + +The first 32 leaves form a complete tile, but tile creation is explicit: + +```mermaid +flowchart TB + N["n = 40"]:::computed + M0["[0, 32)
complete and eligible, still memory only"]:::memory + M1["[32, 40)
incomplete frontier, memory only"]:::memory + D["Disk
still empty until flush()"]:::empty + + N --> M0 + M0 --> M1 + M1 --> D + + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef empty fill:#f9fafb,stroke:#9ca3af,color:#374151 +``` + +### Snapshot B: 40 leaves, after `flush()` + +The default `flush()` writes the full prefix but does not remove it from +memory: + +```mermaid +flowchart TB + F["flush() succeeds
flushed_size() = 32"]:::computed + B["[0, 32)
tile/0/000 + resident memory"]:::both + M["[32, 40)
resident memory only"]:::memory + + F --> B + B --> M + + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +At this point a proof may be answered entirely from memory even though a tile +copy exists. + +### Snapshot B: 40 leaves, after compaction + +With zero retention, compaction drops old leaf detail while preserving leaf 31 +as the rollback boundary: + +```mermaid +flowchart TB + R["R[0, 40)
current in-memory root"]:::computed + P["R[0, 32)
prefix represented by compacted summaries"]:::summary + C["[0, 31)
not leaf-addressable in memory"]:::summary + B["h31
retained boundary leaf"]:::both + M["R[32, 40)
fully resident frontier"]:::memory + T["tile/0/000
proof detail for [0, 32)"]:::tile + + R --> P + P --> C + P --> B + R --> M + P -.->|subtree and leaf detail| T + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef summary fill:#e5e7eb,stroke:#6b7280,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +There are now three logical ownership ranges: + +| Leaf range | Proof detail available from | +|---|---| +| `[0, 31)` | tiles only | +| `[31, 32)` | tiles and memory | +| `[32, 40)` | memory only | + +The compacted in-memory summaries still contribute to `root()`. "Tiles only" +means that a request for a leaf or complete subtree in that range must use the +tile source; it does not mean the in-memory root forgot the prefix hash. + +### Snapshot C: grow from 40 to 72 leaves + +Assume the tree was flushed and compacted at size 40, then 32 more leaves were +appended. + +Before the second flush: + +```mermaid +flowchart TB + N["n = 72
flushed_size() is still 32"]:::computed + T["[0, 31)
tiles only"]:::tile + B["[31, 32)
boundary leaf in both"]:::both + E["[32, 64)
complete and eligible, but still memory only"]:::memory + F["[64, 72)
incomplete memory frontier"]:::memory + + N --> T + T --> B + B --> E + E --> F + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +After the second flush and compaction: + +```mermaid +flowchart TB + F["flush() writes tile/0/001
flushed_size() = 64"]:::computed + T0["tile/0/000 covers [0, 32)"]:::tile + T1["tile/0/001 covers [32, 64)"]:::tile + C["[0, 63)
tiles only after compaction"]:::tile + B["[63, 64)
new boundary leaf in both"]:::both + M["[64, 72)
memory-only frontier"]:::memory + + F --> T0 + T0 --> T1 + T1 --> C + C --> B + B --> M + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +### Snapshot D: 1,030 leaves + +This is the first snapshot with a full illustrative level-1 tile: + +```mermaid +flowchart TB + N["n = 1,030"]:::computed + T0["Level 0
32 files cover [0, 1024)"]:::tile + T1["Level 1
tile/1/000 contains their 32 roots"]:::tile + C["After compaction
[0, 1023) uses tiles for proof detail"]:::tile + B["h1023
boundary leaf in both"]:::both + M["[1024, 1030)
memory-only frontier"]:::memory + + N --> T0 + T0 --> T1 + T1 --> C + C --> B + B --> M + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +### Snapshot summary + +This table assumes each snapshot has just completed a successful flush and +compaction with zero retention: + +| Tree size | `flushed_size()` | Files written | Tiles only | Tiles + memory | Memory only | +|---:|---:|---|---|---|---| +| 20 | 0 | none | none | none | `[0, 20)` | +| 40 | 32 | `tile/0/000` | `[0, 31)` | `[31, 32)` | `[32, 40)` | +| 72 | 64 | `tile/0/000..001` | `[0, 63)` | `[63, 64)` | `[64, 72)` | +| 1,030 | 1,024 | 32 level-0 tiles + `tile/1/000` | `[0, 1023)` | `[1023, 1024)` | `[1024, 1030)` | + +Again, multiply the tile geometry back to 256 for production. In particular, +the production level-1 example starts at 65,536 leaves, not 1,024. + +## How a proof finds a subtree root + +`TiledTree` gives `ProofEngine` a combined source. It tries the resident tree +first because that avoids I/O, then falls back to tiles: + +```mermaid +flowchart TB + Q["ProofEngine requests R[a, b)"]:::computed + L{"Is the range one leaf?"}:::decision + A{"Otherwise, is its width a power of two
and is the range aligned to that width?"}:::decision + P{"Is this complete subtree
fully resident in memory?"}:::decision + M["Return the in-memory hash"]:::memory + T{"Can the tile source resolve it
inside flushed_size()?"}:::decision + D["Read the appropriate tile entries
and roll them up"]:::tile + S["Split the range into smaller subtrees
and resolve each one"]:::computed + E["Fail if no source can resolve a required leaf"]:::error + + Q --> L + L -->|yes| P + L -->|no| A + A -->|yes| P + A -->|no| S + P -->|yes| M + P -->|no| T + T -->|yes| D + T -->|no, and range has multiple leaves| S + T -->|no, and range is one leaf| E + S -->|smaller range| Q + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef decision fill:#fff7ed,stroke:#c2410c,color:#111827 + classDef error fill:#fee2e2,stroke:#dc2626,color:#111827 +``` + +For example, after compacting the 40-leaf tree: + +- `R[0, 32)` is not fully resident, so memory declines it and tiles return it. +- `R[32, 36)` is resident, so memory returns it without touching disk. +- `R[24, 40)` crosses the boundary and is not one complete aligned subtree. + The proof engine splits it into resolvable pieces. + +## Inclusion proof 1: entirely from one tile + +Consider a proof against tree size 32 after `tile/0/000` has been written and +the old leaves have been compacted. This may be the current size or a historical +prefix of a larger tree. We want to prove leaf 5. + +Every required hash is reconstructed from `tile/0/000`: + +```mermaid +flowchart TB + R032["R[0, 32)"]:::tile + R016["R[0, 16)"]:::tile + P1632["R[16, 32)
proof"]:::proofTile + R08["R[0, 8)"]:::tile + P816["R[8, 16)
proof"]:::proofTile + P04["R[0, 4)
proof"]:::proofTile + R48["R[4, 8)"]:::tile + R46["R[4, 6)"]:::tile + P68["R[6, 8)
proof"]:::proofTile + P4["h4
proof"]:::proofTile + X5["h5
target leaf"]:::target + + R032 --> R016 + R032 --> P1632 + R016 --> R08 + R016 --> P816 + R08 --> P04 + R08 --> R48 + R48 --> R46 + R48 --> P68 + R46 --> P4 + R46 --> X5 + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef target fill:#fed7aa,stroke:#ea580c,stroke-width:3px,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 +``` + +The proof payload is ordered from the leaf toward the root: + +| Order | Proof hash | Position relative to the running hash | Source | +|---:|---|---|---| +| 1 | `h4` | left | `tile/0/000` | +| 2 | `R[6, 8)` | right | `tile/0/000` | +| 3 | `R[0, 4)` | left | `tile/0/000` | +| 4 | `R[8, 16)` | right | `tile/0/000` | +| 5 | `R[16, 32)` | right | `tile/0/000` | + +The internal roots in this table are computed on demand from the tile's leaf +hashes. They are not additional files. + +Verification starts with `h5`, combines the five proof hashes in order, and +arrives at `R[0, 32)`. + +## Inclusion proof 2: tiles and memory together + +Return to the compacted 40-leaf tree and prove leaf 36 against the current root +`R[0, 40)`. + +The target and its nearby siblings are in the resident frontier. The old +32-leaf prefix is supplied as one tile-backed subtree root: + +```mermaid +flowchart TB + R040["R[0, 40)"]:::computed + P032["R[0, 32)
proof from tile"]:::proofTile + R3240["R[32, 40)
resident frontier"]:::memory + P3236["R[32, 36)
proof from memory"]:::proofMemory + R3640["R[36, 40)"]:::memory + R3638["R[36, 38)"]:::memory + P3840["R[38, 40)
proof from memory"]:::proofMemory + X36["h36
target leaf"]:::target + P37["h37
proof from memory"]:::proofMemory + + R040 --> P032 + R040 --> R3240 + R3240 --> P3236 + R3240 --> R3640 + R3640 --> R3638 + R3640 --> P3840 + R3638 --> X36 + R3638 --> P37 + + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef target fill:#fed7aa,stroke:#ea580c,stroke-width:3px,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 + classDef proofMemory fill:#fef3c7,stroke:#16a34a,stroke-width:3px,color:#111827 +``` + +The mixed proof payload is: + +| Order | Proof hash | Position | Source | +|---:|---|---|---| +| 1 | `h37` | right | memory | +| 2 | `R[38, 40)` | right | memory | +| 3 | `R[32, 36)` | left | memory | +| 4 | `R[0, 32)` | left | `tile/0/000` | + +The caller sees one ordinary `merkle::Path`. Source selection is internal; the +proof format does not mark some hashes as "tile" and others as "memory". + +Proving an old leaf in the current tree is mixed in the opposite direction. +For example, a proof for leaf 5 at size 40 gets its target and lower siblings +from `tile/0/000`, then gets the final sibling `R[32, 40)` from memory. + +## Consistency proofs: the idea + +An inclusion proof answers: + +> Is this leaf part of this tree root? + +A consistency proof answers: + +> Can the tree with `m` leaves be extended, without changing its first `m` +> leaves, to produce the tree with `n` leaves? + +The verifier already knows: + +- `m` and the old root `R[0, m)`; +- `n` and the new root `R[0, n)`. + +The proof supplies enough complete subtree roots to reconstruct both roots +through a shared history. + +The producer recursively follows the part of the new tree that contains the +old boundary and emits the sibling subtree at each split: + +```mermaid +flowchart TB + A["Start with [0, n) and old size m"]:::computed + B["Split at the largest power of two
smaller than the current range"]:::computed + C{"Which side contains
the old boundary?"}:::decision + D["Recurse into that side"]:::computed + E["Emit the other side's root
as a proof hash"]:::proof + F{"Reached exactly
the old boundary?"}:::decision + G["Return proof hashes
from deepest to highest"]:::proof + + A --> B + B --> C + C --> D + D --> E + E --> F + F -->|no| B + F -->|yes| G + + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef decision fill:#fff7ed,stroke:#c2410c,color:#111827 + classDef proof fill:#fef3c7,stroke:#b45309,color:#111827 +``` + +Each emitted range is resolved through the same memory-first, tile-second +source used by inclusion proofs. + +## Consistency proof 1: a perfect old tree + +First prove that the 32-leaf tree is a prefix of the 40-leaf tree: + +```cpp +auto proof = log.consistency_proof(32, 40); +``` + +Because 32 is a power of two, the old root is already one complete left +subtree. The proof needs only the new right-hand range: + +```mermaid +flowchart TB + OLD["Known old root
R[0, 32)"]:::tile + EXT["proof[0]
R[32, 40) from memory"]:::proofMemory + JOIN["H(R[0, 32), R[32, 40))"]:::computed + NEW["Expected new root
R[0, 40)"]:::result + + OLD --> JOIN + EXT --> JOIN + JOIN --> NEW + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef proofMemory fill:#fef3c7,stroke:#16a34a,stroke-width:3px,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef result fill:#dcfce7,stroke:#15803d,stroke-width:3px,color:#111827 +``` + +The old root may have been calculated from `tile/0/000`; the extension is +resident in memory. Verification combines the known old root with the single +proof hash and compares the result with the known new root. + +## Consistency proof 2: a non-perfect old tree + +Now prove that the 20-leaf tree is a prefix of the 40-leaf tree: + +```cpp +auto proof = log.consistency_proof(20, 40); +``` + +Size 20 is not a power of two, so the old root does not line up with a single +node in the 40-leaf tree. The proof decomposes the relevant ranges: + +```mermaid +flowchart TB + R040["R[0, 40)"]:::computed + R032["R[0, 32)"]:::tile + P3240["P4 = R[32, 40)
memory"]:::proofMemory + P016["P3 = R[0, 16)
tile"]:::proofTile + R1632["R[16, 32)"]:::tile + R1624["R[16, 24)"]:::tile + P2432["P2 = R[24, 32)
tile"]:::proofTile + P1620["P0 = R[16, 20)
tile seed"]:::proofTile + P2024["P1 = R[20, 24)
tile"]:::proofTile + + R040 --> R032 + R040 --> P3240 + R032 --> P016 + R032 --> R1632 + R1632 --> R1624 + R1632 --> P2432 + R1624 --> P1620 + R1624 --> P2024 + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 + classDef proofMemory fill:#fef3c7,stroke:#16a34a,stroke-width:3px,color:#111827 +``` + +The proof vector contains hashes only; the range labels are shown here to make +the algorithm visible. Given `m = 20` and `n = 40`, the verifier derives where +each hash belongs. + +| Order | Illustrative range | Source | Why it is needed | +|---:|---|---|---| +| `P0` | `R[16, 20)` | tile | Seed shared by old and new reconstructions | +| `P1` | `R[20, 24)` | tile | Extend only the new reconstruction | +| `P2` | `R[24, 32)` | tile | Extend only the new reconstruction | +| `P3` | `R[0, 16)` | tile | Complete both the old and new left sides | +| `P4` | `R[32, 40)` | memory | Extend the new reconstruction to size 40 | + +Verification evolves two accumulators: + +The verifier uses the bit structure of `m` and `n` to decide which accumulator +each proof hash updates. Intuitively, `P0` seeds a subtree shared by both +histories and `P3` completes that shared old-tree boundary. `P1`, `P2`, and +`P4` cover leaves at or beyond the old size, so they extend only the new +accumulator. + +```mermaid +flowchart TB + S["Seed both accumulators with P0
old = new = R[16, 20)"]:::proofTile + A["Combine P1 on the right
new = R[16, 24)"]:::tile + B["Combine P2 on the right
new = R[16, 32)"]:::tile + C["Combine P3 on the left
old = R[0, 20)
new = R[0, 32)"]:::tile + D["Combine P4 on the right
new = R[0, 40)"]:::memory + V["Compare both reconstructed roots
with the caller's old and new roots"]:::result + + S --> A + A --> B + B --> C + C --> D + D --> V + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 + classDef result fill:#dcfce7,stroke:#15803d,stroke-width:3px,color:#111827 +``` + +This example is mixed in a useful way: + +- The old 20-leaf state can be reconstructed from the first tile even though + the live in-memory tree has compacted those leaves. +- The newly appended range `[32, 40)` comes from memory. +- The proof is still an ordinary vector of hashes, independent of where each + hash was found. + +## The complete mental model + +```mermaid +flowchart TB + A["Append leaf hashes"]:::memory + B["In-memory left-balanced Merkle tree"]:::memory + C["A 32-entry range becomes complete
(256 entries in production)"]:::both + D["flush() publishes immutable full tiles"]:::tile + E["compact() optionally drops old resident detail"]:::summary + F["MemoryHashSource
serves whatever remains resident"]:::memory + G["TileHashSource
serves the flushed prefix"]:::tile + H["CombinedHashSource
tries memory, then tiles"]:::both + I["ProofEngine"]:::computed + J["Current or historical root"]:::result + K["Inclusion proof"]:::result + L["Consistency proof"]:::result + + A --> B + B --> C + C --> D + D -.->|optional| E + D --> F + D --> G + E --> F + F --> H + G --> H + H --> I + I --> J + I --> K + I --> L + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef summary fill:#e5e7eb,stroke:#6b7280,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef result fill:#dcfce7,stroke:#15803d,stroke-width:3px,color:#111827 +``` + +The important boundary is always the last successfully flushed full tile: + +- Below it, immutable tiles can preserve proof detail after compaction. +- Above it, the incomplete frontier must remain resident in memory. +- A proof may resolve several component subtrees from either side of the + boundary, but the caller receives one normal proof. +- None of these rules depends on the illustrative width of 32. Production uses + the same model with 256-entry tiles. diff --git a/merklecpp.h b/merklecpp.h index 1170b1c..c4114ab 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -142,8 +143,9 @@ namespace merkle { uint8_t high = 0; uint8_t low = 0; - if (!decode_hex_digit(s[2 * i], high) || - !decode_hex_digit(s[2 * i + 1], low)) + if ( + !decode_hex_digit(s[2 * i], high) || + !decode_hex_digit(s[2 * i + 1], low)) { throw std::runtime_error("invalid hash string"); } @@ -294,10 +296,14 @@ namespace merkle { public: /// @brief Path direction - using Direction = enum { PATH_LEFT, PATH_RIGHT }; + enum Direction + { + PATH_LEFT, + PATH_RIGHT + }; /// @brief Path element - using Element = struct + struct Element { /// @brief The hash of the path element HashT hash; @@ -454,14 +460,13 @@ namespace merkle /// @brief The size of the serialised path in number of bytes [[nodiscard]] size_t serialised_size() const { - return sizeof(_leaf) + - sizeof(uint64_t) + // leaf index - sizeof(uint64_t) + // max index - sizeof(uint64_t) + // number of elements - elements.size() * ( - sizeof(Element::hash) + // hash - sizeof(uint8_t) // direction - ); + return sizeof(_leaf) + sizeof(uint64_t) + // leaf index + sizeof(uint64_t) + // max index + sizeof(uint64_t) + // number of elements + elements.size() * + (sizeof(Element::hash) + // hash + sizeof(uint8_t) // direction + ); } /// @brief Index of the leaf of the path @@ -798,9 +803,9 @@ namespace merkle /// @param hash Hash to insert void insert(const Hash& hash) { - MERKLECPP_TRACE(MERKLECPP_TOUT << "> insert " - << hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << "> insert " << hash.to_string(TRACE_HASH_SIZE) + << std::endl;); uninserted_leaf_nodes.push_back(Node::make(hash)); statistics.num_insert++; } @@ -842,10 +847,10 @@ namespace merkle walk_to(index, false, [this](Node*& n, bool go_right) { if (go_right && n->left) { - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - conflate " - << n->left->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - conflate " + << n->left->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); if (n->left && n->left->dirty) { hash(n->left); @@ -900,10 +905,10 @@ namespace merkle n->dirty = true; if (go_left && n->right) { - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - eliminate " - << n->right->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - eliminate " + << n->right->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); bool is_root = n == _root; Node* old_left = n->left; @@ -923,18 +928,18 @@ namespace merkle if (is_root) { - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - new root: " - << n->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT + << " - new root: " << n->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); assert(_root == n); } assert(n->invariant()); - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - after elimination: " << std::endl - << to_string(TRACE_HASH_SIZE) << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - after elimination: " << std::endl + << to_string(TRACE_HASH_SIZE) << std::endl;); return false; } return true; @@ -1007,9 +1012,9 @@ namespace merkle statistics.num_root++; compute_root(); assert(_root && !_root->dirty); - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - root: " << _root->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - root: " << _root->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); return _root->hash; } @@ -1079,12 +1084,12 @@ namespace merkle { walk_stack.push_back(cur); } - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - at " << cur->hash.to_string(TRACE_HASH_SIZE) - << " (" << cur->size << "/" << (unsigned)cur->height - << ")" - << " (" << (go_right ? "R" : "L") << ")" - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - at " << cur->hash.to_string(TRACE_HASH_SIZE) + << " (" << cur->size << "/" << (unsigned)cur->height + << ")" + << " (" << (go_right ? "R" : "L") << ")" + << std::endl;); if (cur->height == height) { if (!f(cur, go_right)) @@ -1139,8 +1144,9 @@ namespace merkle /// tree to @p as_of and then extracting the path of @p index. std::shared_ptr past_path(size_t index, size_t as_of) { - MERKLECPP_TRACE(MERKLECPP_TOUT << "> past_path from " << index - << " as of " << as_of << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << "> past_path from " << index << " as of " << as_of + << std::endl;); statistics.num_past_paths++; if ( @@ -1179,24 +1185,24 @@ namespace merkle bool const go_right_i = ((it_i >> (8 * sizeof(it_i) - 1)) & 0x01) != 0U; bool const go_right_a = ((it_a >> (8 * sizeof(it_a) - 1)) & 0x01) != 0U; - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - at " << (unsigned)height << ": " - << cur_i->hash.to_string(TRACE_HASH_SIZE) << " (" - << cur_i->size << "/" << (unsigned)cur_i->height - << "/" << (go_right_i ? "R" : "L") << ")" - << " / " << cur_a->hash.to_string(TRACE_HASH_SIZE) - << " (" << cur_a->size << "/" - << (unsigned)cur_a->height << "/" - << (go_right_a ? "R" : "L") << ")" << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - at " << (unsigned)height << ": " + << cur_i->hash.to_string(TRACE_HASH_SIZE) << " (" + << cur_i->size << "/" << (unsigned)cur_i->height << "/" + << (go_right_i ? "R" : "L") << ")" + << " / " << cur_a->hash.to_string(TRACE_HASH_SIZE) + << " (" << cur_a->size << "/" + << (unsigned)cur_a->height << "/" + << (go_right_a ? "R" : "L") << ")" << std::endl;); if (!fork_node && go_right_i != go_right_a) { assert(cur_i == cur_a); assert(!go_right_i && go_right_a); - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - split at " - << cur_i->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - split at " + << cur_i->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); fork_node = cur_i; } @@ -1316,6 +1322,82 @@ namespace merkle leaf_node(index)->hash, index, std::move(path), as_of); } + /// @brief Extracts the root hash of a complete subtree resident in memory + /// @param level The height of the subtree (it spans 2**level leaves) + /// @param index The index of the subtree at that height + /// @param out Set to the subtree root hash on success + /// @return Whether the subtree is a complete (balanced) subtree fully + /// resident in memory + /// @note This is read-only and does not change the hashing of the tree: it + /// returns an existing node hash (computing it on demand exactly as root() + /// and path() do). It returns false if any leaf of the subtree has been + /// flushed, if the subtree extends past the last leaf, or if the node at + /// that position is not a full subtree. The subtree spans leaf indices + /// [index << level, (index + 1) << level). + bool subtree_root(uint8_t level, size_t index, Hash& out) + { + const size_t leaves = num_leaves(); + if (leaves == 0 || level >= std::numeric_limits::digits) + { + return false; + } + if (index > (std::numeric_limits::max() >> level)) + { + return false; + } + + const size_t lo = index << level; + const size_t count = (size_t)1 << level; + + if (lo < min_index() || count > leaves || lo > leaves - count) + { + return false; + } + + if (level == 0) + { + out = leaf(lo); + return true; + } + + compute_root(); + + const uint8_t target_height = level + 1; + if (!_root || _root->height < target_height) + { + return false; + } + + Node* cur = _root; + size_t it = lo << (sizeof(lo) * 8 - _root->height + 1); + for (uint8_t height = _root->height; height > target_height;) + { + const bool go_right = ((it >> (8 * sizeof(it) - 1)) & 0x01) != 0U; + if (cur->height == height) + { + Node* next = go_right ? cur->right : cur->left; + if (!next) + { + return false; // conflated/flushed: not resident + } + cur = next; + } + it <<= 1; + height--; + } + + if (cur->height != target_height || !cur->is_full()) + { + return false; + } + if (cur->dirty) + { + hash(cur); + } + out = cur->hash; + return true; + } + /// @brief Serialises the tree /// @param bytes The vector of bytes to serialise to void serialise(std::vector& bytes) @@ -1340,8 +1422,8 @@ namespace merkle compute_root(); - MERKLECPP_TRACE(MERKLECPP_TOUT << to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << to_string(TRACE_HASH_SIZE) << std::endl;); std::vector extras; walk_to(min_index(), false, [&extras](Node*& n, bool go_right) { @@ -1365,8 +1447,9 @@ namespace merkle /// @param bytes The vector of bytes to serialise to void serialise(size_t from, size_t to, std::vector& bytes) { - MERKLECPP_TRACE(MERKLECPP_TOUT << "> serialise from " << from << " to " - << to << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << "> serialise from " << from << " to " << to + << std::endl;); validate_partial_range(from, to); @@ -1383,8 +1466,8 @@ namespace merkle compute_root(); - MERKLECPP_TRACE(MERKLECPP_TOUT << to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << to_string(TRACE_HASH_SIZE) << std::endl;); std::vector extras; walk_to(from, false, [&extras](Node*& n, bool go_right) { @@ -1439,19 +1522,24 @@ namespace merkle // Restore extra hashes on the left edge of the tree if ((it & 0x01) != 0U) { + const uint8_t restored_height = level_no + 1; + if (restored_height >= std::numeric_limits::digits) + { + throw std::runtime_error("invalid serialised tree height"); + } Hash h(bytes, position); MERKLECPP_TRACE(MERKLECPP_TOUT << "+";); auto n = Node::make(h); - n->height = level_no + 1; - n->size = (1 << n->height) - 1; + n->height = restored_height; + n->size = (static_cast(1) << n->height) - 1; assert(n->invariant()); level.insert(level.begin(), n); } - MERKLECPP_TRACE(for (auto& n - : level) MERKLECPP_TOUT - << " " << n->hash.to_string(TRACE_HASH_SIZE); - MERKLECPP_TOUT << std::endl;); + MERKLECPP_TRACE( + for (auto& n : level) MERKLECPP_TOUT + << " " << n->hash.to_string(TRACE_HASH_SIZE); + MERKLECPP_TOUT << std::endl;); // Rebuild the level for (size_t i = 0; i < level.size(); i += 2) @@ -1709,8 +1797,7 @@ namespace merkle protected: void validate_partial_range(size_t from, size_t to) const { - if ( - empty() || !(min_index() <= from && from <= to && to <= max_index())) + if (empty() || !(min_index() <= from && from <= to && to <= max_index())) { throw std::runtime_error("invalid leaf indices"); } @@ -1877,9 +1964,9 @@ namespace merkle { while (true) { - MERKLECPP_TRACE(MERKLECPP_TOUT << " @ " - << n->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " @ " << n->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); assert(n->invariant()); if (n->is_full()) @@ -1950,10 +2037,10 @@ namespace merkle if (!complete && !result->is_full()) { - MERKLECPP_TRACE(MERKLECPP_TOUT - << " X save " - << result->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " X save " + << result->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); return result; } } @@ -1968,9 +2055,9 @@ namespace merkle /// @param n New leaf node to insert void insert_leaf(Node*& root, Node* n) { - MERKLECPP_TRACE(MERKLECPP_TOUT << " - insert_leaf " - << n->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - insert_leaf " + << n->hash.to_string(TRACE_HASH_SIZE) << std::endl;); leaf_nodes.push_back(n); if (insertion_stack.empty() && !root) { @@ -1990,9 +2077,9 @@ namespace merkle { if (!uninserted_leaf_nodes.empty()) { - MERKLECPP_TRACE(MERKLECPP_TOUT - << "* insert_leaves " << leaf_nodes.size() << " +" - << uninserted_leaf_nodes.size() << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << "* insert_leaves " << leaf_nodes.size() << " +" + << uninserted_leaf_nodes.size() << std::endl;); // Potential future improvement: make this go fast when there are many // leaves to insert. for (auto& n : uninserted_leaf_nodes) diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h new file mode 100644 index 0000000..1e3ce90 --- /dev/null +++ b/merklecpp_tiles.h @@ -0,0 +1,1904 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "merklecpp.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include +#else +# include +# include +#endif + +// Tiled storage for merklecpp trees, following the full-tile geometry and path +// encoding of the C2SP tlog-tiles layout (https://c2sp.org/tlog-tiles). Only +// complete, immutable tiles are stored. Their hash values are produced by the +// tree's existing HASH_FUNCTION, so tile-derived proofs are byte-identical to +// those produced by merkle::TreeT (see doc/design/tlog-tiles.md). +// +// Thread safety: types in this header do not synchronize access internally. +// Callers must serialize all operations on shared objects and store prefixes, +// including const proof operations that update the tile cache. + +namespace merkle // NOLINT(modernize-concat-nested-namespaces) +{ + namespace tiles + { + /// @brief Number of tree levels spanned by a single tile. + static constexpr uint16_t TILE_HEIGHT = 8; + + /// @brief Number of hashes in a full tile (2**TILE_HEIGHT). + static constexpr uint16_t TILE_WIDTH = 256; + + namespace detail + { + template + uint64_t contiguous_prefix_length(uint64_t limit, const Present& present) + { + uint64_t length = 0; + while (length < limit && present(length)) + { + length++; + } + return length; + } + } + + /// @brief Encodes a tile index as tlog-tiles path elements. + /// @param n The tile index + /// @return The index as zero-padded, '/'-separated 3-digit groups, where + /// all but the last group are prefixed with 'x'. For example, 1234067 is + /// encoded as "x001/x234/067" and 5 as "005". + static inline std::string encode_tile_index(uint64_t n) + { + std::vector parts; + char buf[8]; + do + { + std::snprintf(buf, sizeof(buf), "%03u", (unsigned)(n % 1000)); + parts.emplace_back(buf); + n /= 1000; + } while (n > 0); + + std::string r; + for (size_t i = 0; i < parts.size(); i++) + { + const std::string& part = parts[parts.size() - 1 - i]; + if (i != 0) + { + r += "/"; + } + if (i + 1 < parts.size()) + { + r += "x"; + } + r += part; + } + return r; + } + + /// @brief Identifies a single (full) tile within a tiled log. + /// @note Only full, TILE_WIDTH-wide tiles are produced and consumed; the + /// incomplete frontier is never tiled (see doc/design/tlog-tiles.md). + struct TileRef + { + /// @brief The level of the tile (0 == leaf hashes). + uint8_t level = 0; + + /// @brief The index of the tile within its level. + uint64_t index = 0; + }; + + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TileWriterT; + + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class EntryBundleWriterT; + + /// @brief Reads and writes tlog-tiles tile files on a local filesystem. + /// @tparam HASH_SIZE Size of each hash in bytes + /// @tparam HASH_FUNCTION The tree's node hash function (carried for use by + /// later components; tile I/O itself does not hash). + /// @warning No internal synchronization is provided. Callers must serialize + /// access to a store and to all stores that share its prefix. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TileStoreT + { + friend class TileWriterT; + friend class EntryBundleWriterT; + + public: + /// @brief The type of hashes stored in tiles. + using Hash = HashT; + + /// @brief Constructs a tile store rooted at @p prefix. + /// @param prefix The directory under which the tiles live. + explicit TileStoreT(std::filesystem::path prefix) : + prefix(std::move(prefix)) + {} + + /// @brief The root directory of the store. + [[nodiscard]] const std::filesystem::path& root() const + { + return prefix; + } + + /// @brief Encodes a tile index (see encode_tile_index). + static std::string encode_index(uint64_t n) + { + return encode_tile_index(n); + } + + /// @brief The filesystem path of a tile. + [[nodiscard]] std::filesystem::path tile_path(const TileRef& ref) const + { + return prefix / + ("tile/" + std::to_string((unsigned)ref.level) + "/" + + encode_tile_index(ref.index)); + } + + /// @brief The filesystem path of an entry bundle. + [[nodiscard]] std::filesystem::path entries_path(uint64_t index) const + { + return prefix / ("tile/entries/" + encode_tile_index(index)); + } + + /// @brief Whether a full tile exists on disk. + [[nodiscard]] bool has_full_tile(uint8_t level, uint64_t index) const + { + std::error_code ec; + const auto path = tile_path(TileRef{level, index}); + if (!std::filesystem::is_regular_file(path, ec) || ec) + { + return false; + } + return std::filesystem::file_size(path, ec) == + (uintmax_t)TILE_WIDTH * HASH_SIZE && + !ec; + } + + /// @brief Writes a tile to disk atomically. + /// @param ref The tile to write + /// @param hashes The tile's hashes (exactly TILE_WIDTH of them) + void write_tile(const TileRef& ref, const std::vector& hashes) + { + if (hashes.size() != TILE_WIDTH) + { + throw std::runtime_error("tile width mismatch"); + } + + std::vector bytes; + bytes.reserve(hashes.size() * HASH_SIZE); + for (const auto& h : hashes) + { + h.serialise(bytes); + } + + write_file_atomically(tile_path(ref), bytes); + } + + /// @brief Reads a tile from disk. + /// @param ref The tile to read + /// @return The tile's hashes (TILE_WIDTH of them) + [[nodiscard]] std::vector read_tile(const TileRef& ref) const + { + std::vector bytes = read_file(tile_path(ref)); + const size_t expected = (size_t)TILE_WIDTH * HASH_SIZE; + if (bytes.size() != expected) + { + throw std::runtime_error("unexpected tile size"); + } + + std::vector hashes; + hashes.reserve(TILE_WIDTH); + size_t position = 0; + for (uint16_t i = 0; i < TILE_WIDTH; i++) + { + hashes.emplace_back(bytes, position); + } + return hashes; + } + + /// @brief Whether a full entry bundle exists on disk. + [[nodiscard]] bool has_entry_bundle(uint64_t index) const + { + std::error_code ec; + const auto path = entries_path(index); + if (!std::filesystem::is_regular_file(path, ec) || ec) + { + return false; + } + try + { + (void)decode_entries(read_file(path), TILE_WIDTH); + return true; + } + catch (const std::exception&) + { + return false; + } + } + + /// @brief Writes a full entry bundle to disk atomically. + /// @param index The bundle index + /// @param entries The raw log entries (exactly TILE_WIDTH of them) + /// @note Entries are stored in the tlog-tiles entry-bundle format: a + /// sequence of big-endian uint16 length-prefixed byte strings. + void write_entry_bundle( + uint64_t index, const std::vector>& entries) + { + if (entries.size() != TILE_WIDTH) + { + throw std::runtime_error("entry bundle width mismatch"); + } + write_file_atomically(entries_path(index), encode_entries(entries)); + } + + /// @brief Reads a full entry bundle from disk. + /// @param index The bundle index + /// @return The raw log entries (TILE_WIDTH of them) + [[nodiscard]] std::vector> read_entry_bundle( + uint64_t index) const + { + return decode_entries(read_file(entries_path(index)), TILE_WIDTH); + } + + /// @brief Encodes log entries into the tlog-tiles entry-bundle format. + static std::vector encode_entries( + const std::vector>& entries) + { + std::vector bytes; + for (const auto& e : entries) + { + if (e.size() > 0xFFFF) + { + throw std::runtime_error( + "entry too large for uint16 length prefix"); + } + bytes.push_back((uint8_t)((e.size() >> 8) & 0xFF)); + bytes.push_back((uint8_t)(e.size() & 0xFF)); + bytes.insert(bytes.end(), e.begin(), e.end()); + } + return bytes; + } + + /// @brief Decodes @p count entries from the entry-bundle format. + static std::vector> decode_entries( + const std::vector& bytes, size_t count) + { + std::vector> out; + out.reserve(count); + size_t pos = 0; + for (size_t i = 0; i < count; i++) + { + if (bytes.size() - pos < 2) + { + throw std::runtime_error("truncated entry bundle"); + } + const auto len = + (uint16_t)(((uint16_t)bytes[pos] << 8) | bytes[pos + 1]); + pos += 2; + if (len > bytes.size() - pos) + { + throw std::runtime_error("truncated entry bundle"); + } + out.emplace_back( + bytes.begin() + static_cast(pos), + bytes.begin() + static_cast(pos + len)); + pos += len; + } + if (pos != bytes.size()) + { + throw std::runtime_error("trailing bytes in entry bundle"); + } + return out; + } + + protected: + using DirectorySync = std::function; + + TileStoreT(std::filesystem::path prefix, DirectorySync directory_sync) : + prefix(std::move(prefix)), directory_sync(std::move(directory_sync)) + {} + + /// @brief The root directory of the store. + std::filesystem::path prefix; + + DirectorySync directory_sync; + std::set durable_directory_entries; + std::set durable_directory_contents; + + [[nodiscard]] bool confirm_full_tile(uint8_t level, uint64_t index) + { + const auto path = tile_path(TileRef{level, index}); + if (!has_full_tile(level, index)) + { + return false; + } + confirm_file_durable(path); + return true; + } + + [[nodiscard]] bool confirm_entry_bundle(uint64_t index) + { + const auto path = entries_path(index); + if (!has_entry_bundle(index)) + { + return false; + } + confirm_file_durable(path); + return true; + } + + void begin_write_attempt() + { + durable_directory_contents.clear(); + } + + /// @brief Reads an entire file into a byte vector. + static std::vector read_file(const std::filesystem::path& path) + { + std::ifstream f(path, std::ios::binary); + if (!f.good()) + { + throw std::runtime_error("cannot open file: " + path.string()); + } + return { + std::istreambuf_iterator(f), std::istreambuf_iterator()}; + } + + /// @brief Writes a file atomically via a synced temporary file. + /// @note Uses unique temp names, cleans them up on errors, and syncs the + /// file before publishing it with an atomic replace. POSIX builds also + /// confirm each directory entry in the path and sync the destination + /// directory after rename. Existing files are only reused after these + /// syncs are re-confirmed by the current write attempt. + void write_file_atomically( + const std::filesystem::path& path, const std::vector& bytes) + { + create_directories_durably(path.parent_path()); + + const std::filesystem::path tmp = temp_path(path); + TempFileGuard guard(tmp); + write_and_sync_file(tmp, bytes); + const auto parent = directory_or_dot(path.parent_path()); + durable_directory_contents.erase(parent); + replace_file(tmp, path); + sync_directory_contents(parent); + guard.dismiss(); + } + + static std::filesystem::path directory_or_dot( + const std::filesystem::path& directory) + { + return directory.empty() ? std::filesystem::path(".") : directory; + } + + void create_directories_durably( + const std::filesystem::path& requested_directory) + { + const auto directory = directory_or_dot(requested_directory); + if (durable_directory_entries.count(directory) != 0) + { + return; + } + + auto parent = directory_or_dot(directory.parent_path()); + if (parent != directory) + { + create_directories_durably(parent); + } + + std::error_code ec; + const bool exists = std::filesystem::exists(directory, ec); + if (ec) + { + throw std::runtime_error( + "cannot inspect directory " + directory.string() + ": " + + ec.message()); + } + if (exists) + { + const bool is_directory = + std::filesystem::is_directory(directory, ec); + if (ec) + { + throw std::runtime_error( + "cannot inspect directory " + directory.string() + ": " + + ec.message()); + } + if (!is_directory) + { + throw std::runtime_error( + "cannot create directory " + directory.string() + + ": path exists and is not a directory"); + } + } + else + { + const bool created = std::filesystem::create_directory(directory, ec); + if (ec) + { + throw std::runtime_error( + "cannot create directory " + directory.string() + ": " + + ec.message()); + } + if (!created && !std::filesystem::is_directory(directory, ec)) + { + throw std::runtime_error( + "cannot create directory " + directory.string() + + (ec ? ": " + ec.message() : "")); + } + } + + if (parent != directory) + { + sync_directory_for_durability(parent); + durable_directory_contents.insert(parent); + } + durable_directory_entries.insert(directory); + } + + void confirm_file_durable(const std::filesystem::path& path) + { + const auto parent = directory_or_dot(path.parent_path()); + create_directories_durably(parent); + sync_directory_contents(parent); + } + + void sync_directory_contents(const std::filesystem::path& directory) + { + const auto path = directory_or_dot(directory); + if (durable_directory_contents.count(path) != 0) + { + return; + } + sync_directory_for_durability(path); + durable_directory_contents.insert(path); + } + + class TempFileGuard + { + public: + explicit TempFileGuard(std::filesystem::path path) : + path(std::move(path)) + {} + + ~TempFileGuard() + { + if (active) + { + std::error_code ec; + std::filesystem::remove(path, ec); + } + } + + void dismiss() + { + active = false; + } + + private: + std::filesystem::path path; + bool active = true; + }; + + static std::filesystem::path temp_path(const std::filesystem::path& path) + { + static std::atomic counter{0}; + const auto stamp = + std::chrono::steady_clock::now().time_since_epoch().count(); + std::filesystem::path tmp = path; + tmp += ".tmp." + std::to_string(process_id()) + "." + + std::to_string((uint64_t)stamp) + "." + + std::to_string(counter.fetch_add(1, std::memory_order_relaxed)); + return tmp; + } + + static uint64_t process_id() + { +#ifdef _WIN32 + return (uint64_t)GetCurrentProcessId(); +#else + return (uint64_t)::getpid(); +#endif + } + + static std::string system_error_message(const std::string& what) + { +#ifdef _WIN32 + return what + ": error " + std::to_string(GetLastError()); +#else + return what + ": " + std::strerror(errno); +#endif + } + + static void require_write_progress( + size_t written, const std::filesystem::path& path) + { + if (written == 0) + { + throw std::runtime_error("short write: " + path.string()); + } + } + + static void write_and_sync_file( + const std::filesystem::path& path, const std::vector& bytes) + { +#ifdef _WIN32 + HANDLE handle = CreateFileW( + path.wstring().c_str(), + GENERIC_WRITE, + 0, + nullptr, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (handle == INVALID_HANDLE_VALUE) + { + throw std::runtime_error("cannot open file: " + path.string()); + } + bool close_handle = true; + try + { + size_t written = 0; + while (written < bytes.size()) + { + const auto remaining = bytes.size() - written; + const auto chunk = (DWORD)std::min( + remaining, (size_t)std::numeric_limits::max()); + DWORD done = 0; + if (!WriteFile( + handle, bytes.data() + written, chunk, &done, nullptr)) + { + throw std::runtime_error( + system_error_message("error writing file " + path.string())); + } + require_write_progress((size_t)done, path); + written += done; + } + if (!FlushFileBuffers(handle)) + { + throw std::runtime_error( + system_error_message("error syncing file " + path.string())); + } + close_handle = false; + if (!CloseHandle(handle)) + { + throw std::runtime_error( + system_error_message("error closing file " + path.string())); + } + } + catch (...) + { + if (close_handle) + { + CloseHandle(handle); + } + throw; + } +#else + int flags = O_WRONLY | O_CREAT | O_TRUNC; +# ifdef O_CLOEXEC + flags |= O_CLOEXEC; +# endif + int fd = ::open(path.c_str(), flags, 0666); + if (fd < 0) + { + throw std::runtime_error( + system_error_message("cannot open file " + path.string())); + } + try + { + size_t written = 0; + while (written < bytes.size()) + { + const ssize_t done = + ::write(fd, bytes.data() + written, bytes.size() - written); + if (done < 0) + { + if (errno == EINTR) + { + continue; + } + throw std::runtime_error( + system_error_message("error writing file " + path.string())); + } + require_write_progress((size_t)done, path); + written += (size_t)done; + } + if (::fsync(fd) != 0) + { + throw std::runtime_error( + system_error_message("error syncing file " + path.string())); + } + if (::close(fd) != 0) + { + fd = -1; + throw std::runtime_error( + system_error_message("error closing file " + path.string())); + } + fd = -1; + } + catch (...) + { + if (fd >= 0) + { + ::close(fd); + } + throw; + } +#endif + } + + static void replace_file( + const std::filesystem::path& tmp, const std::filesystem::path& path) + { +#ifdef _WIN32 + if (!MoveFileExW( + tmp.wstring().c_str(), + path.wstring().c_str(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) + { + throw std::runtime_error(system_error_message( + "cannot rename temp file " + tmp.string() + " to " + + path.string())); + } +#else + std::error_code ec; + std::filesystem::rename(tmp, path, ec); + if (ec) + { + throw std::runtime_error( + "cannot rename temp file " + tmp.string() + " to " + path.string() + + ": " + ec.message()); + } +#endif + } + + void sync_directory_for_durability(const std::filesystem::path& path) + { + if (directory_sync) + { + directory_sync(path); + return; + } + sync_directory_on_disk(path); + } + + static void sync_directory_on_disk(const std::filesystem::path& path) + { +#ifndef _WIN32 + int flags = O_RDONLY; +# ifdef O_DIRECTORY + flags |= O_DIRECTORY; +# endif +# ifdef O_CLOEXEC + flags |= O_CLOEXEC; +# endif + const int fd = ::open(path.c_str(), flags); + if (fd < 0) + { + throw std::runtime_error( + system_error_message("cannot open directory " + path.string())); + } + if (::fsync(fd) != 0) + { + const std::string message = + system_error_message("error syncing directory " + path.string()); + ::close(fd); + throw std::runtime_error(message); + } + if (::close(fd) != 0) + { + throw std::runtime_error( + system_error_message("error closing directory " + path.string())); + } +#else + (void)path; +#endif + } + }; + + /// @brief Computes the Merkle Tree Hash of a perfect (balanced) subtree. + /// @param leaves The subtree's leaves; the count MUST be a power of two. + /// @return The subtree root, computed with the tree's HASH_FUNCTION. + /// @note This is exactly a merkle::TreeT full-node hash, which is why tile + /// entries (such roots) are immutable: an unbalanced subtree would still + /// change as leaves are added and must therefore never be tiled. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + inline HashT perfect_root( + const std::vector>& leaves) + { + if (leaves.empty()) + { + throw std::runtime_error("perfect_root requires at least one leaf"); + } + if ((leaves.size() & (leaves.size() - 1)) != 0) + { + throw std::runtime_error( + "perfect_root requires a power-of-two number of leaves"); + } + + std::vector> level = leaves; + while (level.size() > 1) + { + std::vector> next; + next.reserve(level.size() / 2); + for (size_t i = 0; i + 1 < level.size(); i += 2) + { + HashT h; + HASH_FUNCTION(level[i], level[i + 1], h); + next.push_back(h); + } + level.swap(next); + } + return level.front(); + } + + /// @brief Computes and persists tlog-tiles tiles for a growing tree. + /// @tparam HASH_SIZE Size of each hash in bytes + /// @tparam HASH_FUNCTION The tree's node hash function + /// @note Only balanced subtrees are tiled: a level-L entry is the root of a + /// complete 2**(8L)-leaf subtree. Only full tiles (256 such entries) are + /// written; they are therefore immutable and written exactly once. Entries + /// beyond the last full-tile boundary remain in memory until a later flush + /// completes the next tile. + /// @warning No internal synchronization is provided. Callers must serialize + /// access to a writer and its store. + /// @warning A writer trusts existing full tiles as output from the same + /// tree and hash function. Callers resuming a store must establish that + /// ownership and restore the matching tree state. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TileWriterT + { + public: + /// @brief The type of hashes stored in tiles. + using Hash = HashT; + + /// @brief The associated tile store type. + using Store = TileStoreT; + + /// @brief Supplies the level-0 leaf hash for a given leaf index. + using LeafFn = std::function; + + /// @brief Counts of work performed by a write_up_to call. + struct Stats + { + /// @brief Number of full tiles written. + uint64_t full_written = 0; + }; + + /// @brief Constructs a writer over @p store. + explicit TileWriterT(Store& store) : store(store) {} + + /// @brief Writes all newly-complete full tiles for a tree of @p size + /// leaves. + /// @param size The current tree size + /// @param leaf_at Returns the level-0 leaf hash for a leaf index in + /// [0, size); only ever queried for leaves of complete subtrees. + /// @return Counts of tiles written + /// @note Incremental: full tiles already on disk are immutable and are + /// never rewritten once validated and confirmed durable. Malformed files + /// are replaced. Entries that do not complete a tile are not written. + /// Tiles are always rolled up at every level (0..63), so the on-disk set + /// always contains the higher-level roll-ups that proof generation relies + /// on. + Stats write_up_to(uint64_t size, const LeafFn& leaf_at) + { + Stats stats; + store.begin_write_attempt(); + + // tlog-tiles defines levels 0..63; the loop stops early once a level + // has no complete entries (see the entries == 0 break below). + for (uint8_t level = 0; level <= 63; level++) + { + // Number of complete (balanced) level-L entries available; this + // deliberately excludes the incomplete frontier subtree. + const uint64_t entries = entries_at_level(size, level); + if (entries == 0) + { + break; + } + ensure_level(level); + + const uint64_t full_tiles = entries / TILE_WIDTH; + + if (cursor_inited[level] == 0) + { + next_full[level] = full_prefix_length(level, full_tiles); + cursor_inited[level] = 1; + } + + for (uint64_t n = next_full[level]; n < full_tiles; n++) + { + if (store.confirm_full_tile(level, n)) + { + continue; // immutable: never rewrite an existing full tile + } + store.write_tile( + TileRef{level, n}, + collect(level, n * TILE_WIDTH, TILE_WIDTH, leaf_at)); + stats.full_written++; + } + if (full_tiles > next_full[level]) + { + next_full[level] = full_tiles; + } + } + + return stats; + } + + protected: + /// @brief The tile store written to. + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + Store& store; + + /// @brief Per-level index of the next full tile to write. + std::vector next_full; + + /// @brief Per-level flag indicating next_full has been initialised. + std::vector cursor_inited; + + /// @brief Number of complete level-@p level entries for a tree of @p + /// size. + static uint64_t entries_at_level(uint64_t size, uint8_t level) + { + const unsigned shift = 8U * (unsigned)level; + return shift >= 64 ? 0 : (size >> shift); + } + + /// @brief Ensures per-level bookkeeping vectors cover @p level. + void ensure_level(uint8_t level) + { + const size_t needed = (size_t)level + 1; + if (next_full.size() < needed) + { + next_full.resize(needed, 0); + cursor_inited.resize(needed, 0); + } + } + + /// @brief Length of the confirmed contiguous prefix, bounded by @p limit. + [[nodiscard]] uint64_t full_prefix_length(uint8_t level, uint64_t limit) + { + return detail::contiguous_prefix_length(limit, [&](uint64_t index) { + return store.confirm_full_tile(level, index); + }); + } + + /// @brief Collects @p count consecutive level-@p level entries, each the + /// root of a complete (balanced) subtree. + std::vector collect( + uint8_t level, + uint64_t first_entry, + uint64_t count, + const LeafFn& leaf_at) + { + std::vector out; + out.reserve(count); + for (uint64_t i = 0; i < count; i++) + { + const uint64_t g = first_entry + i; + if (level == 0) + { + out.push_back(leaf_at(g)); + } + else + { + // Roll up the complete child full tile (256 complete entries). + out.push_back( + perfect_root( + store.read_tile(TileRef{(uint8_t)(level - 1), g}))); + } + } + return out; + } + }; + + /// @brief Abstract source of Merkle subtree roots for proof generation. + /// @note Implementations resolve the root of a complete (balanced) subtree + /// from tiles, from an in-memory tree, or from a combination of the two. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + struct HashSourceT + { + /// @brief The type of hashes resolved. + using Hash = HashT; + + virtual ~HashSourceT() = default; + + /// @brief Resolves MTH(D[index << level : (index + 1) << level]). + /// @param level The subtree height (the subtree spans 2**level leaves) + /// @param index The subtree index at that height + /// @param out Set to the subtree root on success + /// @return Whether the complete, balanced subtree could be resolved + virtual bool subtree_root( + uint8_t level, uint64_t index, Hash& out) const = 0; + + /// @brief Resolves the level-0 leaf hash at @p index. + virtual bool leaf(uint64_t index, Hash& out) const + { + return subtree_root(0, index, out); + } + }; + + /// @brief Resolves subtree roots from tlog-tiles tile files. + /// @note @p available_size is rounded down to a whole number of full tiles: + /// only complete, durably-written full tiles are read. A complete subtree + /// within that full-tile prefix is resolvable; anything reaching into the + /// incomplete frontier yields false so that a proof builder can fall back + /// to another source (e.g. an in-memory tree). + /// @warning No internal synchronization is provided. Even const operations + /// update the internal LRU cache, so callers must serialize all access to a + /// shared source. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TileHashSourceT : public HashSourceT + { + public: + using Hash = HashT; + using Store = TileStoreT; + + /// @brief Constructs a source over @p store for trees up to + /// @p available_size leaves. @p available_size is rounded down to a whole + /// number of full tiles, since only full tiles are durable. + TileHashSourceT(const Store& store, uint64_t available_size) : + store(store), available_size((available_size / TILE_WIDTH) * TILE_WIDTH) + { + tile_cache.reserve(TILE_CACHE_SIZE); + } + + bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override + { + // The subtree covers leaves [index << level, (index + 1) << level). It + // is resolvable only when it lies entirely within the full-tile-covered + // prefix; the incomplete frontier is served from another source. + if (level >= 64 || index >= (available_size >> level)) + { + return false; + } + resolve(level, index, out); + return true; + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Store& store; + uint64_t available_size; // full-tile prefix length (a multiple of WIDTH) + + /// @brief Combines the @p span entries at @p off of @p tile into a root. + static Hash roll_up( + const std::vector& tile, uint64_t off, uint64_t span) + { + if (span == 1) + { + return tile.at(off); + } + return perfect_root(std::vector( + tile.begin() + (std::ptrdiff_t)off, + tile.begin() + (std::ptrdiff_t)(off + span))); + } + + /// @brief Resolves a complete subtree known to lie within the full-tile + /// prefix, reading the highest-level full tile that holds it (and rolling + /// up); descends to lower full tiles when a higher-level full tile has not + /// completed. Terminates because full level-0 tiles always cover the + /// prefix. + void resolve(uint8_t level, uint64_t index, Hash& out) const + { + if (level <= TILE_HEIGHT) + { + // Spans 2**level <= TILE_WIDTH leaves: held by one level-0 tile. + const uint64_t span = (uint64_t)1 << level; + const uint64_t start = index << level; + const std::vector tile = + read_tile(TileRef{0, start / TILE_WIDTH}); + out = roll_up(tile, start % TILE_WIDTH, span); + return; + } + + const uint8_t L = level / TILE_HEIGHT; + const uint8_t r = level % TILE_HEIGHT; + const uint64_t first = index << r; // first level-L entry + const uint64_t n = first / TILE_WIDTH; // level-L tile index + const unsigned full_shift = 8U * ((unsigned)L + 1U); + const uint64_t full_tiles = + full_shift >= 64 ? 0 : (available_size >> full_shift); + + if (n < full_tiles) + { + // One full level-L tile holds all 2**r entries of this subtree. + const std::vector tile = read_tile(TileRef{L, n}); + out = roll_up(tile, first % TILE_WIDTH, (uint64_t)1 << r); + return; + } + + // No full level-L tile here: split into two level-(level-1) subtrees. + Hash lo; + Hash hi; + resolve((uint8_t)(level - 1), index * 2, lo); + resolve((uint8_t)(level - 1), index * 2 + 1, hi); + HASH_FUNCTION(lo, hi, out); + } + + struct TileCacheEntry + { + TileRef ref; + std::vector hashes; + }; + + static constexpr size_t TILE_CACHE_SIZE = 64; + mutable std::vector tile_cache; + + std::vector read_tile(const TileRef& ref) const + { + for (auto it = tile_cache.begin(); it != tile_cache.end(); it++) + { + if (it->ref.level == ref.level && it->ref.index == ref.index) + { + TileCacheEntry entry = std::move(*it); + tile_cache.erase(it); + std::vector hashes = entry.hashes; + tile_cache.push_back(std::move(entry)); + return hashes; + } + } + + if (tile_cache.size() >= TILE_CACHE_SIZE) + { + tile_cache.erase(tile_cache.begin()); + } + tile_cache.push_back(TileCacheEntry{ref, store.read_tile(ref)}); + return tile_cache.back().hashes; + } + }; + + /// @brief Builds and verifies inclusion and consistency proofs. + /// @note Proofs are assembled from a HashSource using the tree's + /// HASH_FUNCTION, so an inclusion proof is byte-identical to the one + /// produced by merkle::TreeT::path()/past_path() and verifies with + /// PathT::verify(). + /// @warning Thread safety is inherited from the supplied HashSource. Callers + /// must serialize operations when the source is shared. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class ProofEngineT + { + public: + using Hash = HashT; + using Path = PathT; + using Source = HashSourceT; + + explicit ProofEngineT(const Source& source) : source(source) {} + + /// @brief The Merkle root of a tree of @p size leaves. + Hash root(uint64_t size) const + { + if (size == 0) + { + throw std::runtime_error("empty tree has no root"); + } + Hash out; + if (!mth_range(0, size, out)) + { + throw std::runtime_error("unresolved subtree while computing root"); + } + return out; + } + + /// @brief Inclusion proof for leaf @p index in a tree of @p size leaves. + /// @note Equivalent to TreeT::path(index) when size == num_leaves(), and + /// to TreeT::past_path(index, size - 1) otherwise. + std::shared_ptr inclusion_proof(uint64_t index, uint64_t size) const + { + if (index >= size) + { + throw std::runtime_error("leaf index out of bounds"); + } + + std::list elements; // leaf -> root order + uint64_t lo = 0; + uint64_t hi = size; + while (hi - lo > 1) + { + const uint64_t k = largest_pow2_lt(hi - lo); + typename Path::Element e; + if (index - lo < k) + { + if (!mth_range(lo + k, hi, e.hash)) + { + throw std::runtime_error("unresolved subtree in inclusion proof"); + } + e.direction = Path::PATH_RIGHT; + hi = lo + k; + } + else + { + if (!mth_range(lo, lo + k, e.hash)) + { + throw std::runtime_error("unresolved subtree in inclusion proof"); + } + e.direction = Path::PATH_LEFT; + lo = lo + k; + } + elements.push_front(std::move(e)); + } + + Hash leaf; + if (!source.leaf(index, leaf)) + { + throw std::runtime_error("unresolved leaf in inclusion proof"); + } + return std::make_shared( + leaf, index, std::move(elements), size - 1); + } + + /// @brief Consistency proof that a tree of @p m leaves is a prefix of a + /// tree of @p n leaves (RFC 6962). + std::vector consistency_proof(uint64_t m, uint64_t n) const + { + if (m == 0 || m > n) + { + throw std::runtime_error("invalid consistency proof sizes"); + } + std::vector proof; + if (m == n) + { + return proof; + } + subproof(m, 0, n, true, proof); + return proof; + } + + /// @brief Consistency proof between the trees whose last leaves are at + /// indices @p first_index and @p second_index (first_index <= + /// second_index). + /// @note Equivalent to consistency_proof(first_index + 1, + /// second_index + 1): it proves the tree of the first first_index + 1 + /// leaves is a prefix of the tree of the first second_index + 1 leaves. + std::vector consistency_proof_from_indices( + uint64_t first_index, uint64_t second_index) const + { + return consistency_proof(first_index + 1, second_index + 1); + } + + /// @brief Verifies an RFC 6962 consistency proof reconciling the roots of + /// trees of @p m and @p n leaves. + static bool verify_consistency( + uint64_t m, + uint64_t n, + const Hash& first_hash, + const Hash& second_hash, + std::vector proof) + { + if (m > n) + { + return false; + } + if (m == n) + { + return proof.empty() && first_hash == second_hash; + } + if (m == 0) + { + return proof.empty(); + } + + if (is_pow2(m)) + { + proof.insert(proof.begin(), first_hash); + } + if (proof.empty()) + { + return false; + } + + uint64_t fn = m - 1; + uint64_t sn = n - 1; + while ((fn & 1) != 0) + { + fn >>= 1; + sn >>= 1; + } + + Hash fr = proof[0]; + Hash sr = proof[0]; + for (size_t i = 1; i < proof.size(); i++) + { + if (sn == 0) + { + return false; + } + const Hash& c = proof[i]; + if ((fn & 1) != 0 || fn == sn) + { + HASH_FUNCTION(c, fr, fr); + HASH_FUNCTION(c, sr, sr); + if ((fn & 1) == 0) + { + while ((fn & 1) == 0 && fn != 0) + { + fn >>= 1; + sn >>= 1; + } + } + } + else + { + HASH_FUNCTION(sr, c, sr); + } + fn >>= 1; + sn >>= 1; + } + + return fr == first_hash && sr == second_hash && sn == 0; + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Source& source; + + static bool is_pow2(uint64_t n) + { + return n != 0 && (n & (n - 1)) == 0; + } + + static uint64_t largest_pow2_lt(uint64_t n) + { + uint64_t k = 1; + while (k <= (n - 1) / 2) + { + k <<= 1; + } + return k; + } + + static uint8_t log2_exact(uint64_t n) + { + uint8_t r = 0; + while (n > 1) + { + n >>= 1; + r++; + } + return r; + } + + /// @brief MTH(D[a:b]) via the source; falls back to splitting when a + /// perfect subtree cannot be resolved directly. + bool mth_range(uint64_t a, uint64_t b, Hash& out) const + { + const uint64_t w = b - a; + if (w == 0) + { + return false; + } + if (w == 1) + { + return source.leaf(a, out); + } + if (is_pow2(w) && (a % w == 0)) + { + if (source.subtree_root(log2_exact(w), a / w, out)) + { + return true; + } + } + const uint64_t k = largest_pow2_lt(w); + Hash left; + Hash right; + if (!mth_range(a, a + k, left) || !mth_range(a + k, b, right)) + { + return false; + } + HASH_FUNCTION(left, right, out); + return true; + } + + void subproof( + uint64_t m, + uint64_t lo, + uint64_t hi, + bool complete, + std::vector& proof) const + { + if (m == hi - lo) + { + if (!complete) + { + Hash h; + if (!mth_range(lo, hi, h)) + { + throw std::runtime_error( + "unresolved subtree in consistency proof"); + } + proof.push_back(h); + } + return; + } + const uint64_t k = largest_pow2_lt(hi - lo); + Hash h; + if (m <= k) + { + subproof(m, lo, lo + k, complete, proof); + if (!mth_range(lo + k, hi, h)) + { + throw std::runtime_error("unresolved subtree in consistency proof"); + } + } + else + { + subproof(m - k, lo + k, hi, false, proof); + if (!mth_range(lo, lo + k, h)) + { + throw std::runtime_error("unresolved subtree in consistency proof"); + } + } + proof.push_back(h); + } + }; + + /// @brief Resolves subtree roots from an in-memory merkle::TreeT. + /// @note Resolves only complete subtrees that are fully resident (not + /// flushed), returning false otherwise so that a builder can fall back to + /// another source. Performs no hashing changes (see TreeT::subtree_root). + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class MemoryHashSourceT : public HashSourceT + { + public: + using Hash = HashT; + using Tree = TreeT; + + explicit MemoryHashSourceT(Tree& tree) : tree(tree) {} + + bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override + { + return tree.subtree_root(level, (size_t)index, out); + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + Tree& tree; + }; + + /// @brief Resolves subtree roots from a primary source, falling back to a + /// secondary source. + /// @note Used to combine an in-memory tree (primary: no I/O, serves the + /// resident frontier) with tile files (secondary: serve the flushed past). + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class CombinedHashSourceT : public HashSourceT + { + public: + using Hash = HashT; + using Source = HashSourceT; + + CombinedHashSourceT(const Source& primary, const Source& secondary) : + primary(primary), secondary(secondary) + {} + + bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override + { + return primary.subtree_root(level, index, out) || + secondary.subtree_root(level, index, out); + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Source& primary; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Source& secondary; + }; + + /// @brief A merkle tree backed by tlog-tiles storage. + /// @note Appends grow an in-memory tree; flush() durably writes only full + /// (balanced) tiles, so the incomplete frontier is never tiled and stays + /// resident in memory. Compaction (dropping from memory the leaves already + /// covered by a full tile) is optional: enable it per flush with + /// Config::compact_on_flush, or call compact() explicitly; it never drops + /// the un-tiled frontier. Proofs are served from the combination of the + /// resident tree (frontier) and the full tiles (compacted past). + /// @note TiledTree creates a new tiled tree and cannot reopen one from tile + /// files alone. Construction rejects a non-empty tile namespace because + /// the files do not identify their tree or record enough state to restore + /// it. Use TileWriter directly only when the caller owns and restores that + /// state. + /// @warning No internal synchronization is provided. Callers must serialize + /// all access to a shared tree, including proof operations. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TiledTreeT + { + public: + using Hash = HashT; + using Tree = TreeT; + using Path = PathT; + using Store = TileStoreT; + using Writer = TileWriterT; + using Stats = typename Writer::Stats; + + /// @brief Configuration for a tiled tree. + struct Config + { + /// @brief Root directory for a new tiled tree. + /// @note The directory itself may exist, but its tile subdirectory must + /// be absent or empty. + std::filesystem::path prefix; + + /// @brief Number of most-recent leaves to keep resident when + /// compacting (i.e. never dropped from memory). + /// @note Compaction may retain one additional tiled boundary leaf so + /// rollback to exactly immutable_size() remains possible. + uint64_t retention_margin = 0; + + /// @brief If set, flush() compacts after writing tiles, dropping + /// from memory the leaves already covered by a full tile. Off by + /// default: tiles are written but the tree keeps every leaf resident. + bool compact_on_flush = false; + }; + + explicit TiledTreeT(Config config) : + config(std::move(config)), store(this->config.prefix), writer(store) + { + require_empty_tile_namespace(); + } + + TiledTreeT(const TiledTreeT&) = delete; + TiledTreeT& operator=(const TiledTreeT&) = delete; + + /// @brief Moves a tiled tree, rebinding its writer to the moved store. + TiledTreeT(TiledTreeT&& other) noexcept : + config(std::move(other.config)), + store(std::move(other.store)), + writer(store), + tree(std::move(other.tree)), + tiles_size(std::exchange(other.tiles_size, 0)), + sealed_size(std::exchange(other.sealed_size, 0)) + {} + + TiledTreeT& operator=(TiledTreeT&&) = delete; + + /// @brief Appends a leaf hash. + void append(const Hash& leaf_hash) + { + tree.insert(leaf_hash); + } + + /// @brief The number of leaves (including flushed ones). + [[nodiscard]] uint64_t size() const + { + return tree.num_leaves(); + } + + /// @brief The current Merkle root. + Hash root() + { + return tree.root(); + } + + /// @brief The number of leaves covered by the last fully successful flush. + /// @note This is always a multiple of TILE_WIDTH. It advances only after + /// every required tile level has been written successfully, and controls + /// proof reads and compaction. + [[nodiscard]] uint64_t flushed_size() const + { + return tiles_size; + } + + /// @brief The rollback seal for ranges a flush may have published. + /// @note A flush seals its full-tile boundary before writing. If the write + /// fails, this may exceed flushed_size(); keep the same tree contents and + /// retry the flush. + [[nodiscard]] uint64_t immutable_size() const + { + return sealed_size; + } + + /// @brief Access to the underlying tree. + /// @warning Mutating the tree directly bypasses tiled-tree bookkeeping. + /// In particular, direct retraction can make flushed_size() and + /// immutable_size() exceed size() and can make flushed_size() regress. + /// Use TiledTreeT operations whenever they are available. + Tree& tree_ref() + { + return tree; + } + + /// @brief Access to the underlying tile store. + /// @warning Files written or changed through this reference are trusted + /// by later flushes without checking that their hashes match this tree. + /// Mismatched files can silently invalidate proofs after compaction. + Store& store_ref() + { + return store; + } + + /// @brief Writes newly-complete full tiles to disk; compacts only if + /// Config::compact_on_flush is set. + /// @return Counts of the full tiles written by this flush + /// @note The full-tile boundary is made immutable before any tile write. + /// Only after every required tile level succeeds does flushed_size() + /// advance to that boundary. On failure, immutable_size() may advance + /// while flushed_size() does not; the tree remains resident and the flush + /// can be retried without rewriting finalized tiles. + Stats flush() + { + Stats stats; + const uint64_t n = tree.num_leaves(); + if (n == 0) + { + return stats; + } + + const uint64_t covered = (n / TILE_WIDTH) * TILE_WIDTH; + if (covered > sealed_size) + { + sealed_size = covered; + } + + stats = writer.write_up_to(n, [this](uint64_t i) -> const Hash& { + return tree.leaf((size_t)i); + }); + tiles_size = covered; + + if (config.compact_on_flush) + { + compact(); + } + return stats; + } + + /// @brief Drops old leaves covered by durably-written full tiles, keeping + /// retention_margin recent leaves and one tiled boundary leaf. + /// @return The new minimum (smallest still-resident) leaf index + /// @note Only leaves covered by a full tile are dropped, so the un-tiled + /// frontier is always retained in memory and inclusion/consistency proofs + /// remain available (the past from tiles, the frontier from memory). The + /// leaf at flushed_size() - 1 also remains resident so retract_to() can + /// represent a tree whose size is exactly immutable_size(). Has no effect + /// until tiling has produced full tiles. + uint64_t compact() + { + const uint64_t covered = (tiles_size / TILE_WIDTH) * TILE_WIDTH; + uint64_t target = covered > config.retention_margin ? + covered - config.retention_margin : + 0; + target = (target / TILE_WIDTH) * TILE_WIDTH; + // TreeT cannot retract below min_index(). Keep the final tiled leaf + // resident so rollback to a size of exactly immutable_size() remains + // representable after compaction. + if (covered > 0 && target == covered) + { + target--; + } + if (target > tree.min_index()) + { + tree.flush_to((size_t)target); + } + return tree.min_index(); + } + + /// @brief Rolls the tree back so that @p index becomes the last leaf, + /// removing all leaves after it (same semantics as TreeT::retract_to). + /// @note Only full tiles are immutable: this throws if the resulting size + /// would be smaller than immutable_size(). A failed flush may advance + /// immutable_size() without advancing flushed_size(). + void retract_to(size_t index) + { + if ((uint64_t)index + 1 < sealed_size) + { + throw std::runtime_error( + "TiledTree::retract_to: cannot roll back entries sealed for " + "immutable tiles (resulting size < immutable size)"); + } + tree.retract_to(index); + } + + /// @brief Inclusion proof for @p index in a tree of @p proof_size leaves. + /// @note Served from tiles (flushed past) combined with the resident tree + /// (recent frontier); @p proof_size may exceed flushed_size(). + std::shared_ptr inclusion_proof(uint64_t index, uint64_t proof_size) + { + return with_engine([&](const auto& engine) { + return engine.inclusion_proof(index, proof_size); + }); + } + + /// @brief Consistency proof between tree sizes @p m and @p n. + std::vector consistency_proof(uint64_t m, uint64_t n) + { + return with_engine( + [&](const auto& engine) { return engine.consistency_proof(m, n); }); + } + + /// @brief Consistency proof between the trees whose last leaves are at + /// indices @p first_index and @p second_index (first_index <= + /// second_index). + /// @note Equivalent to consistency_proof(first_index + 1, + /// second_index + 1). + std::vector consistency_proof_from_indices( + uint64_t first_index, uint64_t second_index) + { + return consistency_proof(first_index + 1, second_index + 1); + } + + protected: + Config config; + Store store; + Writer writer; + Tree tree; + uint64_t tiles_size = 0; + uint64_t sealed_size = 0; + + void require_empty_tile_namespace() const + { + const auto tile_root = store.root() / "tile"; + std::error_code ec; + const bool exists = std::filesystem::exists(tile_root, ec); + if (ec) + { + throw std::runtime_error( + "TiledTree: cannot inspect tile namespace " + tile_root.string() + + ": " + ec.message()); + } + if (!exists) + { + return; + } + + const bool is_directory = std::filesystem::is_directory(tile_root, ec); + if (ec) + { + throw std::runtime_error( + "TiledTree: cannot inspect tile namespace " + tile_root.string() + + ": " + ec.message()); + } + const bool is_empty = + is_directory && std::filesystem::is_empty(tile_root, ec); + if (ec) + { + throw std::runtime_error( + "TiledTree: cannot inspect tile namespace " + tile_root.string() + + ": " + ec.message()); + } + if (!is_empty) + { + throw std::runtime_error( + "TiledTree: tile namespace is not empty; reopening an existing " + "tiled tree is not supported"); + } + } + + /// @brief Builds a proof engine over the combined resident-tree + /// (frontier) and full-tile (flushed past) source, and invokes @p fn with + /// it. + /// @note The sources and engine are stack-local; @p fn must consume the + /// engine before returning (proofs are returned by value, holding hash + /// copies, so the result outlives the engine). + template + auto with_engine(Fn fn) + { + MemoryHashSourceT mem(tree); + TileHashSourceT tile_src(store, tiles_size); + CombinedHashSourceT combined(mem, tile_src); + ProofEngineT engine(combined); + return fn(engine); + } + }; + + /// @brief Writes tlog-tiles entry bundles (raw log entries) for a growing + /// log. + /// @note Entry bundles are level-0 only and application-owned: merklecpp + /// stores leaf hashes, while the raw entries (and the leaf-hash derivation + /// linking each entry to its level-0 tile hash) are the application's + /// responsibility. Only full bundles (TILE_WIDTH entries) are written; they + /// are immutable and written exactly once. The incomplete tail stays with + /// the application until it grows into a full bundle, mirroring the + /// un-tiled Merkle frontier. + /// @warning No internal synchronization is provided. Callers must serialize + /// access to a writer and its store. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class EntryBundleWriterT + { + public: + using Store = TileStoreT; + + /// @brief Supplies the raw bytes of the log entry at a given index. + using EntryFn = std::function(uint64_t)>; + + /// @brief Counts of work performed by a write_up_to call. + struct Stats + { + /// @brief Number of full bundles written. + uint64_t full_written = 0; + }; + + explicit EntryBundleWriterT(Store& store) : store(store) {} + + /// @brief Writes all newly-complete full bundles for a log of @p size + /// entries. + /// @param size The current number of entries + /// @param entry_at Returns the raw bytes of the entry at an index in + /// [0, size); only ever queried for entries of complete bundles. + /// @return Counts of bundles written + /// @note Incremental: full bundles already on disk are immutable and are + /// never rewritten once validated and confirmed durable. Malformed files + /// are replaced. The incomplete tail is never bundled. + Stats write_up_to(uint64_t size, const EntryFn& entry_at) + { + Stats stats; + store.begin_write_attempt(); + const uint64_t full = size / TILE_WIDTH; + + if (!cursor_inited) + { + next_full = full_prefix_length(full); + cursor_inited = true; + } + + for (uint64_t n = next_full; n < full; n++) + { + if (store.confirm_entry_bundle(n)) + { + continue; // immutable: never rewrite an existing full bundle + } + store.write_entry_bundle( + n, collect(n * TILE_WIDTH, TILE_WIDTH, entry_at)); + stats.full_written++; + } + if (full > next_full) + { + next_full = full; + } + return stats; + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + Store& store; + uint64_t next_full = 0; + bool cursor_inited = false; + + std::vector> collect( + uint64_t first, uint64_t count, const EntryFn& entry_at) + { + std::vector> out; + out.reserve(count); + for (uint64_t i = 0; i < count; i++) + { + out.push_back(entry_at(first + i)); + } + return out; + } + + [[nodiscard]] uint64_t full_prefix_length(uint64_t limit) + { + return detail::contiguous_prefix_length(limit, [&](uint64_t index) { + return store.confirm_entry_bundle(index); + }); + } + }; + + /// @brief Default tile store (SHA256, default hash function). + using TileStore = TileStoreT<32, sha256_compress>; + + /// @brief Default tile writer (SHA256, default hash function). + using TileWriter = TileWriterT<32, sha256_compress>; + + /// @brief Default abstract hash source (SHA256, default hash function). + using HashSource = HashSourceT<32, sha256_compress>; + + /// @brief Default tile-backed hash source (SHA256, default hash function). + using TileHashSource = TileHashSourceT<32, sha256_compress>; + + /// @brief Default proof engine (SHA256, default hash function). + using ProofEngine = ProofEngineT<32, sha256_compress>; + + /// @brief Default in-memory hash source (SHA256, default hash function). + using MemoryHashSource = MemoryHashSourceT<32, sha256_compress>; + + /// @brief Default combined hash source (SHA256, default hash function). + using CombinedHashSource = CombinedHashSourceT<32, sha256_compress>; + + /// @brief Default tiled tree (SHA256, default hash function). + using TiledTree = TiledTreeT<32, sha256_compress>; + + /// @brief Default entry-bundle writer (SHA256, default hash function). + using EntryBundleWriter = EntryBundleWriterT<32, sha256_compress>; + +#ifdef HAVE_OPENSSL + /// @brief SHA384 tile store. + using TileStore384 = TileStoreT<48, sha384_openssl>; + + /// @brief SHA512 tile store. + using TileStore512 = TileStoreT<64, sha512_openssl>; + + /// @brief SHA384 tile writer. + using TileWriter384 = TileWriterT<48, sha384_openssl>; + + /// @brief SHA512 tile writer. + using TileWriter512 = TileWriterT<64, sha512_openssl>; + + /// @brief SHA384 hash source, tile-backed source and proof engine. + using HashSource384 = HashSourceT<48, sha384_openssl>; + using TileHashSource384 = TileHashSourceT<48, sha384_openssl>; + using ProofEngine384 = ProofEngineT<48, sha384_openssl>; + + /// @brief SHA512 hash source, tile-backed source and proof engine. + using HashSource512 = HashSourceT<64, sha512_openssl>; + using TileHashSource512 = TileHashSourceT<64, sha512_openssl>; + using ProofEngine512 = ProofEngineT<64, sha512_openssl>; + + /// @brief SHA384 memory/combined sources and tiled tree. + using MemoryHashSource384 = MemoryHashSourceT<48, sha384_openssl>; + using CombinedHashSource384 = CombinedHashSourceT<48, sha384_openssl>; + using TiledTree384 = TiledTreeT<48, sha384_openssl>; + + /// @brief SHA512 memory/combined sources and tiled tree. + using MemoryHashSource512 = MemoryHashSourceT<64, sha512_openssl>; + using CombinedHashSource512 = CombinedHashSourceT<64, sha512_openssl>; + using TiledTree512 = TiledTreeT<64, sha512_openssl>; + + /// @brief SHA384/512 entry-bundle writers. + using EntryBundleWriter384 = EntryBundleWriterT<48, sha384_openssl>; + using EntryBundleWriter512 = EntryBundleWriterT<64, sha512_openssl>; +#endif + } +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e90a87c..2a0572b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -29,6 +29,50 @@ add_merklecpp_test(serialisation serialisation.cpp) add_merklecpp_test(partial_serialisation partial_serialisation.cpp) add_merklecpp_test(serialise_to_file serialise_to_file.cpp) add_merklecpp_test(coverage coverage.cpp) +add_merklecpp_test(tiles_store tiles_store.cpp) +add_merklecpp_test(tiles_writer tiles_writer.cpp) +add_merklecpp_test(tiles_proofs tiles_proofs.cpp) +add_merklecpp_test(tiles_tree tiles_tree.cpp) +add_merklecpp_test(tiles_entries tiles_entries.cpp) + +if(OPENSSL) + add_merklecpp_test(tiles_hashes tiles_hashes.cpp) +endif() + +if(LONG_TESTS) + add_merklecpp_test(tiles_level2 tiles_level2.cpp) + add_merklecpp_test(time_tiles time_tiles.cpp) + set_tests_properties( + ${MERKLECPP_TEST_PREFIX}tiles_level2 + ${MERKLECPP_TEST_PREFIX}time_tiles + PROPERTIES TIMEOUT 900 + ) +endif() + +find_path(DOCTEST_DIR doctest.h) +if(NOT DOCTEST_DIR) + set(DOCTEST_VERSION 2.4.11) + set(DOCTEST_DIR "${CMAKE_BINARY_DIR}/_deps/doctest-${DOCTEST_VERSION}") + set(DOCTEST_HEADER "${DOCTEST_DIR}/doctest.h") + + message(STATUS "doctest.h not found; downloading doctest v${DOCTEST_VERSION}") + file(MAKE_DIRECTORY "${DOCTEST_DIR}") + file( + DOWNLOAD + "https://raw.githubusercontent.com/doctest/doctest/v${DOCTEST_VERSION}/doctest/doctest.h" + "${DOCTEST_HEADER}" + EXPECTED_HASH SHA256=44faa038e9c3f9728efbda143748d01124ea0a27f4bf78f35a15d8fab2e039fb + STATUS DOCTEST_DOWNLOAD_STATUS + TLS_VERIFY ON + ) + + list(GET DOCTEST_DOWNLOAD_STATUS 0 DOCTEST_DOWNLOAD_CODE) + list(GET DOCTEST_DOWNLOAD_STATUS 1 DOCTEST_DOWNLOAD_MESSAGE) + if(NOT DOCTEST_DOWNLOAD_CODE EQUAL 0) + message(FATAL_ERROR "failed to download doctest.h: ${DOCTEST_DOWNLOAD_MESSAGE}") + endif() +endif() +add_merklecpp_test(unit_tests unit_tests.cpp) if(TARGET evercrypt.host) add_merklecpp_test(compare_evercrypt compare_evercrypt.cpp) @@ -39,9 +83,3 @@ if(OPENSSL ) add_merklecpp_test(compare_hash_functions compare_hash_functions.cpp) endif() - -find_path(DOCTEST_DIR doctest.h) - -if(DOCTEST_DIR) - add_merklecpp_test(unit_tests unit_tests.cpp) -endif() diff --git a/test/compare_hash_functions.cpp b/test/compare_hash_functions.cpp index 55b9245..9c4b0c4 100644 --- a/test/compare_hash_functions.cpp +++ b/test/compare_hash_functions.cpp @@ -125,7 +125,6 @@ void compare_compression_hashes() #ifdef HAVE_EVERCRYPT compare_roots(mt, mte, "EverCrypt"); #endif - } std::cout << num_trees << " trees, " << total_inserts << " inserts, " @@ -182,7 +181,6 @@ void compare_full_hashes() # ifdef HAVE_EVERCRYPT compare_roots(mto, mte, "OpenSSL"); # endif - } std::cout << num_trees << " trees, " << total_inserts << " inserts, " @@ -210,7 +208,9 @@ void bench( mt.root(); auto stop = std::chrono::high_resolution_clock::now(); const double seconds = - static_cast(std::chrono::duration_cast(stop - start).count()) / + static_cast( + std::chrono::duration_cast(stop - start) + .count()) / 1e9; std::cout << std::left << std::setw(10) << name << ": " << mt.statistics.num_insert << " insertions, " @@ -238,7 +238,9 @@ void benchT( mt.root(); auto stop = std::chrono::high_resolution_clock::now(); const double seconds = - static_cast(std::chrono::duration_cast(stop - start).count()) / + static_cast( + std::chrono::duration_cast(stop - start) + .count()) / 1e9; std::cout << std::left << std::setw(10) << name << ": " << mt.statistics.num_insert << " insertions, " diff --git a/test/coverage.cpp b/test/coverage.cpp index a930d17..b5cf21b 100644 --- a/test/coverage.cpp +++ b/test/coverage.cpp @@ -121,7 +121,9 @@ namespace mixed_case_hex[0] = 'A'; mixed_case_hex[1] = 'f'; const merkle::Hash mixed_case_hash(mixed_case_hex); - require(mixed_case_hash.bytes[0] == 0xAF, "mixed-case hash string parsed incorrectly"); + require( + mixed_case_hash.bytes[0] == 0xAF, + "mixed-case hash string parsed incorrectly"); require_throws( [] { merkle::Hash(std::string(64, 'z')); }, @@ -255,6 +257,18 @@ namespace "partial serialised_size should reject retracted to index"); } + void test_deserialise_rejects_invalid_flushed_height() + { + std::vector buffer; + merkle::serialise_uint64_t(0, buffer); + merkle::serialise_uint64_t(uint64_t{1} << 63, buffer); + + merkle::Tree tree; + require_throws( + [&] { tree.deserialise(buffer); }, + "deserialise should reject impossible flushed subtree heights"); + } + void test_tree_assignment_and_moves() { const auto source_hashes = make_hashes(5); @@ -267,9 +281,14 @@ namespace merkle::Tree copy_assigned(hash_with_byte(0xEE)); copy_assigned = source; - require(copy_assigned.root() == expected_root, "copy assignment root mismatch"); - require(copy_assigned.min_index() == expected_min, "copy assignment min mismatch"); - require(copy_assigned.max_index() == expected_max, "copy assignment max mismatch"); + require( + copy_assigned.root() == expected_root, "copy assignment root mismatch"); + require( + copy_assigned.min_index() == expected_min, + "copy assignment min mismatch"); + require( + copy_assigned.max_index() == expected_max, + "copy assignment max mismatch"); require( copy_assigned.leaf(expected_min) == source_hashes[expected_min], "copy-assigned leaf mismatch"); @@ -312,6 +331,7 @@ int main() test_hash_string_parsing(); test_path_metadata_and_equality(); test_tree_partial_serialisation_bounds(); + test_deserialise_rejects_invalid_flushed_height(); test_tree_assignment_and_moves(); } catch (const std::exception& ex) diff --git a/test/demo_tree.cpp b/test/demo_tree.cpp index 05d6f87..3ec9fcb 100644 --- a/test/demo_tree.cpp +++ b/test/demo_tree.cpp @@ -41,9 +41,9 @@ int main() std::cout << "P" << std::setw(2) << std::setfill('0') << i << ": " << path->to_string(PRINT_HASH_SIZE) << " " << '\n'; if (!path->verify(root)) - { - throw std::runtime_error("root hash mismatch"); - } + { + throw std::runtime_error("root hash mismatch"); + } const std::vector chk = *path; } @@ -52,9 +52,9 @@ int main() mt.serialise(buffer); merkle::Tree dmt(buffer); if (mt.root() != dmt.root()) - { - throw std::runtime_error("root hash mismatch"); - } + { + throw std::runtime_error("root hash mismatch"); + } std::cout << '\n'; } @@ -74,7 +74,6 @@ int main() /// SNIPPET_END: OpenSSL-SHA256 } #endif - } catch (std::exception& ex) { diff --git a/test/flush.cpp b/test/flush.cpp index f3e9d28..bd78746 100644 --- a/test/flush.cpp +++ b/test/flush.cpp @@ -36,8 +36,8 @@ int main() for (size_t k = 0; k < num_trees && !timed_out(timeout, test_start_time); k++) { - const auto num_leaves = - static_cast(1 + (std::rand() / (double)RAND_MAX) * max_num_leaves); + const auto num_leaves = static_cast( + 1 + (std::rand() / (double)RAND_MAX) * max_num_leaves); total_leaves += num_leaves; auto hashes = make_hashes(num_leaves); diff --git a/test/partial_serialisation.cpp b/test/partial_serialisation.cpp index 411a2ae..d75d575 100644 --- a/test/partial_serialisation.cpp +++ b/test/partial_serialisation.cpp @@ -1,16 +1,15 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +#include "util.h" + #include #include #include #include #include -#include - #include - -#include "util.h" +#include constexpr size_t PRINT_HASH_SIZE = 3; @@ -41,7 +40,8 @@ int main() for (size_t k = 0; k < num_trees && !timed_out(timeout, test_start_time); k++) { - const auto num_leaves = static_cast(1 + (std::rand() / (double)RAND_MAX) * max_num_leaves); + const auto num_leaves = static_cast( + 1 + (std::rand() / (double)RAND_MAX) * max_num_leaves); const auto num_subtrees = static_cast( 1 + (std::rand() / (double)RAND_MAX) * max_num_subtrees); total_leaves += num_leaves; diff --git a/test/past_paths.cpp b/test/past_paths.cpp index 6553b91..137510f 100644 --- a/test/past_paths.cpp +++ b/test/past_paths.cpp @@ -1,15 +1,14 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +#include "util.h" + #include #include #include #include - #include -#include "util.h" - constexpr size_t PSZ = 3; std::shared_ptr past_root_spec( @@ -58,8 +57,10 @@ int main() bool is_timed_out = false; for (size_t l = 0; l < num_trees && !is_timed_out; l++) { - const auto num_leaves = static_cast(1 + (std::rand() / (double)RAND_MAX) * max_num_leaves); - const auto num_paths = static_cast(1 + (std::rand() / (double)RAND_MAX) * max_num_paths); + const auto num_leaves = static_cast( + 1 + (std::rand() / (double)RAND_MAX) * max_num_leaves); + const auto num_paths = static_cast( + 1 + (std::rand() / (double)RAND_MAX) * max_num_paths); total_leaves += num_leaves; total_paths += num_paths; @@ -118,12 +119,9 @@ int main() auto pp_root = past_path->root(); #ifdef MERKLECPP_TRACE_ENABLED - MERKLECPP_TOUT << "Past root: " << past_root->to_string(PSZ) - << '\n'; - MERKLECPP_TOUT << "Past path: " << past_path->to_string(PSZ) - << '\n'; - MERKLECPP_TOUT << "Computed root: " << pp_root->to_string(PSZ) - << '\n'; + MERKLECPP_TOUT << "Past root: " << past_root->to_string(PSZ) << '\n'; + MERKLECPP_TOUT << "Past path: " << past_path->to_string(PSZ) << '\n'; + MERKLECPP_TOUT << "Computed root: " << pp_root->to_string(PSZ) << '\n'; #endif if (*past_path_spec != *past_path) diff --git a/test/past_root.cpp b/test/past_root.cpp index f03600d..848b5fc 100644 --- a/test/past_root.cpp +++ b/test/past_root.cpp @@ -1,16 +1,15 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +#include "util.h" + #include #include #include #include #include -#include - #include - -#include "util.h" +#include constexpr size_t PRINT_HASH_SIZE = 3; @@ -80,7 +79,8 @@ int main() k++) { std::map past_roots; - const auto num_leaves = static_cast(1 + (std::rand() / (double)RAND_MAX) * max_num_leaves); + const auto num_leaves = static_cast( + 1 + (std::rand() / (double)RAND_MAX) * max_num_leaves); total_leaves += num_leaves; auto hashes = make_hashes(num_leaves); @@ -112,8 +112,7 @@ int main() if (*pr != kv.second) { std::cout << pr->to_string(PRINT_HASH_SIZE) - << " != " << kv.second.to_string(PRINT_HASH_SIZE) - << '\n'; + << " != " << kv.second.to_string(PRINT_HASH_SIZE) << '\n'; throw std::runtime_error("past root hash mismatch"); } } diff --git a/test/paths.cpp b/test/paths.cpp index 77624fb..a6fb74f 100644 --- a/test/paths.cpp +++ b/test/paths.cpp @@ -1,15 +1,14 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +#include "util.h" + #include #include #include #include - #include -#include "util.h" - constexpr size_t PRINT_HASH_SIZE = 3; int main() @@ -37,8 +36,10 @@ int main() for (size_t l = 0; l < num_trees && !timed_out(timeout, test_start_time); l++) { - const auto num_leaves = static_cast(1 + (std::rand() / (double)RAND_MAX) * max_num_leaves); - const auto num_paths = static_cast(1 + (std::rand() / (double)RAND_MAX) * max_num_paths); + const auto num_leaves = static_cast( + 1 + (std::rand() / (double)RAND_MAX) * max_num_leaves); + const auto num_paths = static_cast( + 1 + (std::rand() / (double)RAND_MAX) * max_num_paths); total_leaves += num_leaves; total_paths += num_paths; @@ -55,7 +56,9 @@ int main() for (size_t p = 0; p < num_paths && !timed_out(timeout, test_start_time); p++) { - const auto i = static_cast((std::rand() / static_cast(RAND_MAX)) * static_cast(num_leaves - 1)); + const auto i = static_cast( + (std::rand() / static_cast(RAND_MAX)) * + static_cast(num_leaves - 1)); auto path = mt.path(i); if (!path->verify(root)) { @@ -65,7 +68,8 @@ int main() path->serialise(serialised_path); if (path->serialised_size() != serialised_path.size()) { - throw std::runtime_error("serialised_size() != serialised_path.size()"); + throw std::runtime_error( + "serialised_size() != serialised_path.size()"); } } } diff --git a/test/retract.cpp b/test/retract.cpp index 30c6973..52b49ce 100644 --- a/test/retract.cpp +++ b/test/retract.cpp @@ -1,15 +1,14 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +#include "util.h" + #include #include #include #include - #include -#include "util.h" - constexpr size_t PRINT_HASH_SIZE = 3; int main() @@ -37,7 +36,8 @@ int main() for (size_t k = 0; k < num_trees && !timed_out(timeout, test_start_time); k++) { - const auto num_leaves = static_cast(1 + (std::rand() / (double)RAND_MAX) * max_num_leaves); + const auto num_leaves = static_cast( + 1 + (std::rand() / (double)RAND_MAX) * max_num_leaves); total_leaves += num_leaves; auto hashes = make_hashes(num_leaves); @@ -55,7 +55,8 @@ int main() mt.retract_to(mt.max_index()); if (mt.max_index() != max_before) { - throw std::runtime_error("retract_to(max_index()) changed max_index"); + throw std::runtime_error( + "retract_to(max_index()) changed max_index"); } } diff --git a/test/serialisation.cpp b/test/serialisation.cpp index 8956ae6..1dd2abc 100644 --- a/test/serialisation.cpp +++ b/test/serialisation.cpp @@ -1,16 +1,15 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +#include "util.h" + #include #include #include #include #include -#include - #include - -#include "util.h" +#include constexpr size_t PRINT_HASH_SIZE = 3; @@ -38,7 +37,8 @@ int main() for (size_t k = 0; k < num_trees && !timed_out(timeout, test_start_time); k++) { - const auto num_leaves = static_cast(1 + (std::rand() / (double)RAND_MAX) * max_num_leaves); + const auto num_leaves = static_cast( + 1 + (std::rand() / (double)RAND_MAX) * max_num_leaves); total_leaves += num_leaves; auto hashes = make_hashes(num_leaves); @@ -80,10 +80,8 @@ int main() mt.serialised_size() != mt2.serialised_size() || mt.size() != mt2.size()) { - std::cout << "before:" << '\n' - << mt.to_string(PRINT_HASH_SIZE) << '\n'; - std::cout << "after:" << '\n' - << mt2.to_string(PRINT_HASH_SIZE) << '\n'; + std::cout << "before:" << '\n' << mt.to_string(PRINT_HASH_SIZE) << '\n'; + std::cout << "after:" << '\n' << mt2.to_string(PRINT_HASH_SIZE) << '\n'; throw std::runtime_error("tree properties mismatch"); } diff --git a/test/tiles_entries.cpp b/test/tiles_entries.cpp new file mode 100644 index 0000000..e48ae37 --- /dev/null +++ b/test/tiles_entries.cpp @@ -0,0 +1,225 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::EntryBundleWriter; +using merkle::tiles::TileRef; +using merkle::tiles::TileStore; +using merkle::tiles::TileWriter; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +static void overwrite_file(const fs::path& p, const std::vector& bytes) +{ + std::ofstream f(p, std::ios::binary | std::ios::trunc); + f.write( + reinterpret_cast(bytes.data()), (std::streamsize)bytes.size()); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + + const fs::path base = fs::temp_directory_path() / + ("merklecpp_tiles_entries_" + std::to_string((unsigned long long)seed) + + "_" + std::to_string(std::rand())); + + try + { + // 1. Encode/decode round-trip with variable-length entries. + { + const std::vector> entries = { + {}, {0x01}, {0x02, 0x03, 0x04}, std::vector(300, 0xAB)}; + const auto enc = TileStore::encode_entries(entries); + // 4 length prefixes (8 bytes) + 0 + 1 + 3 + 300 payload bytes. + expect(enc.size() == 8 + 0 + 1 + 3 + 300, "encoded size"); + const auto dec = TileStore::decode_entries(enc, entries.size()); + expect(dec == entries, "encode/decode round-trip"); + + auto trailing = enc; + trailing.push_back(0xFF); + bool trailing_threw = false; + try + { + (void)TileStore::decode_entries(trailing, entries.size()); + } + catch (const std::exception&) + { + trailing_threw = true; + } + expect(trailing_threw, "decode rejects trailing bytes"); + } + + const auto hashes = make_hashes(70000); + const auto entry_at = [&](uint64_t i) -> std::vector { + return hashes[i]; // HashT -> vector + }; + + // 2. size 256: a single full bundle, no tail. + { + TileStore store(base / "e256"); + EntryBundleWriter writer(store); + const auto s = writer.write_up_to(256, entry_at); + expect(s.full_written == 1, "256 counts"); + expect(store.has_entry_bundle(0), "256 full bundle"); + expect(!store.has_entry_bundle(1), "256 no second bundle"); + + const auto b0 = store.read_entry_bundle(0); + expect(b0.size() == 256, "256 bundle width"); + for (size_t i = 0; i < 256; i++) + { + expect(Hash(b0[i]) == hashes[i], "256 entry round-trip"); + } + } + + // 3. size 70000: 273 full bundles (the 112-entry tail is not bundled), + // plus tile linkage. + { + TileStore store(base / "e70k"); + EntryBundleWriter writer(store); + const auto s = writer.write_up_to(70000, entry_at); + expect(s.full_written == 273, "70000 counts"); + expect(store.has_entry_bundle(0), "70000 bundle 0"); + expect(store.has_entry_bundle(272), "70000 bundle 272"); + expect(!store.has_entry_bundle(273), "70000 no bundle 273"); + + // Level-0 tile entries (leaf hashes) correspond to the bundle entries + // under the identity leaf-hash used here. + TileWriter tw(store); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + tw.write_up_to(70000, leaf_at); + const auto tile0 = store.read_tile(TileRef{0, 0}); + const auto bundle0 = store.read_entry_bundle(0); + for (size_t i = 0; i < 256; i++) + { + expect(Hash(bundle0[i]) == tile0[i], "bundle<->tile linkage"); + } + } + + // 4. Incremental writes: full bundles are immutable, the tail is not + // bundled. + { + TileStore store(base / "einc"); + EntryBundleWriter writer(store); + expect(writer.write_up_to(256, entry_at).full_written == 1, "inc s1"); + const auto s2 = writer.write_up_to(256, entry_at); + expect(s2.full_written == 0, "inc immutable rerun"); + overwrite_file(store.entries_path(0), {0x00, 0x05, 0x01}); + expect(!store.has_entry_bundle(0), "inc corrupt bundle not durable"); + EntryBundleWriter resumed(store); + expect( + resumed.write_up_to(256, entry_at).full_written == 1, + "inc rewrite corrupt"); + expect(store.has_entry_bundle(0), "inc rewritten bundle durable"); + const auto s3 = writer.write_up_to(600, entry_at); + expect(s3.full_written == 1, "inc s3 full"); + expect(store.has_entry_bundle(1), "inc bundle 1"); + expect(!store.has_entry_bundle(2), "inc no tail bundle"); + } + + // 5. Resume repairs an interior malformed bundle even when later files are + // valid. + { + const fs::path dir = base / "eholes"; + { + TileStore store(dir); + EntryBundleWriter writer(store); + expect( + writer.write_up_to(2048, entry_at).full_written == 8, + "holes initial bundles"); + overwrite_file(store.entries_path(3), {0x00, 0x05, 0x01}); + expect(!store.has_entry_bundle(3), "holes interior bundle malformed"); + expect(store.has_entry_bundle(7), "holes later bundle remains valid"); + } + + TileStore store(dir); + EntryBundleWriter writer(store); + expect( + writer.write_up_to(2048, entry_at).full_written == 1, + "holes rewrites interior bundle"); + std::vector> expected; + expected.reserve(merkle::tiles::TILE_WIDTH); + constexpr uint64_t hole_begin = (uint64_t)merkle::tiles::TILE_WIDTH * 3; + constexpr uint64_t hole_end = (uint64_t)merkle::tiles::TILE_WIDTH * 4; + for (uint64_t i = hole_begin; i < hole_end; i++) + { + expected.push_back(entry_at(i)); + } + expect( + store.read_entry_bundle(3) == expected, + "holes repaired bundle contents"); + } + + // 6. Recovery examines only bundle indices relevant to the requested size. + { + const fs::path dir = base / "esparse"; + const std::vector> bundle( + merkle::tiles::TILE_WIDTH, {0x42}); + { + TileStore store(dir); + store.write_entry_bundle(0, bundle); + for (uint64_t index = 1;; index <<= 1) + { + store.write_entry_bundle(index, bundle); + if (index == (uint64_t{1} << 63)) + { + break; + } + } + } + + TileStore store(dir); + EntryBundleWriter writer(store); + expect( + writer.write_up_to(merkle::tiles::TILE_WIDTH, entry_at).full_written == + 0, + "sparse bounded recovery"); + expect( + store.has_entry_bundle(0), "sparse requested bundle remains valid"); + } + + std::cout << "tiles_entries: OK" << '\n'; + + std::error_code ec; + fs::remove_all(base, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + + return 0; +} diff --git a/test/tiles_hashes.cpp b/test/tiles_hashes.cpp new file mode 100644 index 0000000..4ac86a5 --- /dev/null +++ b/test/tiles_hashes.cpp @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +template < + size_t HASH_SIZE, + typename Tree, + typename TiledTree, + typename ProofEngine> +static void exercise_tiled_hash(const fs::path& prefix, const std::string& name) +{ + const auto hashes = make_hashesT(300); + Tree reference; + typename TiledTree::Config config; + config.prefix = prefix; + config.compact_on_flush = true; + TiledTree tree(config); + + for (const auto& hash : hashes) + { + reference.insert(hash); + tree.append(hash); + } + + expect(tree.flush().full_written == 1, name + " full tile written"); + expect(tree.flushed_size() == 256, name + " flushed size"); + expect(tree.root() == reference.root(), name + " root"); + expect( + fs::file_size(prefix / "tile" / "0" / "000") == + (uintmax_t)merkle::tiles::TILE_WIDTH * HASH_SIZE, + name + " tile byte size"); + + const auto inclusion = tree.inclusion_proof(0, tree.size()); + expect(inclusion->verify(reference.root()), name + " inclusion proof"); + + const auto consistency = tree.consistency_proof(256, tree.size()); + expect( + ProofEngine::verify_consistency( + 256, + tree.size(), + *reference.past_root(255), + reference.root(), + consistency), + name + " consistency proof"); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + + const fs::path base = fs::temp_directory_path() / + ("merklecpp_tiles_hashes_" + std::to_string((unsigned long long)seed) + + "_" + std::to_string(std::rand())); + + try + { + exercise_tiled_hash< + 48, + merkle::Tree384, + merkle::tiles::TiledTree384, + merkle::tiles::ProofEngine384>(base / "sha384", "SHA384"); + exercise_tiled_hash< + 64, + merkle::Tree512, + merkle::tiles::TiledTree512, + merkle::tiles::ProofEngine512>(base / "sha512", "SHA512"); + + std::cout << "tiles_hashes: OK" << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + } + catch (const std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + + return 0; +} diff --git a/test/tiles_level2.cpp b/test/tiles_level2.cpp new file mode 100644 index 0000000..e1ea541 --- /dev/null +++ b/test/tiles_level2.cpp @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// End-to-end coverage of the level-2 tile paths. A full level-2 tile requires +// 256^3 == 16,777,216 leaves, so this is a deliberately large test: it writes +// the tiles for that many leaves (~65k tile files) from a deterministic leaf +// source -- no in-memory tree is built -- and cross-checks the writer's +// level-by-level roll-up against TileHashSourceT::resolve (which reads the +// level-2 tile) and against the underlying leaves. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TILE_WIDTH; +using merkle::tiles::TileHashSource; +using merkle::tiles::TileRef; +using merkle::tiles::TileStore; +using merkle::tiles::TileWriter; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +// Roll up a perfect (power-of-two) set of hashes with the default combiner. +static Hash rollup(const std::vector& hashes) +{ + return merkle::tiles::perfect_root<32, merkle::sha256_compress>(hashes); +} + +int main() +{ + const fs::path dir = fs::temp_directory_path() / + ("merklecpp_tiles_level2_" + + std::to_string((unsigned long long)std::time(nullptr))); + + try + { + // 256^3 leaves == exactly one full level-2 tile. + const uint64_t n = + (uint64_t)TILE_WIDTH * (uint64_t)TILE_WIDTH * (uint64_t)TILE_WIDTH; + + TileStore store(dir); + TileWriter writer(store); + + // Deterministic leaf hash derived from the index (low 8 bytes); avoids + // materialising a 16.7M-entry vector. + Hash leaf; + const auto leaf_at = [&](uint64_t i) -> const Hash& { + leaf.zero(); + for (int b = 0; b < 8; b++) + { + leaf.bytes[b] = (uint8_t)(i >> (8 * b)); + } + return leaf; + }; + + const auto stats = writer.write_up_to(n, leaf_at); + + // 65536 full L0 tiles + 256 full L1 tiles + 1 full L2 tile. + expect( + stats.full_written == (uint64_t)TILE_WIDTH * TILE_WIDTH + TILE_WIDTH + 1, + "level2: tile counts"); + expect(store.has_full_tile(2, 0), "level2: L2 tile present"); + expect(!store.has_full_tile(2, 1), "level2: no second L2 tile"); + expect(!store.has_full_tile(3, 0), "level2: no L3 tile"); + + const auto l2 = store.read_tile(TileRef{2, 0}); + expect(l2.size() == TILE_WIDTH, "level2: L2 tile width"); + + const TileHashSource src(store, n); + + // Each level-2 entry j is the root of level-1 tile j, which rolls up + // level-0 tiles, which are the leaves verbatim. Cross-check the writer's + // roll-up, resolve's level-2 read, and the leaf chain on a sample of + // indices. + for (const uint64_t j : + {(uint64_t)0, (uint64_t)1, (uint64_t)200, (uint64_t)255}) + { + const auto l1j = store.read_tile(TileRef{1, j}); + expect(l2[j] == rollup(l1j), "level2: L2[j] == rollup(L1 tile j)"); + + // resolve reads the level-2 tile for the 2^16-leaf subtree at index j. + Hash r16; + expect( + src.subtree_root(16, j, r16), "level2: subtree_root(16,j) resolves"); + expect(r16 == l2[j], "level2: resolve(16,j) == L2[j]"); + + // Anchor to leaves: L1 tile j entry 0 == root of L0 tile (j*256), whose + // first entry is leaf (j * 65536). + const auto l0 = store.read_tile(TileRef{0, j * TILE_WIDTH}); + expect(l1j[0] == rollup(l0), "level2: L1[j][0] == rollup(L0 tile)"); + expect( + l0[0] == leaf_at(j * (uint64_t)TILE_WIDTH * TILE_WIDTH), + "level2: L0 entry == leaf"); + } + + // Intra-tile roll-up: the 2^17-leaf subtree at 0 hashes L2[0] and L2[1]. + Hash r17; + expect(src.subtree_root(17, 0, r17), "level2: subtree_root(17,0) resolves"); + expect( + r17 == rollup({l2[0], l2[1]}), "level2: resolve(17,0) == H(L2[0],L2[1])"); + + std::cout << "tiles_level2: OK" << '\n'; + + std::error_code ec; + fs::remove_all(dir, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + + return 0; +} diff --git a/test/tiles_proofs.cpp b/test/tiles_proofs.cpp new file mode 100644 index 0000000..2bcbe0a --- /dev/null +++ b/test/tiles_proofs.cpp @@ -0,0 +1,254 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::CombinedHashSource; +using merkle::tiles::MemoryHashSource; +using merkle::tiles::ProofEngine; +using merkle::tiles::TileHashSource; +using merkle::tiles::TileStore; +using merkle::tiles::TileWriter; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +// Exercises tile-derived proofs for a tree of `n` leaves against the existing +// library (which acts as the oracle: proofs must be byte-identical). Full tiles +// serve the covered prefix and an in-memory tree serves the un-tiled frontier, +// exactly as TiledTree combines them. +static void check_size( + const fs::path& dir, uint64_t n, const std::vector& hashes) +{ + const std::string at = " @n=" + std::to_string(n); + + TileStore store(dir); + TileWriter writer(store); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + writer.write_up_to(n, leaf_at); + + // Oracle: a full, never-flushed tree with the same leaves. + merkle::Tree tree; + for (uint64_t i = 0; i < n; i++) + { + tree.insert(hashes[i]); + } + const Hash root = tree.root(); + + // Production-shaped source: full tiles serve the covered prefix, an in-memory + // tree serves the un-tiled frontier. Drop the tiled past from the frontier + // tree so proofs over it are genuinely served from the tiles. merklecpp keeps + // at least one resident leaf, so never flush the whole tree. + const uint64_t covered = (n / 256) * 256; // 256 == TILE_WIDTH + merkle::Tree frontier; + for (uint64_t i = 0; i < n; i++) + { + frontier.insert(hashes[i]); + } + uint64_t drop_to = covered; + if (n > 0 && drop_to >= n) + { + drop_to = n - 1; + } + if (drop_to > 0) + { + frontier.flush_to((size_t)drop_to); + } + const MemoryHashSource mem(frontier); + const TileHashSource tiles(store, covered); + const CombinedHashSource source(mem, tiles); + const ProofEngine engine(source); + + // Root recomputed from tiles equals the library root. + expect(engine.root(n) == root, "root" + at); + + // Indices to probe: all of them for small trees, else a spread. + std::vector indices; + if (n <= 16) + { + for (uint64_t i = 0; i < n; i++) + { + indices.push_back(i); + } + } + else + { + for (const uint64_t i : + {(uint64_t)0, (uint64_t)1, n / 3, n / 2, n - 2, n - 1}) + { + indices.push_back(i); + } + } + + // Inclusion proofs are identical to TreeT::path and verify. + for (const uint64_t i : indices) + { + if (i >= n) + { + continue; + } + const auto p = engine.inclusion_proof(i, n); + expect(*p == *tree.path(i), "inclusion==path i=" + std::to_string(i) + at); + expect(p->verify(root), "inclusion verify i=" + std::to_string(i) + at); + } + + // Consistency pairs: exhaustive for small trees, else a fixed spread. + std::vector> pairs; + if (n <= 16) + { + for (uint64_t m = 1; m < n; m++) + { + for (uint64_t k = m + 1; k <= n; k++) + { + pairs.emplace_back(m, k); + } + } + } + else + { + pairs = {{1, n}, {n / 2, n}, {n - 1, n}, {1, 2}}; + // Tile-boundary crossings. + if (n > 256) + { + pairs.emplace_back(256, n); + pairs.emplace_back(257, n); + } + } + + for (const auto& pr : pairs) + { + const uint64_t m = pr.first; + const uint64_t k = pr.second; + if (m == 0 || m >= k || k > n) + { + continue; + } + + const Hash rm = engine.root(m); + const Hash rk = engine.root(k); + expect(rm == *tree.past_root(m - 1), "past_root m" + at); + expect(rk == *tree.past_root(k - 1), "past_root k" + at); + + // Past inclusion proof matches TreeT::past_path. + const uint64_t i = m / 2; + const auto pp = engine.inclusion_proof(i, m); + expect( + *pp == *tree.past_path(i, m - 1), + "inclusion(m)==past_path i=" + std::to_string(i) + at); + expect(pp->verify(rm), "inclusion(m) verify" + at); + + // Consistency proof reconciles the two roots. + const auto cp = engine.consistency_proof(m, k); + expect( + ProofEngine::verify_consistency(m, k, rm, rk, cp), + "consistency " + std::to_string(m) + "->" + std::to_string(k) + at); + + // The index-based variant is consistency_proof(i+1, j+1). + expect( + engine.consistency_proof_from_indices(m - 1, k - 1) == cp, + "consistency index variant" + at); + + // Tampering with a proof element or a root is rejected. + auto bad = cp; + bad[0].bytes[0] ^= 0xFFU; + expect( + !ProofEngine::verify_consistency(m, k, rm, rk, bad), + "consistency tamper rejected" + at); + + Hash wrong = rk; + wrong.bytes[0] ^= 0xFFU; + expect( + !ProofEngine::verify_consistency(m, k, rm, wrong, cp), + "consistency wrong root rejected" + at); + } + + std::error_code ec; + fs::remove_all(dir, ec); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + + const fs::path base = fs::temp_directory_path() / + ("merklecpp_tiles_proofs_" + std::to_string((unsigned long long)seed) + + "_" + std::to_string(std::rand())); + + try + { + const auto hashes = make_hashes(300000); + + for (const uint64_t n : + {(uint64_t)1, + (uint64_t)2, + (uint64_t)3, + (uint64_t)4, + (uint64_t)5, + (uint64_t)7, + (uint64_t)8, + (uint64_t)13, + (uint64_t)16, + (uint64_t)255, + (uint64_t)256, + (uint64_t)257, + (uint64_t)1000}) + { + check_size(base / ("n" + std::to_string(n)), n, hashes); + } + std::cout << "small/medium sizes: OK" << '\n'; + + // Large trees. 65536 == 256 full L0 tiles == one full L1 tile (exact L1 + // boundary); 65537 is one past it; 70000 exercises a full L1 tile plus an + // in-memory frontier; 300000 forces proofs over height->=16 subtrees, so + // TileHashSource::resolve descends through level-2 logic (full_shift = 24) + // before reaching the level-1 tiles -- the only coverage of the L>=2 path. + for (const uint64_t n : + {(uint64_t)65536, (uint64_t)65537, (uint64_t)70000, (uint64_t)300000}) + { + check_size(base / ("big" + std::to_string(n)), n, hashes); + std::cout << "size " << n << ": OK" << '\n'; + } + + std::cout << "tiles_proofs: OK" << '\n'; + + std::error_code ec; + fs::remove_all(base, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + + return 0; +} diff --git a/test/tiles_store.cpp b/test/tiles_store.cpp new file mode 100644 index 0000000..6d45d01 --- /dev/null +++ b/test/tiles_store.cpp @@ -0,0 +1,451 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TileRef; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +static void expect_eq( + const std::string& got, const std::string& expected, const std::string& what) +{ + if (got != expected) + { + throw std::runtime_error( + what + ": got '" + got + "', expected '" + expected + "'"); + } +} + +static std::string rel(const merkle::tiles::TileStore& store, const fs::path& p) +{ + return p.lexically_relative(store.root()).generic_string(); +} + +static bool any_tmp_files(const fs::path& dir) +{ + if (!fs::exists(dir)) + { + return false; + } + return std::any_of( + fs::recursive_directory_iterator(dir), + fs::recursive_directory_iterator(), + [](const auto& entry) { + return entry.path().filename().string().find(".tmp") != std::string::npos; + }); +} + +class TileStoreProbe : public merkle::tiles::TileStore +{ +public: + using Store = merkle::tiles::TileStore; + + static void check_write_progress(size_t written) + { + Store::require_write_progress(written, "test"); + } +}; + +struct SyncFault +{ + fs::path fail_path; + size_t failures_remaining = 0; + size_t matching_calls_before_failure = 0; + std::vector calls; + + void sync(const fs::path& path) + { + const auto normalised = path.lexically_normal(); + calls.push_back(normalised); + if (failures_remaining > 0 && normalised == fail_path.lexically_normal()) + { + if (matching_calls_before_failure > 0) + { + matching_calls_before_failure--; + return; + } + failures_remaining--; + throw std::runtime_error("injected directory sync failure"); + } + } + + [[nodiscard]] size_t call_count(const fs::path& path) const + { + const auto normalised = path.lexically_normal(); + return (size_t)std::count(calls.begin(), calls.end(), normalised); + } +}; + +class FaultInjectingTileStore : public merkle::tiles::TileStore +{ +public: + using Store = merkle::tiles::TileStore; + + FaultInjectingTileStore( + fs::path prefix, const std::shared_ptr& fault) : + Store( + std::move(prefix), [fault](const fs::path& path) { fault->sync(path); }) + {} +}; + +// Overwrites a file with exactly the given bytes (to simulate corruption). +static void overwrite_file(const fs::path& p, const std::vector& bytes) +{ + std::ofstream f(p, std::ios::binary | std::ios::trunc); + f.write( + reinterpret_cast(bytes.data()), (std::streamsize)bytes.size()); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + + const fs::path dir = fs::temp_directory_path() / + ("merklecpp_tiles_" + std::to_string((unsigned long long)seed) + "_" + + std::to_string(std::rand())); + + try + { + // 1. Tile index encoding vectors (tlog-tiles examples). + expect_eq(merkle::tiles::encode_tile_index(0), "000", "encode 0"); + expect_eq(merkle::tiles::encode_tile_index(5), "005", "encode 5"); + expect_eq(merkle::tiles::encode_tile_index(255), "255", "encode 255"); + expect_eq(merkle::tiles::encode_tile_index(999), "999", "encode 999"); + expect_eq( + merkle::tiles::encode_tile_index(1000), "x001/000", "encode 1000"); + expect_eq( + merkle::tiles::encode_tile_index(1234067), + "x001/x234/067", + "encode 1234067"); + + merkle::tiles::TileStore store(dir); + const size_t hsz = Hash().size(); + const auto full = make_hashes(merkle::tiles::TILE_WIDTH); + + // 2. Resource path layout (full tiles and bundles only). + expect_eq( + rel(store, store.tile_path(TileRef{0, 0})), + "tile/0/000", + "tile_path L0 N0"); + expect_eq( + rel(store, store.tile_path(TileRef{1, 1234067})), + "tile/1/x001/x234/067", + "tile_path L1 big index"); + expect_eq( + rel(store, store.entries_path(5)), "tile/entries/005", "entries full"); + + // 2b. Production writes sync each directory link and the destination + // directory in order. + { + const fs::path prefix = dir / "durable"; + const auto fault = std::make_shared(); + FaultInjectingTileStore durable_store(prefix, fault); + durable_store.write_tile(TileRef{1, 0}, full); + const std::vector expected = { + dir.parent_path(), dir, prefix, prefix / "tile", prefix / "tile" / "1"}; + expect( + fault->calls.size() >= expected.size() && + std::equal( + expected.begin(), + expected.end(), + fault->calls.end() - (std::ptrdiff_t)expected.size()), + "directory parents and destination synced in order"); + } + + // 2c. A successful write must make progress. This shared guard prevents + // the Windows WriteFile loop from spinning if it reports zero bytes. + { + bool zero_threw = false; + try + { + TileStoreProbe::check_write_progress(0); + } + catch (const std::exception&) + { + zero_threw = true; + } + expect(zero_threw, "zero-byte write rejected"); + TileStoreProbe::check_write_progress(1); + } + + // 2d. A failed destination-directory sync after rename is retried before + // another already-open writer trusts the visible tile. + { + const fs::path prefix = dir / "retry_destination"; + const fs::path destination = prefix / "tile" / "0"; + const auto fault = std::make_shared(); + fault->fail_path = destination; + constexpr uint64_t growing_size = (uint64_t)merkle::tiles::TILE_WIDTH * 2; + const auto growing = make_hashes((size_t)growing_size); + const auto leaf_at = [&](uint64_t i) -> const Hash& { + return growing[i]; + }; + + FaultInjectingTileStore retry_store(prefix, fault); + merkle::tiles::TileWriter retry_writer(retry_store); + expect( + retry_writer.write_up_to(merkle::tiles::TILE_WIDTH, leaf_at) + .full_written == 1, + "first writer publishes initial tile"); + + fault->matching_calls_before_failure = 1; + fault->failures_remaining = 1; + { + FaultInjectingTileStore failed_store(prefix, fault); + merkle::tiles::TileWriter writer(failed_store); + bool sync_threw = false; + try + { + (void)writer.write_up_to(growing_size, leaf_at); + } + catch (const std::exception&) + { + sync_threw = true; + } + expect(sync_threw, "destination sync failure propagated"); + expect( + failed_store.has_full_tile(0, 1), + "renamed tile remains visible after sync failure"); + } + + expect( + retry_writer.write_up_to(growing_size, leaf_at).full_written == 0, + "retry certifies visible tile without rewriting"); + expect( + fault->call_count(destination) == 4, + "destination directory sync retried"); + const std::vector second( + growing.begin() + (std::ptrdiff_t)merkle::tiles::TILE_WIDTH, + growing.end()); + expect( + retry_store.read_tile(TileRef{0, 1}) == second, + "retried tile remains intact"); + } + + // 2e. A failed ancestor-parent sync is retried even though the directory + // created before the failure remains visible. + { + const fs::path prefix = dir / "retry_ancestor"; + const auto fault = std::make_shared(); + fault->fail_path = prefix; + fault->failures_remaining = 1; + + { + FaultInjectingTileStore failed_store(prefix, fault); + merkle::tiles::TileWriter writer(failed_store); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return full[i]; }; + bool sync_threw = false; + try + { + (void)writer.write_up_to(merkle::tiles::TILE_WIDTH, leaf_at); + } + catch (const std::exception&) + { + sync_threw = true; + } + expect(sync_threw, "ancestor sync failure propagated"); + expect( + fs::is_directory(prefix / "tile"), + "directory remains visible after parent sync failure"); + expect( + !failed_store.has_full_tile(0, 0), + "tile not published before ancestor sync succeeds"); + } + + FaultInjectingTileStore retry_store(prefix, fault); + merkle::tiles::TileWriter retry_writer(retry_store); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return full[i]; }; + expect( + retry_writer.write_up_to(merkle::tiles::TILE_WIDTH, leaf_at) + .full_written == 1, + "retry writes tile after certifying existing ancestor"); + expect(fault->call_count(prefix) == 2, "ancestor parent sync retried"); + expect(retry_store.has_full_tile(0, 0), "retry publishes full tile"); + } + + // 3a. Full tile byte round-trip. + const TileRef full_ref{0, 0}; + store.write_tile(full_ref, full); + expect(store.has_full_tile(0, 0), "has_full_tile after write"); + expect(!store.has_full_tile(0, 5), "missing full tile"); + expect( + fs::file_size(store.tile_path(full_ref)) == + (uintmax_t)merkle::tiles::TILE_WIDTH * hsz, + "full tile file size"); + const auto full_rt = store.read_tile(full_ref); + expect(full_rt.size() == full.size(), "full tile width round-trip"); + for (size_t i = 0; i < full.size(); i++) + { + expect(full_rt[i] == full[i], "full tile hash round-trip"); + } + + // 3b. Wrong-width writes are rejected (only 256-wide tiles are valid). + const std::vector three(full.begin(), full.begin() + 3); + bool threw = false; + try + { + store.write_tile(TileRef{0, 2}, three); // 3 hashes for a full tile + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "width mismatch rejected"); + + // 3c. Corrupt / truncated files are rejected on read (integrity check), so + // a torn write can never be served as a valid tile or bundle. + { + const auto expect_throws = + [](const std::function& fn, const std::string& what) { + bool t = false; + try + { + fn(); + } + catch (const std::exception&) + { + t = true; + } + expect(t, what); + }; + + // Truncated tile: fewer bytes than a full tile. + overwrite_file(store.tile_path(full_ref), std::vector(hsz, 0)); + expect(!store.has_full_tile(0, 0), "truncated tile is not durable"); + expect_throws( + [&] { (void)store.read_tile(full_ref); }, "truncated tile rejected"); + store.write_tile(full_ref, full); + expect(store.has_full_tile(0, 0), "truncated tile rewritten"); + expect(store.read_tile(full_ref) == full, "rewritten tile round-trip"); + + // Oversized tile: more bytes than a full tile. + overwrite_file( + store.tile_path(full_ref), + std::vector((merkle::tiles::TILE_WIDTH + 1) * hsz, 0)); + expect(!store.has_full_tile(0, 0), "oversized tile is not durable"); + expect_throws( + [&] { (void)store.read_tile(full_ref); }, "oversized tile rejected"); + + // A valid full entry bundle whose file is then cut short. + std::vector> entries(merkle::tiles::TILE_WIDTH); + for (size_t i = 0; i < entries.size(); i++) + { + entries[i] = {(uint8_t)i, 0x7F}; + } + store.write_entry_bundle(0, entries); + expect(store.has_entry_bundle(0), "bundle durable before truncation"); + expect( + store.read_entry_bundle(0).size() == merkle::tiles::TILE_WIDTH, + "bundle valid before truncation"); + // Claims a 5-byte entry but supplies only one trailing byte. + overwrite_file(store.entries_path(0), {0x00, 0x05, 0x01}); + expect(!store.has_entry_bundle(0), "truncated bundle is not durable"); + expect_throws( + [&] { (void)store.read_entry_bundle(0); }, + "truncated entry bundle rejected"); + + // A syntactically complete bundle with trailing bytes is not durable. + auto encoded = merkle::tiles::TileStore::encode_entries(entries); + encoded.push_back(0xFF); + overwrite_file(store.entries_path(0), encoded); + expect(!store.has_entry_bundle(0), "trailing bundle is not durable"); + expect_throws( + [&] { (void)store.read_entry_bundle(0); }, + "trailing entry bundle rejected"); + + // decode_entries directly: a length prefix that overruns the buffer. + expect_throws( + [] { + (void)merkle::tiles::TileStore::decode_entries({0xFF, 0xFF, 0x00}, 1); + }, + "decode_entries oversized length rejected"); + expect_throws( + [] { + (void)merkle::tiles::TileStore::decode_entries({0x00, 0x00, 0xFF}, 1); + }, + "decode_entries trailing bytes rejected"); + } + +// Windows file replacement semantics can reject racing same-path replaces even +// when the final tile remains valid, so keep this stress check POSIX-only. +#ifndef _WIN32 + // 3d. Concurrent same-tile writes use unique temp files and leave no + // temporary files behind after success. Each thread owns its store object; + // sharing one object requires caller-provided synchronization. + { + const TileRef concurrent_ref{0, 42}; + std::atomic ok{true}; + std::vector threads; + for (size_t i = 0; i < 8; i++) + { + threads.emplace_back([&] { + try + { + merkle::tiles::TileStore thread_store(dir); + thread_store.write_tile(concurrent_ref, full); + } + catch (...) + { + ok = false; + } + }); + } + for (auto& thread : threads) + { + thread.join(); + } + expect(ok, "concurrent writes succeeded"); + expect(store.read_tile(concurrent_ref) == full, "concurrent tile valid"); + expect(!any_tmp_files(dir), "no temp files left after writes"); + } +#endif + + std::cout << "tiles_store: OK" << '\n'; + + std::error_code ec; + fs::remove_all(dir, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + + return 0; +} diff --git a/test/tiles_tree.cpp b/test/tiles_tree.cpp new file mode 100644 index 0000000..8676081 --- /dev/null +++ b/test/tiles_tree.cpp @@ -0,0 +1,797 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::MemoryHashSource; +using merkle::tiles::ProofEngine; +using merkle::tiles::TiledTree; + +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_copy_assignable_v); +static_assert(std::is_nothrow_move_constructible_v); +static_assert(!std::is_move_assignable_v); + +class ProofEngineProbe : public ProofEngine +{ +public: + using ProofEngine::largest_pow2_lt; +}; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +// Validates the TreeT::subtree_root accessor via memory-only proofs: they must +// match the library exactly. +static void check_memory_source(uint64_t n, const std::vector& hashes) +{ + const std::string at = " @n=" + std::to_string(n); + + merkle::Tree tree; + for (uint64_t i = 0; i < n; i++) + { + tree.insert(hashes[i]); + } + const Hash root = tree.root(); + + const MemoryHashSource source(tree); + const ProofEngine engine(source); + + expect(engine.root(n) == root, "mem root" + at); + + std::vector indices; + if (n <= 16) + { + for (uint64_t i = 0; i < n; i++) + { + indices.push_back(i); + } + } + else + { + for (const uint64_t i : {(uint64_t)0, (uint64_t)1, n / 2, n - 1}) + { + indices.push_back(i); + } + } + + for (const uint64_t i : indices) + { + const auto p = engine.inclusion_proof(i, n); + expect( + *p == *tree.path(i), "mem inclusion==path i=" + std::to_string(i) + at); + expect(p->verify(root), "mem inclusion verify i=" + std::to_string(i) + at); + } + + std::vector> pairs; + if (n <= 16) + { + for (uint64_t m = 1; m < n; m++) + { + for (uint64_t k = m + 1; k <= n; k++) + { + pairs.emplace_back(m, k); + } + } + } + else + { + pairs = {{1, n}, {n / 2, n}, {n - 1, n}}; + } + + for (const auto& pr : pairs) + { + const uint64_t m = pr.first; + const uint64_t k = pr.second; + const Hash rm = engine.root(m); + const Hash rk = engine.root(k); + expect(rm == *tree.past_root(m - 1), "mem past_root m" + at); + + const auto pp = engine.inclusion_proof(m / 2, m); + expect( + *pp == *tree.past_path(m / 2, m - 1), "mem inclusion(m)==past_path" + at); + + const auto cp = engine.consistency_proof(m, k); + expect( + ProofEngine::verify_consistency(m, k, rm, rk, cp), + "mem consistency" + at); + } +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + + const fs::path base = fs::temp_directory_path() / + ("merklecpp_tiles_tree_" + std::to_string((unsigned long long)seed) + "_" + + std::to_string(std::rand())); + + try + { + const auto hashes = make_hashes(1500); + + // ---- Part 0: the empty tree. flush()/compact() are no-ops and there is no + // root. + { + TiledTree::Config ecfg; + ecfg.prefix = base / "tt_empty"; + TiledTree ett(ecfg); + expect(ett.size() == 0, "empty: size 0"); + expect(ett.flush().full_written == 0, "empty: flush writes nothing"); + expect(ett.flushed_size() == 0, "empty: flushed size 0"); + expect(ett.immutable_size() == 0, "empty: immutable size 0"); + expect(ett.compact() == 0, "empty: compact no-op"); + bool ethrew = false; + try + { + (void)ett.root(); + } + catch (const std::exception&) + { + ethrew = true; + } + expect(ethrew, "empty: root throws"); + std::cout << "empty tree: OK" << '\n'; + } + + // ---- Part 0b: moving a tree rebinds its writer to the destination store. + { + TiledTree::Config move_cfg; + move_cfg.prefix = base / "tt_move_expected"; + TiledTree source(move_cfg); + for (uint64_t i = 0; i < 300; i++) + { + source.append(hashes[i]); + } + source.flush(); + + const fs::path work = base / "tt_move_cwd"; + fs::create_directories(work); + const fs::path previous_cwd = fs::current_path(); + fs::current_path(work); + try + { + TiledTree moved(std::move(source)); + expect(moved.flushed_size() == 256, "move: flushed size retained"); + expect(moved.immutable_size() == 256, "move: immutable size retained"); + expect( + moved.store_ref().root() == move_cfg.prefix, + "move: destination store retained"); + for (uint64_t i = 300; i < 512; i++) + { + moved.append(hashes[i]); + } + expect(moved.flush().full_written == 1, "move: next tile written"); + expect( + fs::is_regular_file(move_cfg.prefix / "tile/0/001"), + "move: tile written to configured store"); + expect( + !fs::exists(work / "tile/0/001"), + "move: no tile written relative to current directory"); + } + catch (...) + { + fs::current_path(previous_cwd); + throw; + } + fs::current_path(previous_cwd); + std::cout << "move construction: OK" << '\n'; + } + + // ---- Part 0c: a TiledTree is fresh-only. It accepts an existing root + // directory, but rejects a tile namespace that may belong to another + // tree rather than silently adopting its immutable files. + { + TiledTree::Config existing_cfg; + existing_cfg.prefix = base / "tt_existing"; + fs::create_directories(existing_cfg.prefix / "tile"); + { + TiledTree first(existing_cfg); + for (uint64_t i = 0; i < 256; i++) + { + first.append(hashes[i]); + } + expect( + first.flush().full_written == 1, + "existing prefix: first tree writes tile"); + } + + bool construction_threw = false; + try + { + const TiledTree second(existing_cfg); + } + catch (const std::exception&) + { + construction_threw = true; + } + expect( + construction_threw, + "existing prefix: second tree rejects existing tiles"); + std::cout << "existing tile namespace: OK" << '\n'; + } + + // ---- Part 1: memory-source proofs (exercises TreeT::subtree_root). + for (const uint64_t n : + {(uint64_t)1, + (uint64_t)2, + (uint64_t)3, + (uint64_t)5, + (uint64_t)8, + (uint64_t)13, + (uint64_t)16, + (uint64_t)256, + (uint64_t)257, + (uint64_t)1000}) + { + check_memory_source(n, hashes); + } + std::cout << "memory source: OK" << '\n'; + + // ---- Part 1b: hostile arithmetic inputs are rejected without UB or + // overflow loops. + { + merkle::Tree tree; + tree.insert(hashes[0]); + Hash out; + expect(!tree.subtree_root(64, 0, out), "subtree_root rejects level 64"); + expect(!tree.subtree_root(100, 0, out), "subtree_root rejects level 100"); + expect( + !tree.subtree_root(1, std::numeric_limits::max(), out), + "subtree_root rejects overflowing index"); + + expect(ProofEngineProbe::largest_pow2_lt(2) == 1, "pow2_lt 2"); + expect( + ProofEngineProbe::largest_pow2_lt((uint64_t)1 << 63) == + ((uint64_t)1 << 62), + "pow2_lt 2^63"); + expect( + ProofEngineProbe::largest_pow2_lt(((uint64_t)1 << 63) + 1) == + ((uint64_t)1 << 63), + "pow2_lt 2^63+1"); + expect( + ProofEngineProbe::largest_pow2_lt( + std::numeric_limits::max()) == ((uint64_t)1 << 63), + "pow2_lt uint64 max"); + std::cout << "hostile arithmetic inputs: OK" << '\n'; + } + + // ---- Part 2: TiledTree flush, proofs over tiles + memory. + const uint64_t n1 = 1000; // first flush size + const uint64_t N = 1500; // final size + + // Reference: a plain (never flushed) tree with the same leaves. + merkle::Tree ref; + for (uint64_t i = 0; i < N; i++) + { + ref.insert(hashes[i]); + } + const Hash ref_root = ref.root(); + + // Default mode: flush() writes tiles but drops nothing from memory; + // an explicit compact() drops only the leaves already covered by a tile. + { + TiledTree::Config dcfg; + dcfg.prefix = base / "tt_default"; + TiledTree dtt(dcfg); + for (uint64_t i = 0; i < n1; i++) + { + dtt.append(hashes[i]); + } + dtt.flush(); + expect(dtt.tree_ref().min_index() == 0, "default: nothing dropped"); + dtt.compact(); + expect( + dtt.tree_ref().min_index() == 767, "compact() keeps boundary leaf"); + } + + TiledTree::Config cfg; + cfg.prefix = base / "tt"; + cfg.retention_margin = 0; + cfg.compact_on_flush = true; + TiledTree tt(cfg); + + for (uint64_t i = 0; i < n1; i++) + { + tt.append(hashes[i]); + } + tt.flush(); // flushes full tiles; flushed_size = covered = 768 + expect(tt.flushed_size() == 768, "flushed size"); + expect(tt.tree_ref().min_index() == 767, "compacted with boundary overlap"); + + for (uint64_t i = n1; i < N; i++) + { + tt.append(hashes[i]); + } + expect(tt.size() == N, "size after appends"); + expect(tt.root() == ref_root, "tiled root == reference root"); + + // Indices that are: flushed (tiles only), in the flushed-but-resident + // overlap, and on the un-flushed resident frontier. + for (const uint64_t i : + {(uint64_t)0, + (uint64_t)767, + (uint64_t)800, + (uint64_t)999, + (uint64_t)1000, + (uint64_t)1499}) + { + const auto p = tt.inclusion_proof(i, N); + expect( + *p == *ref.path(i), "combined inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(ref_root), + "combined inclusion verify i=" + std::to_string(i)); + } + + // The resident tree alone cannot prove a flushed index. + bool threw = false; + try + { + (void)tt.tree_ref().path(0); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "memory-only path throws for flushed index"); + + // Consistency across the flush boundary: flushed size -> current size, + // and a flushed-era size -> current size. + { + const auto cp = tt.consistency_proof(n1, N); + expect( + ProofEngine::verify_consistency( + n1, N, *ref.past_root(n1 - 1), ref_root, cp), + "consistency n1->N across flush"); + + const uint64_t m = 500; // flushed era + const auto cp2 = tt.consistency_proof(m, N); + expect( + ProofEngine::verify_consistency( + m, N, *ref.past_root(m - 1), ref_root, cp2), + "consistency m->N across flush"); + + // Index-based variant: consistency_proof_from_indices(i, j) == + // consistency_proof(i + 1, j + 1), spanning tiles and the live tree. + const auto cpi = tt.consistency_proof_from_indices(m - 1, N - 1); + expect(cpi == cp2, "consistency index variant across flush"); + expect( + ProofEngine::verify_consistency( + m, N, *ref.past_root(m - 1), ref_root, cpi), + "consistency index variant verifies"); + } + + // A second flush writes further tiles; proofs for now-flushed indices still + // work (this also confirms the writer never reads a flushed leaf). + tt.flush(); // flushes full tiles; flushed_size = covered = 1280 + expect(tt.flushed_size() == 1280, "second flushed size"); + expect(tt.tree_ref().min_index() == 1279, "second boundary leaf retained"); + + for (const uint64_t i : + {(uint64_t)0, + (uint64_t)1000, + (uint64_t)1279, + (uint64_t)1280, + (uint64_t)1499}) + { + const auto p = tt.inclusion_proof(i, N); + expect( + *p == *ref.path(i), "post-2nd inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(ref_root), + "post-2nd inclusion verify i=" + std::to_string(i)); + } + + std::cout << "tiled tree (flush + combination): OK" << '\n'; + + // ---- Part 2b: compaction when the size is an exact multiple of TILE_WIDTH + // (here 512). flush_to cannot drain the whole tree, so compact() keeps + // one resident leaf rather than throwing; proofs still resolve. + { + TiledTree::Config mcfg; + mcfg.prefix = base / "tt_multiple"; + mcfg.compact_on_flush = true; + TiledTree mtt(mcfg); + for (uint64_t i = 0; i < 512; i++) + { + mtt.append(hashes[i]); + } + mtt.flush(); // covered == size == 512; compaction must not throw + expect(mtt.flushed_size() == 512, "multiple: flushed size 512"); + expect(mtt.tree_ref().min_index() == 511, "multiple: one leaf retained"); + + merkle::Tree mref; + for (uint64_t i = 0; i < 512; i++) + { + mref.insert(hashes[i]); + } + const Hash mroot = mref.root(); + expect(mtt.root() == mroot, "multiple: root matches reference"); + for (const uint64_t i : {(uint64_t)0, (uint64_t)256, (uint64_t)511}) + { + const auto p = mtt.inclusion_proof(i, 512); + expect(*p == *mref.path(i), "multiple: inclusion==ref"); + expect(p->verify(mroot), "multiple: inclusion verify"); + } + std::cout << "tiled tree (exact-multiple compaction): OK" << '\n'; + } + + // ---- Part 2c: compaction with a non-zero retention_margin keeps the most + // recent leaves resident while flushed indices are still served from + // tiles. The immutable prefix remains the full-tile prefix, regardless + // of the margin. + { + TiledTree::Config rcfg; + rcfg.prefix = base / "tt_margin"; + rcfg.retention_margin = 300; + rcfg.compact_on_flush = true; + TiledTree rtt(rcfg); + for (uint64_t i = 0; i < n1; i++) // n1 == 1000 + { + rtt.append(hashes[i]); + } + rtt.flush(); // covered = 768; target = floor((768 - 300) / 256) * 256 = + // 256 + expect(rtt.flushed_size() == 768, "margin: flushed size 768"); + expect( + rtt.tree_ref().min_index() == 256, + "margin: retained >= 300 recent leaves"); + + for (uint64_t i = n1; i < N; i++) + { + rtt.append(hashes[i]); + } + expect(rtt.root() == ref_root, "margin: root matches reference"); + + // Flushed-only (0, 255), flushed-but-resident overlap (256, 767), and the + // un-flushed frontier (1000, 1499). + for (const uint64_t i : + {(uint64_t)0, + (uint64_t)255, + (uint64_t)256, + (uint64_t)767, + (uint64_t)1000, + (uint64_t)1499}) + { + const auto p = rtt.inclusion_proof(i, N); + expect( + *p == *ref.path(i), "margin inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(ref_root), + "margin inclusion verify i=" + std::to_string(i)); + } + + bool threw = false; + try + { + rtt.retract_to(700); // size 701 < immutable_size 768 + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "margin: retract below flushed prefix throws"); + + std::cout << "tiled tree (retention margin): OK" << '\n'; + } + + // ---- Part 3: rollback. Tiles are immutable, so only un-tiled + // (post-flush) entries may be rolled back. + { + // 3a. Before any flush nothing is tiled, so rollback is + // unrestricted. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_pre"; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 50; i++) + { + rb.append(hashes[i]); + } + rb.retract_to(29); // tiles_size == 0 -> allowed + expect(rb.size() == 30, "rb pre-flush retract allowed"); + } + + // 3b. After a flush, the un-tiled frontier can be rolled back and + // the + // tiled region stays consistent and provable; committed entries + // can't. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb"; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 300; i++) + { + rb.append(hashes[i]); + } + rb.flush(); // flushed_size = covered = 256 + for (uint64_t i = 300; i < 400; i++) + { + rb.append(hashes[i]); + } + rb.retract_to(349); // keep [0,349] + expect(rb.size() == 350, "rb retracted to 350"); + for (uint64_t i = 350; i < 400; i++) + { + rb.append(hashes[1000 + i]); // re-append DIFFERENT leaves + } + rb.flush(); // flushed_size = covered = 256 (tile [0,256) already + // written) + + // Reference tree of the exact post-rollback state. + merkle::Tree exp_tree; + for (uint64_t i = 0; i < 350; i++) + { + exp_tree.insert(hashes[i]); + } + for (uint64_t i = 350; i < 400; i++) + { + exp_tree.insert(hashes[1000 + i]); + } + const Hash exp_root = exp_tree.root(); + expect(rb.root() == exp_root, "rb root matches reference"); + + // Proofs for a tiled index and a frontier index match the reference. + for (const uint64_t i : {(uint64_t)100, (uint64_t)299, (uint64_t)399}) + { + const auto p = rb.inclusion_proof(i, 400); + expect( + *p == *exp_tree.path(i), + "rb inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(exp_root), "rb inclusion verify i=" + std::to_string(i)); + } + const auto cp = rb.consistency_proof(300, 400); + expect( + ProofEngine::verify_consistency( + 300, 400, *exp_tree.past_root(299), exp_root, cp), + "rb consistency 300->400"); + + // Only the immutable full-tile prefix [0,256) is protected: rolling + // back into it is refused, while rolling back within the un-tiled + // frontier (>= immutable_size()) is allowed. + expect(rb.flushed_size() == 256, "rb flushed to full-tile prefix"); + + bool threw = false; + try + { + rb.retract_to(100); // size 101 < 256 + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "rb retract below tiled prefix throws"); + threw = false; + try + { + rb.retract_to(254); // size 255 < 256 + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "rb retract just below tiled prefix throws"); + rb.retract_to(255); // size 256 == immutable_size(): drops only frontier + expect(rb.size() == 256, "rb retract to tiled prefix allowed"); + } + + // 3c. Compaction keeps the final tiled leaf resident, so rollback to a + // tree size of exactly immutable_size() remains representable. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_exact_boundary"; + cfg.compact_on_flush = true; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 1000; i++) + { + rb.append(hashes[i]); + } + rb.flush(); + expect(rb.flushed_size() == 768, "rb boundary flushed size"); + expect(rb.immutable_size() == 768, "rb boundary immutable size"); + expect( + rb.tree_ref().min_index() == 767, + "rb boundary final tiled leaf retained"); + + rb.retract_to(767); + expect(rb.size() == 768, "rb exact immutable boundary allowed"); + + merkle::Tree expected; + for (uint64_t i = 0; i < 768; i++) + { + expected.insert(hashes[i]); + } + const Hash expected_root = expected.root(); + expect( + rb.root() == expected_root, "rb boundary root matches reference"); + const auto proof = rb.inclusion_proof(0, 768); + expect( + *proof == *expected.path(0), + "rb boundary inclusion matches reference"); + expect(proof->verify(expected_root), "rb boundary inclusion verifies"); + } + + // 3d. Rollback interacts correctly with compaction (flushed + tiled + // past). + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_compact"; + cfg.compact_on_flush = true; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 1000; i++) + { + rb.append(hashes[i]); + } + rb.flush(); // flushed_size = covered = 768; retain boundary leaf 767 + expect( + rb.tree_ref().min_index() == 767, + "rb compact boundary leaf retained"); + for (uint64_t i = 1000; i < 1200; i++) + { + rb.append(hashes[i]); + } + // Frontier rollback (>= immutable_size()) is allowed. + rb.retract_to(1099); + expect(rb.size() == 1100, "rb compact frontier retract ok"); + + merkle::Tree exp_tree; + for (uint64_t i = 0; i < 1100; i++) + { + exp_tree.insert(hashes[i]); + } + const Hash exp_root = exp_tree.root(); + const auto p = rb.inclusion_proof(0, 1100); // leaf 0 is flushed + tiled + expect(*p == *exp_tree.path(0), "rb compact flushed inclusion==ref"); + expect(p->verify(exp_root), "rb compact flushed inclusion verify"); + + bool threw = false; + try + { + rb.retract_to(500); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "rb compact retract below tiled throws"); + } + + // 3e. A failed flush may publish an immutable full tile before a later + // write fails. The attempted full-tile boundary is sealed against + // rollback, while flushed_size advances only after a successful retry. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_interrupted"; + TiledTree interrupted(cfg); + for (uint64_t i = 0; i < 512; i++) + { + interrupted.append(hashes[i]); + } + + const fs::path blocker = cfg.prefix / "tile/0/001"; + fs::create_directories(blocker); + bool flush_threw = false; + try + { + interrupted.flush(); + } + catch (const std::exception&) + { + flush_threw = true; + } + + expect(flush_threw, "interrupted flush throws"); + expect( + fs::is_regular_file(cfg.prefix / "tile/0/000"), + "interrupted flush published first full tile"); + expect( + interrupted.flushed_size() == 0, + "interrupted flush does not advance flushed size"); + expect( + interrupted.immutable_size() == 512, + "interrupted flush seals attempted boundary"); + expect( + interrupted.compact() == 0, + "interrupted flush does not permit compaction"); + + TiledTree recovered(std::move(interrupted)); + expect( + recovered.immutable_size() == 512, + "interrupted flush seal survives move"); + + bool retract_threw = false; + try + { + recovered.retract_to(0); + } + catch (const std::exception&) + { + retract_threw = true; + } + expect( + retract_threw, + "interrupted flush rejects rollback across attempted tiles"); + expect(recovered.size() == 512, "interrupted rollback changes nothing"); + + fs::remove(blocker); + expect( + recovered.flush().full_written == 1, + "interrupted flush retry writes only missing tile"); + expect( + recovered.flushed_size() == 512, + "interrupted flush retry advances flushed size"); + expect( + recovered.immutable_size() == 512, + "interrupted flush retry preserves immutable size"); + recovered.compact(); + + merkle::Tree expected; + for (uint64_t i = 0; i < 512; i++) + { + expected.insert(hashes[i]); + } + const Hash expected_root = expected.root(); + expect( + recovered.root() == expected_root, + "interrupted flush root matches reference"); + for (const uint64_t i : + {(uint64_t)0, (uint64_t)255, (uint64_t)256, (uint64_t)511}) + { + const auto proof = recovered.inclusion_proof(i, 512); + expect( + *proof == *expected.path(i), + "interrupted flush inclusion matches reference"); + expect( + proof->verify(expected_root), + "interrupted flush inclusion verifies"); + } + } + } + std::cout << "rollback: OK" << '\n'; + + std::cout << "tiles_tree: OK" << '\n'; + + std::error_code ec; + fs::remove_all(base, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + + return 0; +} diff --git a/test/tiles_writer.cpp b/test/tiles_writer.cpp new file mode 100644 index 0000000..05e847f --- /dev/null +++ b/test/tiles_writer.cpp @@ -0,0 +1,337 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TileRef; +using merkle::tiles::TileStore; +using merkle::tiles::TileWriter; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +static size_t tile_file_count(const TileStore& store) +{ + const fs::path tiles = store.root() / "tile"; + if (!fs::exists(tiles)) + { + return 0; + } + const fs::recursive_directory_iterator it(tiles); + return (size_t)std::count_if( + begin(it), end(it), [](const fs::directory_entry& e) { + return e.is_regular_file(); + }); +} + +// Roll up a full level-0 tile and compare with a level-1 tile entry. +static Hash rollup(const std::vector& leaves) +{ + return merkle::tiles::perfect_root<32, merkle::sha256_compress>(leaves); +} + +static void overwrite_file( + const fs::path& path, const std::vector& bytes) +{ + std::ofstream file(path, std::ios::binary | std::ios::trunc); + file.write( + reinterpret_cast(bytes.data()), (std::streamsize)bytes.size()); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + + const fs::path base = fs::temp_directory_path() / + ("merklecpp_tiles_writer_" + std::to_string((unsigned long long)seed) + + "_" + std::to_string(std::rand())); + + try + { + // ---- A. size 256: exactly one full L0 tile and no L1 tile (the L1 root is + // not yet a full tile, so it stays in memory). The level-1 entry it + // would hold equals the real merkle::Tree root. + { + const auto hashes = make_hashes(256); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + + TileStore store(base / "a"); + TileWriter writer(store); + const auto s = writer.write_up_to(256, leaf_at); + + expect(s.full_written == 1, "A full_written"); + + expect(store.has_full_tile(0, 0), "A L0 full tile"); + expect( + fs::file_size(store.tile_path(TileRef{0, 0})) == 256U * Hash().size(), + "A L0 full tile size"); + expect(!store.has_full_tile(1, 0), "A no L1 full tile"); + expect(tile_file_count(store) == 1, "A exact tile file count"); + + // Level-0 tile is the leaf hashes verbatim. + const auto l0 = store.read_tile(TileRef{0, 0}); + for (size_t i = 0; i < hashes.size(); i++) + { + expect(l0[i] == hashes[i], "A L0 entry == leaf"); + } + + // The (un-tiled) level-1 entry == root of the equivalent merkle::Tree. + merkle::Tree tree; + for (const auto& h : hashes) + { + tree.insert(h); + } + expect(rollup(l0) == tree.root(), "A rollup(L0) == tree root"); + + std::cout << "A (size 256): OK" << '\n'; + } + + // ---- B. size 70000: full tiles only (273 full L0 + 1 full L1 = 274); the + // incomplete frontier at every level is left untiled. + { + const auto hashes = make_hashes(70000); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + + TileStore store(base / "b"); + TileWriter writer(store); + const auto s = writer.write_up_to(70000, leaf_at); + + expect(s.full_written == 274, "B full_written"); + + expect(store.has_full_tile(0, 0), "B L0 tile 0"); + expect(store.has_full_tile(0, 272), "B L0 tile 272"); + expect(!store.has_full_tile(0, 273), "B no L0 tile 273"); + + expect(store.has_full_tile(1, 0), "B L1 tile 0"); + expect(!store.has_full_tile(1, 1), "B no L1 tile 1"); + + expect(!fs::exists(store.root() / "tile" / "2"), "B no level 2"); + expect(!fs::exists(store.root() / "tile" / "3"), "B no level 3"); + expect(tile_file_count(store) == 274, "B exact tile file count"); + + // Higher-level entries are roll-ups of the complete child tiles. + const auto l1 = store.read_tile(TileRef{1, 0}); + expect(l1.size() == 256, "B L1 full width"); + expect(l1[0] == rollup(store.read_tile(TileRef{0, 0})), "B L1[0]"); + expect(l1[255] == rollup(store.read_tile(TileRef{0, 255})), "B L1[255]"); + + std::cout << "B (size 70000): OK" << '\n'; + } + + // ---- C. incremental writes preserve immutability and idempotency. + { + const auto hashes = make_hashes(1024); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + + TileStore store(base / "c"); + TileWriter writer(store); + + const auto s1 = writer.write_up_to(256, leaf_at); + expect(s1.full_written == 1, "C s1 full"); + + // Re-running at the same size writes nothing (full tiles are immutable). + const auto s2 = writer.write_up_to(256, leaf_at); + expect(s2.full_written == 0, "C s2 full immutable"); + + // Grow to 600: one new full L0 tile (covering 512), nothing else. + const auto s3 = writer.write_up_to(600, leaf_at); + expect(s3.full_written == 1, "C s3 full"); + + expect(store.has_full_tile(0, 0), "C L0 tile 0"); + expect(store.has_full_tile(0, 1), "C L0 tile 1"); + expect(!store.has_full_tile(0, 2), "C no L0 tile 2"); + expect(!store.has_full_tile(1, 0), "C no L1 full tile"); + expect(tile_file_count(store) == 2, "C exact tile file count"); + + std::cout << "C (incremental): OK" << '\n'; + } + + // ---- D. crossing into a full level-1 tile: the roll-up appears as a full + // tile and prior full tiles are never rewritten. + { + const auto hashes = make_hashes(65536); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + + TileStore store(base / "d"); + TileWriter writer(store); + + // One leaf short of a full L1 tile: 255 full L0 tiles, no L1 yet. + const auto s1 = writer.write_up_to(65535, leaf_at); + expect(s1.full_written == 255, "D s1 255 full L0"); + expect(!store.has_full_tile(1, 0), "D no L1 before completion"); + expect(tile_file_count(store) == 255, "D initial tile file count"); + + // Completing the 256th L0 tile yields one new L0 tile and one full L1. + const auto s2 = writer.write_up_to(65536, leaf_at); + expect(s2.full_written == 2, "D s2 new L0 + new L1"); + expect(store.has_full_tile(0, 255), "D L0 tile 255"); + expect(store.has_full_tile(1, 0), "D L1 tile 0"); + expect(!store.has_full_tile(1, 1), "D no L1 tile 1"); + expect(!store.has_full_tile(2, 0), "D no L2 tile"); + expect(tile_file_count(store) == 257, "D final tile file count"); + + const auto l1 = store.read_tile(TileRef{1, 0}); + expect(l1[0] == rollup(store.read_tile(TileRef{0, 0})), "D L1[0]"); + + // Re-running writes nothing (everything already full and immutable). + const auto s3 = writer.write_up_to(65536, leaf_at); + expect(s3.full_written == 0, "D s3 immutable rerun"); + + std::cout << "D (full L1 tile): OK" << '\n'; + } + + // ---- E. resume: a fresh writer over an existing store rebuilds its cursor + // from disk via full_prefix_length, never rewriting full tiles and + // writing only the newly-complete ones. + { + const auto hashes = make_hashes(70000); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + + const fs::path dir = base / "e"; + { + TileStore store(dir); + TileWriter writer(store); + expect( + writer.write_up_to(600, leaf_at).full_written == 2, + "E first writer 2 L0 tiles"); // 600 / 256 == 2 full L0 tiles + } + + // A brand-new store + writer over the same directory: its cursor must + // resume from what is already on disk. + TileStore store(dir); + TileWriter writer(store); + expect( + writer.write_up_to(600, leaf_at).full_written == 0, + "E resume rewrites nothing"); + + // Extending to 70000 writes only the missing tiles: 273 - 2 == 271 new L0 + // plus 1 new L1 (which rolls up the L0 tiles the first writer wrote). + expect( + writer.write_up_to(70000, leaf_at).full_written == 272, + "E resume writes only new tiles"); + expect(store.has_full_tile(0, 0), "E L0 tile 0 still present"); + expect(store.has_full_tile(0, 272), "E L0 tile 272"); + expect(store.has_full_tile(1, 0), "E L1 tile 0"); + expect( + store.read_tile(TileRef{1, 0})[0] == + rollup(store.read_tile(TileRef{0, 0})), + "E L1[0] rolled up from resumed L0"); + expect(tile_file_count(store) == 274, "E exact tile file count"); + + // A third fresh writer confirms full idempotence after a resume. + TileStore store3(dir); + TileWriter writer3(store3); + expect( + writer3.write_up_to(70000, leaf_at).full_written == 0, + "E second resume idempotent"); + + std::cout << "E (writer resume): OK" << '\n'; + } + + // ---- F. Resume discovers the first hole rather than trusting later valid + // files as proof that the prefix is contiguous. + { + const auto hashes = make_hashes(2048); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + const fs::path dir = base / "f"; + + { + TileStore store(dir); + TileWriter writer(store); + expect( + writer.write_up_to(2048, leaf_at).full_written == 8, + "F initial tiles"); + overwrite_file(store.tile_path(TileRef{0, 3}), {0}); + expect(!store.has_full_tile(0, 3), "F interior tile corrupt"); + expect(store.has_full_tile(0, 7), "F later tile remains valid"); + } + + TileStore store(dir); + TileWriter writer(store); + expect( + writer.write_up_to(2048, leaf_at).full_written == 1, + "F rewrites interior hole"); + const std::vector expected( + hashes.begin() + (std::ptrdiff_t)merkle::tiles::TILE_WIDTH * 3, + hashes.begin() + (std::ptrdiff_t)merkle::tiles::TILE_WIDTH * 4); + expect( + store.read_tile(TileRef{0, 3}) == expected, "F repaired tile contents"); + expect(tile_file_count(store) == 8, "F exact tile file count"); + + std::cout << "F (interior recovery): OK" << '\n'; + } + + // ---- G. Recovery is bounded by the requested tree size, so sparse files + // at geometrically increasing indices cannot overflow its search. + { + const fs::path dir = base / "g"; + const auto hashes = make_hashes(merkle::tiles::TILE_WIDTH); + { + TileStore store(dir); + store.write_tile(TileRef{0, 0}, hashes); + for (uint64_t index = 1;; index <<= 1) + { + store.write_tile(TileRef{0, index}, hashes); + if (index == (uint64_t{1} << 63)) + { + break; + } + } + } + + TileStore store(dir); + TileWriter writer(store); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + expect( + writer.write_up_to(merkle::tiles::TILE_WIDTH, leaf_at).full_written == + 0, + "G bounded sparse recovery"); + expect(store.has_full_tile(0, 0), "G requested tile remains valid"); + + std::cout << "G (bounded sparse recovery): OK" << '\n'; + } + + std::cout << "tiles_writer: OK" << '\n'; + + std::error_code ec; + fs::remove_all(base, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + + return 0; +} diff --git a/test/time_large_trees.cpp b/test/time_large_trees.cpp index f5ebfa1..3ab8b3f 100644 --- a/test/time_large_trees.cpp +++ b/test/time_large_trees.cpp @@ -42,8 +42,9 @@ int main() mt.root(); auto stop = std::chrono::high_resolution_clock::now(); const double seconds = - static_cast(std::chrono::duration_cast(stop - start) - .count()) / + static_cast( + std::chrono::duration_cast(stop - start) + .count()) / 1e9; std::cout << "NEW: " << mt.statistics.to_string() << " in " << seconds << " sec" << '\n'; @@ -66,8 +67,10 @@ int main() mt384.root(); auto stop384 = std::chrono::high_resolution_clock::now(); const double seconds384 = - static_cast(std::chrono::duration_cast(stop384 - start384) - .count()) / + static_cast( + std::chrono::duration_cast( + stop384 - start384) + .count()) / 1e9; std::cout << "SHA384: " << mt384.statistics.to_string() << " in " << seconds384 << " sec" << '\n'; @@ -90,8 +93,10 @@ int main() mt512.root(); auto stop512 = std::chrono::high_resolution_clock::now(); const double seconds512 = - static_cast(std::chrono::duration_cast(stop512 - start512) - .count()) / + static_cast( + std::chrono::duration_cast( + stop512 - start512) + .count()) / 1e9; std::cout << "SHA512: " << mt512.statistics.to_string() << " in " << seconds512 << " sec" << '\n'; @@ -121,8 +126,9 @@ int main() } mt_get_root(ec_mt, ec_root); stop = std::chrono::high_resolution_clock::now(); - const double ec_seconds = std::chrono::duration_cast(stop - start) - .count() / + const double ec_seconds = + std::chrono::duration_cast(stop - start) + .count() / 1e9; std::cout << "EC :" << " num_insert=" << ec_hashes.size() diff --git a/test/time_tiles.cpp b/test/time_tiles.cpp new file mode 100644 index 0000000..dce7d8d --- /dev/null +++ b/test/time_tiles.cpp @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TiledTree; + +static double secs_since( + const std::chrono::high_resolution_clock::time_point& start) +{ + const auto stop = std::chrono::high_resolution_clock::now(); + return (double)std::chrono::duration_cast( + stop - start) + .count() / + 1e9; +} + +// Uniform random in [0, bound). +static uint64_t rand_below(uint64_t bound) +{ + if (bound == 0) + { + return 0; + } + return (uint64_t)((std::rand() / (RAND_MAX + 1.0)) * (double)bound); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + +#ifndef NDEBUG + const uint64_t num_leaves = 50000; + const uint64_t num_proofs = 100; +#else + const uint64_t num_leaves = 1000000; + const uint64_t num_proofs = 10000; +#endif + + const fs::path dir = fs::temp_directory_path() / + ("merklecpp_time_tiles_" + std::to_string((unsigned long long)seed)); + + // Accumulator that consumes each proof, so the work is not optimised away. + volatile uint64_t sink = 0; + + std::cout << std::fixed << std::setprecision(3); + + try + { + const auto hashes = make_hashes(num_leaves); + + TiledTree::Config cfg; + cfg.prefix = dir; + TiledTree log(cfg); + + // 1. Append: grow the in-memory tree. + auto t = std::chrono::high_resolution_clock::now(); + for (uint64_t i = 0; i < num_leaves; i++) + { + log.append(hashes[i]); + } + log.root(); + const double append_s = secs_since(t); + std::cout << "append : " << num_leaves << " leaves in " + << append_s << " sec (" + << (uint64_t)((double)num_leaves / append_s) << " leaves/sec)\n"; + + // 2. Flush: write newly-complete full tiles to disk. + t = std::chrono::high_resolution_clock::now(); + const auto stats = log.flush(); + const double flush_s = secs_since(t); + std::cout << "flush (to disk) : " << stats.full_written + << " full tiles in " << flush_s << " sec (" + << (uint64_t)((double)stats.full_written / flush_s) + << " tiles/sec)\n"; + + const uint64_t n = log.size(); + + // 3. Inclusion proofs while everything is still resident (memory path). + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += log.inclusion_proof(rand_below(n), n)->size(); + } + const double inc_mem_s = secs_since(t); + std::cout << "inclusion (memory) : " << num_proofs << " proofs in " + << inc_mem_s << " sec (" + << (uint64_t)((double)num_proofs / inc_mem_s) << " proofs/sec)\n"; + + // 4. Consistency proofs while resident (memory path). + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += log.consistency_proof(1 + rand_below(n - 1), n).size(); + } + const double con_mem_s = secs_since(t); + std::cout << "consistency(memory): " << num_proofs << " proofs in " + << con_mem_s << " sec (" + << (uint64_t)((double)num_proofs / con_mem_s) << " proofs/sec)\n"; + + // 5. Compact: drop tiled leaves from memory. + t = std::chrono::high_resolution_clock::now(); + const uint64_t min_idx = log.compact(); + const double compact_s = secs_since(t); + std::cout << "compact : dropped " << min_idx + << " leaves from memory in " << compact_s << " sec\n"; + + // 6. Inclusion proofs for evicted leaves: served from the on-disk tiles. + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += log.inclusion_proof(rand_below(min_idx), n)->size(); + } + const double inc_tile_s = secs_since(t); + std::cout << "inclusion (tiles) : " << num_proofs << " proofs in " + << inc_tile_s << " sec (" + << (uint64_t)((double)num_proofs / inc_tile_s) + << " proofs/sec)\n"; + + // 7. Consistency proofs spanning the tiled (evicted) past. + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += log.consistency_proof(1 + rand_below(min_idx - 1), n).size(); + } + const double con_tile_s = secs_since(t); + std::cout << "consistency(tiles) : " << num_proofs << " proofs in " + << con_tile_s << " sec (" + << (uint64_t)((double)num_proofs / con_tile_s) + << " proofs/sec)\n"; + + // 8. Baseline: identical inclusion proofs from a plain in-memory Tree. + merkle::Tree ref; + for (uint64_t i = 0; i < num_leaves; i++) + { + ref.insert(hashes[i]); + } + ref.root(); + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += ref.path(rand_below(n))->size(); + } + const double inc_ref_s = secs_since(t); + std::cout << "inclusion (Tree) : " << num_proofs << " proofs in " + << inc_ref_s << " sec (" + << (uint64_t)((double)num_proofs / inc_ref_s) << " proofs/sec)\n"; + + // Sanity: a tile-served proof still verifies against the reference root. + if (!log.inclusion_proof(0, n)->verify(ref.root())) + { + throw std::runtime_error("benchmark proof failed to verify"); + } + + std::cout << "time_tiles: OK (checksum " << sink << ")\n"; + + std::error_code ec; + fs::remove_all(dir, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + + return 0; +} diff --git a/test/unit_tests.cpp b/test/unit_tests.cpp index e6f6967..c72cac1 100644 --- a/test/unit_tests.cpp +++ b/test/unit_tests.cpp @@ -149,7 +149,7 @@ TEST_CASE("PathT equality") const auto path0a = tree.path(0); const auto path0b = tree.path(0); // same path extracted twice - const auto path1 = tree.path(1); // path to a different leaf + const auto path1 = tree.path(1); // path to a different leaf // Two paths to the same leaf should be equal REQUIRE(*path0a == *path0b); @@ -166,7 +166,7 @@ TEST_CASE("PathT equality") h3.bytes[0] = 3; tree_diff.insert(h3); - const auto path_orig = tree.path(0); // h0 leaf, element has h1 + const auto path_orig = tree.path(0); // h0 leaf, element has h1 const auto path_diff = tree_diff.path(0); // h0 leaf, element has h3 REQUIRE_FALSE(*path_orig == *path_diff); REQUIRE(*path_orig != *path_diff); @@ -175,7 +175,7 @@ TEST_CASE("PathT equality") TEST_CASE("TreeT to_string") { // Empty tree produces "" - merkle::Tree empty_tree; + const merkle::Tree empty_tree; const std::string empty_str = empty_tree.to_string(); REQUIRE(empty_str.find("") != std::string::npos); @@ -192,7 +192,7 @@ TEST_CASE("TreeT to_string") TEST_CASE("TreeT leaf bounds and uninserted leaves") { // leaf() out of bounds on empty tree - merkle::Tree empty_tree; + const merkle::Tree empty_tree; REQUIRE_THROWS(empty_tree.leaf(0)); // Access leaf before insertion is flushed (uninserted_leaf_nodes path) @@ -216,7 +216,7 @@ TEST_CASE("TreeT size with uninserted leaves") { merkle::Tree tree; // size() when tree has uninserted leaves triggers insert_leaves() - merkle::Tree::Hash h; + const merkle::Tree::Hash h; tree.insert(h); // size() forces lazy insertion const size_t sz = tree.size(); diff --git a/test/util.h b/test/util.h index 2abfc77..196f8b9 100644 --- a/test/util.h +++ b/test/util.h @@ -38,9 +38,9 @@ inline std::vector> make_hashesT( inline size_t random_index(merkle::Tree& mt) { - return (size_t)( - mt.min_index() + - (std::rand() / (double)RAND_MAX) * (mt.max_index() - mt.min_index())); + return (size_t)(mt.min_index() + + (std::rand() / (double)RAND_MAX) * + (mt.max_index() - mt.min_index())); } inline double get_timeout()