From fa0d9716c840e11ef3ba786e5086bcfb7a1c9d7d Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 11:29:39 +0000 Subject: [PATCH 01/22] Add tile formats and durable TileStore Introduce merklecpp_tiles.h with the tlog-tiles tile geometry constants, tile index encoding, TileRef, and a hardened TileStoreT: strict full-tile and entry-bundle codecs/I/O, atomic replacement, directory/file durability, validation, and temp-file cleanup, with no internal synchronization (callers must serialize access). Add focused tiles_store tests covering path layout, atomic durable writes, corruption/truncation handling, and concurrent same-tile writes. Store durability-retry tests that exercise TileWriter are deferred to the next branch, which also adds the entry-bundle writer. Wire tiles_store into the test suite, add merklecpp_tiles.h to the Doxygen input set, and run clang-tidy on it in CI. Start the normative tiled-storage design doc with goals, background, the tile/merklecpp mapping, file layout, architecture overview, the header/TileStoreT API sections, and the entry-bundle format (its writer arrives in the next branch). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 4 +- Doxyfile | 3 +- doc/design/tlog-tiles.md | 368 +++++++++++++++++++ merklecpp_tiles.h | 740 +++++++++++++++++++++++++++++++++++++++ test/CMakeLists.txt | 1 + test/tiles_store.cpp | 356 +++++++++++++++++++ 6 files changed, 1470 insertions(+), 2 deletions(-) create mode 100644 doc/design/tlog-tiles.md create mode 100644 merklecpp_tiles.h create mode 100644 test/tiles_store.cpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b12b1bf..aa69062 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,7 +64,9 @@ jobs: - name: Clang-Tidy Header if: matrix.os == 'ubuntu-latest' shell: bash - run: clang-tidy merklecpp.h -p build/${{ matrix.build_type }} --warnings-as-errors='*' + run: | + clang-tidy merklecpp.h -p build/${{ matrix.build_type }} --warnings-as-errors='*' + clang-tidy merklecpp_tiles.h -p build/${{ matrix.build_type }} --warnings-as-errors='*' - name: Test working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} diff --git a/Doxyfile b/Doxyfile index 33229bb..d045b46 100644 --- a/Doxyfile +++ b/Doxyfile @@ -790,7 +790,8 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = merklecpp.h +INPUT = merklecpp.h \ + merklecpp_tiles.h # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md new file mode 100644 index 0000000..d36bf12 --- /dev/null +++ b/doc/design/tlog-tiles.md @@ -0,0 +1,368 @@ +# Design: Tiled storage and tile-backed proofs for merklecpp + +Status: Proposal / Draft +Audience: merklecpp maintainers and contributors +Scope: Extend the header-only library to (a) persist the Merkle tree as +[tlog-tiles](https://c2sp.org/tlog-tiles) tile files progressively (on +compaction), and (b) serve **inclusion** and **consistency** proofs from those +tiles, from the in-memory tree, or from a combination of the two. + +--- + +## 1. Goals and non-goals + +### Goals + +1. Write the tree to disk as a set of immutable, cacheable **tile files** using + the [tlog-tiles](https://c2sp.org/tlog-tiles) file/directory layout and tile + geometry (256-wide tiles, 8 tree levels per tile, path encoding). Only + **full, balanced** tiles are produced. The remaining frontier stays in the + in-memory tree until it crosses the next full-tile boundary. +2. Produce tiles **progressively**: each batch/flush writes only the newly + completed tiles, and the write integrates with the existing `flush_to()` + "compaction" so that flushed (evicted) subtrees are durably persisted before + their nodes are freed from memory. +3. Provide an API to retrieve, **after compaction**, an **inclusion path** and a + **consistency path**: + - from the in-memory tree alone (existing behaviour), + - from full tiles alone, for the tiled (full-tile-covered) prefix, or + - from a combination of full tiles (old, flushed part) and the in-memory + tree (recent frontier) — the normal case. +4. Keep everything **header-only** and templated, matching the existing + `TreeT` style. + +### Full-tile-only policy + +Partial tiles are out of scope: merklecpp never emits or reads them. During a +flush, a tile becomes eligible only after all 256 entries are complete and +final. Entries beyond that boundary remain in memory. Once published, a full +tile is immutable and is never rewritten. + +### Thread-safety policy + +The tiled-storage API provides no internal synchronization. Every object is +single-threaded unless its caller serializes all access. This requirement also +applies to separate objects that share a store prefix and to methods declared +`const`, because proof reads update the `TileHashSourceT` LRU cache. Locking and +reader/writer coordination are deliberately left to the application. + +### Hard constraints (from the request) + +- **Hashing logic is unchanged.** No modification to `HASH_FUNCTION`, to node + hashing, or to leaf handling. +- **Proofs remain compatible with the existing library.** A proof produced from + tiles MUST be identical to (and verify against the same root as) the proof the + existing library would produce via `Tree::path()` / `Tree::past_path()` / + `Tree::root()` / `Tree::past_root()`. + +### Non-goals + +- Byte-level interoperability with *external* RFC 6962 / Go-ecosystem tlog + clients. See [§2.3](#23-compatibility-statement): such interop is only + possible if the consumer instantiates `TreeT` with an RFC 6962 combiner, which + is **out of scope** here and not required by this design. +- An HTTP server. We define the on-disk layout and the read/write/proof APIs; + serving the static files is an application concern. + +--- + +## 2. Background + +### 2.1 tlog-tiles, the parts we adopt + +A tiled log exposes the Merkle tree as a set of static resources: + +- **Tiles** at `/tile//`, `application/octet-stream`. + - `` = level, decimal `0..63`, no leading zeros. + - `` = tile index within the level, encoded as zero-padded 3-digit path + elements where **all but the last element are prefixed with `x`**. Example: + `1234067` → `x001/x234/067`. +- A **full tile** is exactly **256 hashes** wide (`256 * HASH_SIZE` bytes). The + spec fixes `HASH_SIZE = 32` (SHA-256); here it is the template parameter. +- Level 0 hashes are **leaf hashes**. At level `L ≥ 1`, each hash is the Merkle + Tree Hash of a *full* tile at level `L-1`. A tile spans **8 tree levels**: its + 256 entries are the leaves of a height-8 perfect subtree, and intra-tile + internal nodes are reconstructed by hashing entries. +- The `n`-th tile at level `l` contains, for `i = 0..255`: + `MTH(D[(n*256+i)*256^l : (n*256+i+1)*256^l])`. + Its **start index** is `n*256^(l+1)`, its **end index** `(n+1)*256^(l+1)`. +- The frontier beyond the last full tile is held in the in-memory tree, so the + on-disk tile set is always a full-tile prefix. +- **Entry bundles** at `/tile/entries/`: big-endian `uint16` + length-prefixed raw entries, 256 per full bundle. (See + [section 6.8](#68-entry-bundles-optional).) +- **Pruning**: a log keeps a *minimum index*; tiles/bundles whose end index is + `≤ minimum index` may be denied. This maps cleanly onto merklecpp's + `flush_to()` / `min_index()`. + +### 2.2 The merklecpp model + +- `TreeT` is a **left-balanced binary Merkle tree**. + Its shape is identical to RFC 6962: a node's left child is the largest perfect + subtree of `2^k` leaves with `2^k < n`. A *full* node of height `h` covers + exactly `2^(h-1)` aligned leaves, and its hash is the root of that perfect + subtree (verified from `walk_to`, `is_full`, `update_sizes`). +- Internal nodes are combined with `HASH_FUNCTION(left, right, out)` over exactly + two child hashes (`merklecpp.h` node hashing). There is **no** domain + separation; the combiner is the only cryptographic operation. +- `flush_to(index)` is the existing **compaction** primitive: it conflates the + left part of the tree into single hash-only nodes and drops `num_flushed` + leaves from memory, raising `min_index()` to `index`. After a flush, paths for + leaves `< index` can no longer be produced from memory. +- Existing proof APIs we must stay compatible with: + - `root()` / `past_root(i)` — current / historical root. + - `path(i)` — inclusion path for leaf `i` in the current tree. + - `past_path(i, as_of)` — inclusion path for leaf `i` as of size `as_of+1`. + - `PathT::verify(root)` / `PathT::root()` — verification via `HASH_FUNCTION`. + - merklecpp has **no** classic two-size consistency proof today; this design + adds one (built from the same combiner, so it reconciles `past_root`s). + +## 3. Tile ↔ merklecpp mapping (the math) + +Let `TILE_HEIGHT = 8` and `TILE_WIDTH = 256 = 2^8`. All combiners below are the +tree's `HASH_FUNCTION`; `MTH` denotes the Merkle Tree Hash computed with it. + +- **Tile entry = perfect subtree root.** Entry `i` of tile `(L, N)` (global + entry index `g = N·256 + i`) is `MTH(D[g·256^L : (g+1)·256^L])`, i.e. the root + of a perfect subtree of `2^(8L)` leaves — the in-memory full node of height + `8L + 1`. +- **Level roll-up.** Because a full level-`L` entry is the root of 256 full + level-`(L-1)` entries, a level-`L` tile can be computed from level-`(L-1)` + tiles alone: `tile(L,N)[i] = perfect_root( tile(L-1, N·256+i)[0..255] )`. So + the **write path needs no tree internals above level 0** — only leaf hashes + plus roll-ups. +- **Intra-tile internal nodes.** A tile's 256 entries are the leaves of a + height-8 perfect subtree. The root of `2^r` (`0 ≤ r ≤ 8`) consecutive, + aligned entries is `perfect_root` of that run. Hence any RFC 6962 subtree root + at tree level `k = 8L + r` is obtained from **one** level-`L` tile by hashing + `2^r` consecutive entries (a single entry when `r = 0`). +- **Full-tile prefix.** Only complete level-`L` tiles are written: + `full_tiles_L = floor( floor(s / 256^L) / 256 )`, together covering the leaf + prefix `covered = floor(s / 256) * 256`. The incomplete frontier `[covered, s)` + is not tiled; it is served from the in-memory tree. When no higher-level full + tile contains a subtree, it is resolved by **descending to the highest + available full tile and rolling up** (ultimately from full level-0 tiles). + +This yields a single read primitive that both proof types are built on: + +``` +subtree_root(level k, index j) = MTH(D[j·2^k : (j+1)·2^k]) # within `covered` + L = k / 8 ; r = k % 8 + first = j << r ; n = first / 256 ; off = first % 256 + if a full level-L tile n exists (first + 2^r within full_tiles_L · 256): + return perfect_root( tile(L, n)[off : off + 2^r] ) # single entry if r==0 + else: # higher-level tile unavailable: descend + return HASH_FUNCTION( subtree_root(k-1, 2j), subtree_root(k-1, 2j+1) ) +``` + +and a range primitive for non-perfect (right-frontier) subtrees: + +``` +mth_range(a, b): // MTH(D[a:b]); a aligned to 2^ceil(log2(b-a)) + w = b - a + if w == 2^k and source can resolve subtree_root(k, a>>k): return it + k = largest power of two < w // RFC 6962 split + return HASH_FUNCTION( mth_range(a, a+k), mth_range(a+k, b) ) +``` + +`mth_range` falls back to splitting when a perfect subtree is not directly +resolvable from the chosen source; the recursion always bottoms out at resolvable +pieces (see the [combined-source invariant](#7-progressive-production--compaction)). + +--- + +## 4. File and directory layout + +Rooted at a configurable `prefix` directory on local disk: + +``` +/ + tile/ + 0/ # level 0 (leaf hashes), full tiles only + 000 001 ... 255 # full tiles (8192 B each for HASH_SIZE=32) + x001/ + 000 ... 255 + 1/ # level 1 (roll-ups of level-0 full tiles) + 000 ... + .../ + entries/ # optional raw entry bundles (full only) + 000 001 ... +``` + +### Index encoding (`encode_tile_index`) + +``` +encode_tile_index(N): + parts = [] + do { parts.push_front(printf("%03d", N % 1000)); N /= 1000 } while (N > 0) + for each part except the last: prepend 'x' + return join(parts, "/") +``` + +Examples: `5 → "005"`, `255 → "255"`, `1000 → "x001/000"`, +`1234067 → "x001/x234/067"`. + +Resource paths: + +- Full tile: `tile//` (the only tiles this + implementation writes) +- Entry bundle: `tile/entries/` (full bundles only) + +Tile byte format: the 256 entries concatenated, each `HASH_SIZE` raw bytes +(`HashT::bytes`); a full tile is `256 * HASH_SIZE` bytes. + +--- + +## 5. Architecture overview + +``` + append(leaf hash) flush() + caller ───────────────▶ TiledTreeT ───────────────▶ TileStoreT (disk I/O) + │ owns Tree ▲ + │ │ read tiles + inclusion/consistency proof requests │ + ▼ │ + ProofEngine ──▶ HashSource ◀────────┘ + ├─ MemoryHashSource (in-memory Tree) + ├─ TileHashSource (TileStore + size) + └─ CombinedHashSource +``` + +New, all in a companion header `merklecpp_tiles.h` (includes `merklecpp.h`), +namespace `merkle::tiles`, every type templated on `` +with default aliases mirroring the bottom of `merklecpp.h`: + +| Component | Responsibility | +|----------------------|-----------------------------------------------------------------------| +| `TileCoord` | pure index math + path encoding (no I/O) | +| `TileStoreT` | read/write tile files on a local filesystem | +| `TileWriterT` | compute & persist newly-complete full tiles (write path) | +| `HashSource` impls | resolve `subtree_root` from tiles, memory, or both | +| `ProofEngineT` | `mth_range`, `inclusion_proof`, `consistency_proof`, verifiers | +| `TiledTreeT` | convenience wrapper: `append` / `flush` / `prove*` / compaction | + +The **only** (optional) change to the core header is a small, read-only, +**non-hashing** accessor (`subtree_root`) used by `MemoryHashSource`; see +[§6.2](#62-optional-core-accessor-non-hashing). + +--- + +## 6. Public API design + +### 6.1 Header, namespace, aliases + +```cpp +// merklecpp_tiles.h +#include "merklecpp.h" +#include +#include + +namespace merkle { +namespace tiles { + +static constexpr uint16_t TILE_HEIGHT = 8; +static constexpr uint16_t TILE_WIDTH = 256; // 2^TILE_HEIGHT + +template &, + const HashT&, + HashT&)> +class TileStoreT { /* ... */ }; +// ... TileWriterT, ProofEngineT, TiledTreeT ... + +} // namespace tiles + +// Convenience aliases (mirror merklecpp.h) +using TileStore = tiles::TileStoreT<32, sha256_compress>; +using TiledTree = tiles::TiledTreeT<32, sha256_compress>; +// + 384/512 variants +} // namespace merkle +``` + +Note the roll-up/proof combiner is the **same** `HASH_FUNCTION` the tree uses, +and it only ever combines two `HASH_SIZE`-byte hashes — so the default +`sha256_compress` single-block path is sufficient and **no OpenSSL dependency is +introduced**. + +### 6.3 `TileStoreT` — disk I/O + +```cpp +struct TileRef { uint8_t level; uint64_t index; }; // full tiles only + +class TileStoreT { +public: + explicit TileStoreT(std::filesystem::path prefix); + + // Path helpers (pure) + static std::string encode_index(uint64_t n); + std::filesystem::path tile_path(const TileRef&) const; + std::filesystem::path entries_path(uint64_t n) const; + + // Tiles + bool has_full_tile(uint8_t level, uint64_t index) const; + std::vector read_tile(const TileRef&) const; // 256 entries + void write_tile(const TileRef&, const std::vector&); // synced atomic replace +}; +``` + +- Writes use a unique temporary file, sync the file contents, publish with an + atomic replace, and on POSIX sync every newly created directory link plus the + destination directory after the rename. Before reusing a visible file, the + writer re-confirms the directory chain and destination directory, so a retry + repeats a failed sync even if directory creation or rename is already + visible. A wrong-size tile is not treated as a durable full tile and is + rewritten by the writer. +- Full tiles are written once and never rewritten (immutability), so every file + under `tile//` is write-once. +- A small in-process cache of recently read tiles avoids repeated I/O during + proof generation. +- `TileStoreT` does not synchronize access. Callers must serialize all + operations on a store and across stores that share a prefix. + +### 6.8 Entry bundles (optional) + +merklecpp never sees raw entries (callers insert pre-computed leaf hashes), so +entry bundles are an **application-owned** add-on, included for completeness: + +- `tile/entries/` stores big-endian `uint16` length-prefixed entries, + 256 per full bundle (full bundles only). +- The application is responsible for the leaf-hash derivation it uses (e.g. + `leaf_hash = H(entry)`); merklecpp stores whatever leaf hash is inserted. +## 10. Implementation plan + +Each phase is independently testable; phases 1–4 require **no** core changes. + +**Phase 0 — Scaffolding.** Add `merklecpp_tiles.h` (includes `merklecpp.h`), +`merkle::tiles` namespace, geometry constants, default aliases. Wire a new test +group in `test/CMakeLists.txt` following `add_merklecpp_test`. + +**Phase 1 — Coordinates & store.** `TileCoord`/`encode_index`, `TileRef`, +`TileStoreT` (path building, atomic `write_tile`, `read_tile`). *Tests:* +index-encoding vectors; tile byte round-trip. + +**Phase 2 — Write path.** `TileWriterT::write_up_to` (level-0 from `leaf_at`, +roll-ups, incremental cursor; full tiles only — the frontier is never tiled). +*Tests:* sizes 256 & 70 000 produce exactly the expected full-tile set; +full-tile immutability (re-running writes nothing new). + +**Phase 3 — Hash sources & proof engine.** `TileHashSource`, `mth_range`, +`ProofEngineT::root/inclusion_proof/consistency_proof/verify_consistency`. +*Tests:* `root(size)` from tiles == `tree.root()`; `inclusion_proof(i,size)` +**equals** `tree.path(i)` (operator==) and verifies; `inclusion_proof(i, m)` +equals `tree.past_path(i, m-1)`; consistency proof reconciles +`tree.past_root(m-1)`/`tree.past_root(n-1)`. + +**Phase 4 — Combination & wrapper.** Optional core `subtree_root` accessor; +`MemoryHashSource`, `CombinedHashSource`, `TiledTreeT` (append/flush/prove +with `flush_to`). *Tests:* build N leaves, flush, then prove +inclusion for a **flushed** index and a **resident** index against a non-flushed +reference tree's root; consistency across a flush boundary. + +**Phase 5 — Optional entry bundles** (`BundleWriter`) and docs (README "Usage" +snippet, link this design doc, add `merklecpp_tiles.h` to Doxygen inputs). + +Deliverables: `merklecpp_tiles.h`; `test/tiles_*.cpp`; CMake wiring; optional +one-method core addition; README/docs updates. + +--- + diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h new file mode 100644 index 0000000..310ac61 --- /dev/null +++ b/merklecpp_tiles.h @@ -0,0 +1,740 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "merklecpp.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include +#else +# include +# include +#endif + +// Tiled storage for merklecpp trees, following the full-tile geometry and path +// encoding of the C2SP tlog-tiles layout (https://c2sp.org/tlog-tiles). Only +// complete, immutable tiles are stored. Their hash values are produced by the +// tree's existing HASH_FUNCTION, so tile-derived proofs are byte-identical to +// those produced by merkle::TreeT (see doc/design/tlog-tiles.md). +// +// Thread safety: types in this header do not synchronize access internally. +// Callers must serialize all operations on shared objects and store prefixes, +// including const proof operations that update the tile cache. + +namespace merkle // NOLINT(modernize-concat-nested-namespaces) +{ + namespace tiles + { + /// @brief Number of tree levels spanned by a single tile. + static constexpr uint16_t TILE_HEIGHT = 8; + + /// @brief Number of hashes in a full tile (2**TILE_HEIGHT). + static constexpr uint16_t TILE_WIDTH = 256; + + namespace detail + { + template + uint64_t contiguous_prefix_length(uint64_t limit, const Present& present) + { + uint64_t length = 0; + while (length < limit && present(length)) + { + length++; + } + return length; + } + } + + /// @brief Encodes a tile index as tlog-tiles path elements. + /// @param n The tile index + /// @return The index as zero-padded, '/'-separated 3-digit groups, where + /// all but the last group are prefixed with 'x'. For example, 1234067 is + /// encoded as "x001/x234/067" and 5 as "005". + static inline std::string encode_tile_index(uint64_t n) + { + std::vector parts; + char buf[8]; + do + { + std::snprintf(buf, sizeof(buf), "%03u", (unsigned)(n % 1000)); + parts.emplace_back(buf); + n /= 1000; + } while (n > 0); + + std::string r; + for (size_t i = 0; i < parts.size(); i++) + { + const std::string& part = parts[parts.size() - 1 - i]; + if (i != 0) + { + r += "/"; + } + if (i + 1 < parts.size()) + { + r += "x"; + } + r += part; + } + return r; + } + + /// @brief Identifies a single (full) tile within a tiled log. + /// @note Only full, TILE_WIDTH-wide tiles are produced and consumed; the + /// incomplete frontier is never tiled (see doc/design/tlog-tiles.md). + struct TileRef + { + /// @brief The level of the tile (0 == leaf hashes). + uint8_t level = 0; + + /// @brief The index of the tile within its level. + uint64_t index = 0; + }; + + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TileWriterT; + + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class EntryBundleWriterT; + + /// @brief Reads and writes tlog-tiles tile files on a local filesystem. + /// @tparam HASH_SIZE Size of each hash in bytes + /// @tparam HASH_FUNCTION The tree's node hash function (carried for use by + /// later components; tile I/O itself does not hash). + /// @warning No internal synchronization is provided. Callers must serialize + /// access to a store and to all stores that share its prefix. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TileStoreT + { + friend class TileWriterT; + friend class EntryBundleWriterT; + + public: + /// @brief The type of hashes stored in tiles. + using Hash = HashT; + + /// @brief Constructs a tile store rooted at @p prefix. + /// @param prefix The directory under which the tiles live. + explicit TileStoreT(std::filesystem::path prefix) : + prefix(std::move(prefix)) + {} + + /// @brief The root directory of the store. + [[nodiscard]] const std::filesystem::path& root() const + { + return prefix; + } + + /// @brief Encodes a tile index (see encode_tile_index). + static std::string encode_index(uint64_t n) + { + return encode_tile_index(n); + } + + /// @brief The filesystem path of a tile. + [[nodiscard]] std::filesystem::path tile_path(const TileRef& ref) const + { + return prefix / + ("tile/" + std::to_string((unsigned)ref.level) + "/" + + encode_tile_index(ref.index)); + } + + /// @brief The filesystem path of an entry bundle. + [[nodiscard]] std::filesystem::path entries_path(uint64_t index) const + { + return prefix / ("tile/entries/" + encode_tile_index(index)); + } + + /// @brief Whether a full tile exists on disk. + [[nodiscard]] bool has_full_tile(uint8_t level, uint64_t index) const + { + std::error_code ec; + const auto path = tile_path(TileRef{level, index}); + if (!std::filesystem::is_regular_file(path, ec) || ec) + { + return false; + } + return std::filesystem::file_size(path, ec) == + (uintmax_t)TILE_WIDTH * HASH_SIZE && + !ec; + } + + /// @brief Writes a tile to disk atomically. + /// @param ref The tile to write + /// @param hashes The tile's hashes (exactly TILE_WIDTH of them) + void write_tile(const TileRef& ref, const std::vector& hashes) + { + if (hashes.size() != TILE_WIDTH) + { + throw std::runtime_error("tile width mismatch"); + } + + std::vector bytes; + bytes.reserve(hashes.size() * HASH_SIZE); + for (const auto& h : hashes) + { + h.serialise(bytes); + } + + write_file_atomically(tile_path(ref), bytes); + } + + /// @brief Reads a tile from disk. + /// @param ref The tile to read + /// @return The tile's hashes (TILE_WIDTH of them) + [[nodiscard]] std::vector read_tile(const TileRef& ref) const + { + std::vector bytes = read_file(tile_path(ref)); + const size_t expected = (size_t)TILE_WIDTH * HASH_SIZE; + if (bytes.size() != expected) + { + throw std::runtime_error("unexpected tile size"); + } + + std::vector hashes; + hashes.reserve(TILE_WIDTH); + size_t position = 0; + for (uint16_t i = 0; i < TILE_WIDTH; i++) + { + hashes.emplace_back(bytes, position); + } + return hashes; + } + + /// @brief Whether a full entry bundle exists on disk. + [[nodiscard]] bool has_entry_bundle(uint64_t index) const + { + std::error_code ec; + const auto path = entries_path(index); + if (!std::filesystem::is_regular_file(path, ec) || ec) + { + return false; + } + try + { + (void)decode_entries(read_file(path), TILE_WIDTH); + return true; + } + catch (const std::exception&) + { + return false; + } + } + + /// @brief Writes a full entry bundle to disk atomically. + /// @param index The bundle index + /// @param entries The raw log entries (exactly TILE_WIDTH of them) + /// @note Entries are stored in the tlog-tiles entry-bundle format: a + /// sequence of big-endian uint16 length-prefixed byte strings. + void write_entry_bundle( + uint64_t index, const std::vector>& entries) + { + if (entries.size() != TILE_WIDTH) + { + throw std::runtime_error("entry bundle width mismatch"); + } + write_file_atomically(entries_path(index), encode_entries(entries)); + } + + /// @brief Reads a full entry bundle from disk. + /// @param index The bundle index + /// @return The raw log entries (TILE_WIDTH of them) + [[nodiscard]] std::vector> read_entry_bundle( + uint64_t index) const + { + return decode_entries(read_file(entries_path(index)), TILE_WIDTH); + } + + /// @brief Encodes log entries into the tlog-tiles entry-bundle format. + static std::vector encode_entries( + const std::vector>& entries) + { + std::vector bytes; + for (const auto& e : entries) + { + if (e.size() > 0xFFFF) + { + throw std::runtime_error( + "entry too large for uint16 length prefix"); + } + bytes.push_back((uint8_t)((e.size() >> 8) & 0xFF)); + bytes.push_back((uint8_t)(e.size() & 0xFF)); + bytes.insert(bytes.end(), e.begin(), e.end()); + } + return bytes; + } + + /// @brief Decodes @p count entries from the entry-bundle format. + static std::vector> decode_entries( + const std::vector& bytes, size_t count) + { + std::vector> out; + out.reserve(count); + size_t pos = 0; + for (size_t i = 0; i < count; i++) + { + if (bytes.size() - pos < 2) + { + throw std::runtime_error("truncated entry bundle"); + } + const auto len = + (uint16_t)(((uint16_t)bytes[pos] << 8) | bytes[pos + 1]); + pos += 2; + if (len > bytes.size() - pos) + { + throw std::runtime_error("truncated entry bundle"); + } + out.emplace_back( + bytes.begin() + static_cast(pos), + bytes.begin() + static_cast(pos + len)); + pos += len; + } + if (pos != bytes.size()) + { + throw std::runtime_error("trailing bytes in entry bundle"); + } + return out; + } + + protected: + using DirectorySync = std::function; + + TileStoreT(std::filesystem::path prefix, DirectorySync directory_sync) : + prefix(std::move(prefix)), directory_sync(std::move(directory_sync)) + {} + + /// @brief The root directory of the store. + std::filesystem::path prefix; + + DirectorySync directory_sync; + std::set durable_directory_entries; + std::set durable_directory_contents; + + [[nodiscard]] bool confirm_full_tile(uint8_t level, uint64_t index) + { + const auto path = tile_path(TileRef{level, index}); + if (!has_full_tile(level, index)) + { + return false; + } + confirm_file_durable(path); + return true; + } + + [[nodiscard]] bool confirm_entry_bundle(uint64_t index) + { + const auto path = entries_path(index); + if (!has_entry_bundle(index)) + { + return false; + } + confirm_file_durable(path); + return true; + } + + void begin_write_attempt() + { + durable_directory_contents.clear(); + } + + /// @brief Reads an entire file into a byte vector. + static std::vector read_file(const std::filesystem::path& path) + { + std::ifstream f(path, std::ios::binary); + if (!f.good()) + { + throw std::runtime_error("cannot open file: " + path.string()); + } + return { + std::istreambuf_iterator(f), std::istreambuf_iterator()}; + } + + /// @brief Writes a file atomically via a synced temporary file. + /// @note Uses unique temp names, cleans them up on errors, and syncs the + /// file before publishing it with an atomic replace. POSIX builds also + /// confirm each directory entry in the path and sync the destination + /// directory after rename. Existing files are only reused after these + /// syncs are re-confirmed by the current write attempt. + void write_file_atomically( + const std::filesystem::path& path, const std::vector& bytes) + { + create_directories_durably(path.parent_path()); + + const std::filesystem::path tmp = temp_path(path); + TempFileGuard guard(tmp); + write_and_sync_file(tmp, bytes); + const auto parent = directory_or_dot(path.parent_path()); + durable_directory_contents.erase(parent); + replace_file(tmp, path); + sync_directory_contents(parent); + guard.dismiss(); + } + + static std::filesystem::path directory_or_dot( + const std::filesystem::path& directory) + { + return directory.empty() ? std::filesystem::path(".") : directory; + } + + void create_directories_durably( + const std::filesystem::path& requested_directory) + { + const auto directory = directory_or_dot(requested_directory); + if (durable_directory_entries.count(directory) != 0) + { + return; + } + + auto parent = directory_or_dot(directory.parent_path()); + if (parent != directory) + { + create_directories_durably(parent); + } + + std::error_code ec; + const bool exists = std::filesystem::exists(directory, ec); + if (ec) + { + throw std::runtime_error( + "cannot inspect directory " + directory.string() + ": " + + ec.message()); + } + if (exists) + { + const bool is_directory = + std::filesystem::is_directory(directory, ec); + if (ec) + { + throw std::runtime_error( + "cannot inspect directory " + directory.string() + ": " + + ec.message()); + } + if (!is_directory) + { + throw std::runtime_error( + "cannot create directory " + directory.string() + + ": path exists and is not a directory"); + } + } + else + { + const bool created = std::filesystem::create_directory(directory, ec); + if (ec) + { + throw std::runtime_error( + "cannot create directory " + directory.string() + ": " + + ec.message()); + } + if (!created && !std::filesystem::is_directory(directory, ec)) + { + throw std::runtime_error( + "cannot create directory " + directory.string() + + (ec ? ": " + ec.message() : "")); + } + } + + if (parent != directory) + { + sync_directory_for_durability(parent); + durable_directory_contents.insert(parent); + } + durable_directory_entries.insert(directory); + } + + void confirm_file_durable(const std::filesystem::path& path) + { + const auto parent = directory_or_dot(path.parent_path()); + create_directories_durably(parent); + sync_directory_contents(parent); + } + + void sync_directory_contents(const std::filesystem::path& directory) + { + const auto path = directory_or_dot(directory); + if (durable_directory_contents.count(path) != 0) + { + return; + } + sync_directory_for_durability(path); + durable_directory_contents.insert(path); + } + + class TempFileGuard + { + public: + explicit TempFileGuard(std::filesystem::path path) : + path(std::move(path)) + {} + + ~TempFileGuard() + { + if (active) + { + std::error_code ec; + std::filesystem::remove(path, ec); + } + } + + void dismiss() + { + active = false; + } + + private: + std::filesystem::path path; + bool active = true; + }; + + static std::filesystem::path temp_path(const std::filesystem::path& path) + { + static std::atomic counter{0}; + const auto stamp = + std::chrono::steady_clock::now().time_since_epoch().count(); + std::filesystem::path tmp = path; + tmp += ".tmp." + std::to_string(process_id()) + "." + + std::to_string((uint64_t)stamp) + "." + + std::to_string(counter.fetch_add(1, std::memory_order_relaxed)); + return tmp; + } + + static uint64_t process_id() + { +#ifdef _WIN32 + return (uint64_t)GetCurrentProcessId(); +#else + return (uint64_t)::getpid(); +#endif + } + + static std::string system_error_message(const std::string& what) + { +#ifdef _WIN32 + return what + ": error " + std::to_string(GetLastError()); +#else + return what + ": " + std::strerror(errno); +#endif + } + + static void require_write_progress( + size_t written, const std::filesystem::path& path) + { + if (written == 0) + { + throw std::runtime_error("short write: " + path.string()); + } + } + + static void write_and_sync_file( + const std::filesystem::path& path, const std::vector& bytes) + { +#ifdef _WIN32 + HANDLE handle = CreateFileW( + path.wstring().c_str(), + GENERIC_WRITE, + 0, + nullptr, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (handle == INVALID_HANDLE_VALUE) + { + throw std::runtime_error("cannot open file: " + path.string()); + } + bool close_handle = true; + try + { + size_t written = 0; + while (written < bytes.size()) + { + const auto remaining = bytes.size() - written; + const auto chunk = (DWORD)std::min( + remaining, (size_t)std::numeric_limits::max()); + DWORD done = 0; + if (!WriteFile( + handle, bytes.data() + written, chunk, &done, nullptr)) + { + throw std::runtime_error( + system_error_message("error writing file " + path.string())); + } + require_write_progress((size_t)done, path); + written += done; + } + if (!FlushFileBuffers(handle)) + { + throw std::runtime_error( + system_error_message("error syncing file " + path.string())); + } + close_handle = false; + if (!CloseHandle(handle)) + { + throw std::runtime_error( + system_error_message("error closing file " + path.string())); + } + } + catch (...) + { + if (close_handle) + { + CloseHandle(handle); + } + throw; + } +#else + int flags = O_WRONLY | O_CREAT | O_TRUNC; +# ifdef O_CLOEXEC + flags |= O_CLOEXEC; +# endif + int fd = ::open(path.c_str(), flags, 0666); + if (fd < 0) + { + throw std::runtime_error( + system_error_message("cannot open file " + path.string())); + } + try + { + size_t written = 0; + while (written < bytes.size()) + { + const ssize_t done = + ::write(fd, bytes.data() + written, bytes.size() - written); + if (done < 0) + { + if (errno == EINTR) + { + continue; + } + throw std::runtime_error( + system_error_message("error writing file " + path.string())); + } + require_write_progress((size_t)done, path); + written += (size_t)done; + } + if (::fsync(fd) != 0) + { + throw std::runtime_error( + system_error_message("error syncing file " + path.string())); + } + if (::close(fd) != 0) + { + fd = -1; + throw std::runtime_error( + system_error_message("error closing file " + path.string())); + } + fd = -1; + } + catch (...) + { + if (fd >= 0) + { + ::close(fd); + } + throw; + } +#endif + } + + static void replace_file( + const std::filesystem::path& tmp, const std::filesystem::path& path) + { +#ifdef _WIN32 + if (!MoveFileExW( + tmp.wstring().c_str(), + path.wstring().c_str(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) + { + throw std::runtime_error(system_error_message( + "cannot rename temp file " + tmp.string() + " to " + + path.string())); + } +#else + std::error_code ec; + std::filesystem::rename(tmp, path, ec); + if (ec) + { + throw std::runtime_error( + "cannot rename temp file " + tmp.string() + " to " + path.string() + + ": " + ec.message()); + } +#endif + } + + void sync_directory_for_durability(const std::filesystem::path& path) + { + if (directory_sync) + { + directory_sync(path); + return; + } + sync_directory_on_disk(path); + } + + static void sync_directory_on_disk(const std::filesystem::path& path) + { +#ifndef _WIN32 + int flags = O_RDONLY; +# ifdef O_DIRECTORY + flags |= O_DIRECTORY; +# endif +# ifdef O_CLOEXEC + flags |= O_CLOEXEC; +# endif + const int fd = ::open(path.c_str(), flags); + if (fd < 0) + { + throw std::runtime_error( + system_error_message("cannot open directory " + path.string())); + } + if (::fsync(fd) != 0) + { + const std::string message = + system_error_message("error syncing directory " + path.string()); + ::close(fd); + throw std::runtime_error(message); + } + if (::close(fd) != 0) + { + throw std::runtime_error( + system_error_message("error closing directory " + path.string())); + } +#else + (void)path; +#endif + } + }; + + /// @brief Default tile store (SHA256, default hash function). + using TileStore = TileStoreT<32, sha256_compress>; + + } +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e90a87c..a47f8a9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -29,6 +29,7 @@ add_merklecpp_test(serialisation serialisation.cpp) add_merklecpp_test(partial_serialisation partial_serialisation.cpp) add_merklecpp_test(serialise_to_file serialise_to_file.cpp) add_merklecpp_test(coverage coverage.cpp) +add_merklecpp_test(tiles_store tiles_store.cpp) if(TARGET evercrypt.host) add_merklecpp_test(compare_evercrypt compare_evercrypt.cpp) diff --git a/test/tiles_store.cpp b/test/tiles_store.cpp new file mode 100644 index 0000000..cf9d2a2 --- /dev/null +++ b/test/tiles_store.cpp @@ -0,0 +1,356 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TileRef; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +static void expect_eq( + const std::string& got, const std::string& expected, const std::string& what) +{ + if (got != expected) + { + throw std::runtime_error( + what + ": got '" + got + "', expected '" + expected + "'"); + } +} + +static std::string rel(const merkle::tiles::TileStore& store, const fs::path& p) +{ + return p.lexically_relative(store.root()).generic_string(); +} + +static bool any_tmp_files(const fs::path& dir) +{ + if (!fs::exists(dir)) + { + return false; + } + return std::any_of( + fs::recursive_directory_iterator(dir), + fs::recursive_directory_iterator(), + [](const auto& entry) { + return entry.path().filename().string().find(".tmp") != std::string::npos; + }); +} + +class TileStoreProbe : public merkle::tiles::TileStore +{ +public: + using Store = merkle::tiles::TileStore; + + static void check_write_progress(size_t written) + { + Store::require_write_progress(written, "test"); + } +}; + +struct SyncFault +{ + fs::path fail_path; + size_t failures_remaining = 0; + size_t matching_calls_before_failure = 0; + std::vector calls; + + void sync(const fs::path& path) + { + const auto normalised = path.lexically_normal(); + calls.push_back(normalised); + if (failures_remaining > 0 && normalised == fail_path.lexically_normal()) + { + if (matching_calls_before_failure > 0) + { + matching_calls_before_failure--; + return; + } + failures_remaining--; + throw std::runtime_error("injected directory sync failure"); + } + } + + [[nodiscard]] size_t call_count(const fs::path& path) const + { + const auto normalised = path.lexically_normal(); + return (size_t)std::count(calls.begin(), calls.end(), normalised); + } +}; + +class FaultInjectingTileStore : public merkle::tiles::TileStore +{ +public: + using Store = merkle::tiles::TileStore; + + FaultInjectingTileStore( + fs::path prefix, const std::shared_ptr& fault) : + Store( + std::move(prefix), [fault](const fs::path& path) { fault->sync(path); }) + {} +}; + +// Overwrites a file with exactly the given bytes (to simulate corruption). +static void overwrite_file(const fs::path& p, const std::vector& bytes) +{ + std::ofstream f(p, std::ios::binary | std::ios::trunc); + f.write( + reinterpret_cast(bytes.data()), (std::streamsize)bytes.size()); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + + const fs::path dir = fs::temp_directory_path() / + ("merklecpp_tiles_" + std::to_string((unsigned long long)seed) + "_" + + std::to_string(std::rand())); + + try + { + // 1. Tile index encoding vectors (tlog-tiles examples). + expect_eq(merkle::tiles::encode_tile_index(0), "000", "encode 0"); + expect_eq(merkle::tiles::encode_tile_index(5), "005", "encode 5"); + expect_eq(merkle::tiles::encode_tile_index(255), "255", "encode 255"); + expect_eq(merkle::tiles::encode_tile_index(999), "999", "encode 999"); + expect_eq( + merkle::tiles::encode_tile_index(1000), "x001/000", "encode 1000"); + expect_eq( + merkle::tiles::encode_tile_index(1234067), + "x001/x234/067", + "encode 1234067"); + + merkle::tiles::TileStore store(dir); + const size_t hsz = Hash().size(); + const auto full = make_hashes(merkle::tiles::TILE_WIDTH); + + // 2. Resource path layout (full tiles and bundles only). + expect_eq( + rel(store, store.tile_path(TileRef{0, 0})), + "tile/0/000", + "tile_path L0 N0"); + expect_eq( + rel(store, store.tile_path(TileRef{1, 1234067})), + "tile/1/x001/x234/067", + "tile_path L1 big index"); + expect_eq( + rel(store, store.entries_path(5)), "tile/entries/005", "entries full"); + + // 2b. Production writes sync each directory link and the destination + // directory in order. + { + const fs::path prefix = dir / "durable"; + const auto fault = std::make_shared(); + FaultInjectingTileStore durable_store(prefix, fault); + durable_store.write_tile(TileRef{1, 0}, full); + const std::vector expected = { + dir.parent_path(), dir, prefix, prefix / "tile", prefix / "tile" / "1"}; + expect( + fault->calls.size() >= expected.size() && + std::equal( + expected.begin(), + expected.end(), + fault->calls.end() - (std::ptrdiff_t)expected.size()), + "directory parents and destination synced in order"); + } + + // 2c. A successful write must make progress. This shared guard prevents + // the Windows WriteFile loop from spinning if it reports zero bytes. + { + bool zero_threw = false; + try + { + TileStoreProbe::check_write_progress(0); + } + catch (const std::exception&) + { + zero_threw = true; + } + expect(zero_threw, "zero-byte write rejected"); + TileStoreProbe::check_write_progress(1); + } + + // 3a. Full tile byte round-trip. + const TileRef full_ref{0, 0}; + store.write_tile(full_ref, full); + expect(store.has_full_tile(0, 0), "has_full_tile after write"); + expect(!store.has_full_tile(0, 5), "missing full tile"); + expect( + fs::file_size(store.tile_path(full_ref)) == + (uintmax_t)merkle::tiles::TILE_WIDTH * hsz, + "full tile file size"); + const auto full_rt = store.read_tile(full_ref); + expect(full_rt.size() == full.size(), "full tile width round-trip"); + for (size_t i = 0; i < full.size(); i++) + { + expect(full_rt[i] == full[i], "full tile hash round-trip"); + } + + // 3b. Wrong-width writes are rejected (only 256-wide tiles are valid). + const std::vector three(full.begin(), full.begin() + 3); + bool threw = false; + try + { + store.write_tile(TileRef{0, 2}, three); // 3 hashes for a full tile + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "width mismatch rejected"); + + // 3c. Corrupt / truncated files are rejected on read (integrity check), so + // a torn write can never be served as a valid tile or bundle. + { + const auto expect_throws = + [](const std::function& fn, const std::string& what) { + bool t = false; + try + { + fn(); + } + catch (const std::exception&) + { + t = true; + } + expect(t, what); + }; + + // Truncated tile: fewer bytes than a full tile. + overwrite_file(store.tile_path(full_ref), std::vector(hsz, 0)); + expect(!store.has_full_tile(0, 0), "truncated tile is not durable"); + expect_throws( + [&] { (void)store.read_tile(full_ref); }, "truncated tile rejected"); + store.write_tile(full_ref, full); + expect(store.has_full_tile(0, 0), "truncated tile rewritten"); + expect(store.read_tile(full_ref) == full, "rewritten tile round-trip"); + + // Oversized tile: more bytes than a full tile. + overwrite_file( + store.tile_path(full_ref), + std::vector((merkle::tiles::TILE_WIDTH + 1) * hsz, 0)); + expect(!store.has_full_tile(0, 0), "oversized tile is not durable"); + expect_throws( + [&] { (void)store.read_tile(full_ref); }, "oversized tile rejected"); + + // A valid full entry bundle whose file is then cut short. + std::vector> entries(merkle::tiles::TILE_WIDTH); + for (size_t i = 0; i < entries.size(); i++) + { + entries[i] = {(uint8_t)i, 0x7F}; + } + store.write_entry_bundle(0, entries); + expect(store.has_entry_bundle(0), "bundle durable before truncation"); + expect( + store.read_entry_bundle(0).size() == merkle::tiles::TILE_WIDTH, + "bundle valid before truncation"); + // Claims a 5-byte entry but supplies only one trailing byte. + overwrite_file(store.entries_path(0), {0x00, 0x05, 0x01}); + expect(!store.has_entry_bundle(0), "truncated bundle is not durable"); + expect_throws( + [&] { (void)store.read_entry_bundle(0); }, + "truncated entry bundle rejected"); + + // A syntactically complete bundle with trailing bytes is not durable. + auto encoded = merkle::tiles::TileStore::encode_entries(entries); + encoded.push_back(0xFF); + overwrite_file(store.entries_path(0), encoded); + expect(!store.has_entry_bundle(0), "trailing bundle is not durable"); + expect_throws( + [&] { (void)store.read_entry_bundle(0); }, + "trailing entry bundle rejected"); + + // decode_entries directly: a length prefix that overruns the buffer. + expect_throws( + [] { + (void)merkle::tiles::TileStore::decode_entries({0xFF, 0xFF, 0x00}, 1); + }, + "decode_entries oversized length rejected"); + expect_throws( + [] { + (void)merkle::tiles::TileStore::decode_entries({0x00, 0x00, 0xFF}, 1); + }, + "decode_entries trailing bytes rejected"); + } + +// Windows file replacement semantics can reject racing same-path replaces even +// when the final tile remains valid, so keep this stress check POSIX-only. +#ifndef _WIN32 + // 3d. Concurrent same-tile writes use unique temp files and leave no + // temporary files behind after success. Each thread owns its store object; + // sharing one object requires caller-provided synchronization. + { + const TileRef concurrent_ref{0, 42}; + std::atomic ok{true}; + std::vector threads; + for (size_t i = 0; i < 8; i++) + { + threads.emplace_back([&] { + try + { + merkle::tiles::TileStore thread_store(dir); + thread_store.write_tile(concurrent_ref, full); + } + catch (...) + { + ok = false; + } + }); + } + for (auto& thread : threads) + { + thread.join(); + } + expect(ok, "concurrent writes succeeded"); + expect(store.read_tile(concurrent_ref) == full, "concurrent tile valid"); + expect(!any_tmp_files(dir), "no temp files left after writes"); + } +#endif + + std::cout << "tiles_store: OK" << '\n'; + + std::error_code ec; + fs::remove_all(dir, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + + return 0; +} From 2b467f31951be262411f71b5ad942e029e21ea32 Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 13:19:43 +0000 Subject: [PATCH 02/22] Address tile store review comments Keep the Windows handle cleanup armed until CloseHandle succeeds, and make test directories unique across concurrent processes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- merklecpp_tiles.h | 2 +- test/tiles_store.cpp | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 310ac61..0cd9fd5 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -595,12 +595,12 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) throw std::runtime_error( system_error_message("error syncing file " + path.string())); } - close_handle = false; if (!CloseHandle(handle)) { throw std::runtime_error( system_error_message("error closing file " + path.string())); } + close_handle = false; } catch (...) { diff --git a/test/tiles_store.cpp b/test/tiles_store.cpp index cf9d2a2..fd2ec83 100644 --- a/test/tiles_store.cpp +++ b/test/tiles_store.cpp @@ -5,9 +5,8 @@ #include #include +#include #include -#include -#include #include #include #include @@ -69,6 +68,11 @@ class TileStoreProbe : public merkle::tiles::TileStore { Store::require_write_progress(written, "test"); } + + static uint64_t current_process_id() + { + return Store::process_id(); + } }; struct SyncFault @@ -123,13 +127,13 @@ static void overwrite_file(const fs::path& p, const std::vector& bytes) int main() { - const auto seed = std::time(nullptr); - std::srand((unsigned)seed); - std::cout << "seed=" << seed << '\n'; - + const auto nonce = + std::chrono::steady_clock::now().time_since_epoch().count(); const fs::path dir = fs::temp_directory_path() / - ("merklecpp_tiles_" + std::to_string((unsigned long long)seed) + "_" + - std::to_string(std::rand())); + ("merklecpp_tiles_" + + std::to_string( + static_cast(TileStoreProbe::current_process_id())) + + "_" + std::to_string(static_cast(nonce))); try { From 47516cef7fcebe10abdd81dcf4a07f0a7d9def17 Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 13:50:08 +0000 Subject: [PATCH 03/22] Namespace tile stores by hash format Keep every tile 256 hashes wide while separating SHA-256, SHA-384, and SHA-512 payloads under algorithm-qualified storage roots. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/design/tlog-tiles.md | 107 ++++++++++++++++++++++----------- merklecpp_tiles.h | 127 +++++++++++++++++++++++++++++++++++---- test/tiles_store.cpp | 62 ++++++++++++++++++- 3 files changed, 249 insertions(+), 47 deletions(-) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index d36bf12..fd13147 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -2,10 +2,12 @@ Status: Proposal / Draft Audience: merklecpp maintainers and contributors -Scope: Extend the header-only library to (a) persist the Merkle tree as -[tlog-tiles](https://c2sp.org/tlog-tiles) tile files progressively (on -compaction), and (b) serve **inclusion** and **consistency** proofs from those -tiles, from the in-memory tree, or from a combination of the two. +Scope: Extend the header-only library to (a) persist the Merkle tree +progressively (on compaction) using the +[tlog-tiles](https://c2sp.org/tlog-tiles) SHA-256 format and an explicitly +namespaced extension for other SHA output sizes, and (b) serve **inclusion** +and **consistency** proofs from those tiles, from the in-memory tree, or from a +combination of the two. --- @@ -14,10 +16,12 @@ tiles, from the in-memory tree, or from a combination of the two. ### Goals 1. Write the tree to disk as a set of immutable, cacheable **tile files** using - the [tlog-tiles](https://c2sp.org/tlog-tiles) file/directory layout and tile - geometry (256-wide tiles, 8 tree levels per tile, path encoding). Only - **full, balanced** tiles are produced. The remaining frontier stays in the - in-memory tree until it crosses the next full-tile boundary. + the [tlog-tiles](https://c2sp.org/tlog-tiles) payload, path encoding, and + geometry (256 hashes per tile and 8 tree levels per tile). SHA-256 follows + the C2SP format; SHA-384 and SHA-512 retain the same 256-hash width as + merklecpp extensions. Only **full, balanced** tiles are produced. The + remaining frontier stays in the in-memory tree until it crosses the next + full-tile boundary. 2. Produce tiles **progressively**: each batch/flush writes only the newly completed tiles, and the write integrates with the existing `flush_to()` "compaction" so that flushed (evicted) subtrees are durably persisted before @@ -57,10 +61,11 @@ reader/writer coordination are deliberately left to the application. ### Non-goals -- Byte-level interoperability with *external* RFC 6962 / Go-ecosystem tlog - clients. See [§2.3](#23-compatibility-statement): such interop is only - possible if the consumer instantiates `TreeT` with an RFC 6962 combiner, which - is **out of scope** here and not required by this design. +- Making the SHA-384 or SHA-512 extension a C2SP standard. C2SP currently + specifies SHA-256 only; the additional namespaces are merklecpp formats. +- Supplying RFC 6962 leaf hashing or domain separation. Exact external tlog + interoperability requires the consumer to instantiate `TreeT` with + compatible leaf and node hashing; this design does not alter hashing. - An HTTP server. We define the on-disk layout and the read/write/proof APIs; serving the static files is an application concern. @@ -77,8 +82,9 @@ A tiled log exposes the Merkle tree as a set of static resources: - `` = tile index within the level, encoded as zero-padded 3-digit path elements where **all but the last element are prefixed with `x`**. Example: `1234067` → `x001/x234/067`. -- A **full tile** is exactly **256 hashes** wide (`256 * HASH_SIZE` bytes). The - spec fixes `HASH_SIZE = 32` (SHA-256); here it is the template parameter. +- A **full tile** is exactly **256 SHA-256 hashes**, or **8,192 bytes**. C2SP + fixes both the algorithm and width. merklecpp keeps the width at 256 and + makes the hash size a template parameter. - Level 0 hashes are **leaf hashes**. At level `L ≥ 1`, each hash is the Merkle Tree Hash of a *full* tile at level `L-1`. A tile spans **8 tree levels**: its 256 entries are the leaves of a height-8 perfect subtree, and intra-tile @@ -117,6 +123,27 @@ A tiled log exposes the Merkle tree as a set of static resources: - merklecpp has **no** classic two-size consistency proof today; this design adds one (built from the same combiner, so it reconciles `past_root`s). +### 2.3 Compatibility statement + +C2SP `tlog-tiles` normatively specifies SHA-256, 32-byte hashes, and 256 hashes +per full tile. The SHA-256 merklecpp layout adopts that tile payload and the +`tile//` resource tree. Exact proof interoperability additionally requires +RFC 6962-compatible leaf and node hashing; merklecpp deliberately continues to +use the caller's existing `HASH_FUNCTION`. + +For SHA-384 and SHA-512, merklecpp extends the payload by concatenating 48-byte +or 64-byte hashes respectively. **The tile width remains 256 hashes for every +algorithm**, so full tiles are 12,288 bytes for SHA-384 and 16,384 bytes for +SHA-512. These wider-hash payloads are not currently defined by C2SP. + +Each format gets a separate root named +`-w`: `sha256-256w`, `sha384-256w`, or +`sha512-256w`. Beneath that root, the C2SP `tile/...` paths are unchanged. This +prevents files with different hash sizes from sharing a tile namespace while +preserving the C2SP layout for each format. Built-in SHA functions select their +short name automatically; explicit custom names must be lowercase path-safe +short names, and recognized SHA names are checked against `HASH_SIZE`. + ## 3. Tile ↔ merklecpp mapping (the math) Let `TILE_HEIGHT = 8` and `TILE_WIDTH = 256 = 2^8`. All combiners below are the @@ -173,20 +200,24 @@ pieces (see the [combined-source invariant](#7-progressive-production--compactio ## 4. File and directory layout -Rooted at a configurable `prefix` directory on local disk: +Rooted below a configurable `prefix` directory on local disk: ``` / - tile/ - 0/ # level 0 (leaf hashes), full tiles only - 000 001 ... 255 # full tiles (8192 B each for HASH_SIZE=32) - x001/ - 000 ... 255 - 1/ # level 1 (roll-ups of level-0 full tiles) - 000 ... - .../ - entries/ # optional raw entry bundles (full only) - 000 001 ... + sha256-256w/ # algorithm + fixed 256-hash width + tile/ + 0/ # level 0 (leaf hashes), full tiles only + 000 001 ... 255 # 8192 B per SHA-256 full tile + x001/ + 000 ... 255 + 1/ # level 1 (roll-ups of level-0 full tiles) + 000 ... + .../ + entries/ # optional raw entry bundles (full only) + 000 001 ... + sha384-256w/ # optional; same width, 12288 B full tiles + tile/ + ... ``` ### Index encoding (`encode_tile_index`) @@ -204,12 +235,16 @@ Examples: `5 → "005"`, `255 → "255"`, `1000 → "x001/000"`, Resource paths: -- Full tile: `tile//` (the only tiles this - implementation writes) -- Entry bundle: `tile/entries/` (full bundles only) +- Full tile: + `/-256w/tile//` (the only tiles + this implementation writes) +- Entry bundle: + `/-256w/tile/entries/` (full bundles + only) Tile byte format: the 256 entries concatenated, each `HASH_SIZE` raw bytes -(`HashT::bytes`); a full tile is `256 * HASH_SIZE` bytes. +(`HashT::bytes`); a full tile is `256 * HASH_SIZE` bytes. The `256w` suffix +counts hashes, not bytes, and therefore does not change for SHA-384 or SHA-512. --- @@ -291,9 +326,13 @@ struct TileRef { uint8_t level; uint64_t index; }; // full tiles only class TileStoreT { public: + // Built-in SHA functions select sha256, sha384, or sha512. explicit TileStoreT(std::filesystem::path prefix); + TileStoreT(std::filesystem::path prefix, + const std::string& hash_algorithm_short_name); // Path helpers (pure) + static std::string storage_directory_name(const std::string& algorithm); static std::string encode_index(uint64_t n); std::filesystem::path tile_path(const TileRef&) const; std::filesystem::path entries_path(uint64_t n) const; @@ -313,7 +352,7 @@ public: visible. A wrong-size tile is not treated as a durable full tile and is rewritten by the writer. - Full tiles are written once and never rewritten (immutability), so every file - under `tile//` is write-once. + under `-256w/tile//` is write-once. - A small in-process cache of recently read tiles avoids repeated I/O during proof generation. - `TileStoreT` does not synchronize access. Callers must serialize all @@ -324,8 +363,8 @@ public: merklecpp never sees raw entries (callers insert pre-computed leaf hashes), so entry bundles are an **application-owned** add-on, included for completeness: -- `tile/entries/` stores big-endian `uint16` length-prefixed entries, - 256 per full bundle (full bundles only). +- `-256w/tile/entries/` stores big-endian `uint16` + length-prefixed entries, 256 per full bundle (full bundles only). - The application is responsible for the leaf-hash derivation it uses (e.g. `leaf_hash = H(entry)`); merklecpp stores whatever leaf hash is inserted. ## 10. Implementation plan @@ -338,7 +377,8 @@ group in `test/CMakeLists.txt` following `add_merklecpp_test`. **Phase 1 — Coordinates & store.** `TileCoord`/`encode_index`, `TileRef`, `TileStoreT` (path building, atomic `write_tile`, `read_tile`). *Tests:* -index-encoding vectors; tile byte round-trip. +index-encoding vectors; algorithm-qualified roots; SHA-256 and SHA-384 both use +256-hash tiles; tile byte round-trip. **Phase 2 — Write path.** `TileWriterT::write_up_to` (level-0 from `leaf_at`, roll-ups, incremental cursor; full tiles only — the frontier is never tiled). @@ -365,4 +405,3 @@ Deliverables: `merklecpp_tiles.h`; `test/tiles_*.cpp`; CMake wiring; optional one-method core addition; README/docs updates. --- - diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 0cd9fd5..52c7f4c 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -34,11 +34,14 @@ # include #endif -// Tiled storage for merklecpp trees, following the full-tile geometry and path -// encoding of the C2SP tlog-tiles layout (https://c2sp.org/tlog-tiles). Only -// complete, immutable tiles are stored. Their hash values are produced by the -// tree's existing HASH_FUNCTION, so tile-derived proofs are byte-identical to -// those produced by merkle::TreeT (see doc/design/tlog-tiles.md). +// Tiled storage for merklecpp trees, following the full-tile geometry, payload, +// and path encoding of the C2SP tlog-tiles layout +// (https://c2sp.org/tlog-tiles). C2SP defines SHA-256; merklecpp preserves the +// 256-hash tile width for other SHA output sizes and separates each format +// under an algorithm-qualified directory such as sha256-256w or sha384-256w. +// Only complete, immutable tiles are stored. Their hash values are produced by +// the tree's existing HASH_FUNCTION, so tile-derived proofs are byte-identical +// to those produced by merkle::TreeT (see doc/design/tlog-tiles.md). // // Thread safety: types in this header do not synchronize access internally. // Callers must serialize all operations on shared objects and store prefixes, @@ -144,18 +147,39 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @brief The type of hashes stored in tiles. using Hash = HashT; - /// @brief Constructs a tile store rooted at @p prefix. - /// @param prefix The directory under which the tiles live. + /// @brief Constructs a tile store below an algorithm-qualified directory + /// under @p prefix. + /// @param prefix The directory under which the format directory lives + /// @note Built-in SHA functions select sha256, sha384, or sha512. explicit TileStoreT(std::filesystem::path prefix) : - prefix(std::move(prefix)) + TileStoreT(std::move(prefix), default_hash_algorithm_short_name()) {} - /// @brief The root directory of the store. + /// @brief Constructs a tile store for an explicitly named hash algorithm. + /// @param prefix The directory under which the format directory lives + /// @param hash_algorithm_short_name Lowercase algorithm short name + TileStoreT( + std::filesystem::path prefix, + const std::string& hash_algorithm_short_name) : + prefix(storage_root(std::move(prefix), hash_algorithm_short_name)) + {} + + /// @brief The algorithm-qualified root directory of the store. [[nodiscard]] const std::filesystem::path& root() const { return prefix; } + /// @brief The format directory for @p hash_algorithm_short_name. + /// @note TILE_WIDTH is fixed at 256 for every hash output size. + static std::string storage_directory_name( + const std::string& hash_algorithm_short_name) + { + validate_hash_algorithm_short_name(hash_algorithm_short_name); + return hash_algorithm_short_name + "-" + std::to_string(TILE_WIDTH) + + "w"; + } + /// @brief Encodes a tile index (see encode_tile_index). static std::string encode_index(uint64_t n) { @@ -331,16 +355,97 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) using DirectorySync = std::function; TileStoreT(std::filesystem::path prefix, DirectorySync directory_sync) : - prefix(std::move(prefix)), directory_sync(std::move(directory_sync)) + TileStoreT( + std::move(prefix), + default_hash_algorithm_short_name(), + std::move(directory_sync)) {} - /// @brief The root directory of the store. + TileStoreT( + std::filesystem::path prefix, + const std::string& hash_algorithm_short_name, + DirectorySync directory_sync) : + prefix(storage_root(std::move(prefix), hash_algorithm_short_name)), + directory_sync(std::move(directory_sync)) + {} + + /// @brief The algorithm-qualified root directory of the store. std::filesystem::path prefix; DirectorySync directory_sync; std::set durable_directory_entries; std::set durable_directory_contents; + static std::string default_hash_algorithm_short_name() + { + if constexpr (HASH_SIZE == 32) + { + if constexpr (HASH_FUNCTION == sha256_compress) + { + return "sha256"; + } +#ifdef HAVE_OPENSSL + if constexpr (HASH_FUNCTION == sha256_openssl) + { + return "sha256"; + } +#endif + } +#ifdef HAVE_OPENSSL + else if constexpr (HASH_SIZE == 48) + { + if constexpr (HASH_FUNCTION == sha384_openssl) + { + return "sha384"; + } + } + else if constexpr (HASH_SIZE == 64) + { + if constexpr (HASH_FUNCTION == sha512_openssl) + { + return "sha512"; + } + } +#endif + throw std::runtime_error( + "TileStoreT requires a hash algorithm short name"); + } + + static void validate_hash_algorithm_short_name( + const std::string& hash_algorithm_short_name) + { + if ( + hash_algorithm_short_name.empty() || + hash_algorithm_short_name.front() == '-' || + hash_algorithm_short_name.back() == '-') + { + throw std::runtime_error("invalid hash algorithm short name"); + } + for (const char c : hash_algorithm_short_name) + { + if ((c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '-') + { + throw std::runtime_error("invalid hash algorithm short name"); + } + } + } + + static std::filesystem::path storage_root( + std::filesystem::path prefix, + const std::string& hash_algorithm_short_name) + { + if ( + (hash_algorithm_short_name == "sha256" && HASH_SIZE != 32) || + (hash_algorithm_short_name == "sha384" && HASH_SIZE != 48) || + (hash_algorithm_short_name == "sha512" && HASH_SIZE != 64)) + { + throw std::runtime_error( + "hash algorithm short name does not match hash size"); + } + prefix /= storage_directory_name(hash_algorithm_short_name); + return prefix; + } + [[nodiscard]] bool confirm_full_tile(uint8_t level, uint64_t index) { const auto path = tile_path(TileRef{level, index}); diff --git a/test/tiles_store.cpp b/test/tiles_store.cpp index fd2ec83..70c5952 100644 --- a/test/tiles_store.cpp +++ b/test/tiles_store.cpp @@ -40,7 +40,8 @@ static void expect_eq( } } -static std::string rel(const merkle::tiles::TileStore& store, const fs::path& p) +template +static std::string rel(const Store& store, const fs::path& p) { return p.lexically_relative(store.root()).generic_string(); } @@ -154,6 +155,10 @@ int main() const auto full = make_hashes(merkle::tiles::TILE_WIDTH); // 2. Resource path layout (full tiles and bundles only). + expect_eq( + store.root().lexically_relative(dir).generic_string(), + "sha256-256w", + "SHA256 storage directory"); expect_eq( rel(store, store.tile_path(TileRef{0, 0})), "tile/0/000", @@ -164,6 +169,54 @@ int main() "tile_path L1 big index"); expect_eq( rel(store, store.entries_path(5)), "tile/entries/005", "entries full"); + expect_eq( + merkle::tiles::TileStore::storage_directory_name("sha384"), + "sha384-256w", + "SHA384 retains 256-hash tile width"); + bool mismatched_hash_size_threw = false; + try + { + (void)merkle::tiles::TileStore(dir, "sha384"); + } + catch (const std::exception&) + { + mismatched_hash_size_threw = true; + } + expect( + mismatched_hash_size_threw, + "algorithm short name must match hash output size"); + +#ifdef HAVE_OPENSSL + { + using TileStore384 = + merkle::tiles::TileStoreT<48, merkle::sha384_openssl>; + TileStore384 store384(dir); + const auto full384 = make_hashesT<48>(merkle::tiles::TILE_WIDTH); + expect_eq( + store384.root().lexically_relative(dir).generic_string(), + "sha384-256w", + "SHA384 storage directory"); + store384.write_tile(TileRef{0, 0}, full384); + expect( + fs::file_size(store384.tile_path(TileRef{0, 0})) == + (uintmax_t)merkle::tiles::TILE_WIDTH * 48, + "SHA384 full tile remains 256 hashes wide"); + + using TileStore512 = + merkle::tiles::TileStoreT<64, merkle::sha512_openssl>; + TileStore512 store512(dir); + const auto full512 = make_hashesT<64>(merkle::tiles::TILE_WIDTH); + expect_eq( + store512.root().lexically_relative(dir).generic_string(), + "sha512-256w", + "SHA512 storage directory"); + store512.write_tile(TileRef{0, 0}, full512); + expect( + fs::file_size(store512.tile_path(TileRef{0, 0})) == + (uintmax_t)merkle::tiles::TILE_WIDTH * 64, + "SHA512 full tile remains 256 hashes wide"); + } +#endif // 2b. Production writes sync each directory link and the destination // directory in order. @@ -173,7 +226,12 @@ int main() FaultInjectingTileStore durable_store(prefix, fault); durable_store.write_tile(TileRef{1, 0}, full); const std::vector expected = { - dir.parent_path(), dir, prefix, prefix / "tile", prefix / "tile" / "1"}; + dir.parent_path(), + dir, + prefix, + prefix / "sha256-256w", + prefix / "sha256-256w" / "tile", + prefix / "sha256-256w" / "tile" / "1"}; expect( fault->calls.size() >= expected.size() && std::equal( From a7b4741e2cea1269e5204fb11aa6e0be2e96cb97 Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 15:17:48 +0000 Subject: [PATCH 04/22] Expand tiled store test coverage Cover format boundaries, validation failures, entry bundle limits, durable retry behavior, and temporary-file cleanup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- test/tiles_store.cpp | 386 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 338 insertions(+), 48 deletions(-) diff --git a/test/tiles_store.cpp b/test/tiles_store.cpp index 70c5952..6e69f88 100644 --- a/test/tiles_store.cpp +++ b/test/tiles_store.cpp @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include #include #include @@ -40,6 +42,27 @@ static void expect_eq( } } +static void expect_throws( + const std::function& fn, const std::string& what) +{ + bool threw = false; + try + { + fn(); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, what); +} + +static void custom_hash(const Hash& lhs, const Hash& rhs, Hash& out) +{ + (void)rhs; + out = lhs; +} + template static std::string rel(const Store& store, const fs::path& p) { @@ -116,6 +139,21 @@ class FaultInjectingTileStore : public merkle::tiles::TileStore Store( std::move(prefix), [fault](const fs::path& path) { fault->sync(path); }) {} + + void begin_attempt() + { + Store::begin_write_attempt(); + } + + [[nodiscard]] bool confirm_tile(uint8_t level, uint64_t index) + { + return Store::confirm_full_tile(level, index); + } + + [[nodiscard]] bool confirm_bundle(uint64_t index) + { + return Store::confirm_entry_bundle(index); + } }; // Overwrites a file with exactly the given bytes (to simulate corruption). @@ -126,6 +164,16 @@ static void overwrite_file(const fs::path& p, const std::vector& bytes) reinterpret_cast(bytes.data()), (std::streamsize)bytes.size()); } +static std::vector read_file(const fs::path& p) +{ + std::ifstream f(p, std::ios::binary); + if (!f.good()) + { + throw std::runtime_error("cannot open test file: " + p.string()); + } + return {std::istreambuf_iterator(f), std::istreambuf_iterator()}; +} + int main() { const auto nonce = @@ -138,7 +186,34 @@ int main() try { - // 1. Tile index encoding vectors (tlog-tiles examples). + // 1. Tile geometry, references, and index encoding. + expect( + merkle::tiles::TILE_WIDTH == (1U << merkle::tiles::TILE_HEIGHT), + "tile width matches tile height"); + const TileRef default_ref; + expect( + default_ref.level == 0 && default_ref.index == 0, + "TileRef defaults to the first leaf tile"); + + size_t prefix_probes = 0; + const auto prefix_length = + merkle::tiles::detail::contiguous_prefix_length(10, [&](uint64_t index) { + prefix_probes++; + return index < 3; + }); + expect( + prefix_length == 3 && prefix_probes == 4, + "contiguous prefix stops at the first gap"); + prefix_probes = 0; + const auto empty_prefix_length = + merkle::tiles::detail::contiguous_prefix_length(0, [&](uint64_t) { + prefix_probes++; + return true; + }); + expect( + empty_prefix_length == 0 && prefix_probes == 0, + "empty contiguous prefix performs no probes"); + expect_eq(merkle::tiles::encode_tile_index(0), "000", "encode 0"); expect_eq(merkle::tiles::encode_tile_index(5), "005", "encode 5"); expect_eq(merkle::tiles::encode_tile_index(255), "255", "encode 255"); @@ -149,6 +224,14 @@ int main() merkle::tiles::encode_tile_index(1234067), "x001/x234/067", "encode 1234067"); + expect_eq( + merkle::tiles::encode_tile_index(std::numeric_limits::max()), + "x018/x446/x744/x073/x709/x551/615", + "encode uint64 max"); + expect_eq( + merkle::tiles::TileStore::encode_index(1000), + "x001/000", + "store index encoder"); merkle::tiles::TileStore store(dir); const size_t hsz = Hash().size(); @@ -173,21 +256,46 @@ int main() merkle::tiles::TileStore::storage_directory_name("sha384"), "sha384-256w", "SHA384 retains 256-hash tile width"); - bool mismatched_hash_size_threw = false; - try + expect_eq( + merkle::tiles::TileStore::storage_directory_name("sha3-256"), + "sha3-256-256w", + "custom algorithm storage directory"); + for (const std::string& invalid_name : + {"", "-sha256", "sha256-", "SHA256", "sha_256", "sha/256"}) { - (void)merkle::tiles::TileStore(dir, "sha384"); + expect_throws( + [&] { + (void)merkle::tiles::TileStore::storage_directory_name(invalid_name); + }, + "invalid algorithm short name rejected"); } - catch (const std::exception&) + for (const std::string& mismatched_name : {"sha384", "sha512"}) { - mismatched_hash_size_threw = true; + expect_throws( + [&] { (void)merkle::tiles::TileStore(dir, mismatched_name); }, + "algorithm short name must match hash output size"); } - expect( - mismatched_hash_size_threw, - "algorithm short name must match hash output size"); + + using CustomStore = merkle::tiles::TileStoreT<32, custom_hash>; + expect_throws( + [&] { (void)CustomStore(dir); }, + "custom hash requires an explicit algorithm name"); + const CustomStore custom_store(dir, "custom-hash"); + expect_eq( + custom_store.root().lexically_relative(dir).generic_string(), + "custom-hash-256w", + "explicit custom hash storage directory"); #ifdef HAVE_OPENSSL { + using TileStore256 = + merkle::tiles::TileStoreT<32, merkle::sha256_openssl>; + const TileStore256 store256(dir); + expect_eq( + store256.root().lexically_relative(dir).generic_string(), + "sha256-256w", + "OpenSSL SHA256 storage directory"); + using TileStore384 = merkle::tiles::TileStoreT<48, merkle::sha384_openssl>; TileStore384 store384(dir); @@ -201,6 +309,10 @@ int main() fs::file_size(store384.tile_path(TileRef{0, 0})) == (uintmax_t)merkle::tiles::TILE_WIDTH * 48, "SHA384 full tile remains 256 hashes wide"); + expect( + store384.has_full_tile(0, 0) && + store384.read_tile(TileRef{0, 0}) == full384, + "SHA384 tile round-trip"); using TileStore512 = merkle::tiles::TileStoreT<64, merkle::sha512_openssl>; @@ -215,6 +327,10 @@ int main() fs::file_size(store512.tile_path(TileRef{0, 0})) == (uintmax_t)merkle::tiles::TILE_WIDTH * 64, "SHA512 full tile remains 256 hashes wide"); + expect( + store512.has_full_tile(0, 0) && + store512.read_tile(TileRef{0, 0}) == full512, + "SHA512 tile round-trip"); } #endif @@ -241,19 +357,105 @@ int main() "directory parents and destination synced in order"); } - // 2c. A successful write must make progress. This shared guard prevents + // 2c. A failed directory-link sync is retried even when the directory + // created before the failure is already visible. + { + const fs::path prefix = dir / "directory_retry"; + const auto fault = std::make_shared(); + fault->fail_path = prefix; + fault->failures_remaining = 1; + FaultInjectingTileStore retry_store(prefix, fault); + const TileRef retry_ref{2, 7}; + expect_throws( + [&] { retry_store.write_tile(retry_ref, full); }, + "directory sync failure propagated"); + expect( + !retry_store.has_full_tile(retry_ref.level, retry_ref.index), + "tile not published before directory chain is durable"); + retry_store.write_tile(retry_ref, full); + expect( + retry_store.read_tile(retry_ref) == full, + "write succeeds after directory sync retry"); + expect( + fault->call_count(prefix) == 2, "failed directory link sync retried"); + } + + // 2d. A failure syncing the destination directory leaves a complete, + // visible file that can be re-confirmed on the next write attempt. + { + const fs::path prefix = dir / "publication_retry"; + const auto fault = std::make_shared(); + FaultInjectingTileStore retry_store(prefix, fault); + const TileRef retry_ref{3, 9}; + const auto tile_directory = + retry_store.tile_path(retry_ref).parent_path(); + fault->fail_path = tile_directory; + fault->failures_remaining = 1; + expect_throws( + [&] { retry_store.write_tile(retry_ref, full); }, + "publication directory sync failure propagated"); + expect( + retry_store.has_full_tile(retry_ref.level, retry_ref.index) && + retry_store.read_tile(retry_ref) == full, + "complete tile remains visible after publication sync failure"); + expect( + !any_tmp_files(prefix), + "publication sync failure leaves no temporary file"); + + const size_t calls_before_confirm = fault->call_count(tile_directory); + retry_store.begin_attempt(); + expect( + retry_store.confirm_tile(retry_ref.level, retry_ref.index), + "visible tile durability re-confirmed"); + expect( + fault->call_count(tile_directory) == calls_before_confirm + 1, + "destination directory sync retried"); + expect( + retry_store.confirm_tile(retry_ref.level, retry_ref.index), + "confirmed tile remains reusable"); + expect( + fault->call_count(tile_directory) == calls_before_confirm + 1, + "directory sync cached within one attempt"); + + retry_store.begin_attempt(); + expect( + !retry_store.confirm_tile(retry_ref.level, retry_ref.index + 1), + "missing tile cannot be confirmed"); + } + + // 2e. Failures before atomic replacement clean up temporary files, and a + // non-directory in the destination path is rejected. + { + const fs::path prefix = dir / "replacement_failure"; + merkle::tiles::TileStore failure_store(prefix); + const TileRef blocked_ref{0, 1}; + fs::create_directories(failure_store.tile_path(blocked_ref)); + expect_throws( + [&] { failure_store.write_tile(blocked_ref, full); }, + "replacement of a directory rejected"); + expect( + !any_tmp_files(prefix), + "failed atomic replacement cleans up temporary file"); + } + { + const fs::path prefix = dir / "directory_conflict"; + fs::create_directories(prefix); + overwrite_file(prefix / "sha256-256w", {0x00}); + merkle::tiles::TileStore failure_store(prefix); + expect_throws( + [&] { failure_store.write_tile(TileRef{0, 0}, full); }, + "non-directory storage root rejected"); + expect( + !any_tmp_files(prefix), + "directory creation failure leaves no temporary file"); + } + + // 2f. A successful write must make progress. This shared guard prevents // the Windows WriteFile loop from spinning if it reports zero bytes. { - bool zero_threw = false; - try - { - TileStoreProbe::check_write_progress(0); - } - catch (const std::exception&) - { - zero_threw = true; - } - expect(zero_threw, "zero-byte write rejected"); + expect_throws( + [] { TileStoreProbe::check_write_progress(0); }, + "zero-byte write rejected"); TileStoreProbe::check_write_progress(1); } @@ -262,10 +464,22 @@ int main() store.write_tile(full_ref, full); expect(store.has_full_tile(0, 0), "has_full_tile after write"); expect(!store.has_full_tile(0, 5), "missing full tile"); + expect_throws( + [&] { (void)store.read_tile(TileRef{0, 5}); }, + "missing tile read rejected"); expect( fs::file_size(store.tile_path(full_ref)) == (uintmax_t)merkle::tiles::TILE_WIDTH * hsz, "full tile file size"); + std::vector expected_tile_bytes; + expected_tile_bytes.reserve(full.size() * hsz); + for (const auto& hash : full) + { + hash.serialise(expected_tile_bytes); + } + expect( + read_file(store.tile_path(full_ref)) == expected_tile_bytes, + "tile file contains concatenated raw hashes"); const auto full_rt = store.read_tile(full_ref); expect(full_rt.size() == full.size(), "full tile width round-trip"); for (size_t i = 0; i < full.size(); i++) @@ -275,34 +489,112 @@ int main() // 3b. Wrong-width writes are rejected (only 256-wide tiles are valid). const std::vector three(full.begin(), full.begin() + 3); - bool threw = false; - try + expect_throws( + [&] { store.write_tile(TileRef{0, 2}, three); }, + "width mismatch rejected"); + + // 4. Entry bundles preserve empty, short, and multi-byte-length entries, + // enforce their width and uint16 length limits, and can be re-confirmed. + std::vector> entries(merkle::tiles::TILE_WIDTH); + entries[1] = {0xA5}; + entries[2] = std::vector(256, 0x5A); + for (size_t i = 3; i < entries.size(); i++) { - store.write_tile(TileRef{0, 2}, three); // 3 hashes for a full tile + entries[i] = {(uint8_t)i, 0x7F}; } - catch (const std::exception&) + const auto encoded_entries = + merkle::tiles::TileStore::encode_entries(entries); + expect( + encoded_entries.size() > 7 && encoded_entries[0] == 0x00 && + encoded_entries[1] == 0x00 && encoded_entries[2] == 0x00 && + encoded_entries[3] == 0x01 && encoded_entries[4] == 0xA5 && + encoded_entries[5] == 0x01 && encoded_entries[6] == 0x00, + "entry lengths use big-endian uint16 prefixes"); + expect( + merkle::tiles::TileStore::decode_entries( + encoded_entries, merkle::tiles::TILE_WIDTH) == entries, + "entry codec round-trip"); + expect( + merkle::tiles::TileStore::decode_entries({}, 0).empty(), + "empty entry sequence round-trip"); + + const std::vector maximum_entry(0xFFFF, 0xC3); + const auto maximum_encoded = + merkle::tiles::TileStore::encode_entries({maximum_entry}); + const auto maximum_decoded = + merkle::tiles::TileStore::decode_entries(maximum_encoded, 1); + expect( + maximum_encoded.size() == maximum_entry.size() + 2 && + maximum_encoded[0] == 0xFF && maximum_encoded[1] == 0xFF && + maximum_decoded.size() == 1 && maximum_decoded[0] == maximum_entry, + "maximum uint16-sized entry round-trip"); + expect_throws( + [] { + (void)merkle::tiles::TileStore::encode_entries( + {std::vector(0x10000)}); + }, + "oversized entry rejected"); + + expect(!store.has_entry_bundle(17), "missing entry bundle"); + expect_throws( + [&] { (void)store.read_entry_bundle(17); }, + "missing entry bundle read rejected"); + const std::vector> short_bundle( + entries.begin(), entries.end() - 1); + expect_throws( + [&] { store.write_entry_bundle(1, short_bundle); }, + "short entry bundle rejected"); + auto oversized_bundle = entries; + oversized_bundle[0].resize(0x10000); + expect_throws( + [&] { store.write_entry_bundle(2, oversized_bundle); }, + "entry bundle containing oversized entry rejected"); + expect( + !fs::exists(store.entries_path(1)) && !fs::exists(store.entries_path(2)), + "invalid entry bundles are not published"); + + store.write_entry_bundle(0, entries); + expect(store.has_entry_bundle(0), "full entry bundle exists"); + expect( + store.read_entry_bundle(0) == entries, + "full entry bundle byte round-trip"); + expect( + fs::file_size(store.entries_path(0)) == encoded_entries.size(), + "entry bundle file size"); + expect( + read_file(store.entries_path(0)) == encoded_entries, + "entry bundle file uses the encoded wire format"); + { - threw = true; + const fs::path prefix = dir / "bundle_confirmation"; + const auto fault = std::make_shared(); + FaultInjectingTileStore confirming_store(prefix, fault); + confirming_store.write_entry_bundle(4, entries); + const auto entries_directory = + confirming_store.entries_path(4).parent_path(); + const size_t calls_before_confirm = fault->call_count(entries_directory); + confirming_store.begin_attempt(); + expect( + confirming_store.confirm_bundle(4), + "visible entry bundle durability confirmed"); + expect( + fault->call_count(entries_directory) == calls_before_confirm + 1, + "entry bundle directory synced for new attempt"); + expect( + confirming_store.confirm_bundle(4), + "confirmed entry bundle remains reusable"); + expect( + fault->call_count(entries_directory) == calls_before_confirm + 1, + "entry bundle sync cached within one attempt"); + confirming_store.begin_attempt(); + expect( + !confirming_store.confirm_bundle(5), + "missing entry bundle cannot be confirmed"); } - expect(threw, "width mismatch rejected"); - // 3c. Corrupt / truncated files are rejected on read (integrity check), so + // 5. Corrupt / truncated files are rejected on read (integrity check), so // a torn write can never be served as a valid tile or bundle. { - const auto expect_throws = - [](const std::function& fn, const std::string& what) { - bool t = false; - try - { - fn(); - } - catch (const std::exception&) - { - t = true; - } - expect(t, what); - }; - // Truncated tile: fewer bytes than a full tile. overwrite_file(store.tile_path(full_ref), std::vector(hsz, 0)); expect(!store.has_full_tile(0, 0), "truncated tile is not durable"); @@ -321,15 +613,10 @@ int main() [&] { (void)store.read_tile(full_ref); }, "oversized tile rejected"); // A valid full entry bundle whose file is then cut short. - std::vector> entries(merkle::tiles::TILE_WIDTH); - for (size_t i = 0; i < entries.size(); i++) - { - entries[i] = {(uint8_t)i, 0x7F}; - } store.write_entry_bundle(0, entries); expect(store.has_entry_bundle(0), "bundle durable before truncation"); expect( - store.read_entry_bundle(0).size() == merkle::tiles::TILE_WIDTH, + store.read_entry_bundle(0) == entries, "bundle valid before truncation"); // Claims a 5-byte entry but supplies only one trailing byte. overwrite_file(store.entries_path(0), {0x00, 0x05, 0x01}); @@ -353,6 +640,9 @@ int main() (void)merkle::tiles::TileStore::decode_entries({0xFF, 0xFF, 0x00}, 1); }, "decode_entries oversized length rejected"); + expect_throws( + [] { (void)merkle::tiles::TileStore::decode_entries({}, 1); }, + "decode_entries missing length prefix rejected"); expect_throws( [] { (void)merkle::tiles::TileStore::decode_entries({0x00, 0x00, 0xFF}, 1); @@ -363,7 +653,7 @@ int main() // Windows file replacement semantics can reject racing same-path replaces even // when the final tile remains valid, so keep this stress check POSIX-only. #ifndef _WIN32 - // 3d. Concurrent same-tile writes use unique temp files and leave no + // 6. Concurrent same-tile writes use unique temp files and leave no // temporary files behind after success. Each thread owns its store object; // sharing one object requires caller-provided synchronization. { From 2370a7a48a47d2ad6fb0192f902272ee8ffe583a Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 15:50:38 +0000 Subject: [PATCH 05/22] Derive tile width from tile height Compute TILE_WIDTH as a constexpr shift so the geometry constants cannot drift out of sync. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/design/tlog-tiles.md | 3 ++- merklecpp_tiles.h | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index fd13147..e787ce8 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -296,7 +296,8 @@ namespace merkle { namespace tiles { static constexpr uint16_t TILE_HEIGHT = 8; -static constexpr uint16_t TILE_WIDTH = 256; // 2^TILE_HEIGHT +static constexpr uint16_t TILE_WIDTH = + uint16_t{1U << TILE_HEIGHT}; template &, diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 52c7f4c..e85ee82 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -55,7 +55,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) static constexpr uint16_t TILE_HEIGHT = 8; /// @brief Number of hashes in a full tile (2**TILE_HEIGHT). - static constexpr uint16_t TILE_WIDTH = 256; + static constexpr uint16_t TILE_WIDTH = uint16_t{1U << TILE_HEIGHT}; namespace detail { From a4ccbe1e7e19e36367f36149dea29a34e7f5b27b Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 16:29:01 +0000 Subject: [PATCH 06/22] Move merklecpp to C++20 formatting Require C++20 for consumers and replace C formatting and chained string construction with std::format and std::format_to. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CMakeLists.txt | 7 +- README.md | 2 + doc/design/tlog-tiles.md | 2 +- merklecpp.h | 44 +++++++----- merklecpp_tiles.h | 148 +++++++++++++++++++++++---------------- 5 files changed, 122 insertions(+), 81 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e35781..0ac550a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,10 +1,12 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +cmake_minimum_required(VERSION 3.12) + project(merklecpp LANGUAGES CXX C ASM) -cmake_minimum_required(VERSION 3.11) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) set(MERKLECPP_DIR ${CMAKE_CURRENT_SOURCE_DIR}) @@ -23,6 +25,7 @@ if(CLANG_TIDY) endif() add_library(merklecpp INTERFACE) +target_compile_features(merklecpp INTERFACE cxx_std_20) target_include_directories(merklecpp INTERFACE .) if(TRACE) diff --git a/README.md b/README.md index 904c442..db01d1c 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ unusual features like flushing, retracting, and tree segment serialisation. ## Usage +merklecpp requires C++20. + #include merkle::Tree::Hash hash("fa8f44eabb728d4020e7f33d1aa973faaef19de6c06679bccdc5100a3c01f54a"); diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index e787ce8..4efcd77 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -225,7 +225,7 @@ Rooted below a configurable `prefix` directory on local disk: ``` encode_tile_index(N): parts = [] - do { parts.push_front(printf("%03d", N % 1000)); N /= 1000 } while (N > 0) + do { parts.push_front(format("{:03}", N % 1000)); N /= 1000 } while (N > 0) for each part except the last: prepend 'x' return join(parts, "/") ``` diff --git a/merklecpp.h b/merklecpp.h index 1170b1c..d3be0f9 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -9,7 +9,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -118,21 +120,21 @@ namespace merkle uint8_t bytes[SIZE]; /// @brief Constructs a Hash with all bytes set to zero - HashT() + HashT() { std::fill(bytes, bytes + SIZE, 0); } /// @brief Constructs a Hash from a byte buffer /// @param bytes Buffer with hash value - HashT(const uint8_t* bytes) + HashT(const uint8_t* bytes) { std::copy(bytes, bytes + SIZE, this->bytes); } /// @brief Constructs a Hash from a string /// @param s String to read the hash value from - HashT(const std::string& s) + HashT(const std::string& s) { if (s.length() != 2 * SIZE) { @@ -153,7 +155,7 @@ namespace merkle /// @brief Deserialises a Hash from a vector of bytes /// @param bytes Vector to read the hash value from - HashT(const std::vector& bytes) + HashT(const std::vector& bytes) { if (bytes.size() < SIZE) { @@ -165,7 +167,7 @@ namespace merkle /// @brief Deserialises a Hash from a vector of bytes /// @param bytes Vector to read the hash value from /// @param position Position of the first byte in @p bytes - HashT(const std::vector& bytes, size_t& position) + HashT(const std::vector& bytes, size_t& position) { if (bytes.size() - position < SIZE) { @@ -176,7 +178,7 @@ namespace merkle /// @brief Deserialises a Hash from an array of bytes /// @param bytes Array to read the hash value from - HashT(const std::array& bytes) + HashT(const std::array& bytes) { std::copy(bytes.data(), bytes.data() + SIZE, this->bytes); } @@ -206,14 +208,20 @@ namespace merkle size_t num_bytes = SIZE, bool lower_case = true) const { size_t const num_chars = 2 * num_bytes; - std::string r(num_chars, '_'); + std::string r; + r.reserve(num_chars); for (size_t i = 0; i < num_bytes; i++) { - snprintf( - const_cast(r.data() + 2 * i), - num_chars + 1 - 2 * i, - lower_case ? "%02x" : "%02X", - bytes[i]); + if (lower_case) + { + std::format_to( + std::back_inserter(r), "{:02x}", static_cast(bytes[i])); + } + else + { + std::format_to( + std::back_inserter(r), "{:02X}", static_cast(bytes[i])); + } } return r; } @@ -1916,8 +1924,10 @@ namespace merkle MERKLECPP_TRACE({ std::string nodes; for (size_t i = 0; i < insertion_stack.size(); i++) - nodes += - " " + insertion_stack.at(i).n->hash.to_string(TRACE_HASH_SIZE); + std::format_to( + std::back_inserter(nodes), + " {}", + insertion_stack.at(i).n->hash.to_string(TRACE_HASH_SIZE)); MERKLECPP_TOUT << " X " << (complete ? "complete" : "continue") << ":" << nodes << std::endl; }); @@ -2104,7 +2114,7 @@ namespace merkle EVP_Digest(&block[0], sizeof(block), out.bytes, nullptr, md, nullptr); if (rc != 1) { - throw std::runtime_error("EVP_Digest failed: " + std::to_string(rc)); + throw std::runtime_error(std::format("EVP_Digest failed: {}", rc)); } } @@ -2126,7 +2136,7 @@ namespace merkle EVP_Digest(&block[0], sizeof(block), out.bytes, nullptr, md, nullptr); if (rc != 1) { - throw std::runtime_error("EVP_Digest failed: " + std::to_string(rc)); + throw std::runtime_error(std::format("EVP_Digest failed: {}", rc)); } } @@ -2148,7 +2158,7 @@ namespace merkle EVP_Digest(&block[0], sizeof(block), out.bytes, nullptr, md, nullptr); if (rc != 1) { - throw std::runtime_error("EVP_Digest failed: " + std::to_string(rc)); + throw std::runtime_error(std::format("EVP_Digest failed: {}", rc)); } } diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index e85ee82..0a8d541 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -10,9 +10,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -78,28 +78,24 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// encoded as "x001/x234/067" and 5 as "005". static inline std::string encode_tile_index(uint64_t n) { - std::vector parts; - char buf[8]; + std::vector parts; do { - std::snprintf(buf, sizeof(buf), "%03u", (unsigned)(n % 1000)); - parts.emplace_back(buf); + parts.emplace_back(static_cast(n % 1000)); n /= 1000; } while (n > 0); std::string r; + r.reserve(parts.size() * 5); for (size_t i = 0; i < parts.size(); i++) { - const std::string& part = parts[parts.size() - 1 - i]; - if (i != 0) - { - r += "/"; - } - if (i + 1 < parts.size()) - { - r += "x"; - } - r += part; + const auto part = parts[parts.size() - 1 - i]; + std::format_to( + std::back_inserter(r), + "{}{}{:03}", + i == 0 ? "" : "/", + i + 1 < parts.size() ? "x" : "", + part); } return r; } @@ -176,8 +172,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const std::string& hash_algorithm_short_name) { validate_hash_algorithm_short_name(hash_algorithm_short_name); - return hash_algorithm_short_name + "-" + std::to_string(TILE_WIDTH) + - "w"; + return std::format("{}-{}w", hash_algorithm_short_name, TILE_WIDTH); } /// @brief Encodes a tile index (see encode_tile_index). @@ -189,15 +184,19 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @brief The filesystem path of a tile. [[nodiscard]] std::filesystem::path tile_path(const TileRef& ref) const { - return prefix / - ("tile/" + std::to_string((unsigned)ref.level) + "/" + - encode_tile_index(ref.index)); + const auto relative_path = std::format( + "tile/{}/{}", + static_cast(ref.level), + encode_tile_index(ref.index)); + return prefix / relative_path; } /// @brief The filesystem path of an entry bundle. [[nodiscard]] std::filesystem::path entries_path(uint64_t index) const { - return prefix / ("tile/entries/" + encode_tile_index(index)); + const auto relative_path = + std::format("tile/entries/{}", encode_tile_index(index)); + return prefix / relative_path; } /// @brief Whether a full tile exists on disk. @@ -479,7 +478,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) std::ifstream f(path, std::ios::binary); if (!f.good()) { - throw std::runtime_error("cannot open file: " + path.string()); + throw std::runtime_error( + std::format("cannot open file: {}", path.string())); } return { std::istreambuf_iterator(f), std::istreambuf_iterator()}; @@ -516,7 +516,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const std::filesystem::path& requested_directory) { const auto directory = directory_or_dot(requested_directory); - if (durable_directory_entries.count(directory) != 0) + if (durable_directory_entries.contains(directory)) { return; } @@ -531,9 +531,10 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const bool exists = std::filesystem::exists(directory, ec); if (ec) { - throw std::runtime_error( - "cannot inspect directory " + directory.string() + ": " + - ec.message()); + throw std::runtime_error(std::format( + "cannot inspect directory {}: {}", + directory.string(), + ec.message())); } if (exists) { @@ -541,15 +542,16 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) std::filesystem::is_directory(directory, ec); if (ec) { - throw std::runtime_error( - "cannot inspect directory " + directory.string() + ": " + - ec.message()); + throw std::runtime_error(std::format( + "cannot inspect directory {}: {}", + directory.string(), + ec.message())); } if (!is_directory) { - throw std::runtime_error( - "cannot create directory " + directory.string() + - ": path exists and is not a directory"); + throw std::runtime_error(std::format( + "cannot create directory {}: path exists and is not a directory", + directory.string())); } } else @@ -557,15 +559,20 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const bool created = std::filesystem::create_directory(directory, ec); if (ec) { - throw std::runtime_error( - "cannot create directory " + directory.string() + ": " + - ec.message()); + throw std::runtime_error(std::format( + "cannot create directory {}: {}", + directory.string(), + ec.message())); } if (!created && !std::filesystem::is_directory(directory, ec)) { throw std::runtime_error( - "cannot create directory " + directory.string() + - (ec ? ": " + ec.message() : "")); + ec ? + std::format( + "cannot create directory {}: {}", + directory.string(), + ec.message()) : + std::format("cannot create directory {}", directory.string())); } } @@ -587,7 +594,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) void sync_directory_contents(const std::filesystem::path& directory) { const auto path = directory_or_dot(directory); - if (durable_directory_contents.count(path) != 0) + if (durable_directory_contents.contains(path)) { return; } @@ -627,9 +634,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const auto stamp = std::chrono::steady_clock::now().time_since_epoch().count(); std::filesystem::path tmp = path; - tmp += ".tmp." + std::to_string(process_id()) + "." + - std::to_string((uint64_t)stamp) + "." + - std::to_string(counter.fetch_add(1, std::memory_order_relaxed)); + tmp += std::format( + ".tmp.{}.{}.{}", + process_id(), + (uint64_t)stamp, + counter.fetch_add(1, std::memory_order_relaxed)); return tmp; } @@ -642,13 +651,27 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) #endif } - static std::string system_error_message(const std::string& what) + template + static std::string system_error_message( + std::format_string format_string, Args&&... args) { #ifdef _WIN32 - return what + ": error " + std::to_string(GetLastError()); + const auto error = GetLastError(); +#else + const auto error = errno; +#endif + std::string message; + std::format_to( + std::back_inserter(message), + format_string, + std::forward(args)...); +#ifdef _WIN32 + std::format_to(std::back_inserter(message), ": error {}", error); #else - return what + ": " + std::strerror(errno); + std::format_to( + std::back_inserter(message), ": {}", std::strerror(error)); #endif + return message; } static void require_write_progress( @@ -656,7 +679,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) { if (written == 0) { - throw std::runtime_error("short write: " + path.string()); + throw std::runtime_error( + std::format("short write: {}", path.string())); } } @@ -674,7 +698,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) nullptr); if (handle == INVALID_HANDLE_VALUE) { - throw std::runtime_error("cannot open file: " + path.string()); + throw std::runtime_error( + std::format("cannot open file: {}", path.string())); } bool close_handle = true; try @@ -690,7 +715,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) handle, bytes.data() + written, chunk, &done, nullptr)) { throw std::runtime_error( - system_error_message("error writing file " + path.string())); + system_error_message("error writing file {}", path.string())); } require_write_progress((size_t)done, path); written += done; @@ -698,12 +723,12 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) if (!FlushFileBuffers(handle)) { throw std::runtime_error( - system_error_message("error syncing file " + path.string())); + system_error_message("error syncing file {}", path.string())); } if (!CloseHandle(handle)) { throw std::runtime_error( - system_error_message("error closing file " + path.string())); + system_error_message("error closing file {}", path.string())); } close_handle = false; } @@ -724,7 +749,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) if (fd < 0) { throw std::runtime_error( - system_error_message("cannot open file " + path.string())); + system_error_message("cannot open file {}", path.string())); } try { @@ -740,7 +765,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) continue; } throw std::runtime_error( - system_error_message("error writing file " + path.string())); + system_error_message("error writing file {}", path.string())); } require_write_progress((size_t)done, path); written += (size_t)done; @@ -748,13 +773,13 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) if (::fsync(fd) != 0) { throw std::runtime_error( - system_error_message("error syncing file " + path.string())); + system_error_message("error syncing file {}", path.string())); } if (::close(fd) != 0) { fd = -1; throw std::runtime_error( - system_error_message("error closing file " + path.string())); + system_error_message("error closing file {}", path.string())); } fd = -1; } @@ -779,17 +804,18 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) { throw std::runtime_error(system_error_message( - "cannot rename temp file " + tmp.string() + " to " + - path.string())); + "cannot rename temp file {} to {}", tmp.string(), path.string())); } #else std::error_code ec; std::filesystem::rename(tmp, path, ec); if (ec) { - throw std::runtime_error( - "cannot rename temp file " + tmp.string() + " to " + path.string() + - ": " + ec.message()); + throw std::runtime_error(std::format( + "cannot rename temp file {} to {}: {}", + tmp.string(), + path.string(), + ec.message())); } #endif } @@ -818,19 +844,19 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) if (fd < 0) { throw std::runtime_error( - system_error_message("cannot open directory " + path.string())); + system_error_message("cannot open directory {}", path.string())); } if (::fsync(fd) != 0) { const std::string message = - system_error_message("error syncing directory " + path.string()); + system_error_message("error syncing directory {}", path.string()); ::close(fd); throw std::runtime_error(message); } if (::close(fd) != 0) { throw std::runtime_error( - system_error_message("error closing directory " + path.string())); + system_error_message("error closing directory {}", path.string())); } #else (void)path; From e2dde8d219f281ef30d35629e446055dfe2827e3 Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 18:02:55 +0000 Subject: [PATCH 07/22] Harden tile file handling and add PAL Use exclusive temporary-file creation, preserve platform errors, and isolate durable file operations in merklecpp_pal.h. Derive tile hash metadata from the core tree types and cover collision handling. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 1 + Doxyfile | 1 + doc/design/tlog-tiles.md | 44 +++--- merklecpp.h | 22 +-- merklecpp_pal.h | 283 +++++++++++++++++++++++++++++++++++++++ merklecpp_tiles.h | 280 +++++--------------------------------- test/tiles_store.cpp | 111 ++++++++++----- 7 files changed, 435 insertions(+), 307 deletions(-) create mode 100644 merklecpp_pal.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa69062..0534e2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,7 @@ jobs: run: | clang-tidy merklecpp.h -p build/${{ matrix.build_type }} --warnings-as-errors='*' clang-tidy merklecpp_tiles.h -p build/${{ matrix.build_type }} --warnings-as-errors='*' + clang-tidy merklecpp_pal.h -p build/${{ matrix.build_type }} --warnings-as-errors='*' - name: Test working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} diff --git a/Doxyfile b/Doxyfile index d045b46..566ab97 100644 --- a/Doxyfile +++ b/Doxyfile @@ -791,6 +791,7 @@ WARN_LOGFILE = # Note: If this tag is empty the current directory is searched. INPUT = merklecpp.h \ + merklecpp_pal.h \ merklecpp_tiles.h # This tag can be used to specify the character encoding of the source files diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 4efcd77..bb198a0 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -263,12 +263,16 @@ counts hashes, not bytes, and therefore does not change for SHA-384 or SHA-512. └─ CombinedHashSource ``` -New, all in a companion header `merklecpp_tiles.h` (includes `merklecpp.h`), -namespace `merkle::tiles`, every type templated on `` -with default aliases mirroring the bottom of `merklecpp.h`: +The public tiled-storage implementation lives in `merklecpp_tiles.h` (which +includes `merklecpp.h`). Low-level operating-system operations are isolated in +the internal `merklecpp_pal.h` platform abstraction (`merkle::pal`). Public +types remain in namespace `merkle::tiles`, templated on +``, with default aliases mirroring the bottom of +`merklecpp.h`: | Component | Responsibility | |----------------------|-----------------------------------------------------------------------| +| PAL helpers | exclusive file creation, replacement, and durability primitives | | `TileCoord` | pure index math + path encoding (no I/O) | | `TileStoreT` | read/write tile files on a local filesystem | | `TileWriterT` | compute & persist newly-complete full tiles (write path) | @@ -289,6 +293,7 @@ The **only** (optional) change to the core header is a small, read-only, ```cpp // merklecpp_tiles.h #include "merklecpp.h" +#include "merklecpp_pal.h" // internal platform abstraction #include #include @@ -306,19 +311,22 @@ template ; -using TiledTree = tiles::TiledTreeT<32, sha256_compress>; +using TileStore = + TileStoreT; +using TiledTree = + TiledTreeT; // + 384/512 variants + +} // namespace tiles } // namespace merkle ``` Note the roll-up/proof combiner is the **same** `HASH_FUNCTION` the tree uses, and it only ever combines two `HASH_SIZE`-byte hashes — so the default `sha256_compress` single-block path is sufficient and **no OpenSSL dependency is -introduced**. +introduced**. Convenience aliases derive both template arguments from the core +tree type, so hash sizes and functions have one source of truth. ### 6.3 `TileStoreT` — disk I/O @@ -372,14 +380,16 @@ entry bundles are an **application-owned** add-on, included for completeness: Each phase is independently testable; phases 1–4 require **no** core changes. -**Phase 0 — Scaffolding.** Add `merklecpp_tiles.h` (includes `merklecpp.h`), -`merkle::tiles` namespace, geometry constants, default aliases. Wire a new test -group in `test/CMakeLists.txt` following `add_merklecpp_test`. +**Phase 0 — Scaffolding.** Add `merklecpp_tiles.h` (includes `merklecpp.h`) and +the internal `merklecpp_pal.h`, `merkle::tiles` namespace, geometry +constants, default aliases. Wire a new test group in `test/CMakeLists.txt` +following `add_merklecpp_test`. **Phase 1 — Coordinates & store.** `TileCoord`/`encode_index`, `TileRef`, -`TileStoreT` (path building, atomic `write_tile`, `read_tile`). *Tests:* -index-encoding vectors; algorithm-qualified roots; SHA-256 and SHA-384 both use -256-hash tiles; tile byte round-trip. +`TileStoreT` (path building, atomic `write_tile`, `read_tile`) and PAL-backed +exclusive temporary-file creation. *Tests:* index-encoding vectors; +algorithm-qualified roots; SHA-256 and SHA-384 both use 256-hash tiles; tile +byte round-trip; existing-file and symlink collision rejection. **Phase 2 — Write path.** `TileWriterT::write_up_to` (level-0 from `leaf_at`, roll-ups, incremental cursor; full tiles only — the frontier is never tiled). @@ -400,9 +410,9 @@ inclusion for a **flushed** index and a **resident** index against a non-flushed reference tree's root; consistency across a flush boundary. **Phase 5 — Optional entry bundles** (`BundleWriter`) and docs (README "Usage" -snippet, link this design doc, add `merklecpp_tiles.h` to Doxygen inputs). +snippet, link this design doc, add the public and PAL headers to Doxygen inputs). -Deliverables: `merklecpp_tiles.h`; `test/tiles_*.cpp`; CMake wiring; optional -one-method core addition; README/docs updates. +Deliverables: `merklecpp_tiles.h`; `merklecpp_pal.h`; `test/tiles_*.cpp`; +CMake wiring; optional one-method core addition; README/docs updates. --- diff --git a/merklecpp.h b/merklecpp.h index d3be0f9..c56bf22 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -116,6 +116,9 @@ namespace merkle template struct HashT { + /// Size of the hash in bytes. + static constexpr size_t size_bytes = SIZE; + /// Holds the hash bytes uint8_t bytes[SIZE]; @@ -186,7 +189,7 @@ namespace merkle /// @brief The size of the hash (in number of bytes) [[nodiscard]] size_t size() const { - return SIZE; + return size_bytes; } /// @brief zeros out all bytes in the hash @@ -198,7 +201,7 @@ namespace merkle /// @brief The size of the serialisation of the hash (in number of bytes) [[nodiscard]] size_t serialised_size() const { - return SIZE; + return size_bytes; } /// @brief Convert a hash to a hex-encoded string @@ -736,6 +739,9 @@ namespace merkle }; public: + /// @brief Hash function used to combine tree nodes. + static constexpr auto hash_function = HASH_FUNCTION; + /// @brief The type of hashes in the tree using Hash = HashT; @@ -2162,18 +2168,12 @@ namespace merkle } } - /// @brief Type of hashes in the SHA384 tree type - using Hash384 = HashT<48>; - /// @brief Type of paths in the SHA384 tree type using Path384 = PathT<48, sha384_openssl>; /// @brief SHA384 tree with OpenSSL hash function using Tree384 = TreeT<48, sha384_openssl>; - /// @brief Type of hashes in the SHA512 tree type - using Hash512 = HashT<64>; - /// @brief Type of paths in the SHA512 tree type using Path512 = PathT<64, sha512_openssl>; @@ -2181,6 +2181,12 @@ namespace merkle using Tree512 = TreeT<64, sha512_openssl>; #endif + /// @brief Type of SHA384-sized hashes + using Hash384 = HashT<48>; + + /// @brief Type of SHA512-sized hashes + using Hash512 = HashT<64>; + /// @brief Type of hashes in the default tree type using Hash = HashT<32>; diff --git a/merklecpp_pal.h b/merklecpp_pal.h new file mode 100644 index 0000000..682ef97 --- /dev/null +++ b/merklecpp_pal.h @@ -0,0 +1,283 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include +#else +# include +# include +#endif + +// Internal platform abstraction for durable file operations. + +namespace merkle // NOLINT(modernize-concat-nested-namespaces) +{ + namespace pal + { +#ifdef _WIN32 + using SystemError = DWORD; +#else + using SystemError = int; +#endif + + static inline SystemError last_system_error() + { +#ifdef _WIN32 + return GetLastError(); +#else + return errno; +#endif + } + + template + std::string system_error_message( + SystemError error, + std::format_string format_string, + Args&&... args) + { + std::string message; + std::format_to( + std::back_inserter(message), + format_string, + std::forward(args)...); +#ifdef _WIN32 + std::format_to(std::back_inserter(message), ": error {}", error); +#else + std::format_to(std::back_inserter(message), ": {}", std::strerror(error)); +#endif + return message; + } + + static inline uint64_t process_id() + { +#ifdef _WIN32 + return static_cast(GetCurrentProcessId()); +#else + return static_cast(::getpid()); +#endif + } + + static inline void require_write_progress( + size_t written, const std::filesystem::path& path) + { + if (written == 0) + { + throw std::runtime_error(std::format("short write: {}", path.string())); + } + } + + static inline void remove_owned_file( + const std::filesystem::path& path) noexcept + { + std::error_code ec; + std::filesystem::remove(path, ec); + } + + /// Writes and syncs a new file, failing if @p path already exists. + /// A file created by this call is removed if a later operation fails. + static inline void write_and_sync_file( + const std::filesystem::path& path, const std::vector& bytes) + { +#ifdef _WIN32 + HANDLE handle = CreateFileW( + path.wstring().c_str(), + GENERIC_WRITE, + 0, + nullptr, + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (handle == INVALID_HANDLE_VALUE) + { + const auto error = last_system_error(); + throw std::runtime_error( + system_error_message(error, "cannot open file {}", path.string())); + } + bool close_handle = true; + try + { + size_t written = 0; + while (written < bytes.size()) + { + const auto remaining = bytes.size() - written; + const auto chunk = static_cast(std::min( + remaining, static_cast(std::numeric_limits::max()))); + DWORD done = 0; + if (!WriteFile(handle, bytes.data() + written, chunk, &done, nullptr)) + { + const auto error = last_system_error(); + throw std::runtime_error(system_error_message( + error, "error writing file {}", path.string())); + } + require_write_progress(static_cast(done), path); + written += done; + } + if (!FlushFileBuffers(handle)) + { + const auto error = last_system_error(); + throw std::runtime_error(system_error_message( + error, "error syncing file {}", path.string())); + } + if (!CloseHandle(handle)) + { + const auto error = last_system_error(); + throw std::runtime_error(system_error_message( + error, "error closing file {}", path.string())); + } + close_handle = false; + } + catch (...) + { + if (close_handle) + { + CloseHandle(handle); + } + remove_owned_file(path); + throw; + } +#else + int flags = O_WRONLY | O_CREAT | O_EXCL; +# ifdef O_CLOEXEC + flags |= O_CLOEXEC; +# endif + int fd = ::open(path.c_str(), flags, 0666); + if (fd < 0) + { + const auto error = last_system_error(); + throw std::runtime_error( + system_error_message(error, "cannot open file {}", path.string())); + } + try + { + size_t written = 0; + while (written < bytes.size()) + { + const ssize_t done = + ::write(fd, bytes.data() + written, bytes.size() - written); + if (done < 0) + { + if (errno == EINTR) + { + continue; + } + const auto error = last_system_error(); + throw std::runtime_error(system_error_message( + error, "error writing file {}", path.string())); + } + require_write_progress(static_cast(done), path); + written += static_cast(done); + } + if (::fsync(fd) != 0) + { + const auto error = last_system_error(); + throw std::runtime_error(system_error_message( + error, "error syncing file {}", path.string())); + } + if (::close(fd) != 0) + { + const auto error = last_system_error(); + fd = -1; + throw std::runtime_error(system_error_message( + error, "error closing file {}", path.string())); + } + fd = -1; + } + catch (...) + { + if (fd >= 0) + { + ::close(fd); + } + remove_owned_file(path); + throw; + } +#endif + } + + static inline void replace_file( + const std::filesystem::path& tmp, const std::filesystem::path& path) + { +#ifdef _WIN32 + if (!MoveFileExW( + tmp.wstring().c_str(), + path.wstring().c_str(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) + { + const auto error = last_system_error(); + throw std::runtime_error(system_error_message( + error, + "cannot rename temp file {} to {}", + tmp.string(), + path.string())); + } +#else + std::error_code ec; + std::filesystem::rename(tmp, path, ec); + if (ec) + { + throw std::runtime_error(std::format( + "cannot rename temp file {} to {}: {}", + tmp.string(), + path.string(), + ec.message())); + } +#endif + } + + static inline void sync_directory_on_disk(const std::filesystem::path& path) + { +#ifndef _WIN32 + int flags = O_RDONLY; +# ifdef O_DIRECTORY + flags |= O_DIRECTORY; +# endif +# ifdef O_CLOEXEC + flags |= O_CLOEXEC; +# endif + const int fd = ::open(path.c_str(), flags); + if (fd < 0) + { + const auto error = last_system_error(); + throw std::runtime_error(system_error_message( + error, "cannot open directory {}", path.string())); + } + if (::fsync(fd) != 0) + { + const auto error = last_system_error(); + const std::string message = system_error_message( + error, "error syncing directory {}", path.string()); + ::close(fd); + throw std::runtime_error(message); + } + if (::close(fd) != 0) + { + const auto error = last_system_error(); + throw std::runtime_error(system_error_message( + error, "error closing directory {}", path.string())); + } +#else + (void)path; +#endif + } + } +} diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 0a8d541..10a7695 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -4,36 +4,23 @@ #pragma once #include "merklecpp.h" +#include "merklecpp_pal.h" -#include #include -#include #include #include -#include #include #include #include #include #include -#include #include #include #include -#include +#include #include #include -#ifdef _WIN32 -# ifndef NOMINMAX -# define NOMINMAX -# endif -# include -#else -# include -# include -#endif - // Tiled storage for merklecpp trees, following the full-tile geometry, payload, // and path encoding of the C2SP tlog-tiles layout // (https://c2sp.org/tlog-tiles). C2SP defines SHA-256; merklecpp preserves the @@ -59,6 +46,10 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) namespace detail { + static constexpr std::string_view SHA256_ALGORITHM_SHORT_NAME = "sha256"; + static constexpr std::string_view SHA384_ALGORITHM_SHORT_NAME = "sha384"; + static constexpr std::string_view SHA512_ALGORITHM_SHORT_NAME = "sha512"; + template uint64_t contiguous_prefix_length(uint64_t limit, const Present& present) { @@ -377,32 +368,32 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) static std::string default_hash_algorithm_short_name() { - if constexpr (HASH_SIZE == 32) + if constexpr (HASH_SIZE == merkle::Tree::Hash::size_bytes) { - if constexpr (HASH_FUNCTION == sha256_compress) + if constexpr (HASH_FUNCTION == merkle::Tree::hash_function) { - return "sha256"; + return std::string(detail::SHA256_ALGORITHM_SHORT_NAME); } #ifdef HAVE_OPENSSL if constexpr (HASH_FUNCTION == sha256_openssl) { - return "sha256"; + return std::string(detail::SHA256_ALGORITHM_SHORT_NAME); } #endif } #ifdef HAVE_OPENSSL - else if constexpr (HASH_SIZE == 48) + else if constexpr (HASH_SIZE == merkle::Tree384::Hash::size_bytes) { - if constexpr (HASH_FUNCTION == sha384_openssl) + if constexpr (HASH_FUNCTION == merkle::Tree384::hash_function) { - return "sha384"; + return std::string(detail::SHA384_ALGORITHM_SHORT_NAME); } } - else if constexpr (HASH_SIZE == 64) + else if constexpr (HASH_SIZE == merkle::Tree512::Hash::size_bytes) { - if constexpr (HASH_FUNCTION == sha512_openssl) + if constexpr (HASH_FUNCTION == merkle::Tree512::hash_function) { - return "sha512"; + return std::string(detail::SHA512_ALGORITHM_SHORT_NAME); } } #endif @@ -434,9 +425,12 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const std::string& hash_algorithm_short_name) { if ( - (hash_algorithm_short_name == "sha256" && HASH_SIZE != 32) || - (hash_algorithm_short_name == "sha384" && HASH_SIZE != 48) || - (hash_algorithm_short_name == "sha512" && HASH_SIZE != 64)) + (hash_algorithm_short_name == detail::SHA256_ALGORITHM_SHORT_NAME && + HASH_SIZE != merkle::Hash::size_bytes) || + (hash_algorithm_short_name == detail::SHA384_ALGORITHM_SHORT_NAME && + HASH_SIZE != merkle::Hash384::size_bytes) || + (hash_algorithm_short_name == detail::SHA512_ALGORITHM_SHORT_NAME && + HASH_SIZE != merkle::Hash512::size_bytes)) { throw std::runtime_error( "hash algorithm short name does not match hash size"); @@ -498,10 +492,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const std::filesystem::path tmp = temp_path(path); TempFileGuard guard(tmp); - write_and_sync_file(tmp, bytes); + merkle::pal::write_and_sync_file(tmp, bytes); + guard.arm(); const auto parent = directory_or_dot(path.parent_path()); durable_directory_contents.erase(parent); - replace_file(tmp, path); + merkle::pal::replace_file(tmp, path); sync_directory_contents(parent); guard.dismiss(); } @@ -618,6 +613,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } } + void arm() + { + active = true; + } + void dismiss() { active = false; @@ -625,7 +625,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) private: std::filesystem::path path; - bool active = true; + bool active = false; }; static std::filesystem::path temp_path(const std::filesystem::path& path) @@ -636,190 +636,12 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) std::filesystem::path tmp = path; tmp += std::format( ".tmp.{}.{}.{}", - process_id(), + merkle::pal::process_id(), (uint64_t)stamp, counter.fetch_add(1, std::memory_order_relaxed)); return tmp; } - static uint64_t process_id() - { -#ifdef _WIN32 - return (uint64_t)GetCurrentProcessId(); -#else - return (uint64_t)::getpid(); -#endif - } - - template - static std::string system_error_message( - std::format_string format_string, Args&&... args) - { -#ifdef _WIN32 - const auto error = GetLastError(); -#else - const auto error = errno; -#endif - std::string message; - std::format_to( - std::back_inserter(message), - format_string, - std::forward(args)...); -#ifdef _WIN32 - std::format_to(std::back_inserter(message), ": error {}", error); -#else - std::format_to( - std::back_inserter(message), ": {}", std::strerror(error)); -#endif - return message; - } - - static void require_write_progress( - size_t written, const std::filesystem::path& path) - { - if (written == 0) - { - throw std::runtime_error( - std::format("short write: {}", path.string())); - } - } - - static void write_and_sync_file( - const std::filesystem::path& path, const std::vector& bytes) - { -#ifdef _WIN32 - HANDLE handle = CreateFileW( - path.wstring().c_str(), - GENERIC_WRITE, - 0, - nullptr, - CREATE_ALWAYS, - FILE_ATTRIBUTE_NORMAL, - nullptr); - if (handle == INVALID_HANDLE_VALUE) - { - throw std::runtime_error( - std::format("cannot open file: {}", path.string())); - } - bool close_handle = true; - try - { - size_t written = 0; - while (written < bytes.size()) - { - const auto remaining = bytes.size() - written; - const auto chunk = (DWORD)std::min( - remaining, (size_t)std::numeric_limits::max()); - DWORD done = 0; - if (!WriteFile( - handle, bytes.data() + written, chunk, &done, nullptr)) - { - throw std::runtime_error( - system_error_message("error writing file {}", path.string())); - } - require_write_progress((size_t)done, path); - written += done; - } - if (!FlushFileBuffers(handle)) - { - throw std::runtime_error( - system_error_message("error syncing file {}", path.string())); - } - if (!CloseHandle(handle)) - { - throw std::runtime_error( - system_error_message("error closing file {}", path.string())); - } - close_handle = false; - } - catch (...) - { - if (close_handle) - { - CloseHandle(handle); - } - throw; - } -#else - int flags = O_WRONLY | O_CREAT | O_TRUNC; -# ifdef O_CLOEXEC - flags |= O_CLOEXEC; -# endif - int fd = ::open(path.c_str(), flags, 0666); - if (fd < 0) - { - throw std::runtime_error( - system_error_message("cannot open file {}", path.string())); - } - try - { - size_t written = 0; - while (written < bytes.size()) - { - const ssize_t done = - ::write(fd, bytes.data() + written, bytes.size() - written); - if (done < 0) - { - if (errno == EINTR) - { - continue; - } - throw std::runtime_error( - system_error_message("error writing file {}", path.string())); - } - require_write_progress((size_t)done, path); - written += (size_t)done; - } - if (::fsync(fd) != 0) - { - throw std::runtime_error( - system_error_message("error syncing file {}", path.string())); - } - if (::close(fd) != 0) - { - fd = -1; - throw std::runtime_error( - system_error_message("error closing file {}", path.string())); - } - fd = -1; - } - catch (...) - { - if (fd >= 0) - { - ::close(fd); - } - throw; - } -#endif - } - - static void replace_file( - const std::filesystem::path& tmp, const std::filesystem::path& path) - { -#ifdef _WIN32 - if (!MoveFileExW( - tmp.wstring().c_str(), - path.wstring().c_str(), - MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) - { - throw std::runtime_error(system_error_message( - "cannot rename temp file {} to {}", tmp.string(), path.string())); - } -#else - std::error_code ec; - std::filesystem::rename(tmp, path, ec); - if (ec) - { - throw std::runtime_error(std::format( - "cannot rename temp file {} to {}: {}", - tmp.string(), - path.string(), - ec.message())); - } -#endif - } - void sync_directory_for_durability(const std::filesystem::path& path) { if (directory_sync) @@ -827,45 +649,13 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) directory_sync(path); return; } - sync_directory_on_disk(path); - } - - static void sync_directory_on_disk(const std::filesystem::path& path) - { -#ifndef _WIN32 - int flags = O_RDONLY; -# ifdef O_DIRECTORY - flags |= O_DIRECTORY; -# endif -# ifdef O_CLOEXEC - flags |= O_CLOEXEC; -# endif - const int fd = ::open(path.c_str(), flags); - if (fd < 0) - { - throw std::runtime_error( - system_error_message("cannot open directory {}", path.string())); - } - if (::fsync(fd) != 0) - { - const std::string message = - system_error_message("error syncing directory {}", path.string()); - ::close(fd); - throw std::runtime_error(message); - } - if (::close(fd) != 0) - { - throw std::runtime_error( - system_error_message("error closing directory {}", path.string())); - } -#else - (void)path; -#endif + merkle::pal::sync_directory_on_disk(path); } }; /// @brief Default tile store (SHA256, default hash function). - using TileStore = TileStoreT<32, sha256_compress>; + using TileStore = + TileStoreT; } } diff --git a/test/tiles_store.cpp b/test/tiles_store.cpp index 6e69f88..9ab8f65 100644 --- a/test/tiles_store.cpp +++ b/test/tiles_store.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -28,7 +30,7 @@ static void expect(bool cond, const std::string& what) { if (!cond) { - throw std::runtime_error("check failed: " + what); + throw std::runtime_error(std::format("check failed: {}", what)); } } @@ -38,7 +40,7 @@ static void expect_eq( if (got != expected) { throw std::runtime_error( - what + ": got '" + got + "', expected '" + expected + "'"); + std::format("{}: got '{}', expected '{}'", what, got, expected)); } } @@ -57,6 +59,20 @@ static void expect_throws( expect(threw, what); } +static std::string expect_throws_message( + const std::function& fn, const std::string& what) +{ + try + { + fn(); + } + catch (const std::exception& ex) + { + return ex.what(); + } + throw std::runtime_error(std::format("check failed: {}", what)); +} + static void custom_hash(const Hash& lhs, const Hash& rhs, Hash& out) { (void)rhs; @@ -83,22 +99,6 @@ static bool any_tmp_files(const fs::path& dir) }); } -class TileStoreProbe : public merkle::tiles::TileStore -{ -public: - using Store = merkle::tiles::TileStore; - - static void check_write_progress(size_t written) - { - Store::require_write_progress(written, "test"); - } - - static uint64_t current_process_id() - { - return Store::process_id(); - } -}; - struct SyncFault { fs::path fail_path; @@ -169,7 +169,8 @@ static std::vector read_file(const fs::path& p) std::ifstream f(p, std::ios::binary); if (!f.good()) { - throw std::runtime_error("cannot open test file: " + p.string()); + throw std::runtime_error( + std::format("cannot open test file: {}", p.string())); } return {std::istreambuf_iterator(f), std::istreambuf_iterator()}; } @@ -179,10 +180,7 @@ int main() const auto nonce = std::chrono::steady_clock::now().time_since_epoch().count(); const fs::path dir = fs::temp_directory_path() / - ("merklecpp_tiles_" + - std::to_string( - static_cast(TileStoreProbe::current_process_id())) + - "_" + std::to_string(static_cast(nonce))); + std::format("merklecpp_tiles_{}_{}", merkle::pal::process_id(), nonce); try { @@ -276,7 +274,8 @@ int main() "algorithm short name must match hash output size"); } - using CustomStore = merkle::tiles::TileStoreT<32, custom_hash>; + using CustomStore = + merkle::tiles::TileStoreT; expect_throws( [&] { (void)CustomStore(dir); }, "custom hash requires an explicit algorithm name"); @@ -288,18 +287,20 @@ int main() #ifdef HAVE_OPENSSL { - using TileStore256 = - merkle::tiles::TileStoreT<32, merkle::sha256_openssl>; + using TileStore256 = merkle::tiles:: + TileStoreT; const TileStore256 store256(dir); expect_eq( store256.root().lexically_relative(dir).generic_string(), "sha256-256w", "OpenSSL SHA256 storage directory"); - using TileStore384 = - merkle::tiles::TileStoreT<48, merkle::sha384_openssl>; + using TileStore384 = merkle::tiles::TileStoreT< + merkle::Tree384::Hash::size_bytes, + merkle::Tree384::hash_function>; TileStore384 store384(dir); - const auto full384 = make_hashesT<48>(merkle::tiles::TILE_WIDTH); + const auto full384 = make_hashesT( + merkle::tiles::TILE_WIDTH); expect_eq( store384.root().lexically_relative(dir).generic_string(), "sha384-256w", @@ -307,17 +308,20 @@ int main() store384.write_tile(TileRef{0, 0}, full384); expect( fs::file_size(store384.tile_path(TileRef{0, 0})) == - (uintmax_t)merkle::tiles::TILE_WIDTH * 48, + (uintmax_t)merkle::tiles::TILE_WIDTH * + merkle::Tree384::Hash::size_bytes, "SHA384 full tile remains 256 hashes wide"); expect( store384.has_full_tile(0, 0) && store384.read_tile(TileRef{0, 0}) == full384, "SHA384 tile round-trip"); - using TileStore512 = - merkle::tiles::TileStoreT<64, merkle::sha512_openssl>; + using TileStore512 = merkle::tiles::TileStoreT< + merkle::Tree512::Hash::size_bytes, + merkle::Tree512::hash_function>; TileStore512 store512(dir); - const auto full512 = make_hashesT<64>(merkle::tiles::TILE_WIDTH); + const auto full512 = make_hashesT( + merkle::tiles::TILE_WIDTH); expect_eq( store512.root().lexically_relative(dir).generic_string(), "sha512-256w", @@ -325,7 +329,8 @@ int main() store512.write_tile(TileRef{0, 0}, full512); expect( fs::file_size(store512.tile_path(TileRef{0, 0})) == - (uintmax_t)merkle::tiles::TILE_WIDTH * 64, + (uintmax_t)merkle::tiles::TILE_WIDTH * + merkle::Tree512::Hash::size_bytes, "SHA512 full tile remains 256 hashes wide"); expect( store512.has_full_tile(0, 0) && @@ -450,13 +455,45 @@ int main() "directory creation failure leaves no temporary file"); } - // 2f. A successful write must make progress. This shared guard prevents + // 2f. Unique temporary-file creation must fail rather than overwrite an + // existing path, including a pre-created symlink on POSIX. + { + const fs::path existing_path = dir / "preexisting-temp"; + const std::vector original = {0x11, 0x22, 0x33}; + overwrite_file(existing_path, original); + const auto collision_error = expect_throws_message( + [&] { merkle::pal::write_and_sync_file(existing_path, {0xAA}); }, + "exclusive temporary-file creation rejects collisions"); + expect( + collision_error.find(existing_path.string()) != std::string::npos, + "temporary-file collision identifies the path"); +#ifdef _WIN32 + expect( + collision_error.find(": error ") != std::string::npos, + "temporary-file collision includes the Windows system error"); +#endif + expect( + read_file(existing_path) == original, + "temporary-file collision does not clobber existing content"); +#ifndef _WIN32 + const fs::path symlink_path = dir / "preexisting-temp-symlink"; + fs::create_symlink(existing_path, symlink_path); + expect_throws( + [&] { merkle::pal::write_and_sync_file(symlink_path, {0xBB}); }, + "exclusive temporary-file creation rejects symlinks"); + expect( + read_file(existing_path) == original, + "temporary-file symlink does not clobber its target"); +#endif + } + + // 2g. A successful write must make progress. This shared guard prevents // the Windows WriteFile loop from spinning if it reports zero bytes. { expect_throws( - [] { TileStoreProbe::check_write_progress(0); }, + [] { merkle::pal::require_write_progress(0, "test"); }, "zero-byte write rejected"); - TileStoreProbe::check_write_progress(1); + merkle::pal::require_write_progress(1, "test"); } // 3a. Full tile byte round-trip. From 9a4a0e08bdede8799ebeebaa35f8bc56faac57e9 Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 19:55:13 +0000 Subject: [PATCH 08/22] Address final tile storage review findings Bound hash formatting and tile levels, retry interrupted durability syncs, and harden Apple and Windows portability. Expand exercised coverage, restore doctest discovery, and align the tiled-storage design with the implementation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/design/tlog-tiles.md | 71 ++++++++++++++++++++-------------- merklecpp.h | 11 ++++-- merklecpp_pal.h | 36 +++++++++++++++-- merklecpp_tiles.h | 18 ++++++++- test/CMakeLists.txt | 6 ++- test/coverage.cpp | 3 ++ test/tiles_store.cpp | 66 ++++++++++++++++++++++++++++++- test/unit_tests.cpp | 7 ++-- test/windows_include_order.cpp | 11 ++++++ 9 files changed, 186 insertions(+), 43 deletions(-) create mode 100644 test/windows_include_order.cpp diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index bb198a0..1d64d80 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -39,8 +39,9 @@ combination of the two. Partial tiles are out of scope: merklecpp never emits or reads them. During a flush, a tile becomes eligible only after all 256 entries are complete and -final. Entries beyond that boundary remain in memory. Once published, a full -tile is immutable and is never rewritten. +final. Entries beyond that boundary remain in memory. Higher-level writers +treat a published full tile as immutable; the low-level store permits atomic +replacement only for repair and idempotent publication. ### Thread-safety policy @@ -96,7 +97,7 @@ A tiled log exposes the Merkle tree as a set of static resources: on-disk tile set is always a full-tile prefix. - **Entry bundles** at `/tile/entries/`: big-endian `uint16` length-prefixed raw entries, 256 per full bundle. (See - [section 6.8](#68-entry-bundles-optional).) + [section 6.3](#63-entry-bundles-optional).) - **Pruning**: a log keeps a *minimum index*; tiles/bundles whose end index is `≤ minimum index` may be denied. This maps cleanly onto merklecpp's `flush_to()` / `min_index()`. @@ -194,7 +195,8 @@ mth_range(a, b): // MTH(D[a:b]); a aligned to 2^ceil(log `mth_range` falls back to splitting when a perfect subtree is not directly resolvable from the chosen source; the recursion always bottoms out at resolvable -pieces (see the [combined-source invariant](#7-progressive-production--compaction)). +pieces supplied by the tile or memory source in the later combined-source +phase. --- @@ -273,16 +275,16 @@ types remain in namespace `merkle::tiles`, templated on | Component | Responsibility | |----------------------|-----------------------------------------------------------------------| | PAL helpers | exclusive file creation, replacement, and durability primitives | -| `TileCoord` | pure index math + path encoding (no I/O) | +| `TileRef` / encoder | identify full tiles and encode their indices (no I/O) | | `TileStoreT` | read/write tile files on a local filesystem | | `TileWriterT` | compute & persist newly-complete full tiles (write path) | | `HashSource` impls | resolve `subtree_root` from tiles, memory, or both | | `ProofEngineT` | `mth_range`, `inclusion_proof`, `consistency_proof`, verifiers | | `TiledTreeT` | convenience wrapper: `append` / `flush` / `prove*` / compaction | -The **only** (optional) change to the core header is a small, read-only, -**non-hashing** accessor (`subtree_root`) used by `MemoryHashSource`; see -[§6.2](#62-optional-core-accessor-non-hashing). +The storage aliases use read-only compile-time metadata exposed by the core +types (`HashT::size_bytes` and `TreeT::hash_function`). A future combined +memory/tile source may also require a non-hashing subtree accessor. --- @@ -303,6 +305,7 @@ namespace tiles { static constexpr uint16_t TILE_HEIGHT = 8; static constexpr uint16_t TILE_WIDTH = uint16_t{1U << TILE_HEIGHT}; +static constexpr uint8_t MAX_TILE_LEVEL = 63; template &, @@ -328,7 +331,7 @@ and it only ever combines two `HASH_SIZE`-byte hashes — so the default introduced**. Convenience aliases derive both template arguments from the core tree type, so hash sizes and functions have one source of truth. -### 6.3 `TileStoreT` — disk I/O +### 6.2 `TileStoreT` — disk I/O ```cpp struct TileRef { uint8_t level; uint64_t index; }; // full tiles only @@ -353,21 +356,24 @@ public: }; ``` -- Writes use a unique temporary file, sync the file contents, publish with an - atomic replace, and on POSIX sync every newly created directory link plus the - destination directory after the rename. Before reusing a visible file, the - writer re-confirms the directory chain and destination directory, so a retry - repeats a failed sync even if directory creation or rename is already - visible. A wrong-size tile is not treated as a durable full tile and is - rewritten by the writer. -- Full tiles are written once and never rewritten (immutability), so every file - under `-256w/tile//` is write-once. -- A small in-process cache of recently read tiles avoids repeated I/O during - proof generation. +- Writes use a unique temporary file, sync the file contents (`F_FULLFSYNC` + where supported on Apple platforms, with an `fsync` fallback), publish with + an atomic replace, and on POSIX sync every newly created directory link plus + the destination directory after the rename. Before reusing a visible file, a + higher-level writer re-confirms the directory chain and destination + directory, so a retry repeats a failed sync even if directory creation or + rename is already visible. A wrong-size tile is not treated as a durable full + tile and can be repaired by a later write. +- Higher-level writers enforce full-tile immutability. The low-level + `TileStoreT` publication primitive deliberately permits atomic replacement + for repair and idempotent concurrent publication, and trusts callers not to + replace a tile with different content. +- The planned `TileHashSource` owns the read cache used during proof + generation; `TileStoreT` itself does not cache reads. - `TileStoreT` does not synchronize access. Callers must serialize all operations on a store and across stores that share a prefix. -### 6.8 Entry bundles (optional) +### 6.3 Entry bundles (optional) merklecpp never sees raw entries (callers insert pre-computed leaf hashes), so entry bundles are an **application-owned** add-on, included for completeness: @@ -376,25 +382,32 @@ entry bundles are an **application-owned** add-on, included for completeness: length-prefixed entries, 256 per full bundle (full bundles only). - The application is responsible for the leaf-hash derivation it uses (e.g. `leaf_hash = H(entry)`); merklecpp stores whatever leaf hash is inserted. -## 10. Implementation plan -Each phase is independently testable; phases 1–4 require **no** core changes. +## 7. Implementation plan + +This PR implements Phases 0 and 1 plus the standalone entry-bundle codec and +storage primitives. The remaining phases are delivered by the later PRs in the +stack. + +Each phase is independently testable. Phases 1–3 require no further core +changes; Phase 4 may add one non-hashing accessor. **Phase 0 — Scaffolding.** Add `merklecpp_tiles.h` (includes `merklecpp.h`) and the internal `merklecpp_pal.h`, `merkle::tiles` namespace, geometry constants, default aliases. Wire a new test group in `test/CMakeLists.txt` following `add_merklecpp_test`. -**Phase 1 — Coordinates & store.** `TileCoord`/`encode_index`, `TileRef`, +**Phase 1 — Coordinates & store.** `TileRef`/`encode_tile_index`, `TileStoreT` (path building, atomic `write_tile`, `read_tile`) and PAL-backed exclusive temporary-file creation. *Tests:* index-encoding vectors; algorithm-qualified roots; SHA-256 and SHA-384 both use 256-hash tiles; tile byte round-trip; existing-file and symlink collision rejection. **Phase 2 — Write path.** `TileWriterT::write_up_to` (level-0 from `leaf_at`, -roll-ups, incremental cursor; full tiles only — the frontier is never tiled). -*Tests:* sizes 256 & 70 000 produce exactly the expected full-tile set; -full-tile immutability (re-running writes nothing new). +roll-ups, incremental cursor) and `EntryBundleWriterT` (incremental full +bundles); full resources only — the frontier is never tiled. *Tests:* sizes 256 +& 70 000 produce exactly the expected full-tile set; full-resource +immutability (re-running writes nothing new). **Phase 3 — Hash sources & proof engine.** `TileHashSource`, `mth_range`, `ProofEngineT::root/inclusion_proof/consistency_proof/verify_consistency`. @@ -409,8 +422,8 @@ with `flush_to`). *Tests:* build N leaves, flush, then prove inclusion for a **flushed** index and a **resident** index against a non-flushed reference tree's root; consistency across a flush boundary. -**Phase 5 — Optional entry bundles** (`BundleWriter`) and docs (README "Usage" -snippet, link this design doc, add the public and PAL headers to Doxygen inputs). +**Phase 5 — User documentation and performance coverage.** Add a README usage +example, link this design, and benchmark tile-backed operation. Deliverables: `merklecpp_tiles.h`; `merklecpp_pal.h`; `test/tiles_*.cpp`; CMake wiring; optional one-method core addition; README/docs updates. diff --git a/merklecpp.h b/merklecpp.h index c56bf22..4a843fe 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -205,11 +205,16 @@ namespace merkle } /// @brief Convert a hash to a hex-encoded string - /// @param num_bytes The maximum number of bytes to convert + /// @param num_bytes The number of bytes to convert /// @param lower_case Enables lower-case hex characters + /// @throws std::out_of_range if @p num_bytes exceeds the hash size [[nodiscard]] std::string to_string( - size_t num_bytes = SIZE, bool lower_case = true) const + size_t num_bytes = size_bytes, bool lower_case = true) const { + if (num_bytes > size_bytes) + { + throw std::out_of_range("hash string byte count exceeds hash size"); + } size_t const num_chars = 2 * num_bytes; std::string r; r.reserve(num_chars); @@ -709,7 +714,7 @@ namespace merkle if (left && right) { size = left->size + right->size + 1; - height = std::max(left->height, right->height) + 1; + height = (std::max)(left->height, right->height) + 1; } else { diff --git a/merklecpp_pal.h b/merklecpp_pal.h index 682ef97..2194bc5 100644 --- a/merklecpp_pal.h +++ b/merklecpp_pal.h @@ -77,6 +77,35 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) #endif } +#ifndef _WIN32 + namespace detail + { + template + static inline int retry_on_eintr(const Operation& operation) + { + int result = 0; + do + { + result = operation(); + } while (result == -1 && errno == EINTR); + return result; + } + + static inline int sync_file_on_disk(int fd) + { +# ifdef F_FULLFSYNC + const int result = + retry_on_eintr([fd]() { return ::fcntl(fd, F_FULLFSYNC); }); + if (result == 0 || (errno != ENOTSUP && errno != EINVAL)) + { + return result; + } +# endif + return retry_on_eintr([fd]() { return ::fsync(fd); }); + } + } +#endif + static inline void require_write_progress( size_t written, const std::filesystem::path& path) { @@ -121,7 +150,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) { const auto remaining = bytes.size() - written; const auto chunk = static_cast(std::min( - remaining, static_cast(std::numeric_limits::max()))); + remaining, + static_cast((std::numeric_limits::max)()))); DWORD done = 0; if (!WriteFile(handle, bytes.data() + written, chunk, &done, nullptr)) { @@ -187,7 +217,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) require_write_progress(static_cast(done), path); written += static_cast(done); } - if (::fsync(fd) != 0) + if (detail::sync_file_on_disk(fd) != 0) { const auto error = last_system_error(); throw std::runtime_error(system_error_message( @@ -261,7 +291,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) throw std::runtime_error(system_error_message( error, "cannot open directory {}", path.string())); } - if (::fsync(fd) != 0) + if (detail::retry_on_eintr([fd]() { return ::fsync(fd); }) != 0) { const auto error = last_system_error(); const std::string message = system_error_message( diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 10a7695..91ae604 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -44,6 +44,9 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @brief Number of hashes in a full tile (2**TILE_HEIGHT). static constexpr uint16_t TILE_WIDTH = uint16_t{1U << TILE_HEIGHT}; + /// @brief Highest tile level permitted by the tlog-tiles layout. + static constexpr uint8_t MAX_TILE_LEVEL = 63; + namespace detail { static constexpr std::string_view SHA256_ALGORITHM_SHORT_NAME = "sha256"; @@ -96,7 +99,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// incomplete frontier is never tiled (see doc/design/tlog-tiles.md). struct TileRef { - /// @brief The level of the tile (0 == leaf hashes). + /// @brief The level of the tile (0 == leaf hashes, maximum 63). uint8_t level = 0; /// @brief The index of the tile within its level. @@ -173,8 +176,13 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } /// @brief The filesystem path of a tile. + /// @throws std::runtime_error if the tile level exceeds 63 [[nodiscard]] std::filesystem::path tile_path(const TileRef& ref) const { + if (ref.level > MAX_TILE_LEVEL) + { + throw std::runtime_error("tile level out of range"); + } const auto relative_path = std::format( "tile/{}/{}", static_cast(ref.level), @@ -260,7 +268,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) (void)decode_entries(read_file(path), TILE_WIDTH); return true; } - catch (const std::exception&) + catch (const std::runtime_error&) { return false; } @@ -313,6 +321,10 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) static std::vector> decode_entries( const std::vector& bytes, size_t count) { + if (count > bytes.size() / 2) + { + throw std::runtime_error("truncated entry bundle"); + } std::vector> out; out.reserve(count); size_t pos = 0; @@ -341,6 +353,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) return out; } + /// @cond INTERNAL protected: using DirectorySync = std::function; @@ -651,6 +664,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } merkle::pal::sync_directory_on_disk(path); } + /// @endcond }; /// @brief Default tile store (SHA256, default hash function). diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a47f8a9..dbc5b39 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -31,6 +31,10 @@ add_merklecpp_test(serialise_to_file serialise_to_file.cpp) add_merklecpp_test(coverage coverage.cpp) add_merklecpp_test(tiles_store tiles_store.cpp) +if(WIN32) + add_merklecpp_test(windows_include_order windows_include_order.cpp) +endif() + if(TARGET evercrypt.host) add_merklecpp_test(compare_evercrypt compare_evercrypt.cpp) endif() @@ -41,7 +45,7 @@ if(OPENSSL add_merklecpp_test(compare_hash_functions compare_hash_functions.cpp) endif() -find_path(DOCTEST_DIR doctest.h) +find_path(DOCTEST_DIR doctest.h PATH_SUFFIXES doctest) if(DOCTEST_DIR) add_merklecpp_test(unit_tests unit_tests.cpp) diff --git a/test/coverage.cpp b/test/coverage.cpp index a930d17..eaa4626 100644 --- a/test/coverage.cpp +++ b/test/coverage.cpp @@ -116,6 +116,9 @@ namespace { require(byte == 0xAA, "valid hash string parsed incorrectly"); } + require_throws( + [&] { (void)valid_hash.to_string(valid_hash.size() + 1); }, + "hash string should reject byte counts beyond the hash size"); std::string mixed_case_hex(64, '0'); mixed_case_hex[0] = 'A'; diff --git a/test/tiles_store.cpp b/test/tiles_store.cpp index 9ab8f65..73f0228 100644 --- a/test/tiles_store.cpp +++ b/test/tiles_store.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -52,7 +54,7 @@ static void expect_throws( { fn(); } - catch (const std::exception&) + catch (const std::runtime_error&) { threw = true; } @@ -66,7 +68,7 @@ static std::string expect_throws_message( { fn(); } - catch (const std::exception& ex) + catch (const std::runtime_error& ex) { return ex.what(); } @@ -248,6 +250,18 @@ int main() rel(store, store.tile_path(TileRef{1, 1234067})), "tile/1/x001/x234/067", "tile_path L1 big index"); + expect_eq( + rel( + store, + store.tile_path(TileRef{merkle::tiles::MAX_TILE_LEVEL, 0})), + "tile/63/000", + "maximum tile level"); + expect_throws( + [&] { + (void)store.tile_path(TileRef{ + static_cast(merkle::tiles::MAX_TILE_LEVEL + 1), 0}); + }, + "tile level above 63 rejected"); expect_eq( rel(store, store.entries_path(5)), "tile/entries/005", "entries full"); expect_eq( @@ -298,6 +312,12 @@ int main() using TileStore384 = merkle::tiles::TileStoreT< merkle::Tree384::Hash::size_bytes, merkle::Tree384::hash_function>; + for (const std::string& mismatched_name : {"sha256", "sha512"}) + { + expect_throws( + [&] { (void)TileStore384(dir, mismatched_name); }, + "SHA384 store rejects mismatched algorithm names"); + } TileStore384 store384(dir); const auto full384 = make_hashesT( merkle::tiles::TILE_WIDTH); @@ -319,6 +339,12 @@ int main() using TileStore512 = merkle::tiles::TileStoreT< merkle::Tree512::Hash::size_bytes, merkle::Tree512::hash_function>; + for (const std::string& mismatched_name : {"sha256", "sha384"}) + { + expect_throws( + [&] { (void)TileStore512(dir, mismatched_name); }, + "SHA512 store rejects mismatched algorithm names"); + } TileStore512 store512(dir); const auto full512 = make_hashesT( merkle::tiles::TILE_WIDTH); @@ -496,6 +522,36 @@ int main() merkle::pal::require_write_progress(1, "test"); } +#ifndef _WIN32 + // 2h. Interrupted sync operations are retried, while other failures are + // returned immediately with errno intact. + { + size_t attempts = 0; + const int retry_result = merkle::pal::detail::retry_on_eintr([&]() { + attempts++; + if (attempts == 1) + { + errno = EINTR; + return -1; + } + return 0; + }); + expect( + retry_result == 0 && attempts == 2, + "interrupted sync operation retried"); + + attempts = 0; + const int error_result = merkle::pal::detail::retry_on_eintr([&]() { + attempts++; + errno = EIO; + return -1; + }); + expect( + error_result == -1 && attempts == 1 && errno == EIO, + "non-interrupted sync failure preserved"); + } +#endif + // 3a. Full tile byte round-trip. const TileRef full_ref{0, 0}; store.write_tile(full_ref, full); @@ -554,6 +610,12 @@ int main() expect( merkle::tiles::TileStore::decode_entries({}, 0).empty(), "empty entry sequence round-trip"); + expect_throws( + [] { + (void)merkle::tiles::TileStore::decode_entries( + {}, std::numeric_limits::max()); + }, + "impossible entry count rejected before allocation"); const std::vector maximum_entry(0xFFFF, 0xC3); const auto maximum_encoded = diff --git a/test/unit_tests.cpp b/test/unit_tests.cpp index e6f6967..b0868e5 100644 --- a/test/unit_tests.cpp +++ b/test/unit_tests.cpp @@ -102,6 +102,7 @@ TEST_CASE("HashT methods") const std::string s2 = h2.to_string(2); REQUIRE(s2.size() == 4); REQUIRE(s2 == "abcd"); + REQUIRE_THROWS_AS((void)h2.to_string(33), std::out_of_range); // operator== and operator!= merkle::Hash ha; @@ -175,7 +176,7 @@ TEST_CASE("PathT equality") TEST_CASE("TreeT to_string") { // Empty tree produces "" - merkle::Tree empty_tree; + const merkle::Tree empty_tree; const std::string empty_str = empty_tree.to_string(); REQUIRE(empty_str.find("") != std::string::npos); @@ -192,7 +193,7 @@ TEST_CASE("TreeT to_string") TEST_CASE("TreeT leaf bounds and uninserted leaves") { // leaf() out of bounds on empty tree - merkle::Tree empty_tree; + const merkle::Tree empty_tree; REQUIRE_THROWS(empty_tree.leaf(0)); // Access leaf before insertion is flushed (uninserted_leaf_nodes path) @@ -216,7 +217,7 @@ TEST_CASE("TreeT size with uninserted leaves") { merkle::Tree tree; // size() when tree has uninserted leaves triggers insert_leaves() - merkle::Tree::Hash h; + const merkle::Tree::Hash h; tree.insert(h); // size() forces lazy insertion const size_t sz = tree.size(); diff --git a/test/windows_include_order.cpp b/test/windows_include_order.cpp new file mode 100644 index 0000000..9cd309a --- /dev/null +++ b/test/windows_include_order.cpp @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include + +#include + +int main() +{ + return 0; +} From 3250b46205208e2c054cd5682f53f3052030802d Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 20:35:28 +0000 Subject: [PATCH 09/22] Condense tiled storage design Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/design/tlog-tiles.md | 538 ++++++++++++++------------------------- 1 file changed, 196 insertions(+), 342 deletions(-) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 1d64d80..120a026 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -1,177 +1,100 @@ # Design: Tiled storage and tile-backed proofs for merklecpp -Status: Proposal / Draft -Audience: merklecpp maintainers and contributors -Scope: Extend the header-only library to (a) persist the Merkle tree -progressively (on compaction) using the -[tlog-tiles](https://c2sp.org/tlog-tiles) SHA-256 format and an explicitly -namespaced extension for other SHA output sizes, and (b) serve **inclusion** -and **consistency** proofs from those tiles, from the in-memory tree, or from a -combination of the two. +**Status:** Proposal / Draft ---- +**Audience:** merklecpp maintainers and contributors -## 1. Goals and non-goals - -### Goals - -1. Write the tree to disk as a set of immutable, cacheable **tile files** using - the [tlog-tiles](https://c2sp.org/tlog-tiles) payload, path encoding, and - geometry (256 hashes per tile and 8 tree levels per tile). SHA-256 follows - the C2SP format; SHA-384 and SHA-512 retain the same 256-hash width as - merklecpp extensions. Only **full, balanced** tiles are produced. The - remaining frontier stays in the in-memory tree until it crosses the next - full-tile boundary. -2. Produce tiles **progressively**: each batch/flush writes only the newly - completed tiles, and the write integrates with the existing `flush_to()` - "compaction" so that flushed (evicted) subtrees are durably persisted before - their nodes are freed from memory. -3. Provide an API to retrieve, **after compaction**, an **inclusion path** and a - **consistency path**: - - from the in-memory tree alone (existing behaviour), - - from full tiles alone, for the tiled (full-tile-covered) prefix, or - - from a combination of full tiles (old, flushed part) and the in-memory - tree (recent frontier) — the normal case. -4. Keep everything **header-only** and templated, matching the existing - `TreeT` style. - -### Full-tile-only policy - -Partial tiles are out of scope: merklecpp never emits or reads them. During a -flush, a tile becomes eligible only after all 256 entries are complete and -final. Entries beyond that boundary remain in memory. Higher-level writers -treat a published full tile as immutable; the low-level store permits atomic -replacement only for repair and idempotent publication. - -### Thread-safety policy - -The tiled-storage API provides no internal synchronization. Every object is -single-threaded unless its caller serializes all access. This requirement also -applies to separate objects that share a store prefix and to methods declared -`const`, because proof reads update the `TileHashSourceT` LRU cache. Locking and -reader/writer coordination are deliberately left to the application. - -### Hard constraints (from the request) - -- **Hashing logic is unchanged.** No modification to `HASH_FUNCTION`, to node - hashing, or to leaf handling. -- **Proofs remain compatible with the existing library.** A proof produced from - tiles MUST be identical to (and verify against the same root as) the proof the - existing library would produce via `Tree::path()` / `Tree::past_path()` / - `Tree::root()` / `Tree::past_root()`. - -### Non-goals - -- Making the SHA-384 or SHA-512 extension a C2SP standard. C2SP currently - specifies SHA-256 only; the additional namespaces are merklecpp formats. -- Supplying RFC 6962 leaf hashing or domain separation. Exact external tlog - interoperability requires the consumer to instantiate `TreeT` with - compatible leaf and node hashing; this design does not alter hashing. -- An HTTP server. We define the on-disk layout and the read/write/proof APIs; - serving the static files is an application concern. +**Scope:** Persist trees progressively during compaction using +[tlog-tiles](https://c2sp.org/tlog-tiles), then serve inclusion and consistency +proofs from tiles, memory, or both. --- -## 2. Background - -### 2.1 tlog-tiles, the parts we adopt - -A tiled log exposes the Merkle tree as a set of static resources: - -- **Tiles** at `/tile//`, `application/octet-stream`. - - `` = level, decimal `0..63`, no leading zeros. - - `` = tile index within the level, encoded as zero-padded 3-digit path - elements where **all but the last element are prefixed with `x`**. Example: - `1234067` → `x001/x234/067`. -- A **full tile** is exactly **256 SHA-256 hashes**, or **8,192 bytes**. C2SP - fixes both the algorithm and width. merklecpp keeps the width at 256 and - makes the hash size a template parameter. -- Level 0 hashes are **leaf hashes**. At level `L ≥ 1`, each hash is the Merkle - Tree Hash of a *full* tile at level `L-1`. A tile spans **8 tree levels**: its - 256 entries are the leaves of a height-8 perfect subtree, and intra-tile - internal nodes are reconstructed by hashing entries. -- The `n`-th tile at level `l` contains, for `i = 0..255`: - `MTH(D[(n*256+i)*256^l : (n*256+i+1)*256^l])`. - Its **start index** is `n*256^(l+1)`, its **end index** `(n+1)*256^(l+1)`. -- The frontier beyond the last full tile is held in the in-memory tree, so the - on-disk tile set is always a full-tile prefix. -- **Entry bundles** at `/tile/entries/`: big-endian `uint16` - length-prefixed raw entries, 256 per full bundle. (See - [section 6.3](#63-entry-bundles-optional).) -- **Pruning**: a log keeps a *minimum index*; tiles/bundles whose end index is - `≤ minimum index` may be denied. This maps cleanly onto merklecpp's - `flush_to()` / `min_index()`. - -### 2.2 The merklecpp model - -- `TreeT` is a **left-balanced binary Merkle tree**. - Its shape is identical to RFC 6962: a node's left child is the largest perfect - subtree of `2^k` leaves with `2^k < n`. A *full* node of height `h` covers - exactly `2^(h-1)` aligned leaves, and its hash is the root of that perfect - subtree (verified from `walk_to`, `is_full`, `update_sizes`). -- Internal nodes are combined with `HASH_FUNCTION(left, right, out)` over exactly - two child hashes (`merklecpp.h` node hashing). There is **no** domain - separation; the combiner is the only cryptographic operation. -- `flush_to(index)` is the existing **compaction** primitive: it conflates the - left part of the tree into single hash-only nodes and drops `num_flushed` - leaves from memory, raising `min_index()` to `index`. After a flush, paths for - leaves `< index` can no longer be produced from memory. -- Existing proof APIs we must stay compatible with: - - `root()` / `past_root(i)` — current / historical root. - - `path(i)` — inclusion path for leaf `i` in the current tree. - - `past_path(i, as_of)` — inclusion path for leaf `i` as of size `as_of+1`. - - `PathT::verify(root)` / `PathT::root()` — verification via `HASH_FUNCTION`. - - merklecpp has **no** classic two-size consistency proof today; this design - adds one (built from the same combiner, so it reconciles `past_root`s). - -### 2.3 Compatibility statement - -C2SP `tlog-tiles` normatively specifies SHA-256, 32-byte hashes, and 256 hashes -per full tile. The SHA-256 merklecpp layout adopts that tile payload and the -`tile//` resource tree. Exact proof interoperability additionally requires -RFC 6962-compatible leaf and node hashing; merklecpp deliberately continues to -use the caller's existing `HASH_FUNCTION`. - -For SHA-384 and SHA-512, merklecpp extends the payload by concatenating 48-byte -or 64-byte hashes respectively. **The tile width remains 256 hashes for every -algorithm**, so full tiles are 12,288 bytes for SHA-384 and 16,384 bytes for -SHA-512. These wider-hash payloads are not currently defined by C2SP. - -Each format gets a separate root named -`-w`: `sha256-256w`, `sha384-256w`, or -`sha512-256w`. Beneath that root, the C2SP `tile/...` paths are unchanged. This -prevents files with different hash sizes from sharing a tile namespace while -preserving the C2SP layout for each format. Built-in SHA functions select their -short name automatically; explicit custom names must be lowercase path-safe -short names, and recognized SHA names are checked against `HASH_SIZE`. - -## 3. Tile ↔ merklecpp mapping (the math) - -Let `TILE_HEIGHT = 8` and `TILE_WIDTH = 256 = 2^8`. All combiners below are the -tree's `HASH_FUNCTION`; `MTH` denotes the Merkle Tree Hash computed with it. - -- **Tile entry = perfect subtree root.** Entry `i` of tile `(L, N)` (global - entry index `g = N·256 + i`) is `MTH(D[g·256^L : (g+1)·256^L])`, i.e. the root - of a perfect subtree of `2^(8L)` leaves — the in-memory full node of height - `8L + 1`. -- **Level roll-up.** Because a full level-`L` entry is the root of 256 full - level-`(L-1)` entries, a level-`L` tile can be computed from level-`(L-1)` - tiles alone: `tile(L,N)[i] = perfect_root( tile(L-1, N·256+i)[0..255] )`. So - the **write path needs no tree internals above level 0** — only leaf hashes - plus roll-ups. -- **Intra-tile internal nodes.** A tile's 256 entries are the leaves of a - height-8 perfect subtree. The root of `2^r` (`0 ≤ r ≤ 8`) consecutive, - aligned entries is `perfect_root` of that run. Hence any RFC 6962 subtree root - at tree level `k = 8L + r` is obtained from **one** level-`L` tile by hashing - `2^r` consecutive entries (a single entry when `r = 0`). -- **Full-tile prefix.** Only complete level-`L` tiles are written: - `full_tiles_L = floor( floor(s / 256^L) / 256 )`, together covering the leaf - prefix `covered = floor(s / 256) * 256`. The incomplete frontier `[covered, s)` - is not tiled; it is served from the in-memory tree. When no higher-level full - tile contains a subtree, it is resolved by **descending to the highest - available full tile and rolling up** (ultimately from full level-0 tiles). - -This yields a single read primitive that both proof types are built on: +## 1. Requirements + +| Area | Contract | +|---|---| +| Format | SHA-256 uses the C2SP payload, paths, and geometry: 256 hashes per tile and 8 tree levels per tile. SHA-384 and SHA-512 use explicitly namespaced merklecpp extensions with the same width. | +| Persistence | A flush writes only newly completed, balanced tiles and makes them durable before `flush_to()` frees their nodes. Partial tiles are never written or read; the incomplete frontier remains in memory. | +| Immutability | Higher-level writers treat published tiles as immutable. The low-level store allows atomic replacement only for repair and idempotent publication. | +| Proofs | Tiled inclusion paths must equal `Tree::path()` / `Tree::past_path()` output and verify against the same `root()` / `past_root()`. Consistency proofs use the same tree and combiner. | +| Hashing | Leaf handling, node hashing, and `HASH_FUNCTION` are unchanged. | +| API | The implementation remains header-only and follows the existing `TreeT` template style. | +| Concurrency | The API performs no synchronization. Callers must serialize access, including across objects sharing a prefix and through `const` proof reads that update the `TileHashSourceT` LRU cache. | + +**Non-goals:** standardizing the SHA-384/512 extensions; adding RFC 6962 leaf +hashing or domain separation; and providing an HTTP server. Applications remain +responsible for compatible hashing and for serving the static resources. + +## 2. Format and compatibility + +### 2.1 Resource model + +| Resource | Layout and meaning | +|---|---| +| Tile | `/-256w/tile//`, `application/octet-stream`; `L` is decimal `0..63` without leading zeros, and `N` uses the grouped encoding in [section 4](#4-storage-layout-and-publication). | +| Full tile | 256 hashes. Level 0 stores leaf hashes; each level-`L` entry for `L >= 1` is the Merkle Tree Hash of one complete level-`L-1` tile. | +| Entry bundle | `/tile/entries/`; 256 raw entries encoded as big-endian `uint16` length-prefixed values. See [section 5.3](#53-entry-bundles-optional). | +| Pruning | Tiles or bundles ending at or before a log's minimum index may be denied, matching `flush_to()` / `min_index()`. | + +A tile spans eight tree levels. Tile `n` at level `l` contains, for +`i = 0..255`: + +`MTH(D[(n*256+i)*256^l : (n*256+i+1)*256^l])` + +Its leaf range is `[n*256^(l+1), (n+1)*256^(l+1))`. Only a full-tile prefix is +stored; the remaining frontier stays in memory. + +### 2.2 merklecpp model + +- `TreeT` is left-balanced with the RFC 6962 shape. + A full node of height `h` covers `2^(h-1)` aligned leaves. +- Internal nodes use `HASH_FUNCTION(left, right, out)` over two child hashes, + without domain separation. +- `flush_to(index)` replaces the compacted prefix with hash-only nodes, removes + its leaves, and raises `min_index()`. Memory alone can no longer prove leaves + below that index. +- Existing proof surfaces are `root()`, `past_root(i)`, `path(i)`, + `past_path(i, as_of)`, and `PathT::verify()` / `root()`. merklecpp does not + currently expose a classic two-size consistency proof; this design adds one + that reconciles historical roots with the same combiner. + +### 2.3 Algorithms and namespaces + +| Algorithm | Hash bytes | Full-tile bytes | Root | Status | +|---|---:|---:|---|---| +| SHA-256 | 32 | 8,192 | `sha256-256w` | C2SP payload and resource layout | +| SHA-384 | 48 | 12,288 | `sha384-256w` | merklecpp extension | +| SHA-512 | 64 | 16,384 | `sha512-256w` | merklecpp extension | + +The width remains 256 hashes for every algorithm; the namespace prevents +different hash sizes from sharing files while retaining the C2SP `tile/...` +layout. Built-in SHA functions select their names automatically. Custom names +must be lowercase, path-safe short names, and recognized SHA names must match +`HASH_SIZE`. + +Exact external interoperability also requires RFC 6962-compatible leaf and node +hashing; merklecpp continues to use the caller's `HASH_FUNCTION`. + +## 3. Tile mapping and proof math + +Let `TILE_HEIGHT = 8`, `TILE_WIDTH = 256 = 2^8`, and `MTH` use the tree's +`HASH_FUNCTION`. + +- Entry `i` of tile `(L, N)`, with `g = N*256+i`, is + `MTH(D[g*256^L : (g+1)*256^L])`: a perfect subtree of `2^(8L)` leaves and an + in-memory full node of height `8L+1`. +- A level-`L` tile rolls up level `L-1` tiles: + `tile(L,N)[i] = perfect_root(tile(L-1, N*256+i)[0..255])`. The writer + therefore needs only level-0 leaf hashes and lower-level tiles. +- For `k = 8L+r`, the root of `2^r` aligned entries (`0 <= r <= 8`) comes from + one level-`L` tile by hashing that run; `r = 0` reads one entry. +- At size `s`, `full_tiles_L = floor(floor(s / 256^L) / 256)` and + `covered = floor(s/256)*256`. `[covered, s)` remains in memory. If a + higher-level tile is absent, lookup descends to available lower-level tiles + and rolls them up. + +Both proof types use one perfect-subtree primitive: ``` subtree_root(level k, index j) = MTH(D[j·2^k : (j+1)·2^k]) # within `covered` @@ -186,246 +109,177 @@ subtree_root(level k, index j) = MTH(D[j·2^k : (j+1)·2^k]) # within `covered and a range primitive for non-perfect (right-frontier) subtrees: ``` -mth_range(a, b): // MTH(D[a:b]); a aligned to 2^ceil(log2(b-a)) +mth_range(a, b): // MTH(D[a:b]); a aligned to 2^ceil(log2(b-a)) w = b - a if w == 2^k and source can resolve subtree_root(k, a>>k): return it - k = largest power of two < w // RFC 6962 split + k = largest power of two < w // RFC 6962 split return HASH_FUNCTION( mth_range(a, a+k), mth_range(a+k, b) ) ``` -`mth_range` falls back to splitting when a perfect subtree is not directly -resolvable from the chosen source; the recursion always bottoms out at resolvable -pieces supplied by the tile or memory source in the later combined-source -phase. +If the selected source cannot resolve a perfect subtree directly, `mth_range` +splits until tile or memory sources can resolve each piece. --- -## 4. File and directory layout +## 4. Storage layout and publication -Rooted below a configurable `prefix` directory on local disk: +All resources live below a configurable local `prefix`: ``` -/ - sha256-256w/ # algorithm + fixed 256-hash width - tile/ - 0/ # level 0 (leaf hashes), full tiles only - 000 001 ... 255 # 8192 B per SHA-256 full tile - x001/ - 000 ... 255 - 1/ # level 1 (roll-ups of level-0 full tiles) - 000 ... - .../ - entries/ # optional raw entry bundles (full only) - 000 001 ... - sha384-256w/ # optional; same width, 12288 B full tiles - tile/ - ... +/-256w/tile/ + 0/ # level-0 full tiles + 000 ... 255 + x001/000 ... 255 + 1/ # roll-ups of level-0 tiles + 000 ... + ... + entries/ # optional full entry bundles + 000 ... ``` -### Index encoding (`encode_tile_index`) +`encode_tile_index` groups decimal indices into zero-padded three-digit +components and prefixes every non-final component with `x`: ``` encode_tile_index(N): - parts = [] - do { parts.push_front(format("{:03}", N % 1000)); N /= 1000 } while (N > 0) - for each part except the last: prepend 'x' + parts = split N into base-1000 groups, each formatted "{:03}" + prefix every group except the last with "x" return join(parts, "/") ``` -Examples: `5 → "005"`, `255 → "255"`, `1000 → "x001/000"`, -`1234067 → "x001/x234/067"`. - -Resource paths: +Examples: `5 -> "005"`, `255 -> "255"`, `1000 -> "x001/000"`, and +`1234067 -> "x001/x234/067"`. -- Full tile: - `/-256w/tile//` (the only tiles - this implementation writes) -- Entry bundle: - `/-256w/tile/entries/` (full bundles - only) +The complete paths are +`/-256w/tile//` and +`/-256w/tile/entries/`. A tile concatenates 256 +`HashT::bytes` values and is therefore `256 * HASH_SIZE` bytes; `256w` counts +hashes, not bytes. -Tile byte format: the 256 entries concatenated, each `HASH_SIZE` raw bytes -(`HashT::bytes`); a full tile is `256 * HASH_SIZE` bytes. The `256w` suffix -counts hashes, not bytes, and therefore does not change for SHA-384 or SHA-512. +### Publication guarantees ---- +- Create a unique temporary file exclusively, sync its contents, and atomically + replace the destination. Apple uses `F_FULLFSYNC` when supported and falls + back to `fsync`. +- On POSIX, sync each newly created directory link and the destination + directory after rename. A higher-level retry rechecks the directory chain and + destination even when the file is already visible. +- Wrong-size files are not durable full tiles and may be repaired. Higher-level + writers enforce immutability; the low-level primitive permits replacement for + repair or idempotent concurrent publication and trusts callers not to change + valid content. -## 5. Architecture overview +## 5. Architecture and API ``` append(leaf hash) flush() - caller ───────────────▶ TiledTreeT ───────────────▶ TileStoreT (disk I/O) - │ owns Tree ▲ - │ │ read tiles - inclusion/consistency proof requests │ + caller ───────────────▶ TiledTreeT ───────────────▶ TileStoreT + │ owns Tree ▲ + inclusion / consistency proofs │ tiles ▼ │ - ProofEngine ──▶ HashSource ◀────────┘ - ├─ MemoryHashSource (in-memory Tree) - ├─ TileHashSource (TileStore + size) - └─ CombinedHashSource + ProofEngine ──▶ HashSource ◀──────────┘ + ├─ MemoryHashSource + ├─ TileHashSource + └─ CombinedHashSource ``` -The public tiled-storage implementation lives in `merklecpp_tiles.h` (which -includes `merklecpp.h`). Low-level operating-system operations are isolated in -the internal `merklecpp_pal.h` platform abstraction (`merkle::pal`). Public -types remain in namespace `merkle::tiles`, templated on -``, with default aliases mirroring the bottom of -`merklecpp.h`: - -| Component | Responsibility | -|----------------------|-----------------------------------------------------------------------| -| PAL helpers | exclusive file creation, replacement, and durability primitives | -| `TileRef` / encoder | identify full tiles and encode their indices (no I/O) | -| `TileStoreT` | read/write tile files on a local filesystem | -| `TileWriterT` | compute & persist newly-complete full tiles (write path) | -| `HashSource` impls | resolve `subtree_root` from tiles, memory, or both | -| `ProofEngineT` | `mth_range`, `inclusion_proof`, `consistency_proof`, verifiers | -| `TiledTreeT` | convenience wrapper: `append` / `flush` / `prove*` / compaction | - -The storage aliases use read-only compile-time metadata exposed by the core -types (`HashT::size_bytes` and `TreeT::hash_function`). A future combined -memory/tile source may also require a non-hashing subtree accessor. +`merklecpp_tiles.h` contains the public `merkle::tiles` API and includes +`merklecpp.h`; internal OS operations live in `merklecpp_pal.h` under +`merkle::pal`. ---- +| Component | Responsibility | +|---|---| +| PAL helpers | Exclusive creation, atomic replacement, and durability | +| `TileRef` / encoder | Full-tile identity and index paths | +| `TileStoreT` | Local tile and entry-bundle I/O | +| `TileWriterT` | Persist newly completed full tiles | +| Hash sources | Resolve subtree roots from memory, tiles, or both | +| `ProofEngineT` | Roots, inclusion/consistency proofs, and verification | +| `TiledTreeT` | `append`, `flush`, proof APIs, and compaction | -## 6. Public API design +The planned `TileHashSource` owns the proof-read LRU cache; `TileStoreT` does +not cache. A combined source may require one read-only, non-hashing core +`subtree_root` accessor. -### 6.1 Header, namespace, aliases +### 5.1 Types and aliases ```cpp -// merklecpp_tiles.h -#include "merklecpp.h" -#include "merklecpp_pal.h" // internal platform abstraction -#include -#include - -namespace merkle { -namespace tiles { +namespace merkle::tiles { static constexpr uint16_t TILE_HEIGHT = 8; -static constexpr uint16_t TILE_WIDTH = - uint16_t{1U << TILE_HEIGHT}; +static constexpr uint16_t TILE_WIDTH = uint16_t{1U << TILE_HEIGHT}; static constexpr uint8_t MAX_TILE_LEVEL = 63; template &, const HashT&, HashT&)> -class TileStoreT { /* ... */ }; -// ... TileWriterT, ProofEngineT, TiledTreeT ... +class TileStoreT; -// Convenience aliases (mirror merklecpp.h) using TileStore = - TileStoreT; + TileStoreT; using TiledTree = - TiledTreeT; -// + 384/512 variants + TiledTreeT; +// Equivalent SHA-384 and SHA-512 aliases. -} // namespace tiles -} // namespace merkle +} ``` -Note the roll-up/proof combiner is the **same** `HASH_FUNCTION` the tree uses, -and it only ever combines two `HASH_SIZE`-byte hashes — so the default -`sha256_compress` single-block path is sufficient and **no OpenSSL dependency is -introduced**. Convenience aliases derive both template arguments from the core -tree type, so hash sizes and functions have one source of truth. +Aliases derive hash size and function from the core tree. Roll-up and proof code +combine two `HASH_SIZE`-byte hashes with that same function, so the default +single-block `sha256_compress` remains sufficient and adds no OpenSSL +dependency. -### 6.2 `TileStoreT` — disk I/O +### 5.2 `TileStoreT` ```cpp struct TileRef { uint8_t level; uint64_t index; }; // full tiles only class TileStoreT { public: - // Built-in SHA functions select sha256, sha384, or sha512. explicit TileStoreT(std::filesystem::path prefix); TileStoreT(std::filesystem::path prefix, const std::string& hash_algorithm_short_name); - // Path helpers (pure) static std::string storage_directory_name(const std::string& algorithm); static std::string encode_index(uint64_t n); std::filesystem::path tile_path(const TileRef&) const; std::filesystem::path entries_path(uint64_t n) const; - // Tiles bool has_full_tile(uint8_t level, uint64_t index) const; - std::vector read_tile(const TileRef&) const; // 256 entries - void write_tile(const TileRef&, const std::vector&); // synced atomic replace + std::vector read_tile(const TileRef&) const; // 256 hashes + void write_tile(const TileRef&, const std::vector&); // durable replace }; ``` -- Writes use a unique temporary file, sync the file contents (`F_FULLFSYNC` - where supported on Apple platforms, with an `fsync` fallback), publish with - an atomic replace, and on POSIX sync every newly created directory link plus - the destination directory after the rename. Before reusing a visible file, a - higher-level writer re-confirms the directory chain and destination - directory, so a retry repeats a failed sync even if directory creation or - rename is already visible. A wrong-size tile is not treated as a durable full - tile and can be repaired by a later write. -- Higher-level writers enforce full-tile immutability. The low-level - `TileStoreT` publication primitive deliberately permits atomic replacement - for repair and idempotent concurrent publication, and trusts callers not to - replace a tile with different content. -- The planned `TileHashSource` owns the read cache used during proof - generation; `TileStoreT` itself does not cache reads. -- `TileStoreT` does not synchronize access. Callers must serialize all - operations on a store and across stores that share a prefix. - -### 6.3 Entry bundles (optional) - -merklecpp never sees raw entries (callers insert pre-computed leaf hashes), so -entry bundles are an **application-owned** add-on, included for completeness: - -- `-256w/tile/entries/` stores big-endian `uint16` - length-prefixed entries, 256 per full bundle (full bundles only). -- The application is responsible for the leaf-hash derivation it uses (e.g. - `leaf_hash = H(entry)`); merklecpp stores whatever leaf hash is inserted. - -## 7. Implementation plan - -This PR implements Phases 0 and 1 plus the standalone entry-bundle codec and -storage primitives. The remaining phases are delivered by the later PRs in the -stack. - -Each phase is independently testable. Phases 1–3 require no further core -changes; Phase 4 may add one non-hashing accessor. - -**Phase 0 — Scaffolding.** Add `merklecpp_tiles.h` (includes `merklecpp.h`) and -the internal `merklecpp_pal.h`, `merkle::tiles` namespace, geometry -constants, default aliases. Wire a new test group in `test/CMakeLists.txt` -following `add_merklecpp_test`. - -**Phase 1 — Coordinates & store.** `TileRef`/`encode_tile_index`, -`TileStoreT` (path building, atomic `write_tile`, `read_tile`) and PAL-backed -exclusive temporary-file creation. *Tests:* index-encoding vectors; -algorithm-qualified roots; SHA-256 and SHA-384 both use 256-hash tiles; tile -byte round-trip; existing-file and symlink collision rejection. - -**Phase 2 — Write path.** `TileWriterT::write_up_to` (level-0 from `leaf_at`, -roll-ups, incremental cursor) and `EntryBundleWriterT` (incremental full -bundles); full resources only — the frontier is never tiled. *Tests:* sizes 256 -& 70 000 produce exactly the expected full-tile set; full-resource -immutability (re-running writes nothing new). - -**Phase 3 — Hash sources & proof engine.** `TileHashSource`, `mth_range`, -`ProofEngineT::root/inclusion_proof/consistency_proof/verify_consistency`. -*Tests:* `root(size)` from tiles == `tree.root()`; `inclusion_proof(i,size)` -**equals** `tree.path(i)` (operator==) and verifies; `inclusion_proof(i, m)` -equals `tree.past_path(i, m-1)`; consistency proof reconciles -`tree.past_root(m-1)`/`tree.past_root(n-1)`. - -**Phase 4 — Combination & wrapper.** Optional core `subtree_root` accessor; -`MemoryHashSource`, `CombinedHashSource`, `TiledTreeT` (append/flush/prove -with `flush_to`). *Tests:* build N leaves, flush, then prove -inclusion for a **flushed** index and a **resident** index against a non-flushed -reference tree's root; consistency across a flush boundary. - -**Phase 5 — User documentation and performance coverage.** Add a README usage -example, link this design, and benchmark tile-backed operation. - -Deliverables: `merklecpp_tiles.h`; `merklecpp_pal.h`; `test/tiles_*.cpp`; -CMake wiring; optional one-method core addition; README/docs updates. - ---- +Built-in hash functions select `sha256`, `sha384`, or `sha512`; custom +algorithms use the explicit-name constructor. The concurrency contract is in +[section 1](#1-requirements), and publication semantics are in +[section 4](#4-storage-layout-and-publication). + +### 5.3 Entry bundles (optional) + +Entry bundles are application-owned because merklecpp receives precomputed leaf +hashes, not raw entries. Each full bundle stores 256 big-endian `uint16` +length-prefixed entries at `-256w/tile/entries/`. The application +defines the leaf derivation (for example, `leaf_hash = H(entry)`); merklecpp +stores the supplied leaf hash unchanged. + +## 6. Delivery plan + +This PR delivers phases 0 and 1 plus the standalone entry-bundle codec and +storage primitives. Later PRs deliver the remaining independently testable +phases; phases 1-3 need no further core changes, while phase 4 may add one +non-hashing accessor. + +| Phase | Scope | Key tests | +|---|---|---| +| 0. Scaffolding | Headers, PAL, namespace, geometry, aliases, and CMake test wiring | Public-header and build integration | +| 1. Coordinates/store | `TileRef`, index/path encoding, `TileStoreT`, durable atomic I/O, entry-bundle primitives | Encoding vectors; algorithm roots; 256-hash SHA-256/384 tiles; round trips; file/symlink collisions | +| 2. Writers | Incremental `TileWriterT::write_up_to` from `leaf_at`, roll-ups, and `EntryBundleWriterT`; full resources only | Sizes 256 and 70,000 produce the exact tile set; repeated writes preserve immutability | +| 3. Proof engine | `TileHashSource`, `mth_range`, roots, inclusion/consistency proofs, and verification | Tile roots equal tree roots; inclusion equals `path()` / `past_path()` and verifies; consistency reconciles `past_root()` values | +| 4. Combined tree | Optional `subtree_root`; memory/combined sources; `TiledTreeT` append, flush, proof, and compaction APIs | Prove flushed and resident leaves against a non-flushed reference; consistency across a flush boundary | +| 5. Documentation/performance | README usage, design link, and tile-backed benchmarks | Documentation and benchmark coverage | + +Deliverables are `merklecpp_tiles.h`, `merklecpp_pal.h`, `test/tiles_*.cpp`, +CMake wiring, the optional core accessor, and README/design updates. From 969ae209bf0fbafbe588b047b4159b78d7548f2e Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 20:46:51 +0000 Subject: [PATCH 10/22] Simplify platform portability handling Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/design/tlog-tiles.md | 5 ++--- merklecpp.h | 1 + merklecpp_pal.h | 26 +++++++------------------- test/CMakeLists.txt | 4 ---- test/windows_include_order.cpp | 11 ----------- 5 files changed, 10 insertions(+), 37 deletions(-) delete mode 100644 test/windows_include_order.cpp diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 120a026..5a8b4b7 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -158,9 +158,8 @@ hashes, not bytes. ### Publication guarantees -- Create a unique temporary file exclusively, sync its contents, and atomically - replace the destination. Apple uses `F_FULLFSYNC` when supported and falls - back to `fsync`. +- Create a unique temporary file exclusively, sync its contents with `fsync`, + and atomically replace the destination. - On POSIX, sync each newly created directory link and the destination directory after rename. A higher-level retry rechecks the directory chain and destination even when the file is already visible. diff --git a/merklecpp.h b/merklecpp.h index 4a843fe..5060bd0 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -714,6 +714,7 @@ namespace merkle if (left && right) { size = left->size + right->size + 1; + // Parentheses prevent expansion of the Windows max macro. height = (std::max)(left->height, right->height) + 1; } else diff --git a/merklecpp_pal.h b/merklecpp_pal.h index 2194bc5..8ddde7f 100644 --- a/merklecpp_pal.h +++ b/merklecpp_pal.h @@ -19,9 +19,6 @@ #include #ifdef _WIN32 -# ifndef NOMINMAX -# define NOMINMAX -# endif # include #else # include @@ -91,18 +88,6 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) return result; } - static inline int sync_file_on_disk(int fd) - { -# ifdef F_FULLFSYNC - const int result = - retry_on_eintr([fd]() { return ::fcntl(fd, F_FULLFSYNC); }); - if (result == 0 || (errno != ENOTSUP && errno != EINVAL)) - { - return result; - } -# endif - return retry_on_eintr([fd]() { return ::fsync(fd); }); - } } #endif @@ -145,13 +130,16 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) bool close_handle = true; try { + // Parentheses prevent expansion of the Windows max macro. + constexpr auto max_write_size = + static_cast((std::numeric_limits::max)()); size_t written = 0; while (written < bytes.size()) { const auto remaining = bytes.size() - written; - const auto chunk = static_cast(std::min( - remaining, - static_cast((std::numeric_limits::max)()))); + // Parentheses prevent expansion of the Windows min macro. + const auto chunk = + static_cast((std::min)(remaining, max_write_size)); DWORD done = 0; if (!WriteFile(handle, bytes.data() + written, chunk, &done, nullptr)) { @@ -217,7 +205,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) require_write_progress(static_cast(done), path); written += static_cast(done); } - if (detail::sync_file_on_disk(fd) != 0) + if (detail::retry_on_eintr([fd]() { return ::fsync(fd); }) != 0) { const auto error = last_system_error(); throw std::runtime_error(system_error_message( diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index dbc5b39..ea07319 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -31,10 +31,6 @@ add_merklecpp_test(serialise_to_file serialise_to_file.cpp) add_merklecpp_test(coverage coverage.cpp) add_merklecpp_test(tiles_store tiles_store.cpp) -if(WIN32) - add_merklecpp_test(windows_include_order windows_include_order.cpp) -endif() - if(TARGET evercrypt.host) add_merklecpp_test(compare_evercrypt compare_evercrypt.cpp) endif() diff --git a/test/windows_include_order.cpp b/test/windows_include_order.cpp deleted file mode 100644 index 9cd309a..0000000 --- a/test/windows_include_order.cpp +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -#include - -#include - -int main() -{ - return 0; -} From 325212e2ad8c15aefbe12395d74edc1377f8320c Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 21:07:58 +0000 Subject: [PATCH 11/22] Convert tile store tests to doctest Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- test/CMakeLists.txt | 2 +- test/tiles_store.cpp | 1146 +++++++++++++++++++----------------------- 2 files changed, 519 insertions(+), 629 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ea07319..3dd41e6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -29,7 +29,6 @@ add_merklecpp_test(serialisation serialisation.cpp) add_merklecpp_test(partial_serialisation partial_serialisation.cpp) add_merklecpp_test(serialise_to_file serialise_to_file.cpp) add_merklecpp_test(coverage coverage.cpp) -add_merklecpp_test(tiles_store tiles_store.cpp) if(TARGET evercrypt.host) add_merklecpp_test(compare_evercrypt compare_evercrypt.cpp) @@ -44,5 +43,6 @@ endif() find_path(DOCTEST_DIR doctest.h PATH_SUFFIXES doctest) if(DOCTEST_DIR) + add_merklecpp_test(tiles_store tiles_store.cpp) add_merklecpp_test(unit_tests unit_tests.cpp) endif() diff --git a/test/tiles_store.cpp b/test/tiles_store.cpp index 73f0228..7cd1f55 100644 --- a/test/tiles_store.cpp +++ b/test/tiles_store.cpp @@ -3,16 +3,16 @@ #include "util.h" +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN #include #include #include #include #include +#include #include #include #include -#include -#include #include #include #include @@ -28,52 +28,39 @@ namespace fs = std::filesystem; using merkle::Hash; using merkle::tiles::TileRef; -static void expect(bool cond, const std::string& what) +class TemporaryDirectory { - if (!cond) - { - throw std::runtime_error(std::format("check failed: {}", what)); - } -} +private: + fs::path path_; -static void expect_eq( - const std::string& got, const std::string& expected, const std::string& what) -{ - if (got != expected) +public: + TemporaryDirectory() { - throw std::runtime_error( - std::format("{}: got '{}', expected '{}'", what, got, expected)); + static std::atomic sequence = 0; + const auto nonce = + std::chrono::steady_clock::now().time_since_epoch().count(); + path_ = fs::temp_directory_path() / + std::format( + "merklecpp_tiles_{}_{}_{}", + merkle::pal::process_id(), + nonce, + sequence++); } -} -static void expect_throws( - const std::function& fn, const std::string& what) -{ - bool threw = false; - try - { - fn(); - } - catch (const std::runtime_error&) + ~TemporaryDirectory() { - threw = true; + std::error_code ec; + fs::remove_all(path_, ec); } - expect(threw, what); -} -static std::string expect_throws_message( - const std::function& fn, const std::string& what) -{ - try - { - fn(); - } - catch (const std::runtime_error& ex) + TemporaryDirectory(const TemporaryDirectory&) = delete; + TemporaryDirectory& operator=(const TemporaryDirectory&) = delete; + + [[nodiscard]] const fs::path& path() const { - return ex.what(); + return path_; } - throw std::runtime_error(std::format("check failed: {}", what)); -} +}; static void custom_hash(const Hash& lhs, const Hash& rhs, Hash& out) { @@ -158,7 +145,6 @@ class FaultInjectingTileStore : public merkle::tiles::TileStore } }; -// Overwrites a file with exactly the given bytes (to simulate corruption). static void overwrite_file(const fs::path& p, const std::vector& bytes) { std::ofstream f(p, std::ios::binary | std::ios::trunc); @@ -177,631 +163,535 @@ static std::vector read_file(const fs::path& p) return {std::istreambuf_iterator(f), std::istreambuf_iterator()}; } -int main() +static std::vector> make_entries() { - const auto nonce = - std::chrono::steady_clock::now().time_since_epoch().count(); - const fs::path dir = fs::temp_directory_path() / - std::format("merklecpp_tiles_{}_{}", merkle::pal::process_id(), nonce); + std::vector> entries(merkle::tiles::TILE_WIDTH); + entries[1] = {0xA5}; + entries[2] = std::vector(256, 0x5A); + for (size_t i = 3; i < entries.size(); i++) + { + entries[i] = {(uint8_t)i, 0x7F}; + } + return entries; +} - try +TEST_CASE("Tile geometry, references, and index encoding") +{ + // 1. Tile geometry, references, and index encoding. + CHECK(merkle::tiles::TILE_WIDTH == (1U << merkle::tiles::TILE_HEIGHT)); + + const TileRef default_ref; + CHECK(default_ref.level == 0); + CHECK(default_ref.index == 0); + + size_t prefix_probes = 0; + const auto prefix_length = + merkle::tiles::detail::contiguous_prefix_length(10, [&](uint64_t index) { + prefix_probes++; + return index < 3; + }); + CHECK(prefix_length == 3); + CHECK(prefix_probes == 4); + + prefix_probes = 0; + const auto empty_prefix_length = + merkle::tiles::detail::contiguous_prefix_length(0, [&](uint64_t) { + prefix_probes++; + return true; + }); + CHECK(empty_prefix_length == 0); + CHECK(prefix_probes == 0); + + CHECK(merkle::tiles::encode_tile_index(0) == "000"); + CHECK(merkle::tiles::encode_tile_index(5) == "005"); + CHECK(merkle::tiles::encode_tile_index(255) == "255"); + CHECK(merkle::tiles::encode_tile_index(999) == "999"); + CHECK(merkle::tiles::encode_tile_index(1000) == "x001/000"); + CHECK(merkle::tiles::encode_tile_index(1234067) == "x001/x234/067"); + CHECK( + merkle::tiles::encode_tile_index(std::numeric_limits::max()) == + "x018/x446/x744/x073/x709/x551/615"); + CHECK(merkle::tiles::TileStore::encode_index(1000) == "x001/000"); +} + +TEST_CASE("Tile store paths and hash namespaces") +{ + // 2. Resource path layout (full tiles and bundles only). + TemporaryDirectory temporary_directory; + const fs::path& dir = temporary_directory.path(); + merkle::tiles::TileStore store(dir); + + CHECK(store.root().lexically_relative(dir).generic_string() == "sha256-256w"); + CHECK(rel(store, store.tile_path(TileRef{0, 0})) == "tile/0/000"); + CHECK( + rel(store, store.tile_path(TileRef{1, 1234067})) == "tile/1/x001/x234/067"); + CHECK( + rel(store, store.tile_path(TileRef{merkle::tiles::MAX_TILE_LEVEL, 0})) == + "tile/63/000"); + CHECK_THROWS_AS( + (store.tile_path( + TileRef{static_cast(merkle::tiles::MAX_TILE_LEVEL + 1), 0})), + std::runtime_error); + CHECK(rel(store, store.entries_path(5)) == "tile/entries/005"); + CHECK( + merkle::tiles::TileStore::storage_directory_name("sha384") == + "sha384-256w"); + CHECK( + merkle::tiles::TileStore::storage_directory_name("sha3-256") == + "sha3-256-256w"); + + for (const std::string& invalid_name : + {"", "-sha256", "sha256-", "SHA256", "sha_256", "sha/256"}) { - // 1. Tile geometry, references, and index encoding. - expect( - merkle::tiles::TILE_WIDTH == (1U << merkle::tiles::TILE_HEIGHT), - "tile width matches tile height"); - const TileRef default_ref; - expect( - default_ref.level == 0 && default_ref.index == 0, - "TileRef defaults to the first leaf tile"); - - size_t prefix_probes = 0; - const auto prefix_length = - merkle::tiles::detail::contiguous_prefix_length(10, [&](uint64_t index) { - prefix_probes++; - return index < 3; - }); - expect( - prefix_length == 3 && prefix_probes == 4, - "contiguous prefix stops at the first gap"); - prefix_probes = 0; - const auto empty_prefix_length = - merkle::tiles::detail::contiguous_prefix_length(0, [&](uint64_t) { - prefix_probes++; - return true; - }); - expect( - empty_prefix_length == 0 && prefix_probes == 0, - "empty contiguous prefix performs no probes"); - - expect_eq(merkle::tiles::encode_tile_index(0), "000", "encode 0"); - expect_eq(merkle::tiles::encode_tile_index(5), "005", "encode 5"); - expect_eq(merkle::tiles::encode_tile_index(255), "255", "encode 255"); - expect_eq(merkle::tiles::encode_tile_index(999), "999", "encode 999"); - expect_eq( - merkle::tiles::encode_tile_index(1000), "x001/000", "encode 1000"); - expect_eq( - merkle::tiles::encode_tile_index(1234067), - "x001/x234/067", - "encode 1234067"); - expect_eq( - merkle::tiles::encode_tile_index(std::numeric_limits::max()), - "x018/x446/x744/x073/x709/x551/615", - "encode uint64 max"); - expect_eq( - merkle::tiles::TileStore::encode_index(1000), - "x001/000", - "store index encoder"); - - merkle::tiles::TileStore store(dir); - const size_t hsz = Hash().size(); - const auto full = make_hashes(merkle::tiles::TILE_WIDTH); - - // 2. Resource path layout (full tiles and bundles only). - expect_eq( - store.root().lexically_relative(dir).generic_string(), - "sha256-256w", - "SHA256 storage directory"); - expect_eq( - rel(store, store.tile_path(TileRef{0, 0})), - "tile/0/000", - "tile_path L0 N0"); - expect_eq( - rel(store, store.tile_path(TileRef{1, 1234067})), - "tile/1/x001/x234/067", - "tile_path L1 big index"); - expect_eq( - rel( - store, - store.tile_path(TileRef{merkle::tiles::MAX_TILE_LEVEL, 0})), - "tile/63/000", - "maximum tile level"); - expect_throws( - [&] { - (void)store.tile_path(TileRef{ - static_cast(merkle::tiles::MAX_TILE_LEVEL + 1), 0}); - }, - "tile level above 63 rejected"); - expect_eq( - rel(store, store.entries_path(5)), "tile/entries/005", "entries full"); - expect_eq( - merkle::tiles::TileStore::storage_directory_name("sha384"), - "sha384-256w", - "SHA384 retains 256-hash tile width"); - expect_eq( - merkle::tiles::TileStore::storage_directory_name("sha3-256"), - "sha3-256-256w", - "custom algorithm storage directory"); - for (const std::string& invalid_name : - {"", "-sha256", "sha256-", "SHA256", "sha_256", "sha/256"}) - { - expect_throws( - [&] { - (void)merkle::tiles::TileStore::storage_directory_name(invalid_name); - }, - "invalid algorithm short name rejected"); - } - for (const std::string& mismatched_name : {"sha384", "sha512"}) - { - expect_throws( - [&] { (void)merkle::tiles::TileStore(dir, mismatched_name); }, - "algorithm short name must match hash output size"); - } + CAPTURE(invalid_name); + CHECK_THROWS_AS( + merkle::tiles::TileStore::storage_directory_name(invalid_name), + std::runtime_error); + } + for (const std::string& mismatched_name : {"sha384", "sha512"}) + { + CAPTURE(mismatched_name); + CHECK_THROWS_AS( + (merkle::tiles::TileStore(dir, mismatched_name)), std::runtime_error); + } - using CustomStore = - merkle::tiles::TileStoreT; - expect_throws( - [&] { (void)CustomStore(dir); }, - "custom hash requires an explicit algorithm name"); - const CustomStore custom_store(dir, "custom-hash"); - expect_eq( - custom_store.root().lexically_relative(dir).generic_string(), - "custom-hash-256w", - "explicit custom hash storage directory"); + using CustomStore = + merkle::tiles::TileStoreT; + CHECK_THROWS_AS((void)CustomStore(dir), std::runtime_error); + const CustomStore custom_store(dir, "custom-hash"); + CHECK( + custom_store.root().lexically_relative(dir).generic_string() == + "custom-hash-256w"); #ifdef HAVE_OPENSSL - { - using TileStore256 = merkle::tiles:: - TileStoreT; - const TileStore256 store256(dir); - expect_eq( - store256.root().lexically_relative(dir).generic_string(), - "sha256-256w", - "OpenSSL SHA256 storage directory"); - - using TileStore384 = merkle::tiles::TileStoreT< - merkle::Tree384::Hash::size_bytes, - merkle::Tree384::hash_function>; - for (const std::string& mismatched_name : {"sha256", "sha512"}) - { - expect_throws( - [&] { (void)TileStore384(dir, mismatched_name); }, - "SHA384 store rejects mismatched algorithm names"); - } - TileStore384 store384(dir); - const auto full384 = make_hashesT( - merkle::tiles::TILE_WIDTH); - expect_eq( - store384.root().lexically_relative(dir).generic_string(), - "sha384-256w", - "SHA384 storage directory"); - store384.write_tile(TileRef{0, 0}, full384); - expect( - fs::file_size(store384.tile_path(TileRef{0, 0})) == - (uintmax_t)merkle::tiles::TILE_WIDTH * - merkle::Tree384::Hash::size_bytes, - "SHA384 full tile remains 256 hashes wide"); - expect( - store384.has_full_tile(0, 0) && - store384.read_tile(TileRef{0, 0}) == full384, - "SHA384 tile round-trip"); - - using TileStore512 = merkle::tiles::TileStoreT< - merkle::Tree512::Hash::size_bytes, - merkle::Tree512::hash_function>; - for (const std::string& mismatched_name : {"sha256", "sha384"}) - { - expect_throws( - [&] { (void)TileStore512(dir, mismatched_name); }, - "SHA512 store rejects mismatched algorithm names"); - } - TileStore512 store512(dir); - const auto full512 = make_hashesT( - merkle::tiles::TILE_WIDTH); - expect_eq( - store512.root().lexically_relative(dir).generic_string(), - "sha512-256w", - "SHA512 storage directory"); - store512.write_tile(TileRef{0, 0}, full512); - expect( - fs::file_size(store512.tile_path(TileRef{0, 0})) == - (uintmax_t)merkle::tiles::TILE_WIDTH * - merkle::Tree512::Hash::size_bytes, - "SHA512 full tile remains 256 hashes wide"); - expect( - store512.has_full_tile(0, 0) && - store512.read_tile(TileRef{0, 0}) == full512, - "SHA512 tile round-trip"); - } + using TileStore256 = + merkle::tiles::TileStoreT; + const TileStore256 store256(dir); + CHECK( + store256.root().lexically_relative(dir).generic_string() == "sha256-256w"); + + using TileStore384 = merkle::tiles::TileStoreT< + merkle::Tree384::Hash::size_bytes, + merkle::Tree384::hash_function>; + for (const std::string& mismatched_name : {"sha256", "sha512"}) + { + CAPTURE(mismatched_name); + CHECK_THROWS_AS((TileStore384(dir, mismatched_name)), std::runtime_error); + } + TileStore384 store384(dir); + const auto full384 = + make_hashesT(merkle::tiles::TILE_WIDTH); + CHECK( + store384.root().lexically_relative(dir).generic_string() == "sha384-256w"); + store384.write_tile(TileRef{0, 0}, full384); + CHECK( + fs::file_size(store384.tile_path(TileRef{0, 0})) == + (uintmax_t)merkle::tiles::TILE_WIDTH * merkle::Tree384::Hash::size_bytes); + CHECK(store384.has_full_tile(0, 0)); + CHECK(store384.read_tile(TileRef{0, 0}) == full384); + + using TileStore512 = merkle::tiles::TileStoreT< + merkle::Tree512::Hash::size_bytes, + merkle::Tree512::hash_function>; + for (const std::string& mismatched_name : {"sha256", "sha384"}) + { + CAPTURE(mismatched_name); + CHECK_THROWS_AS((TileStore512(dir, mismatched_name)), std::runtime_error); + } + TileStore512 store512(dir); + const auto full512 = + make_hashesT(merkle::tiles::TILE_WIDTH); + CHECK( + store512.root().lexically_relative(dir).generic_string() == "sha512-256w"); + store512.write_tile(TileRef{0, 0}, full512); + CHECK( + fs::file_size(store512.tile_path(TileRef{0, 0})) == + (uintmax_t)merkle::tiles::TILE_WIDTH * merkle::Tree512::Hash::size_bytes); + CHECK(store512.has_full_tile(0, 0)); + CHECK(store512.read_tile(TileRef{0, 0}) == full512); #endif +} - // 2b. Production writes sync each directory link and the destination - // directory in order. - { - const fs::path prefix = dir / "durable"; - const auto fault = std::make_shared(); - FaultInjectingTileStore durable_store(prefix, fault); - durable_store.write_tile(TileRef{1, 0}, full); - const std::vector expected = { - dir.parent_path(), - dir, - prefix, - prefix / "sha256-256w", - prefix / "sha256-256w" / "tile", - prefix / "sha256-256w" / "tile" / "1"}; - expect( - fault->calls.size() >= expected.size() && - std::equal( - expected.begin(), - expected.end(), - fault->calls.end() - (std::ptrdiff_t)expected.size()), - "directory parents and destination synced in order"); - } +TEST_CASE("Tile writes sync directory links in order") +{ + // 2b. Production writes sync each directory link and the destination + // directory in order. + TemporaryDirectory temporary_directory; + const fs::path& dir = temporary_directory.path(); + const fs::path prefix = dir / "durable"; + const auto fault = std::make_shared(); + FaultInjectingTileStore store(prefix, fault); + store.write_tile(TileRef{1, 0}, make_hashes(merkle::tiles::TILE_WIDTH)); + + const std::vector expected = { + dir.parent_path(), + dir, + prefix, + prefix / "sha256-256w", + prefix / "sha256-256w" / "tile", + prefix / "sha256-256w" / "tile" / "1"}; + REQUIRE(fault->calls.size() >= expected.size()); + CHECK(std::equal( + expected.begin(), + expected.end(), + fault->calls.end() - (std::ptrdiff_t)expected.size())); +} - // 2c. A failed directory-link sync is retried even when the directory - // created before the failure is already visible. - { - const fs::path prefix = dir / "directory_retry"; - const auto fault = std::make_shared(); - fault->fail_path = prefix; - fault->failures_remaining = 1; - FaultInjectingTileStore retry_store(prefix, fault); - const TileRef retry_ref{2, 7}; - expect_throws( - [&] { retry_store.write_tile(retry_ref, full); }, - "directory sync failure propagated"); - expect( - !retry_store.has_full_tile(retry_ref.level, retry_ref.index), - "tile not published before directory chain is durable"); - retry_store.write_tile(retry_ref, full); - expect( - retry_store.read_tile(retry_ref) == full, - "write succeeds after directory sync retry"); - expect( - fault->call_count(prefix) == 2, "failed directory link sync retried"); - } +TEST_CASE("Failed directory-link sync is retried") +{ + // 2c. A failed directory-link sync is retried even when the directory + // created before the failure is already visible. + TemporaryDirectory temporary_directory; + const fs::path prefix = temporary_directory.path() / "directory_retry"; + const auto fault = std::make_shared(); + fault->fail_path = prefix; + fault->failures_remaining = 1; + FaultInjectingTileStore store(prefix, fault); + const TileRef ref{2, 7}; + const auto full = make_hashes(merkle::tiles::TILE_WIDTH); + + CHECK_THROWS_AS((store.write_tile(ref, full)), std::runtime_error); + CHECK_FALSE(store.has_full_tile(ref.level, ref.index)); + store.write_tile(ref, full); + CHECK(store.read_tile(ref) == full); + CHECK(fault->call_count(prefix) == 2); +} - // 2d. A failure syncing the destination directory leaves a complete, - // visible file that can be re-confirmed on the next write attempt. - { - const fs::path prefix = dir / "publication_retry"; - const auto fault = std::make_shared(); - FaultInjectingTileStore retry_store(prefix, fault); - const TileRef retry_ref{3, 9}; - const auto tile_directory = - retry_store.tile_path(retry_ref).parent_path(); - fault->fail_path = tile_directory; - fault->failures_remaining = 1; - expect_throws( - [&] { retry_store.write_tile(retry_ref, full); }, - "publication directory sync failure propagated"); - expect( - retry_store.has_full_tile(retry_ref.level, retry_ref.index) && - retry_store.read_tile(retry_ref) == full, - "complete tile remains visible after publication sync failure"); - expect( - !any_tmp_files(prefix), - "publication sync failure leaves no temporary file"); - - const size_t calls_before_confirm = fault->call_count(tile_directory); - retry_store.begin_attempt(); - expect( - retry_store.confirm_tile(retry_ref.level, retry_ref.index), - "visible tile durability re-confirmed"); - expect( - fault->call_count(tile_directory) == calls_before_confirm + 1, - "destination directory sync retried"); - expect( - retry_store.confirm_tile(retry_ref.level, retry_ref.index), - "confirmed tile remains reusable"); - expect( - fault->call_count(tile_directory) == calls_before_confirm + 1, - "directory sync cached within one attempt"); - - retry_store.begin_attempt(); - expect( - !retry_store.confirm_tile(retry_ref.level, retry_ref.index + 1), - "missing tile cannot be confirmed"); - } +TEST_CASE("Visible publication can be confirmed after sync failure") +{ + // 2d. A failure syncing the destination directory leaves a complete, + // visible file that can be re-confirmed on the next write attempt. + TemporaryDirectory temporary_directory; + const fs::path prefix = temporary_directory.path() / "publication_retry"; + const auto fault = std::make_shared(); + FaultInjectingTileStore store(prefix, fault); + const TileRef ref{3, 9}; + const auto full = make_hashes(merkle::tiles::TILE_WIDTH); + const auto tile_directory = store.tile_path(ref).parent_path(); + fault->fail_path = tile_directory; + fault->failures_remaining = 1; + + CHECK_THROWS_AS((store.write_tile(ref, full)), std::runtime_error); + CHECK(store.has_full_tile(ref.level, ref.index)); + CHECK(store.read_tile(ref) == full); + CHECK_FALSE(any_tmp_files(prefix)); + + const size_t calls_before_confirm = fault->call_count(tile_directory); + store.begin_attempt(); + CHECK(store.confirm_tile(ref.level, ref.index)); + CHECK(fault->call_count(tile_directory) == calls_before_confirm + 1); + CHECK(store.confirm_tile(ref.level, ref.index)); + CHECK(fault->call_count(tile_directory) == calls_before_confirm + 1); + + store.begin_attempt(); + CHECK_FALSE(store.confirm_tile(ref.level, ref.index + 1)); +} - // 2e. Failures before atomic replacement clean up temporary files, and a - // non-directory in the destination path is rejected. - { - const fs::path prefix = dir / "replacement_failure"; - merkle::tiles::TileStore failure_store(prefix); - const TileRef blocked_ref{0, 1}; - fs::create_directories(failure_store.tile_path(blocked_ref)); - expect_throws( - [&] { failure_store.write_tile(blocked_ref, full); }, - "replacement of a directory rejected"); - expect( - !any_tmp_files(prefix), - "failed atomic replacement cleans up temporary file"); - } - { - const fs::path prefix = dir / "directory_conflict"; - fs::create_directories(prefix); - overwrite_file(prefix / "sha256-256w", {0x00}); - merkle::tiles::TileStore failure_store(prefix); - expect_throws( - [&] { failure_store.write_tile(TileRef{0, 0}, full); }, - "non-directory storage root rejected"); - expect( - !any_tmp_files(prefix), - "directory creation failure leaves no temporary file"); - } +TEST_CASE("Replacement and directory conflicts clean up temporary files") +{ + // 2e. Failures before atomic replacement clean up temporary files, and a + // non-directory in the destination path is rejected. + TemporaryDirectory temporary_directory; + const fs::path& dir = temporary_directory.path(); + const auto full = make_hashes(merkle::tiles::TILE_WIDTH); + + const fs::path replacement_prefix = dir / "replacement_failure"; + merkle::tiles::TileStore replacement_store(replacement_prefix); + const TileRef blocked_ref{0, 1}; + fs::create_directories(replacement_store.tile_path(blocked_ref)); + CHECK_THROWS_AS( + (replacement_store.write_tile(blocked_ref, full)), std::runtime_error); + CHECK_FALSE(any_tmp_files(replacement_prefix)); + + const fs::path conflict_prefix = dir / "directory_conflict"; + fs::create_directories(conflict_prefix); + overwrite_file(conflict_prefix / "sha256-256w", {0x00}); + merkle::tiles::TileStore conflict_store(conflict_prefix); + CHECK_THROWS_AS( + (conflict_store.write_tile(TileRef{0, 0}, full)), std::runtime_error); + CHECK_FALSE(any_tmp_files(conflict_prefix)); +} - // 2f. Unique temporary-file creation must fail rather than overwrite an - // existing path, including a pre-created symlink on POSIX. - { - const fs::path existing_path = dir / "preexisting-temp"; - const std::vector original = {0x11, 0x22, 0x33}; - overwrite_file(existing_path, original); - const auto collision_error = expect_throws_message( - [&] { merkle::pal::write_and_sync_file(existing_path, {0xAA}); }, - "exclusive temporary-file creation rejects collisions"); - expect( - collision_error.find(existing_path.string()) != std::string::npos, - "temporary-file collision identifies the path"); +TEST_CASE("Exclusive file creation rejects file and symlink collisions") +{ + // 2f. Unique temporary-file creation must fail rather than overwrite an + // existing path, including a pre-created symlink on POSIX. + TemporaryDirectory temporary_directory; + const fs::path& dir = temporary_directory.path(); + fs::create_directories(dir); + const fs::path existing_path = dir / "preexisting-temp"; + const std::vector original = {0x11, 0x22, 0x33}; + overwrite_file(existing_path, original); + + std::string collision_error; + try + { + merkle::pal::write_and_sync_file(existing_path, {0xAA}); + } + catch (const std::runtime_error& ex) + { + collision_error = ex.what(); + } + REQUIRE_FALSE(collision_error.empty()); + CHECK(collision_error.find(existing_path.string()) != std::string::npos); #ifdef _WIN32 - expect( - collision_error.find(": error ") != std::string::npos, - "temporary-file collision includes the Windows system error"); + CHECK(collision_error.find(": error ") != std::string::npos); #endif - expect( - read_file(existing_path) == original, - "temporary-file collision does not clobber existing content"); + CHECK(read_file(existing_path) == original); + #ifndef _WIN32 - const fs::path symlink_path = dir / "preexisting-temp-symlink"; - fs::create_symlink(existing_path, symlink_path); - expect_throws( - [&] { merkle::pal::write_and_sync_file(symlink_path, {0xBB}); }, - "exclusive temporary-file creation rejects symlinks"); - expect( - read_file(existing_path) == original, - "temporary-file symlink does not clobber its target"); + const fs::path symlink_path = dir / "preexisting-temp-symlink"; + fs::create_symlink(existing_path, symlink_path); + CHECK_THROWS_AS( + (merkle::pal::write_and_sync_file(symlink_path, {0xBB})), + std::runtime_error); + CHECK(read_file(existing_path) == original); #endif - } +} - // 2g. A successful write must make progress. This shared guard prevents - // the Windows WriteFile loop from spinning if it reports zero bytes. - { - expect_throws( - [] { merkle::pal::require_write_progress(0, "test"); }, - "zero-byte write rejected"); - merkle::pal::require_write_progress(1, "test"); - } +TEST_CASE("Successful writes must make progress") +{ + // 2g. A successful write must make progress. This shared guard prevents + // the Windows WriteFile loop from spinning if it reports zero bytes. + CHECK_THROWS_AS( + merkle::pal::require_write_progress(0, "test"), std::runtime_error); + CHECK_NOTHROW(merkle::pal::require_write_progress(1, "test")); +} #ifndef _WIN32 - // 2h. Interrupted sync operations are retried, while other failures are - // returned immediately with errno intact. +TEST_CASE("Interrupted sync operations are retried") +{ + // 2h. Interrupted sync operations are retried, while other failures are + // returned immediately with errno intact. + size_t attempts = 0; + const int retry_result = merkle::pal::detail::retry_on_eintr([&]() { + attempts++; + if (attempts == 1) { - size_t attempts = 0; - const int retry_result = merkle::pal::detail::retry_on_eintr([&]() { - attempts++; - if (attempts == 1) - { - errno = EINTR; - return -1; - } - return 0; - }); - expect( - retry_result == 0 && attempts == 2, - "interrupted sync operation retried"); - - attempts = 0; - const int error_result = merkle::pal::detail::retry_on_eintr([&]() { - attempts++; - errno = EIO; - return -1; - }); - expect( - error_result == -1 && attempts == 1 && errno == EIO, - "non-interrupted sync failure preserved"); + errno = EINTR; + return -1; } + return 0; + }); + CHECK(retry_result == 0); + CHECK(attempts == 2); + + attempts = 0; + const int error_result = merkle::pal::detail::retry_on_eintr([&]() { + attempts++; + errno = EIO; + return -1; + }); + CHECK(error_result == -1); + CHECK(attempts == 1); + CHECK(errno == EIO); +} #endif - // 3a. Full tile byte round-trip. - const TileRef full_ref{0, 0}; - store.write_tile(full_ref, full); - expect(store.has_full_tile(0, 0), "has_full_tile after write"); - expect(!store.has_full_tile(0, 5), "missing full tile"); - expect_throws( - [&] { (void)store.read_tile(TileRef{0, 5}); }, - "missing tile read rejected"); - expect( - fs::file_size(store.tile_path(full_ref)) == - (uintmax_t)merkle::tiles::TILE_WIDTH * hsz, - "full tile file size"); - std::vector expected_tile_bytes; - expected_tile_bytes.reserve(full.size() * hsz); - for (const auto& hash : full) - { - hash.serialise(expected_tile_bytes); - } - expect( - read_file(store.tile_path(full_ref)) == expected_tile_bytes, - "tile file contains concatenated raw hashes"); - const auto full_rt = store.read_tile(full_ref); - expect(full_rt.size() == full.size(), "full tile width round-trip"); - for (size_t i = 0; i < full.size(); i++) - { - expect(full_rt[i] == full[i], "full tile hash round-trip"); - } +TEST_CASE("Full tiles use raw bytes and round-trip") +{ + // 3a. Full tile byte round-trip. + TemporaryDirectory temporary_directory; + merkle::tiles::TileStore store(temporary_directory.path()); + const size_t hash_size = Hash().size(); + const auto full = make_hashes(merkle::tiles::TILE_WIDTH); + const TileRef full_ref{0, 0}; + + store.write_tile(full_ref, full); + CHECK(store.has_full_tile(0, 0)); + CHECK_FALSE(store.has_full_tile(0, 5)); + CHECK_THROWS_AS((store.read_tile(TileRef{0, 5})), std::runtime_error); + CHECK( + fs::file_size(store.tile_path(full_ref)) == + (uintmax_t)merkle::tiles::TILE_WIDTH * hash_size); + + std::vector expected_tile_bytes; + expected_tile_bytes.reserve(full.size() * hash_size); + for (const auto& hash : full) + { + hash.serialise(expected_tile_bytes); + } + CHECK(read_file(store.tile_path(full_ref)) == expected_tile_bytes); - // 3b. Wrong-width writes are rejected (only 256-wide tiles are valid). - const std::vector three(full.begin(), full.begin() + 3); - expect_throws( - [&] { store.write_tile(TileRef{0, 2}, three); }, - "width mismatch rejected"); - - // 4. Entry bundles preserve empty, short, and multi-byte-length entries, - // enforce their width and uint16 length limits, and can be re-confirmed. - std::vector> entries(merkle::tiles::TILE_WIDTH); - entries[1] = {0xA5}; - entries[2] = std::vector(256, 0x5A); - for (size_t i = 3; i < entries.size(); i++) - { - entries[i] = {(uint8_t)i, 0x7F}; - } - const auto encoded_entries = - merkle::tiles::TileStore::encode_entries(entries); - expect( - encoded_entries.size() > 7 && encoded_entries[0] == 0x00 && - encoded_entries[1] == 0x00 && encoded_entries[2] == 0x00 && - encoded_entries[3] == 0x01 && encoded_entries[4] == 0xA5 && - encoded_entries[5] == 0x01 && encoded_entries[6] == 0x00, - "entry lengths use big-endian uint16 prefixes"); - expect( - merkle::tiles::TileStore::decode_entries( - encoded_entries, merkle::tiles::TILE_WIDTH) == entries, - "entry codec round-trip"); - expect( - merkle::tiles::TileStore::decode_entries({}, 0).empty(), - "empty entry sequence round-trip"); - expect_throws( - [] { - (void)merkle::tiles::TileStore::decode_entries( - {}, std::numeric_limits::max()); - }, - "impossible entry count rejected before allocation"); - - const std::vector maximum_entry(0xFFFF, 0xC3); - const auto maximum_encoded = - merkle::tiles::TileStore::encode_entries({maximum_entry}); - const auto maximum_decoded = - merkle::tiles::TileStore::decode_entries(maximum_encoded, 1); - expect( - maximum_encoded.size() == maximum_entry.size() + 2 && - maximum_encoded[0] == 0xFF && maximum_encoded[1] == 0xFF && - maximum_decoded.size() == 1 && maximum_decoded[0] == maximum_entry, - "maximum uint16-sized entry round-trip"); - expect_throws( - [] { - (void)merkle::tiles::TileStore::encode_entries( - {std::vector(0x10000)}); - }, - "oversized entry rejected"); - - expect(!store.has_entry_bundle(17), "missing entry bundle"); - expect_throws( - [&] { (void)store.read_entry_bundle(17); }, - "missing entry bundle read rejected"); - const std::vector> short_bundle( - entries.begin(), entries.end() - 1); - expect_throws( - [&] { store.write_entry_bundle(1, short_bundle); }, - "short entry bundle rejected"); - auto oversized_bundle = entries; - oversized_bundle[0].resize(0x10000); - expect_throws( - [&] { store.write_entry_bundle(2, oversized_bundle); }, - "entry bundle containing oversized entry rejected"); - expect( - !fs::exists(store.entries_path(1)) && !fs::exists(store.entries_path(2)), - "invalid entry bundles are not published"); - - store.write_entry_bundle(0, entries); - expect(store.has_entry_bundle(0), "full entry bundle exists"); - expect( - store.read_entry_bundle(0) == entries, - "full entry bundle byte round-trip"); - expect( - fs::file_size(store.entries_path(0)) == encoded_entries.size(), - "entry bundle file size"); - expect( - read_file(store.entries_path(0)) == encoded_entries, - "entry bundle file uses the encoded wire format"); + const auto full_roundtrip = store.read_tile(full_ref); + REQUIRE(full_roundtrip.size() == full.size()); + for (size_t i = 0; i < full.size(); i++) + { + CAPTURE(i); + CHECK(full_roundtrip[i] == full[i]); + } - { - const fs::path prefix = dir / "bundle_confirmation"; - const auto fault = std::make_shared(); - FaultInjectingTileStore confirming_store(prefix, fault); - confirming_store.write_entry_bundle(4, entries); - const auto entries_directory = - confirming_store.entries_path(4).parent_path(); - const size_t calls_before_confirm = fault->call_count(entries_directory); - confirming_store.begin_attempt(); - expect( - confirming_store.confirm_bundle(4), - "visible entry bundle durability confirmed"); - expect( - fault->call_count(entries_directory) == calls_before_confirm + 1, - "entry bundle directory synced for new attempt"); - expect( - confirming_store.confirm_bundle(4), - "confirmed entry bundle remains reusable"); - expect( - fault->call_count(entries_directory) == calls_before_confirm + 1, - "entry bundle sync cached within one attempt"); - confirming_store.begin_attempt(); - expect( - !confirming_store.confirm_bundle(5), - "missing entry bundle cannot be confirmed"); - } + // 3b. Wrong-width writes are rejected (only 256-wide tiles are valid). + const std::vector three(full.begin(), full.begin() + 3); + CHECK_THROWS_AS((store.write_tile(TileRef{0, 2}, three)), std::runtime_error); +} - // 5. Corrupt / truncated files are rejected on read (integrity check), so - // a torn write can never be served as a valid tile or bundle. - { - // Truncated tile: fewer bytes than a full tile. - overwrite_file(store.tile_path(full_ref), std::vector(hsz, 0)); - expect(!store.has_full_tile(0, 0), "truncated tile is not durable"); - expect_throws( - [&] { (void)store.read_tile(full_ref); }, "truncated tile rejected"); - store.write_tile(full_ref, full); - expect(store.has_full_tile(0, 0), "truncated tile rewritten"); - expect(store.read_tile(full_ref) == full, "rewritten tile round-trip"); - - // Oversized tile: more bytes than a full tile. - overwrite_file( - store.tile_path(full_ref), - std::vector((merkle::tiles::TILE_WIDTH + 1) * hsz, 0)); - expect(!store.has_full_tile(0, 0), "oversized tile is not durable"); - expect_throws( - [&] { (void)store.read_tile(full_ref); }, "oversized tile rejected"); - - // A valid full entry bundle whose file is then cut short. - store.write_entry_bundle(0, entries); - expect(store.has_entry_bundle(0), "bundle durable before truncation"); - expect( - store.read_entry_bundle(0) == entries, - "bundle valid before truncation"); - // Claims a 5-byte entry but supplies only one trailing byte. - overwrite_file(store.entries_path(0), {0x00, 0x05, 0x01}); - expect(!store.has_entry_bundle(0), "truncated bundle is not durable"); - expect_throws( - [&] { (void)store.read_entry_bundle(0); }, - "truncated entry bundle rejected"); - - // A syntactically complete bundle with trailing bytes is not durable. - auto encoded = merkle::tiles::TileStore::encode_entries(entries); - encoded.push_back(0xFF); - overwrite_file(store.entries_path(0), encoded); - expect(!store.has_entry_bundle(0), "trailing bundle is not durable"); - expect_throws( - [&] { (void)store.read_entry_bundle(0); }, - "trailing entry bundle rejected"); - - // decode_entries directly: a length prefix that overruns the buffer. - expect_throws( - [] { - (void)merkle::tiles::TileStore::decode_entries({0xFF, 0xFF, 0x00}, 1); - }, - "decode_entries oversized length rejected"); - expect_throws( - [] { (void)merkle::tiles::TileStore::decode_entries({}, 1); }, - "decode_entries missing length prefix rejected"); - expect_throws( - [] { - (void)merkle::tiles::TileStore::decode_entries({0x00, 0x00, 0xFF}, 1); - }, - "decode_entries trailing bytes rejected"); - } +TEST_CASE("Entry encoding enforces bounds and round-trips") +{ + // 4. Entry bundles preserve empty, short, and multi-byte-length entries, + // enforce their width and uint16 length limits, and can be re-confirmed. + const auto entries = make_entries(); + const auto encoded_entries = + merkle::tiles::TileStore::encode_entries(entries); + REQUIRE(encoded_entries.size() > 7); + CHECK(encoded_entries[0] == 0x00); + CHECK(encoded_entries[1] == 0x00); + CHECK(encoded_entries[2] == 0x00); + CHECK(encoded_entries[3] == 0x01); + CHECK(encoded_entries[4] == 0xA5); + CHECK(encoded_entries[5] == 0x01); + CHECK(encoded_entries[6] == 0x00); + CHECK( + merkle::tiles::TileStore::decode_entries( + encoded_entries, merkle::tiles::TILE_WIDTH) == entries); + CHECK(merkle::tiles::TileStore::decode_entries({}, 0).empty()); + CHECK_THROWS_AS( + (merkle::tiles::TileStore::decode_entries( + {}, std::numeric_limits::max())), + std::runtime_error); + + const std::vector maximum_entry(0xFFFF, 0xC3); + const auto maximum_encoded = + merkle::tiles::TileStore::encode_entries({maximum_entry}); + const auto maximum_decoded = + merkle::tiles::TileStore::decode_entries(maximum_encoded, 1); + CHECK(maximum_encoded.size() == maximum_entry.size() + 2); + CHECK(maximum_encoded[0] == 0xFF); + CHECK(maximum_encoded[1] == 0xFF); + REQUIRE(maximum_decoded.size() == 1); + CHECK(maximum_decoded[0] == maximum_entry); + CHECK_THROWS_AS( + (merkle::tiles::TileStore::encode_entries({std::vector(0x10000)})), + std::runtime_error); +} + +TEST_CASE("Entry bundle storage validates and confirms publications") +{ + TemporaryDirectory temporary_directory; + const fs::path& dir = temporary_directory.path(); + merkle::tiles::TileStore store(dir); + const auto entries = make_entries(); + const auto encoded_entries = + merkle::tiles::TileStore::encode_entries(entries); + + CHECK_FALSE(store.has_entry_bundle(17)); + CHECK_THROWS_AS(store.read_entry_bundle(17), std::runtime_error); + const std::vector> short_bundle( + entries.begin(), entries.end() - 1); + CHECK_THROWS_AS( + (store.write_entry_bundle(1, short_bundle)), std::runtime_error); + auto oversized_bundle = entries; + oversized_bundle[0].resize(0x10000); + CHECK_THROWS_AS( + (store.write_entry_bundle(2, oversized_bundle)), std::runtime_error); + CHECK_FALSE(fs::exists(store.entries_path(1))); + CHECK_FALSE(fs::exists(store.entries_path(2))); + + store.write_entry_bundle(0, entries); + CHECK(store.has_entry_bundle(0)); + CHECK(store.read_entry_bundle(0) == entries); + CHECK(fs::file_size(store.entries_path(0)) == encoded_entries.size()); + CHECK(read_file(store.entries_path(0)) == encoded_entries); + + const fs::path prefix = dir / "bundle_confirmation"; + const auto fault = std::make_shared(); + FaultInjectingTileStore confirming_store(prefix, fault); + confirming_store.write_entry_bundle(4, entries); + const auto entries_directory = confirming_store.entries_path(4).parent_path(); + const size_t calls_before_confirm = fault->call_count(entries_directory); + confirming_store.begin_attempt(); + CHECK(confirming_store.confirm_bundle(4)); + CHECK(fault->call_count(entries_directory) == calls_before_confirm + 1); + CHECK(confirming_store.confirm_bundle(4)); + CHECK(fault->call_count(entries_directory) == calls_before_confirm + 1); + confirming_store.begin_attempt(); + CHECK_FALSE(confirming_store.confirm_bundle(5)); +} + +TEST_CASE("Corrupt tiles and entry bundles are rejected") +{ + // 5. Corrupt / truncated files are rejected on read (integrity check), so + // a torn write can never be served as a valid tile or bundle. + TemporaryDirectory temporary_directory; + merkle::tiles::TileStore store(temporary_directory.path()); + const size_t hash_size = Hash().size(); + const auto full = make_hashes(merkle::tiles::TILE_WIDTH); + const TileRef full_ref{0, 0}; + const auto entries = make_entries(); + store.write_tile(full_ref, full); + + overwrite_file(store.tile_path(full_ref), std::vector(hash_size, 0)); + CHECK_FALSE(store.has_full_tile(0, 0)); + CHECK_THROWS_AS(store.read_tile(full_ref), std::runtime_error); + store.write_tile(full_ref, full); + CHECK(store.has_full_tile(0, 0)); + CHECK(store.read_tile(full_ref) == full); + + overwrite_file( + store.tile_path(full_ref), + std::vector((merkle::tiles::TILE_WIDTH + 1) * hash_size, 0)); + CHECK_FALSE(store.has_full_tile(0, 0)); + CHECK_THROWS_AS(store.read_tile(full_ref), std::runtime_error); + + store.write_entry_bundle(0, entries); + CHECK(store.has_entry_bundle(0)); + CHECK(store.read_entry_bundle(0) == entries); + overwrite_file(store.entries_path(0), {0x00, 0x05, 0x01}); + CHECK_FALSE(store.has_entry_bundle(0)); + CHECK_THROWS_AS(store.read_entry_bundle(0), std::runtime_error); + + auto encoded = merkle::tiles::TileStore::encode_entries(entries); + encoded.push_back(0xFF); + overwrite_file(store.entries_path(0), encoded); + CHECK_FALSE(store.has_entry_bundle(0)); + CHECK_THROWS_AS(store.read_entry_bundle(0), std::runtime_error); + + CHECK_THROWS_AS( + (merkle::tiles::TileStore::decode_entries({0xFF, 0xFF, 0x00}, 1)), + std::runtime_error); + CHECK_THROWS_AS( + (merkle::tiles::TileStore::decode_entries({}, 1)), std::runtime_error); + CHECK_THROWS_AS( + (merkle::tiles::TileStore::decode_entries({0x00, 0x00, 0xFF}, 1)), + std::runtime_error); +} // Windows file replacement semantics can reject racing same-path replaces even // when the final tile remains valid, so keep this stress check POSIX-only. #ifndef _WIN32 - // 6. Concurrent same-tile writes use unique temp files and leave no - // temporary files behind after success. Each thread owns its store object; - // sharing one object requires caller-provided synchronization. - { - const TileRef concurrent_ref{0, 42}; - std::atomic ok{true}; - std::vector threads; - for (size_t i = 0; i < 8; i++) +TEST_CASE("Concurrent same-tile writes leave a valid tile") +{ + // 6. Concurrent same-tile writes use unique temp files and leave no + // temporary files behind after success. Each thread owns its store object; + // sharing one object requires caller-provided synchronization. + TemporaryDirectory temporary_directory; + const fs::path& dir = temporary_directory.path(); + merkle::tiles::TileStore store(dir); + const auto full = make_hashes(merkle::tiles::TILE_WIDTH); + const TileRef concurrent_ref{0, 42}; + std::atomic ok{true}; + std::vector threads; + for (size_t i = 0; i < 8; i++) + { + threads.emplace_back([&] { + try { - threads.emplace_back([&] { - try - { - merkle::tiles::TileStore thread_store(dir); - thread_store.write_tile(concurrent_ref, full); - } - catch (...) - { - ok = false; - } - }); + merkle::tiles::TileStore thread_store(dir); + thread_store.write_tile(concurrent_ref, full); } - for (auto& thread : threads) + catch (...) { - thread.join(); + ok = false; } - expect(ok, "concurrent writes succeeded"); - expect(store.read_tile(concurrent_ref) == full, "concurrent tile valid"); - expect(!any_tmp_files(dir), "no temp files left after writes"); - } -#endif - - std::cout << "tiles_store: OK" << '\n'; - - std::error_code ec; - fs::remove_all(dir, ec); - } - catch (std::exception& ex) - { - std::cout << "Error: " << ex.what() << '\n'; - std::error_code ec; - fs::remove_all(dir, ec); - return 1; + }); } - catch (...) + for (auto& thread : threads) { - std::cout << "Error" << '\n'; - std::error_code ec; - fs::remove_all(dir, ec); - return 1; + thread.join(); } - return 0; + CHECK(ok.load()); + CHECK(store.read_tile(concurrent_ref) == full); + CHECK_FALSE(any_tmp_files(dir)); } +#endif From 6a303824f6995e9a3ea91b9a8522625b6749d16a Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Mon, 13 Jul 2026 22:11:39 +0100 Subject: [PATCH 12/22] Apply suggestion from @achamayou --- doc/design/tlog-tiles.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 5a8b4b7..355e963 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -1,8 +1,5 @@ # Design: Tiled storage and tile-backed proofs for merklecpp -**Status:** Proposal / Draft - -**Audience:** merklecpp maintainers and contributors **Scope:** Persist trees progressively during compaction using [tlog-tiles](https://c2sp.org/tlog-tiles), then serve inclusion and consistency From e7373f539f0ee54ccfa790493862255bf2a5b552 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Mon, 13 Jul 2026 22:13:24 +0100 Subject: [PATCH 13/22] Apply suggestion from @achamayou --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3dd41e6..bae673a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -40,7 +40,7 @@ if(OPENSSL add_merklecpp_test(compare_hash_functions compare_hash_functions.cpp) endif() -find_path(DOCTEST_DIR doctest.h PATH_SUFFIXES doctest) +find_path(DOCTEST_DIR doctest.h) if(DOCTEST_DIR) add_merklecpp_test(tiles_store tiles_store.cpp) From 638ccab8e77c0a444ce1fb3dd4d9fb6b01b52a78 Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 21:33:48 +0000 Subject: [PATCH 14/22] Fetch doctest consistently with CMake Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- CMakeLists.txt | 2 +- test/CMakeLists.txt | 24 +++++++++++++----------- test/tiles_store.cpp | 40 ++++++++++++++++++++-------------------- test/unit_tests.cpp | 2 +- 5 files changed, 36 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0534e2b..d078891 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: - name: Install packages run: | sudo apt-get update - sudo apt-get -y install doctest-dev clang clang-tidy + sudo apt-get -y install clang clang-tidy if: matrix.os == 'ubuntu-latest' - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ac550a..9953557 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -cmake_minimum_required(VERSION 3.12) +cmake_minimum_required(VERSION 3.14) project(merklecpp LANGUAGES CXX C ASM) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bae673a..dcb8c1b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,16 +3,22 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +include(FetchContent) +FetchContent_Declare( + doctest + GIT_REPOSITORY https://github.com/doctest/doctest.git + GIT_TAG ae7a13539fb71f270b87eb2e874fbac80bc8dda2 # v2.4.11 +) +FetchContent_MakeAvailable(doctest) + function(add_merklecpp_test target source) add_unit_test(${MERKLECPP_TEST_PREFIX}${target} ${source}) + target_link_libraries( + ${MERKLECPP_TEST_PREFIX}${target} PRIVATE doctest::doctest + ) set_tests_properties( ${MERKLECPP_TEST_PREFIX}${target} PROPERTIES LABELS "merklecpp" ) - if (DOCTEST_DIR) - target_include_directories( - ${MERKLECPP_TEST_PREFIX}${target} PRIVATE ${DOCTEST_DIR} - ) - endif() if (WIN32) target_compile_definitions(${MERKLECPP_TEST_PREFIX}${target} PRIVATE _CRT_SECURE_NO_WARNINGS) endif() @@ -40,9 +46,5 @@ if(OPENSSL add_merklecpp_test(compare_hash_functions compare_hash_functions.cpp) endif() -find_path(DOCTEST_DIR doctest.h) - -if(DOCTEST_DIR) - add_merklecpp_test(tiles_store tiles_store.cpp) - add_merklecpp_test(unit_tests unit_tests.cpp) -endif() +add_merklecpp_test(tiles_store tiles_store.cpp) +add_merklecpp_test(unit_tests unit_tests.cpp) diff --git a/test/tiles_store.cpp b/test/tiles_store.cpp index 7cd1f55..4def4bd 100644 --- a/test/tiles_store.cpp +++ b/test/tiles_store.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include @@ -217,9 +217,9 @@ TEST_CASE("Tile geometry, references, and index encoding") TEST_CASE("Tile store paths and hash namespaces") { // 2. Resource path layout (full tiles and bundles only). - TemporaryDirectory temporary_directory; + const TemporaryDirectory temporary_directory; const fs::path& dir = temporary_directory.path(); - merkle::tiles::TileStore store(dir); + const merkle::tiles::TileStore store(dir); CHECK(store.root().lexically_relative(dir).generic_string() == "sha256-256w"); CHECK(rel(store, store.tile_path(TileRef{0, 0})) == "tile/0/000"); @@ -229,7 +229,7 @@ TEST_CASE("Tile store paths and hash namespaces") rel(store, store.tile_path(TileRef{merkle::tiles::MAX_TILE_LEVEL, 0})) == "tile/63/000"); CHECK_THROWS_AS( - (store.tile_path( + ((void)store.tile_path( TileRef{static_cast(merkle::tiles::MAX_TILE_LEVEL + 1), 0})), std::runtime_error); CHECK(rel(store, store.entries_path(5)) == "tile/entries/005"); @@ -316,7 +316,7 @@ TEST_CASE("Tile writes sync directory links in order") { // 2b. Production writes sync each directory link and the destination // directory in order. - TemporaryDirectory temporary_directory; + const TemporaryDirectory temporary_directory; const fs::path& dir = temporary_directory.path(); const fs::path prefix = dir / "durable"; const auto fault = std::make_shared(); @@ -341,7 +341,7 @@ TEST_CASE("Failed directory-link sync is retried") { // 2c. A failed directory-link sync is retried even when the directory // created before the failure is already visible. - TemporaryDirectory temporary_directory; + const TemporaryDirectory temporary_directory; const fs::path prefix = temporary_directory.path() / "directory_retry"; const auto fault = std::make_shared(); fault->fail_path = prefix; @@ -361,7 +361,7 @@ TEST_CASE("Visible publication can be confirmed after sync failure") { // 2d. A failure syncing the destination directory leaves a complete, // visible file that can be re-confirmed on the next write attempt. - TemporaryDirectory temporary_directory; + const TemporaryDirectory temporary_directory; const fs::path prefix = temporary_directory.path() / "publication_retry"; const auto fault = std::make_shared(); FaultInjectingTileStore store(prefix, fault); @@ -391,7 +391,7 @@ TEST_CASE("Replacement and directory conflicts clean up temporary files") { // 2e. Failures before atomic replacement clean up temporary files, and a // non-directory in the destination path is rejected. - TemporaryDirectory temporary_directory; + const TemporaryDirectory temporary_directory; const fs::path& dir = temporary_directory.path(); const auto full = make_hashes(merkle::tiles::TILE_WIDTH); @@ -416,7 +416,7 @@ TEST_CASE("Exclusive file creation rejects file and symlink collisions") { // 2f. Unique temporary-file creation must fail rather than overwrite an // existing path, including a pre-created symlink on POSIX. - TemporaryDirectory temporary_directory; + const TemporaryDirectory temporary_directory; const fs::path& dir = temporary_directory.path(); fs::create_directories(dir); const fs::path existing_path = dir / "preexisting-temp"; @@ -491,7 +491,7 @@ TEST_CASE("Interrupted sync operations are retried") TEST_CASE("Full tiles use raw bytes and round-trip") { // 3a. Full tile byte round-trip. - TemporaryDirectory temporary_directory; + const TemporaryDirectory temporary_directory; merkle::tiles::TileStore store(temporary_directory.path()); const size_t hash_size = Hash().size(); const auto full = make_hashes(merkle::tiles::TILE_WIDTH); @@ -500,7 +500,7 @@ TEST_CASE("Full tiles use raw bytes and round-trip") store.write_tile(full_ref, full); CHECK(store.has_full_tile(0, 0)); CHECK_FALSE(store.has_full_tile(0, 5)); - CHECK_THROWS_AS((store.read_tile(TileRef{0, 5})), std::runtime_error); + CHECK_THROWS_AS((void)store.read_tile(TileRef{0, 5}), std::runtime_error); CHECK( fs::file_size(store.tile_path(full_ref)) == (uintmax_t)merkle::tiles::TILE_WIDTH * hash_size); @@ -567,7 +567,7 @@ TEST_CASE("Entry encoding enforces bounds and round-trips") TEST_CASE("Entry bundle storage validates and confirms publications") { - TemporaryDirectory temporary_directory; + const TemporaryDirectory temporary_directory; const fs::path& dir = temporary_directory.path(); merkle::tiles::TileStore store(dir); const auto entries = make_entries(); @@ -575,7 +575,7 @@ TEST_CASE("Entry bundle storage validates and confirms publications") merkle::tiles::TileStore::encode_entries(entries); CHECK_FALSE(store.has_entry_bundle(17)); - CHECK_THROWS_AS(store.read_entry_bundle(17), std::runtime_error); + CHECK_THROWS_AS((void)store.read_entry_bundle(17), std::runtime_error); const std::vector> short_bundle( entries.begin(), entries.end() - 1); CHECK_THROWS_AS( @@ -612,7 +612,7 @@ TEST_CASE("Corrupt tiles and entry bundles are rejected") { // 5. Corrupt / truncated files are rejected on read (integrity check), so // a torn write can never be served as a valid tile or bundle. - TemporaryDirectory temporary_directory; + const TemporaryDirectory temporary_directory; merkle::tiles::TileStore store(temporary_directory.path()); const size_t hash_size = Hash().size(); const auto full = make_hashes(merkle::tiles::TILE_WIDTH); @@ -622,7 +622,7 @@ TEST_CASE("Corrupt tiles and entry bundles are rejected") overwrite_file(store.tile_path(full_ref), std::vector(hash_size, 0)); CHECK_FALSE(store.has_full_tile(0, 0)); - CHECK_THROWS_AS(store.read_tile(full_ref), std::runtime_error); + CHECK_THROWS_AS((void)store.read_tile(full_ref), std::runtime_error); store.write_tile(full_ref, full); CHECK(store.has_full_tile(0, 0)); CHECK(store.read_tile(full_ref) == full); @@ -631,20 +631,20 @@ TEST_CASE("Corrupt tiles and entry bundles are rejected") store.tile_path(full_ref), std::vector((merkle::tiles::TILE_WIDTH + 1) * hash_size, 0)); CHECK_FALSE(store.has_full_tile(0, 0)); - CHECK_THROWS_AS(store.read_tile(full_ref), std::runtime_error); + CHECK_THROWS_AS((void)store.read_tile(full_ref), std::runtime_error); store.write_entry_bundle(0, entries); CHECK(store.has_entry_bundle(0)); CHECK(store.read_entry_bundle(0) == entries); overwrite_file(store.entries_path(0), {0x00, 0x05, 0x01}); CHECK_FALSE(store.has_entry_bundle(0)); - CHECK_THROWS_AS(store.read_entry_bundle(0), std::runtime_error); + CHECK_THROWS_AS((void)store.read_entry_bundle(0), std::runtime_error); auto encoded = merkle::tiles::TileStore::encode_entries(entries); encoded.push_back(0xFF); overwrite_file(store.entries_path(0), encoded); CHECK_FALSE(store.has_entry_bundle(0)); - CHECK_THROWS_AS(store.read_entry_bundle(0), std::runtime_error); + CHECK_THROWS_AS((void)store.read_entry_bundle(0), std::runtime_error); CHECK_THROWS_AS( (merkle::tiles::TileStore::decode_entries({0xFF, 0xFF, 0x00}, 1)), @@ -664,9 +664,9 @@ TEST_CASE("Concurrent same-tile writes leave a valid tile") // 6. Concurrent same-tile writes use unique temp files and leave no // temporary files behind after success. Each thread owns its store object; // sharing one object requires caller-provided synchronization. - TemporaryDirectory temporary_directory; + const TemporaryDirectory temporary_directory; const fs::path& dir = temporary_directory.path(); - merkle::tiles::TileStore store(dir); + const merkle::tiles::TileStore store(dir); const auto full = make_hashes(merkle::tiles::TILE_WIDTH); const TileRef concurrent_ref{0, 42}; std::atomic ok{true}; diff --git a/test/unit_tests.cpp b/test/unit_tests.cpp index b0868e5..c3a6e2d 100644 --- a/test/unit_tests.cpp +++ b/test/unit_tests.cpp @@ -4,7 +4,7 @@ #include "util.h" #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN -#include +#include #include TEST_CASE("HashT constructors and error paths") From b9512e399c4567f265134d55a56dd60f77aeeffb Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 21:45:33 +0000 Subject: [PATCH 15/22] Fix doctest support with CMake 4 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- test/CMakeLists.txt | 2 +- test/unit_tests.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index dcb8c1b..aa92ab2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -7,7 +7,7 @@ include(FetchContent) FetchContent_Declare( doctest GIT_REPOSITORY https://github.com/doctest/doctest.git - GIT_TAG ae7a13539fb71f270b87eb2e874fbac80bc8dda2 # v2.4.11 + GIT_TAG 2d0a9359a60c51affe2a9bebb1be1dca47868151 # v2.5.3 ) FetchContent_MakeAvailable(doctest) diff --git a/test/unit_tests.cpp b/test/unit_tests.cpp index c3a6e2d..073e7e0 100644 --- a/test/unit_tests.cpp +++ b/test/unit_tests.cpp @@ -105,7 +105,7 @@ TEST_CASE("HashT methods") REQUIRE_THROWS_AS((void)h2.to_string(33), std::out_of_range); // operator== and operator!= - merkle::Hash ha; + const merkle::Hash ha; merkle::Hash hb; REQUIRE(ha == hb); REQUIRE_FALSE(ha != hb); From b051c59df2836e5b4dbec81918271d09655f49e1 Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 21:54:25 +0000 Subject: [PATCH 16/22] Harden tile file reads Bound malformed file reads, keep fetched doctest test-only, and align the documented entry-bundle path with the algorithm namespace. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/design/tlog-tiles.md | 3 +-- merklecpp_tiles.h | 52 +++++++++++++++++++++++++++++++++------- test/CMakeLists.txt | 2 ++ 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 355e963..ed58a0b 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -1,6 +1,5 @@ # Design: Tiled storage and tile-backed proofs for merklecpp - **Scope:** Persist trees progressively during compaction using [tlog-tiles](https://c2sp.org/tlog-tiles), then serve inclusion and consistency proofs from tiles, memory, or both. @@ -31,7 +30,7 @@ responsible for compatible hashing and for serving the static resources. |---|---| | Tile | `/-256w/tile//`, `application/octet-stream`; `L` is decimal `0..63` without leading zeros, and `N` uses the grouped encoding in [section 4](#4-storage-layout-and-publication). | | Full tile | 256 hashes. Level 0 stores leaf hashes; each level-`L` entry for `L >= 1` is the Merkle Tree Hash of one complete level-`L-1` tile. | -| Entry bundle | `/tile/entries/`; 256 raw entries encoded as big-endian `uint16` length-prefixed values. See [section 5.3](#53-entry-bundles-optional). | +| Entry bundle | `/-256w/tile/entries/`; 256 raw entries encoded as big-endian `uint16` length-prefixed values. See [section 5.3](#53-entry-bundles-optional). | | Pruning | Tiles or bundles ending at or before a log's minimum index may be denied, matching `flush_to()` / `min_index()`. | A tile spans eight tree levels. Tile `n` at level `l` contains, for diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 91ae604..4efba6d 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -6,6 +6,7 @@ #include "merklecpp.h" #include "merklecpp_pal.h" +#include #include #include #include @@ -52,6 +53,10 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) static constexpr std::string_view SHA256_ALGORITHM_SHORT_NAME = "sha256"; static constexpr std::string_view SHA384_ALGORITHM_SHORT_NAME = "sha384"; static constexpr std::string_view SHA512_ALGORITHM_SHORT_NAME = "sha512"; + static constexpr size_t ENTRY_LENGTH_PREFIX_SIZE = 2; + static constexpr size_t MAX_ENTRY_SIZE = 0xFFFF; + static constexpr size_t MAX_ENTRY_BUNDLE_SIZE = + TILE_WIDTH * (ENTRY_LENGTH_PREFIX_SIZE + MAX_ENTRY_SIZE); template uint64_t contiguous_prefix_length(uint64_t limit, const Present& present) @@ -237,8 +242,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @return The tile's hashes (TILE_WIDTH of them) [[nodiscard]] std::vector read_tile(const TileRef& ref) const { - std::vector bytes = read_file(tile_path(ref)); const size_t expected = (size_t)TILE_WIDTH * HASH_SIZE; + std::vector bytes = read_file(tile_path(ref), expected); if (bytes.size() != expected) { throw std::runtime_error("unexpected tile size"); @@ -265,7 +270,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } try { - (void)decode_entries(read_file(path), TILE_WIDTH); + (void)decode_entries( + read_file(path, detail::MAX_ENTRY_BUNDLE_SIZE), TILE_WIDTH); return true; } catch (const std::runtime_error&) @@ -295,7 +301,9 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) [[nodiscard]] std::vector> read_entry_bundle( uint64_t index) const { - return decode_entries(read_file(entries_path(index)), TILE_WIDTH); + return decode_entries( + read_file(entries_path(index), detail::MAX_ENTRY_BUNDLE_SIZE), + TILE_WIDTH); } /// @brief Encodes log entries into the tlog-tiles entry-bundle format. @@ -305,7 +313,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) std::vector bytes; for (const auto& e : entries) { - if (e.size() > 0xFFFF) + if (e.size() > detail::MAX_ENTRY_SIZE) { throw std::runtime_error( "entry too large for uint16 length prefix"); @@ -321,7 +329,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) static std::vector> decode_entries( const std::vector& bytes, size_t count) { - if (count > bytes.size() / 2) + if (count > bytes.size() / detail::ENTRY_LENGTH_PREFIX_SIZE) { throw std::runtime_error("truncated entry bundle"); } @@ -480,7 +488,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } /// @brief Reads an entire file into a byte vector. - static std::vector read_file(const std::filesystem::path& path) + static std::vector read_file( + const std::filesystem::path& path, size_t max_size) { std::ifstream f(path, std::ios::binary); if (!f.good()) @@ -488,8 +497,35 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) throw std::runtime_error( std::format("cannot open file: {}", path.string())); } - return { - std::istreambuf_iterator(f), std::istreambuf_iterator()}; + + std::vector bytes; + std::array buffer{}; + while (f) + { + const size_t remaining = max_size - bytes.size(); + const size_t request = + remaining < buffer.size() ? remaining + 1 : buffer.size(); + f.read( + reinterpret_cast(buffer.data()), + static_cast(request)); + const auto count = static_cast(f.gcount()); + if (count == 0) + { + break; + } + if (count > remaining) + { + throw std::runtime_error( + std::format("file exceeds maximum size: {}", path.string())); + } + bytes.insert(bytes.end(), buffer.begin(), buffer.begin() + count); + } + if (f.bad()) + { + throw std::runtime_error( + std::format("error reading file: {}", path.string())); + } + return bytes; } /// @brief Writes a file atomically via a synced temporary file. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index aa92ab2..63709b2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -9,6 +9,8 @@ FetchContent_Declare( GIT_REPOSITORY https://github.com/doctest/doctest.git GIT_TAG 2d0a9359a60c51affe2a9bebb1be1dca47868151 # v2.5.3 ) +set(DOCTEST_NO_INSTALL ON) +set(DOCTEST_WITH_MAIN_IN_STATIC_LIB OFF) FetchContent_MakeAvailable(doctest) function(add_merklecpp_test target source) From d7825a4f36bfdc9b60af709d00b21d4159bb7692 Mon Sep 17 00:00:00 2001 From: achamayou Date: Tue, 14 Jul 2026 07:38:40 +0000 Subject: [PATCH 17/22] Prevent Windows min/max macro leakage Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- merklecpp.h | 3 +-- merklecpp_pal.h | 14 ++++++++++---- test/tiles_store.cpp | 9 +++++++-- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/merklecpp.h b/merklecpp.h index 5060bd0..775aeef 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -714,8 +714,7 @@ namespace merkle if (left && right) { size = left->size + right->size + 1; - // Parentheses prevent expansion of the Windows max macro. - height = (std::max)(left->height, right->height) + 1; + height = std::max(left->height, right->height) + 1; } else { diff --git a/merklecpp_pal.h b/merklecpp_pal.h index 8ddde7f..7d688d0 100644 --- a/merklecpp_pal.h +++ b/merklecpp_pal.h @@ -19,7 +19,15 @@ #include #ifdef _WIN32 +# ifndef NOMINMAX +# define NOMINMAX +# define MERKLECPP_UNDEF_NOMINMAX +# endif # include +# ifdef MERKLECPP_UNDEF_NOMINMAX +# undef MERKLECPP_UNDEF_NOMINMAX +# undef NOMINMAX +# endif #else # include # include @@ -130,16 +138,14 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) bool close_handle = true; try { - // Parentheses prevent expansion of the Windows max macro. constexpr auto max_write_size = - static_cast((std::numeric_limits::max)()); + static_cast(std::numeric_limits::max()); size_t written = 0; while (written < bytes.size()) { const auto remaining = bytes.size() - written; - // Parentheses prevent expansion of the Windows min macro. const auto chunk = - static_cast((std::min)(remaining, max_write_size)); + static_cast(std::min(remaining, max_write_size)); DWORD done = 0; if (!WriteFile(handle, bytes.data() + written, chunk, &done, nullptr)) { diff --git a/test/tiles_store.cpp b/test/tiles_store.cpp index 4def4bd..993fa69 100644 --- a/test/tiles_store.cpp +++ b/test/tiles_store.cpp @@ -1,9 +1,12 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN + +#include + #include "util.h" -#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN #include #include #include @@ -17,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -28,6 +30,9 @@ namespace fs = std::filesystem; using merkle::Hash; using merkle::tiles::TileRef; +static_assert(std::min(1, 2) == 1); +static_assert(std::max(1, 2) == 2); + class TemporaryDirectory { private: From 382d3b1f8b4d0f78f0a475a257acdabb19d9ac07 Mon Sep 17 00:00:00 2001 From: achamayou Date: Tue, 14 Jul 2026 08:46:41 +0000 Subject: [PATCH 18/22] Address review and build consistency issues Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- CMakeLists.txt | 12 ++++++++---- merklecpp.h | 5 +++-- merklecpp_tiles.h | 27 ++++++++++++++++++++++----- test/tiles_store.cpp | 20 ++++++++++++++++++-- test/unit_tests.cpp | 11 +++++++++++ 6 files changed, 63 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d078891..73d7b0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,7 +53,7 @@ jobs: if [ "$RUNNER_OS" == "Linux" ]; then cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=ON -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON else - cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON + cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=ON fi - name: Build diff --git a/CMakeLists.txt b/CMakeLists.txt index 9953557..543a9e4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,8 @@ cmake_minimum_required(VERSION 3.14) project(merklecpp LANGUAGES CXX C ASM) +include(GNUInstallDirs) + set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -33,8 +35,10 @@ if(TRACE) endif() install(TARGETS merklecpp) - -set(CMAKE_FIND_LIBRARY_SUFFIXES ".a" ".so") +install( + FILES merklecpp.h merklecpp_pal.h merklecpp_tiles.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) if(EVERCRYPT) if(NOT EVERCRYPT_DIR) @@ -56,9 +60,9 @@ if(EVERCRYPT) endif() if(OPENSSL) - find_package(OpenSSL) + find_package(OpenSSL REQUIRED COMPONENTS Crypto) target_compile_definitions(merklecpp INTERFACE HAVE_OPENSSL) - target_link_libraries(merklecpp INTERFACE crypto) + target_link_libraries(merklecpp INTERFACE OpenSSL::Crypto) endif() if(TESTS) diff --git a/merklecpp.h b/merklecpp.h index 775aeef..a1c6e3c 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -3,6 +3,7 @@ #pragma once +#include #include #include #include @@ -172,7 +173,7 @@ namespace merkle /// @param position Position of the first byte in @p bytes HashT(const std::vector& bytes, size_t& position) { - if (bytes.size() - position < SIZE) + if (position > bytes.size() || bytes.size() - position < SIZE) { throw std::runtime_error("not enough bytes"); } @@ -270,7 +271,7 @@ namespace merkle void deserialise(const std::vector& buffer, size_t& position) { MERKLECPP_TRACE(MERKLECPP_TOUT << "> HashT::deserialise " << std::endl); - if (buffer.size() - position < SIZE) + if (position > buffer.size() || buffer.size() - position < SIZE) { throw std::runtime_error("not enough bytes"); } diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 4efba6d..9d503c8 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -206,6 +206,10 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @brief Whether a full tile exists on disk. [[nodiscard]] bool has_full_tile(uint8_t level, uint64_t index) const { + if (level > MAX_TILE_LEVEL) + { + return false; + } std::error_code ec; const auto path = tile_path(TileRef{level, index}); if (!std::filesystem::is_regular_file(path, ec) || ec) @@ -243,10 +247,15 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) [[nodiscard]] std::vector read_tile(const TileRef& ref) const { const size_t expected = (size_t)TILE_WIDTH * HASH_SIZE; - std::vector bytes = read_file(tile_path(ref), expected); + const auto path = tile_path(ref); + std::vector bytes = read_file(path, expected); if (bytes.size() != expected) { - throw std::runtime_error("unexpected tile size"); + throw std::runtime_error(std::format( + "unexpected tile size for {}: expected {} bytes, got {}", + path.string(), + expected, + bytes.size())); } std::vector hashes; @@ -301,9 +310,17 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) [[nodiscard]] std::vector> read_entry_bundle( uint64_t index) const { - return decode_entries( - read_file(entries_path(index), detail::MAX_ENTRY_BUNDLE_SIZE), - TILE_WIDTH); + const auto path = entries_path(index); + const auto bytes = read_file(path, detail::MAX_ENTRY_BUNDLE_SIZE); + try + { + return decode_entries(bytes, TILE_WIDTH); + } + catch (const std::runtime_error& error) + { + throw std::runtime_error(std::format( + "invalid entry bundle {}: {}", path.string(), error.what())); + } } /// @brief Encodes log entries into the tlog-tiles entry-bundle format. diff --git a/test/tiles_store.cpp b/test/tiles_store.cpp index 993fa69..07b17af 100644 --- a/test/tiles_store.cpp +++ b/test/tiles_store.cpp @@ -237,6 +237,8 @@ TEST_CASE("Tile store paths and hash namespaces") ((void)store.tile_path( TileRef{static_cast(merkle::tiles::MAX_TILE_LEVEL + 1), 0})), std::runtime_error); + CHECK_FALSE(store.has_full_tile( + static_cast(merkle::tiles::MAX_TILE_LEVEL + 1), 0)); CHECK(rel(store, store.entries_path(5)) == "tile/entries/005"); CHECK( merkle::tiles::TileStore::storage_directory_name("sha384") == @@ -627,7 +629,15 @@ TEST_CASE("Corrupt tiles and entry bundles are rejected") overwrite_file(store.tile_path(full_ref), std::vector(hash_size, 0)); CHECK_FALSE(store.has_full_tile(0, 0)); - CHECK_THROWS_AS((void)store.read_tile(full_ref), std::runtime_error); + const auto short_tile_error = std::format( + "unexpected tile size for {}: expected {} bytes, got {}", + store.tile_path(full_ref).string(), + merkle::tiles::TILE_WIDTH * hash_size, + hash_size); + CHECK_THROWS_WITH_AS( + (void)store.read_tile(full_ref), + short_tile_error.c_str(), + std::runtime_error); store.write_tile(full_ref, full); CHECK(store.has_full_tile(0, 0)); CHECK(store.read_tile(full_ref) == full); @@ -643,7 +653,13 @@ TEST_CASE("Corrupt tiles and entry bundles are rejected") CHECK(store.read_entry_bundle(0) == entries); overwrite_file(store.entries_path(0), {0x00, 0x05, 0x01}); CHECK_FALSE(store.has_entry_bundle(0)); - CHECK_THROWS_AS((void)store.read_entry_bundle(0), std::runtime_error); + const auto invalid_bundle_error = std::format( + "invalid entry bundle {}: truncated entry bundle", + store.entries_path(0).string()); + CHECK_THROWS_WITH_AS( + (void)store.read_entry_bundle(0), + invalid_bundle_error.c_str(), + std::runtime_error); auto encoded = merkle::tiles::TileStore::encode_entries(entries); encoded.push_back(0xFF); diff --git a/test/unit_tests.cpp b/test/unit_tests.cpp index 073e7e0..9c009a5 100644 --- a/test/unit_tests.cpp +++ b/test/unit_tests.cpp @@ -72,6 +72,11 @@ TEST_CASE("HashT constructors and error paths") size_t pos2 = 10; const std::vector too_short(41, 0); // only 31 bytes from position 10 REQUIRE_THROWS(merkle::Hash(too_short, pos2)); + + // Vector+position constructor: position beyond the buffer throws + size_t beyond_end = too_short.size() + 1; + REQUIRE_THROWS(merkle::Hash(too_short, beyond_end)); + REQUIRE(beyond_end == too_short.size() + 1); } TEST_CASE("HashT methods") @@ -86,6 +91,12 @@ TEST_CASE("HashT methods") REQUIRE(b == 0); } + // deserialise: position beyond the buffer throws without advancing + const std::vector empty; + size_t beyond_end = 1; + REQUIRE_THROWS(h.deserialise(empty, beyond_end)); + REQUIRE(beyond_end == 1); + // to_string (lower case) merkle::Hash h2; h2.bytes[0] = 0xAB; From abb389e3b1d34e1e6ac940804fdc2364739ab366 Mon Sep 17 00:00:00 2001 From: achamayou Date: Tue, 14 Jul 2026 09:25:22 +0000 Subject: [PATCH 19/22] Exclude internal PAL from Doxygen Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Doxyfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Doxyfile b/Doxyfile index 566ab97..d045b46 100644 --- a/Doxyfile +++ b/Doxyfile @@ -791,7 +791,6 @@ WARN_LOGFILE = # Note: If this tag is empty the current directory is searched. INPUT = merklecpp.h \ - merklecpp_pal.h \ merklecpp_tiles.h # This tag can be used to specify the character encoding of the source files From e11168d5053affa02c9acc71bebecdda0db1a37b Mon Sep 17 00:00:00 2001 From: achamayou Date: Tue, 14 Jul 2026 09:45:05 +0000 Subject: [PATCH 20/22] Clarify tile concurrency contract Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/design/tlog-tiles.md | 14 ++++++++------ merklecpp_tiles.h | 10 +++++++--- test/tiles_store.cpp | 41 ---------------------------------------- 3 files changed, 15 insertions(+), 50 deletions(-) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index ed58a0b..f80b37b 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -16,11 +16,12 @@ proofs from tiles, memory, or both. | Proofs | Tiled inclusion paths must equal `Tree::path()` / `Tree::past_path()` output and verify against the same `root()` / `past_root()`. Consistency proofs use the same tree and combiner. | | Hashing | Leaf handling, node hashing, and `HASH_FUNCTION` are unchanged. | | API | The implementation remains header-only and follows the existing `TreeT` template style. | -| Concurrency | The API performs no synchronization. Callers must serialize access, including across objects sharing a prefix and through `const` proof reads that update the `TileHashSourceT` LRU cache. | +| Concurrency | The API performs no synchronization. Callers must serialize access to each object, including `const` proof reads that update the `TileHashSourceT` LRU cache, and all writers sharing a prefix. Independent objects may read while the serialized writer publishes tiles atomically. Concurrent writers are out of scope. | **Non-goals:** standardizing the SHA-384/512 extensions; adding RFC 6962 leaf -hashing or domain separation; and providing an HTTP server. Applications remain -responsible for compatible hashing and for serving the static resources. +hashing or domain separation; coordinating concurrent writers; and providing an +HTTP server. Applications remain responsible for compatible hashing and for +serving the static resources. ## 2. Format and compatibility @@ -155,14 +156,15 @@ hashes, not bytes. ### Publication guarantees - Create a unique temporary file exclusively, sync its contents with `fsync`, - and atomically replace the destination. + and atomically replace the destination. Independent readers observe a complete + old or new resource while the serialized writer publishes. - On POSIX, sync each newly created directory link and the destination directory after rename. A higher-level retry rechecks the directory chain and destination even when the file is already visible. - Wrong-size files are not durable full tiles and may be repaired. Higher-level writers enforce immutability; the low-level primitive permits replacement for - repair or idempotent concurrent publication and trusts callers not to change - valid content. + repair or idempotent publication and trusts callers not to change valid + content. ## 5. Architecture and API diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 9d503c8..e5fc8c9 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -32,8 +32,10 @@ // to those produced by merkle::TreeT (see doc/design/tlog-tiles.md). // // Thread safety: types in this header do not synchronize access internally. -// Callers must serialize all operations on shared objects and store prefixes, -// including const proof operations that update the tile cache. +// Callers must serialize access to each shared object, including const proof +// operations that update the tile cache, and all writers sharing a store +// prefix. Independent store objects may read while the serialized writer +// publishes tiles atomically. namespace merkle // NOLINT(modernize-concat-nested-namespaces) { @@ -128,7 +130,9 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @tparam HASH_FUNCTION The tree's node hash function (carried for use by /// later components; tile I/O itself does not hash). /// @warning No internal synchronization is provided. Callers must serialize - /// access to a store and to all stores that share its prefix. + /// access to each store object and all writers sharing its prefix. + /// Independent store objects may read while the serialized writer publishes + /// tiles atomically. template < size_t HASH_SIZE, void HASH_FUNCTION( diff --git a/test/tiles_store.cpp b/test/tiles_store.cpp index 07b17af..6dbe5d2 100644 --- a/test/tiles_store.cpp +++ b/test/tiles_store.cpp @@ -23,7 +23,6 @@ #include #include #include -#include #include namespace fs = std::filesystem; @@ -676,43 +675,3 @@ TEST_CASE("Corrupt tiles and entry bundles are rejected") (merkle::tiles::TileStore::decode_entries({0x00, 0x00, 0xFF}, 1)), std::runtime_error); } - -// Windows file replacement semantics can reject racing same-path replaces even -// when the final tile remains valid, so keep this stress check POSIX-only. -#ifndef _WIN32 -TEST_CASE("Concurrent same-tile writes leave a valid tile") -{ - // 6. Concurrent same-tile writes use unique temp files and leave no - // temporary files behind after success. Each thread owns its store object; - // sharing one object requires caller-provided synchronization. - const TemporaryDirectory temporary_directory; - const fs::path& dir = temporary_directory.path(); - const merkle::tiles::TileStore store(dir); - const auto full = make_hashes(merkle::tiles::TILE_WIDTH); - const TileRef concurrent_ref{0, 42}; - std::atomic ok{true}; - std::vector threads; - for (size_t i = 0; i < 8; i++) - { - threads.emplace_back([&] { - try - { - merkle::tiles::TileStore thread_store(dir); - thread_store.write_tile(concurrent_ref, full); - } - catch (...) - { - ok = false; - } - }); - } - for (auto& thread : threads) - { - thread.join(); - } - - CHECK(ok.load()); - CHECK(store.read_tile(concurrent_ref) == full); - CHECK_FALSE(any_tmp_files(dir)); -} -#endif From 96fbe5606803e46e6dc68f30ce325f3ebbe6e387 Mon Sep 17 00:00:00 2001 From: achamayou Date: Tue, 14 Jul 2026 11:00:45 +0000 Subject: [PATCH 21/22] Use built-in SHA256 by default --- .github/workflows/ci.yml | 7 +- doc/design/tlog-tiles.md | 3 +- doc/index.rst | 14 +- merklecpp.h | 339 ++++++++++++++++++-------------- test/compare_evercrypt.cpp | 11 +- test/compare_hash_functions.cpp | 132 ++++--------- test/unit_tests.cpp | 20 +- test/util.h | 6 +- 8 files changed, 268 insertions(+), 264 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73d7b0b..fd8a941 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,13 +22,16 @@ jobs: os: [ubuntu-latest] build_type: [Debug, Release] compiler: [g++, clang++] + openssl: ['OFF', 'ON'] include: - os: windows-latest build_type: Debug compiler: msvc + openssl: 'OFF' - os: windows-latest build_type: Release compiler: msvc + openssl: 'OFF' steps: - name: Install packages @@ -51,9 +54,9 @@ jobs: working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} run: | if [ "$RUNNER_OS" == "Linux" ]; then - cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=ON -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=${{ matrix.openssl }} -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON else - cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=ON + cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=${{ matrix.openssl }} fi - name: Build diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index f80b37b..f264f8e 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -224,8 +224,7 @@ using TiledTree = Aliases derive hash size and function from the core tree. Roll-up and proof code combine two `HASH_SIZE`-byte hashes with that same function, so the default -single-block `sha256_compress` remains sufficient and adds no OpenSSL -dependency. +built-in `sha256` function adds no OpenSSL dependency. ### 5.2 `TileStoreT` diff --git a/doc/index.rst b/doc/index.rst index fa02e49..1aaf272 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -7,9 +7,9 @@ allows us to create a Merkle tree with a compile-time configurable hash size and function. A default implementation without further dependencies is provided as -:cpp:type:`merkle::Tree`, which uses the SHA256 compression function -(:cpp:func:`merkle::sha256_compress`). merklecpp also provides bindings -for the respective OpenSSL functions (see `Hash functions`_), +:cpp:type:`merkle::Tree`, which uses the built-in SHA256 function +(:cpp:func:`merkle::sha256`). merklecpp also provides bindings for the +respective OpenSSL functions (see `Hash functions`_), which can be specified as a template parameter as illustrated by the following example: @@ -46,13 +46,13 @@ Paths Hash functions -------------- -By default, merklecpp uses the SHA256 compression function -(:cpp:func:`merkle::sha256_compress`) for node hashes. For convenience, -it also provides bindings to the SHA256 implementation in OpenSSL. +By default, merklecpp uses its built-in SHA256 function +(:cpp:func:`merkle::sha256`) for node hashes. For convenience, merklecpp also +provides bindings to the SHA256 implementation in OpenSSL. To enable these bindings, merklecpp requires the compiler macro :code:`HAVE_OPENSSL` to be defined. -.. doxygenfunction:: merkle::sha256_compress +.. doxygenfunction:: merkle::sha256 :project: merklecpp .. doxygenfunction:: merkle::sha256_openssl diff --git a/merklecpp.h b/merklecpp.h index a1c6e3c..5709da7 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -23,7 +23,6 @@ #ifdef HAVE_OPENSSL # include -# include #endif #ifdef MERKLECPP_TRACE_ENABLED @@ -148,8 +147,9 @@ namespace merkle { uint8_t high = 0; uint8_t low = 0; - if (!decode_hex_digit(s[2 * i], high) || - !decode_hex_digit(s[2 * i + 1], low)) + if ( + !decode_hex_digit(s[2 * i], high) || + !decode_hex_digit(s[2 * i + 1], low)) { throw std::runtime_error("invalid hash string"); } @@ -471,14 +471,13 @@ namespace merkle /// @brief The size of the serialised path in number of bytes [[nodiscard]] size_t serialised_size() const { - return sizeof(_leaf) + - sizeof(uint64_t) + // leaf index - sizeof(uint64_t) + // max index - sizeof(uint64_t) + // number of elements - elements.size() * ( - sizeof(Element::hash) + // hash - sizeof(uint8_t) // direction - ); + return sizeof(_leaf) + sizeof(uint64_t) + // leaf index + sizeof(uint64_t) + // max index + sizeof(uint64_t) + // number of elements + elements.size() * + (sizeof(Element::hash) + // hash + sizeof(uint8_t) // direction + ); } /// @brief Index of the leaf of the path @@ -818,9 +817,9 @@ namespace merkle /// @param hash Hash to insert void insert(const Hash& hash) { - MERKLECPP_TRACE(MERKLECPP_TOUT << "> insert " - << hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << "> insert " << hash.to_string(TRACE_HASH_SIZE) + << std::endl;); uninserted_leaf_nodes.push_back(Node::make(hash)); statistics.num_insert++; } @@ -862,10 +861,10 @@ namespace merkle walk_to(index, false, [this](Node*& n, bool go_right) { if (go_right && n->left) { - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - conflate " - << n->left->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - conflate " + << n->left->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); if (n->left && n->left->dirty) { hash(n->left); @@ -920,10 +919,10 @@ namespace merkle n->dirty = true; if (go_left && n->right) { - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - eliminate " - << n->right->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - eliminate " + << n->right->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); bool is_root = n == _root; Node* old_left = n->left; @@ -943,18 +942,18 @@ namespace merkle if (is_root) { - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - new root: " - << n->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT + << " - new root: " << n->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); assert(_root == n); } assert(n->invariant()); - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - after elimination: " << std::endl - << to_string(TRACE_HASH_SIZE) << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - after elimination: " << std::endl + << to_string(TRACE_HASH_SIZE) << std::endl;); return false; } return true; @@ -1027,9 +1026,9 @@ namespace merkle statistics.num_root++; compute_root(); assert(_root && !_root->dirty); - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - root: " << _root->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - root: " << _root->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); return _root->hash; } @@ -1099,12 +1098,12 @@ namespace merkle { walk_stack.push_back(cur); } - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - at " << cur->hash.to_string(TRACE_HASH_SIZE) - << " (" << cur->size << "/" << (unsigned)cur->height - << ")" - << " (" << (go_right ? "R" : "L") << ")" - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - at " << cur->hash.to_string(TRACE_HASH_SIZE) + << " (" << cur->size << "/" << (unsigned)cur->height + << ")" + << " (" << (go_right ? "R" : "L") << ")" + << std::endl;); if (cur->height == height) { if (!f(cur, go_right)) @@ -1159,8 +1158,9 @@ namespace merkle /// tree to @p as_of and then extracting the path of @p index. std::shared_ptr past_path(size_t index, size_t as_of) { - MERKLECPP_TRACE(MERKLECPP_TOUT << "> past_path from " << index - << " as of " << as_of << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << "> past_path from " << index << " as of " << as_of + << std::endl;); statistics.num_past_paths++; if ( @@ -1199,24 +1199,24 @@ namespace merkle bool const go_right_i = ((it_i >> (8 * sizeof(it_i) - 1)) & 0x01) != 0U; bool const go_right_a = ((it_a >> (8 * sizeof(it_a) - 1)) & 0x01) != 0U; - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - at " << (unsigned)height << ": " - << cur_i->hash.to_string(TRACE_HASH_SIZE) << " (" - << cur_i->size << "/" << (unsigned)cur_i->height - << "/" << (go_right_i ? "R" : "L") << ")" - << " / " << cur_a->hash.to_string(TRACE_HASH_SIZE) - << " (" << cur_a->size << "/" - << (unsigned)cur_a->height << "/" - << (go_right_a ? "R" : "L") << ")" << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - at " << (unsigned)height << ": " + << cur_i->hash.to_string(TRACE_HASH_SIZE) << " (" + << cur_i->size << "/" << (unsigned)cur_i->height << "/" + << (go_right_i ? "R" : "L") << ")" + << " / " << cur_a->hash.to_string(TRACE_HASH_SIZE) + << " (" << cur_a->size << "/" + << (unsigned)cur_a->height << "/" + << (go_right_a ? "R" : "L") << ")" << std::endl;); if (!fork_node && go_right_i != go_right_a) { assert(cur_i == cur_a); assert(!go_right_i && go_right_a); - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - split at " - << cur_i->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - split at " + << cur_i->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); fork_node = cur_i; } @@ -1360,8 +1360,8 @@ namespace merkle compute_root(); - MERKLECPP_TRACE(MERKLECPP_TOUT << to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << to_string(TRACE_HASH_SIZE) << std::endl;); std::vector extras; walk_to(min_index(), false, [&extras](Node*& n, bool go_right) { @@ -1385,8 +1385,9 @@ namespace merkle /// @param bytes The vector of bytes to serialise to void serialise(size_t from, size_t to, std::vector& bytes) { - MERKLECPP_TRACE(MERKLECPP_TOUT << "> serialise from " << from << " to " - << to << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << "> serialise from " << from << " to " << to + << std::endl;); validate_partial_range(from, to); @@ -1403,8 +1404,8 @@ namespace merkle compute_root(); - MERKLECPP_TRACE(MERKLECPP_TOUT << to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << to_string(TRACE_HASH_SIZE) << std::endl;); std::vector extras; walk_to(from, false, [&extras](Node*& n, bool go_right) { @@ -1468,10 +1469,10 @@ namespace merkle level.insert(level.begin(), n); } - MERKLECPP_TRACE(for (auto& n - : level) MERKLECPP_TOUT - << " " << n->hash.to_string(TRACE_HASH_SIZE); - MERKLECPP_TOUT << std::endl;); + MERKLECPP_TRACE( + for (auto& n : level) MERKLECPP_TOUT + << " " << n->hash.to_string(TRACE_HASH_SIZE); + MERKLECPP_TOUT << std::endl;); // Rebuild the level for (size_t i = 0; i < level.size(); i += 2) @@ -1729,8 +1730,7 @@ namespace merkle protected: void validate_partial_range(size_t from, size_t to) const { - if ( - empty() || !(min_index() <= from && from <= to && to <= max_index())) + if (empty() || !(min_index() <= from && from <= to && to <= max_index())) { throw std::runtime_error("invalid leaf indices"); } @@ -1897,9 +1897,9 @@ namespace merkle { while (true) { - MERKLECPP_TRACE(MERKLECPP_TOUT << " @ " - << n->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " @ " << n->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); assert(n->invariant()); if (n->is_full()) @@ -1972,10 +1972,10 @@ namespace merkle if (!complete && !result->is_full()) { - MERKLECPP_TRACE(MERKLECPP_TOUT - << " X save " - << result->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " X save " + << result->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); return result; } } @@ -1990,9 +1990,9 @@ namespace merkle /// @param n New leaf node to insert void insert_leaf(Node*& root, Node* n) { - MERKLECPP_TRACE(MERKLECPP_TOUT << " - insert_leaf " - << n->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - insert_leaf " + << n->hash.to_string(TRACE_HASH_SIZE) << std::endl;); leaf_nodes.push_back(n); if (insertion_stack.empty() && !root) { @@ -2012,9 +2012,9 @@ namespace merkle { if (!uninserted_leaf_nodes.empty()) { - MERKLECPP_TRACE(MERKLECPP_TOUT - << "* insert_leaves " << leaf_nodes.size() << " +" - << uninserted_leaf_nodes.size() << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << "* insert_leaves " << leaf_nodes.size() << " +" + << uninserted_leaf_nodes.size() << std::endl;); // Potential future improvement: make this go fast when there are many // leaves to insert. for (auto& n : uninserted_leaf_nodes) @@ -2030,88 +2030,131 @@ namespace merkle } }; - // clang-format off - /// @brief SHA256 compression function for tree node hashes + namespace detail + { + static inline std::array sha256_initial_state() + { + return { + 0x6a09e667, + 0xbb67ae85, + 0x3c6ef372, + 0xa54ff53a, + 0x510e527f, + 0x9b05688c, + 0x1f83d9ab, + 0x5be0cd19}; + } + + static inline void sha256_transform( + const uint8_t block[64], std::array& state) + { + static constexpr std::array constants = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, + 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, + 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, + 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, + 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + + std::array schedule = {}; + for (size_t i = 0; i < 16; ++i) + { + const size_t offset = i * 4; + schedule[i] = (static_cast(block[offset]) << 24) | + (static_cast(block[offset + 1]) << 16) | + (static_cast(block[offset + 2]) << 8) | + static_cast(block[offset + 3]); + } + + for (size_t i = 16; i < 64; ++i) + { + const uint32_t word15 = schedule[i - 15]; + const uint32_t word2 = schedule[i - 2]; + const uint32_t sigma0 = (word15 >> 7 | word15 << 25) ^ + (word15 >> 18 | word15 << 14) ^ (word15 >> 3); + const uint32_t sigma1 = (word2 >> 17 | word2 << 15) ^ + (word2 >> 19 | word2 << 13) ^ (word2 >> 10); + schedule[i] = schedule[i - 16] + sigma0 + schedule[i - 7] + sigma1; + } + + auto working = state; + for (size_t i = 0; i < 64; ++i) + { + const uint32_t choice = + (working[4] & working[5]) ^ (~working[4] & working[6]); + const uint32_t majority = (working[0] & working[1]) ^ + (working[0] & working[2]) ^ (working[1] & working[2]); + const uint32_t sigma0 = (working[0] >> 2 | working[0] << 30) ^ + (working[0] >> 13 | working[0] << 19) ^ + (working[0] >> 22 | working[0] << 10); + const uint32_t sigma1 = (working[4] >> 6 | working[4] << 26) ^ + (working[4] >> 11 | working[4] << 21) ^ + (working[4] >> 25 | working[4] << 7); + const uint32_t temporary1 = + working[7] + sigma1 + choice + constants[i] + schedule[i]; + const uint32_t temporary2 = sigma0 + majority; + + working[7] = working[6]; + working[6] = working[5]; + working[5] = working[4]; + working[4] = working[3] + temporary1; + working[3] = working[2]; + working[2] = working[1]; + working[1] = working[0]; + working[0] = temporary1 + temporary2; + } + + for (size_t i = 0; i < state.size(); ++i) + { + state[i] += working[i]; + } + } + + static inline void sha256_write_digest( + const std::array& state, HashT<32>& out) + { + for (size_t i = 0; i < state.size(); ++i) + { + out.bytes[i * 4] = static_cast(state[i] >> 24); + out.bytes[i * 4 + 1] = static_cast(state[i] >> 16); + out.bytes[i * 4 + 2] = static_cast(state[i] >> 8); + out.bytes[i * 4 + 3] = static_cast(state[i]); + } + } + } + + /// @brief Built-in SHA256 function for tree node hashes /// @param l Left node hash /// @param r Right node hash /// @param out Output node hash - /// @details This function is the compression function of SHA256, which, for - /// the special case of hashing two hashes, is more efficient than a full - /// SHA256 while providing similar guarantees. - static inline void sha256_compress(const HashT<32> &l, const HashT<32> &r, HashT<32> &out) { - static const uint32_t constants[] = { - 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, - 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, - 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, - 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, - 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, - 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, - 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, - 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 - }; - + /// @details Computes SHA256 over the 64-byte concatenation of @p l and @p r, + /// including standard SHA256 message padding. + static inline void sha256( + const HashT<32>& l, const HashT<32>& r, HashT<32>& out) + { uint8_t block[32 * 2]; memcpy(&block[0], l.bytes, 32); memcpy(&block[32], r.bytes, 32); - static const uint32_t s[8] = { 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, - 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19 }; - - uint32_t cws[64] = {0}; - - for (int i=0; i < 16; i++) { - cws[i] = convert_endianness(reinterpret_cast(block)[i]); -} - - for (int i = 16; i < 64; i++) { - uint32_t const t16 = cws[i - 16]; - uint32_t const t15 = cws[i - 15]; - uint32_t const t7 = cws[i - 7]; - uint32_t const t2 = cws[i - 2]; - uint32_t const s1 = (t2 >> 17 | t2 << 15) ^ ((t2 >> 19 | t2 << 13) ^ t2 >> 10); - uint32_t const s0 = (t15 >> 7 | t15 << 25) ^ ((t15 >> 18 | t15 << 14) ^ t15 >> 3); - cws[i] = (s1 + t7 + s0 + t16); - } - - uint32_t h[8]; - for (int i=0; i < 8; i++) { - h[i] = s[i]; -} - - for (int i=0; i < 64; i++) { - const uint32_t a0 = h[0]; - const uint32_t b0 = h[1]; - const uint32_t c0 = h[2]; - const uint32_t d0 = h[3]; - const uint32_t e0 = h[4]; - const uint32_t f0 = h[5]; - const uint32_t g0 = h[6]; - const uint32_t h03 = h[7]; - uint32_t const w = cws[i]; - uint32_t const t1 = h03 + ((e0 >> 6 | e0 << 26) ^ ((e0 >> 11 | e0 << 21) ^ (e0 >> 25 | e0 << 7))) + ((e0 & f0) ^ (~e0 & g0)) + constants[i] + w; - uint32_t const t2 = ((a0 >> 2 | a0 << 30) ^ ((a0 >> 13 | a0 << 19) ^ (a0 >> 22 | a0 << 10))) + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); - h[0] = t1 + t2; - h[1] = a0; - h[2] = b0; - h[3] = c0; - h[4] = d0 + t1; - h[5] = e0; - h[6] = f0; - h[7] = g0; - } - - for (int i=0; i < 8; i++) { - reinterpret_cast(out.bytes)[i] = convert_endianness(s[i] + h[i]); -} + auto state = detail::sha256_initial_state(); + detail::sha256_transform(block, state); + + uint8_t padding[64] = {0x80}; + padding[62] = 0x02; + detail::sha256_transform(padding, state); + detail::sha256_write_digest(state, out); } - // clang-format on #ifdef HAVE_OPENSSL /// @brief OpenSSL SHA256 /// @param l Left node hash /// @param r Right node hash /// @param out Output node hash - /// @note Some versions of OpenSSL may not provide SHA256_Transform. static inline void sha256_openssl( const merkle::HashT<32>& l, const merkle::HashT<32>& r, @@ -2197,8 +2240,8 @@ namespace merkle using Hash = HashT<32>; /// @brief Type of paths in the default tree type - using Path = PathT<32, sha256_compress>; + using Path = PathT<32, sha256>; /// @brief Default tree with default hash size and function - using Tree = TreeT<32, sha256_compress>; + using Tree = TreeT<32, sha256>; }; diff --git a/test/compare_evercrypt.cpp b/test/compare_evercrypt.cpp index b637e76..60f30d1 100644 --- a/test/compare_evercrypt.cpp +++ b/test/compare_evercrypt.cpp @@ -3,6 +3,7 @@ #include "util.h" +#include #include #include #include @@ -11,6 +12,14 @@ #define HSZ 32 #define PRNTSZ 3 +void sha256_evercrypt(uint8_t* lhs, uint8_t* rhs, uint8_t* out) +{ + uint8_t block[HSZ * 2]; + memcpy(&block[0], lhs, HSZ); + memcpy(&block[HSZ], rhs, HSZ); + Hacl_Hash_SHA2_hash_256(block, sizeof(block), out); +} + void dump_ec_tree(merkle_tree* mt) { std::cout << "hs=" << std::endl; @@ -91,7 +100,7 @@ int main() memcpy(ec_hash, h.bytes, HSZ); if (!ec_mt) - ec_mt = mt_create(ec_hash); + ec_mt = mt_create_custom(HSZ, ec_hash, sha256_evercrypt); else mt_insert(ec_mt, ec_hash); diff --git a/test/compare_hash_functions.cpp b/test/compare_hash_functions.cpp index 55b9245..b521edf 100644 --- a/test/compare_hash_functions.cpp +++ b/test/compare_hash_functions.cpp @@ -8,21 +8,12 @@ #include #include -constexpr size_t HSZ = 32; constexpr size_t PRNTSZ = 3; #ifdef HAVE_EVERCRYPT # include # include -void sha256_compress_evercrypt( - const merkle::HashT<32>& l, - const merkle::HashT<32>& r, - merkle::HashT<32>& out) -{ - mt_sha256_compress((uint8_t*)l.bytes, (uint8_t*)r.bytes, (uint8_t*)out.bytes); -} - void sha256_evercrypt( const merkle::HashT<32>& l, const merkle::HashT<32>& r, @@ -42,12 +33,11 @@ void mt_sha256_evercrypt(uint8_t* src1, uint8_t* src2, uint8_t* dst) Hacl_Hash_SHA2_hash_256(block, sizeof(block), dst); } -typedef merkle::TreeT<32, sha256_compress_evercrypt> EverCryptTree; -typedef merkle::TreeT<32, sha256_evercrypt> EverCryptFullTree; +using EverCryptTree = merkle::TreeT<32, sha256_evercrypt>; #endif #ifdef HAVE_OPENSSL -using OpenSSLFullTree = merkle::TreeT<32, merkle::sha256_openssl>; +using OpenSSLTree = merkle::TreeT<32, merkle::sha256_openssl>; #endif template < @@ -77,7 +67,7 @@ void compare_roots( } } -void compare_compression_hashes() +void compare_sha256_hashes() { #ifndef NDEBUG const size_t num_trees = 1024; @@ -94,6 +84,10 @@ void compare_compression_hashes() { merkle::Tree mt; +#ifdef HAVE_OPENSSL + OpenSSLTree mto; +#endif + #ifdef HAVE_EVERCRYPT EverCryptTree mte; #endif @@ -106,6 +100,10 @@ void compare_compression_hashes() { mt.insert(h); +#ifdef HAVE_OPENSSL + mto.insert(h); +#endif + #ifdef HAVE_EVERCRYPT mte.insert(h); #endif @@ -114,6 +112,9 @@ void compare_compression_hashes() if ((j++ % root_interval) == 0) { +#ifdef HAVE_OPENSSL + compare_roots(mt, mto, "OpenSSL"); +#endif #ifdef HAVE_EVERCRYPT compare_roots(mt, mte, "EverCrypt"); #endif @@ -122,74 +123,18 @@ void compare_compression_hashes() } } +#ifdef HAVE_OPENSSL + compare_roots(mt, mto, "OpenSSL"); +#endif #ifdef HAVE_EVERCRYPT compare_roots(mt, mte, "EverCrypt"); #endif - } std::cout << num_trees << " trees, " << total_inserts << " inserts, " - << total_roots << " roots with SHA256 compression function: OK" - << '\n'; + << total_roots << " roots with SHA256: OK" << '\n'; } -#if defined(HAVE_OPENSSL) && defined(HAVE_EVERCRYPT) -void compare_full_hashes() -{ -# ifndef NDEBUG - const size_t num_trees = 1024; - const size_t root_interval = 31; -# else - const size_t num_trees = 4096; - const size_t root_interval = 128; -# endif - - size_t total_inserts = 0; - size_t total_roots = 0; - - for (size_t k = 0; k < num_trees; k++) - { - OpenSSLFullTree mto; - -# ifdef HAVE_EVERCRYPT - merkle::TreeT<32, sha256_evercrypt> mte; -# endif - - // Build trees with k+1 leaves - int j = 0; - auto hashes = make_hashes(k + 1); - - for (const auto h : hashes) - { - mto.insert(h); - -# ifdef HAVE_EVERCRYPT - mte.insert(h); -# endif - - total_inserts++; - - if ((j++ % root_interval) == 0) - { -# ifdef HAVE_EVERCRYPT - compare_roots(mto, mte, "EverCrypt"); -# endif - - total_roots++; - } - } - -# ifdef HAVE_EVERCRYPT - compare_roots(mto, mte, "OpenSSL"); -# endif - - } - - std::cout << num_trees << " trees, " << total_inserts << " inserts, " - << total_roots << " roots with full SHA256: OK" << '\n'; -} -#endif - template void bench( const std::vector& hashes, @@ -210,7 +155,9 @@ void bench( mt.root(); auto stop = std::chrono::high_resolution_clock::now(); const double seconds = - static_cast(std::chrono::duration_cast(stop - start).count()) / + static_cast( + std::chrono::duration_cast(stop - start) + .count()) / 1e9; std::cout << std::left << std::setw(10) << name << ": " << mt.statistics.num_insert << " insertions, " @@ -238,7 +185,9 @@ void benchT( mt.root(); auto stop = std::chrono::high_resolution_clock::now(); const double seconds = - static_cast(std::chrono::duration_cast(stop - start).count()) / + static_cast( + std::chrono::duration_cast(stop - start) + .count()) / 1e9; std::cout << std::left << std::setw(10) << name << ": " << mt.statistics.num_insert << " insertions, " @@ -288,11 +237,7 @@ int main() // std::srand(0); std::srand(std::time(nullptr)); - compare_compression_hashes(); - -#if defined(HAVE_EVERCRYPT) && defined(HAVE_OPENSSL) - compare_full_hashes(); -#endif + compare_sha256_hashes(); #ifndef NDEBUG const size_t num_leaves = static_cast(128) * 1024; @@ -304,39 +249,32 @@ int main() auto hashes = make_hashes(num_leaves); - std::cout << "--- merklecpp trees with SHA256 compression function: " - << '\n'; + std::cout << "--- merklecpp trees with SHA256: " << '\n'; bench(hashes, "merklecpp", root_interval); -#ifdef HAVE_EVERCRYPT - bench(hashes, "EverCrypt", root_interval); +#ifdef HAVE_OPENSSL + bench(hashes, "OpenSSL", root_interval); #endif - std::cout << "--- merklecpp trees with full SHA256: " << '\n'; - -#ifdef HAVE_OPENSSL - bench(hashes, "OpenSSL", root_interval); +#ifdef HAVE_EVERCRYPT + bench(hashes, "EverCrypt", root_interval); #endif #ifdef HAVE_OPENSSL { - std::cout << "--- merklecpp trees with full SHA384: " << '\n'; + std::cout << "--- merklecpp trees with SHA384: " << '\n'; auto hashes384 = make_hashesT<48>(num_leaves); benchT(hashes384, "OpenSSL", root_interval); } { - std::cout << "--- merklecpp trees with full SHA512: " << '\n'; + std::cout << "--- merklecpp trees with SHA512: " << '\n'; auto hashes512 = make_hashesT<64>(num_leaves); benchT(hashes512, "OpenSSL", root_interval); } #endif -#ifdef HAVE_EVERCRYPT - bench(hashes, "EverCrypt", root_interval); -#endif - #ifdef HAVE_EVERCRYPT std::vector ec_hashes; for (auto& h : hashes) @@ -345,11 +283,7 @@ int main() memcpy(ec_hashes.back(), h.bytes, 32); } - std::cout << "--- EverCrypt trees with SHA256 compression function: " - << '\n'; - bench_evercrypt(ec_hashes, "EverCrypt", root_interval); - - std::cout << "--- EverCrypt trees with full SHA256: " << '\n'; + std::cout << "--- EverCrypt trees with SHA256: " << '\n'; bench_evercrypt(ec_hashes, "EverCrypt", root_interval); for (auto h : ec_hashes) diff --git a/test/unit_tests.cpp b/test/unit_tests.cpp index 9c009a5..ce7ce8d 100644 --- a/test/unit_tests.cpp +++ b/test/unit_tests.cpp @@ -7,6 +7,22 @@ #include #include +TEST_CASE("Built-in SHA256 hashes complete messages") +{ + const merkle::Hash zero; + merkle::Hash digest; + merkle::sha256(zero, zero, digest); + + REQUIRE( + digest.to_string() == + "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b"); + + merkle::Tree tree; + tree.insert(zero); + tree.insert(zero); + REQUIRE(tree.root() == digest); +} + TEST_CASE("HashT constructors and error paths") { // Default constructor: all bytes zero @@ -161,7 +177,7 @@ TEST_CASE("PathT equality") const auto path0a = tree.path(0); const auto path0b = tree.path(0); // same path extracted twice - const auto path1 = tree.path(1); // path to a different leaf + const auto path1 = tree.path(1); // path to a different leaf // Two paths to the same leaf should be equal REQUIRE(*path0a == *path0b); @@ -178,7 +194,7 @@ TEST_CASE("PathT equality") h3.bytes[0] = 3; tree_diff.insert(h3); - const auto path_orig = tree.path(0); // h0 leaf, element has h1 + const auto path_orig = tree.path(0); // h0 leaf, element has h1 const auto path_diff = tree_diff.path(0); // h0 leaf, element has h3 REQUIRE_FALSE(*path_orig == *path_diff); REQUIRE(*path_orig != *path_diff); diff --git a/test/util.h b/test/util.h index 2abfc77..196f8b9 100644 --- a/test/util.h +++ b/test/util.h @@ -38,9 +38,9 @@ inline std::vector> make_hashesT( inline size_t random_index(merkle::Tree& mt) { - return (size_t)( - mt.min_index() + - (std::rand() / (double)RAND_MAX) * (mt.max_index() - mt.min_index())); + return (size_t)(mt.min_index() + + (std::rand() / (double)RAND_MAX) * + (mt.max_index() - mt.min_index())); } inline double get_timeout() From b74d4d985a7e999f53816a804c861c58d5edf062 Mon Sep 17 00:00:00 2001 From: achamayou Date: Tue, 14 Jul 2026 11:06:40 +0000 Subject: [PATCH 22/22] Reserve entry bundle encoding capacity --- merklecpp_tiles.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index e5fc8c9..8485e24 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -331,7 +331,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) static std::vector encode_entries( const std::vector>& entries) { - std::vector bytes; + size_t encoded_size = 0; for (const auto& e : entries) { if (e.size() > detail::MAX_ENTRY_SIZE) @@ -339,6 +339,13 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) throw std::runtime_error( "entry too large for uint16 length prefix"); } + encoded_size += detail::ENTRY_LENGTH_PREFIX_SIZE + e.size(); + } + + std::vector bytes; + bytes.reserve(encoded_size); + for (const auto& e : entries) + { bytes.push_back((uint8_t)((e.size() >> 8) & 0xFF)); bytes.push_back((uint8_t)(e.size() & 0xFF)); bytes.insert(bytes.end(), e.begin(), e.end());