diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8638695..7917457 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,19 +22,22 @@ jobs: os: [ubuntu-latest] build_type: [Debug, Release] compiler: [g++, clang++] + openssl: ['OFF', 'ON'] include: - os: windows-latest build_type: Debug compiler: msvc + openssl: 'OFF' - os: windows-latest build_type: Release compiler: msvc + openssl: 'OFF' steps: - name: Install packages run: | sudo apt-get update - sudo apt-get -y install doctest-dev clang clang-tidy + sudo apt-get -y install clang clang-tidy if: matrix.os == 'ubuntu-latest' - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -51,9 +54,9 @@ jobs: working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} run: | if [ "$RUNNER_OS" == "Linux" ]; then - cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=ON -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=${{ matrix.openssl }} -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON else - cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON + cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=${{ matrix.openssl }} fi - name: Build @@ -64,7 +67,10 @@ jobs: - name: Clang-Tidy Header if: matrix.os == 'ubuntu-latest' shell: bash - run: clang-tidy merklecpp.h -p build/${{ matrix.build_type }} --warnings-as-errors='*' + run: | + clang-tidy merklecpp.h -p build/${{ matrix.build_type }} --warnings-as-errors='*' + clang-tidy merklecpp_tiles.h -p build/${{ matrix.build_type }} --warnings-as-errors='*' + clang-tidy merklecpp_pal.h -p build/${{ matrix.build_type }} --warnings-as-errors='*' - name: Test working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 8acf629..b6f5ad2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,10 +1,14 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +cmake_minimum_required(VERSION 3.14) + project(merklecpp LANGUAGES CXX) -cmake_minimum_required(VERSION 3.11) -set(CMAKE_CXX_STANDARD 17) +include(GNUInstallDirs) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) set(MERKLECPP_DIR ${CMAKE_CURRENT_SOURCE_DIR}) @@ -22,6 +26,7 @@ if(CLANG_TIDY) endif() add_library(merklecpp INTERFACE) +target_compile_features(merklecpp INTERFACE cxx_std_20) target_include_directories(merklecpp INTERFACE .) if(TRACE) @@ -29,13 +34,15 @@ if(TRACE) endif() install(TARGETS merklecpp) - -set(CMAKE_FIND_LIBRARY_SUFFIXES ".a" ".so") +install( + FILES merklecpp.h merklecpp_pal.h merklecpp_tiles.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) if(OPENSSL) - find_package(OpenSSL) + find_package(OpenSSL REQUIRED COMPONENTS Crypto) target_compile_definitions(merklecpp INTERFACE HAVE_OPENSSL) - target_link_libraries(merklecpp INTERFACE crypto) + target_link_libraries(merklecpp INTERFACE OpenSSL::Crypto) endif() if(TESTS) diff --git a/Doxyfile b/Doxyfile index 33229bb..d045b46 100644 --- a/Doxyfile +++ b/Doxyfile @@ -790,7 +790,8 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = merklecpp.h +INPUT = merklecpp.h \ + merklecpp_tiles.h # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/README.md b/README.md index 904c442..db01d1c 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ unusual features like flushing, retracting, and tree segment serialisation. ## Usage +merklecpp requires C++20. + #include merkle::Tree::Hash hash("fa8f44eabb728d4020e7f33d1aa973faaef19de6c06679bccdc5100a3c01f54a"); diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md new file mode 100644 index 0000000..f264f8e --- /dev/null +++ b/doc/design/tlog-tiles.md @@ -0,0 +1,281 @@ +# Design: Tiled storage and tile-backed proofs for merklecpp + +**Scope:** Persist trees progressively during compaction using +[tlog-tiles](https://c2sp.org/tlog-tiles), then serve inclusion and consistency +proofs from tiles, memory, or both. + +--- + +## 1. Requirements + +| Area | Contract | +|---|---| +| Format | SHA-256 uses the C2SP payload, paths, and geometry: 256 hashes per tile and 8 tree levels per tile. SHA-384 and SHA-512 use explicitly namespaced merklecpp extensions with the same width. | +| Persistence | A flush writes only newly completed, balanced tiles and makes them durable before `flush_to()` frees their nodes. Partial tiles are never written or read; the incomplete frontier remains in memory. | +| Immutability | Higher-level writers treat published tiles as immutable. The low-level store allows atomic replacement only for repair and idempotent publication. | +| Proofs | Tiled inclusion paths must equal `Tree::path()` / `Tree::past_path()` output and verify against the same `root()` / `past_root()`. Consistency proofs use the same tree and combiner. | +| Hashing | Leaf handling, node hashing, and `HASH_FUNCTION` are unchanged. | +| API | The implementation remains header-only and follows the existing `TreeT` template style. | +| Concurrency | The API performs no synchronization. Callers must serialize access to each object, including `const` proof reads that update the `TileHashSourceT` LRU cache, and all writers sharing a prefix. Independent objects may read while the serialized writer publishes tiles atomically. Concurrent writers are out of scope. | + +**Non-goals:** standardizing the SHA-384/512 extensions; adding RFC 6962 leaf +hashing or domain separation; coordinating concurrent writers; and providing an +HTTP server. Applications remain responsible for compatible hashing and for +serving the static resources. + +## 2. Format and compatibility + +### 2.1 Resource model + +| Resource | Layout and meaning | +|---|---| +| Tile | `/-256w/tile//`, `application/octet-stream`; `L` is decimal `0..63` without leading zeros, and `N` uses the grouped encoding in [section 4](#4-storage-layout-and-publication). | +| Full tile | 256 hashes. Level 0 stores leaf hashes; each level-`L` entry for `L >= 1` is the Merkle Tree Hash of one complete level-`L-1` tile. | +| Entry bundle | `/-256w/tile/entries/`; 256 raw entries encoded as big-endian `uint16` length-prefixed values. See [section 5.3](#53-entry-bundles-optional). | +| Pruning | Tiles or bundles ending at or before a log's minimum index may be denied, matching `flush_to()` / `min_index()`. | + +A tile spans eight tree levels. Tile `n` at level `l` contains, for +`i = 0..255`: + +`MTH(D[(n*256+i)*256^l : (n*256+i+1)*256^l])` + +Its leaf range is `[n*256^(l+1), (n+1)*256^(l+1))`. Only a full-tile prefix is +stored; the remaining frontier stays in memory. + +### 2.2 merklecpp model + +- `TreeT` is left-balanced with the RFC 6962 shape. + A full node of height `h` covers `2^(h-1)` aligned leaves. +- Internal nodes use `HASH_FUNCTION(left, right, out)` over two child hashes, + without domain separation. +- `flush_to(index)` replaces the compacted prefix with hash-only nodes, removes + its leaves, and raises `min_index()`. Memory alone can no longer prove leaves + below that index. +- Existing proof surfaces are `root()`, `past_root(i)`, `path(i)`, + `past_path(i, as_of)`, and `PathT::verify()` / `root()`. merklecpp does not + currently expose a classic two-size consistency proof; this design adds one + that reconciles historical roots with the same combiner. + +### 2.3 Algorithms and namespaces + +| Algorithm | Hash bytes | Full-tile bytes | Root | Status | +|---|---:|---:|---|---| +| SHA-256 | 32 | 8,192 | `sha256-256w` | C2SP payload and resource layout | +| SHA-384 | 48 | 12,288 | `sha384-256w` | merklecpp extension | +| SHA-512 | 64 | 16,384 | `sha512-256w` | merklecpp extension | + +The width remains 256 hashes for every algorithm; the namespace prevents +different hash sizes from sharing files while retaining the C2SP `tile/...` +layout. Built-in SHA functions select their names automatically. Custom names +must be lowercase, path-safe short names, and recognized SHA names must match +`HASH_SIZE`. + +Exact external interoperability also requires RFC 6962-compatible leaf and node +hashing; merklecpp continues to use the caller's `HASH_FUNCTION`. + +## 3. Tile mapping and proof math + +Let `TILE_HEIGHT = 8`, `TILE_WIDTH = 256 = 2^8`, and `MTH` use the tree's +`HASH_FUNCTION`. + +- Entry `i` of tile `(L, N)`, with `g = N*256+i`, is + `MTH(D[g*256^L : (g+1)*256^L])`: a perfect subtree of `2^(8L)` leaves and an + in-memory full node of height `8L+1`. +- A level-`L` tile rolls up level `L-1` tiles: + `tile(L,N)[i] = perfect_root(tile(L-1, N*256+i)[0..255])`. The writer + therefore needs only level-0 leaf hashes and lower-level tiles. +- For `k = 8L+r`, the root of `2^r` aligned entries (`0 <= r <= 8`) comes from + one level-`L` tile by hashing that run; `r = 0` reads one entry. +- At size `s`, `full_tiles_L = floor(floor(s / 256^L) / 256)` and + `covered = floor(s/256)*256`. `[covered, s)` remains in memory. If a + higher-level tile is absent, lookup descends to available lower-level tiles + and rolls them up. + +Both proof types use one perfect-subtree primitive: + +``` +subtree_root(level k, index j) = MTH(D[j·2^k : (j+1)·2^k]) # within `covered` + L = k / 8 ; r = k % 8 + first = j << r ; n = first / 256 ; off = first % 256 + if a full level-L tile n exists (first + 2^r within full_tiles_L · 256): + return perfect_root( tile(L, n)[off : off + 2^r] ) # single entry if r==0 + else: # higher-level tile unavailable: descend + return HASH_FUNCTION( subtree_root(k-1, 2j), subtree_root(k-1, 2j+1) ) +``` + +and a range primitive for non-perfect (right-frontier) subtrees: + +``` +mth_range(a, b): // MTH(D[a:b]); a aligned to 2^ceil(log2(b-a)) + w = b - a + if w == 2^k and source can resolve subtree_root(k, a>>k): return it + k = largest power of two < w // RFC 6962 split + return HASH_FUNCTION( mth_range(a, a+k), mth_range(a+k, b) ) +``` + +If the selected source cannot resolve a perfect subtree directly, `mth_range` +splits until tile or memory sources can resolve each piece. + +--- + +## 4. Storage layout and publication + +All resources live below a configurable local `prefix`: + +``` +/-256w/tile/ + 0/ # level-0 full tiles + 000 ... 255 + x001/000 ... 255 + 1/ # roll-ups of level-0 tiles + 000 ... + ... + entries/ # optional full entry bundles + 000 ... +``` + +`encode_tile_index` groups decimal indices into zero-padded three-digit +components and prefixes every non-final component with `x`: + +``` +encode_tile_index(N): + parts = split N into base-1000 groups, each formatted "{:03}" + prefix every group except the last with "x" + return join(parts, "/") +``` + +Examples: `5 -> "005"`, `255 -> "255"`, `1000 -> "x001/000"`, and +`1234067 -> "x001/x234/067"`. + +The complete paths are +`/-256w/tile//` and +`/-256w/tile/entries/`. A tile concatenates 256 +`HashT::bytes` values and is therefore `256 * HASH_SIZE` bytes; `256w` counts +hashes, not bytes. + +### Publication guarantees + +- Create a unique temporary file exclusively, sync its contents with `fsync`, + and atomically replace the destination. Independent readers observe a complete + old or new resource while the serialized writer publishes. +- On POSIX, sync each newly created directory link and the destination + directory after rename. A higher-level retry rechecks the directory chain and + destination even when the file is already visible. +- Wrong-size files are not durable full tiles and may be repaired. Higher-level + writers enforce immutability; the low-level primitive permits replacement for + repair or idempotent publication and trusts callers not to change valid + content. + +## 5. Architecture and API + +``` + append(leaf hash) flush() + caller ───────────────▶ TiledTreeT ───────────────▶ TileStoreT + │ owns Tree ▲ + inclusion / consistency proofs │ tiles + ▼ │ + ProofEngine ──▶ HashSource ◀──────────┘ + ├─ MemoryHashSource + ├─ TileHashSource + └─ CombinedHashSource +``` + +`merklecpp_tiles.h` contains the public `merkle::tiles` API and includes +`merklecpp.h`; internal OS operations live in `merklecpp_pal.h` under +`merkle::pal`. + +| Component | Responsibility | +|---|---| +| PAL helpers | Exclusive creation, atomic replacement, and durability | +| `TileRef` / encoder | Full-tile identity and index paths | +| `TileStoreT` | Local tile and entry-bundle I/O | +| `TileWriterT` | Persist newly completed full tiles | +| Hash sources | Resolve subtree roots from memory, tiles, or both | +| `ProofEngineT` | Roots, inclusion/consistency proofs, and verification | +| `TiledTreeT` | `append`, `flush`, proof APIs, and compaction | + +The planned `TileHashSource` owns the proof-read LRU cache; `TileStoreT` does +not cache. A combined source may require one read-only, non-hashing core +`subtree_root` accessor. + +### 5.1 Types and aliases + +```cpp +namespace merkle::tiles { + +static constexpr uint16_t TILE_HEIGHT = 8; +static constexpr uint16_t TILE_WIDTH = uint16_t{1U << TILE_HEIGHT}; +static constexpr uint8_t MAX_TILE_LEVEL = 63; + +template &, + const HashT&, + HashT&)> +class TileStoreT; + +using TileStore = + TileStoreT; +using TiledTree = + TiledTreeT; +// Equivalent SHA-384 and SHA-512 aliases. + +} +``` + +Aliases derive hash size and function from the core tree. Roll-up and proof code +combine two `HASH_SIZE`-byte hashes with that same function, so the default +built-in `sha256` function adds no OpenSSL dependency. + +### 5.2 `TileStoreT` + +```cpp +struct TileRef { uint8_t level; uint64_t index; }; // full tiles only + +class TileStoreT { +public: + explicit TileStoreT(std::filesystem::path prefix); + TileStoreT(std::filesystem::path prefix, + const std::string& hash_algorithm_short_name); + + static std::string storage_directory_name(const std::string& algorithm); + static std::string encode_index(uint64_t n); + std::filesystem::path tile_path(const TileRef&) const; + std::filesystem::path entries_path(uint64_t n) const; + + bool has_full_tile(uint8_t level, uint64_t index) const; + std::vector read_tile(const TileRef&) const; // 256 hashes + void write_tile(const TileRef&, const std::vector&); // durable replace +}; +``` + +Built-in hash functions select `sha256`, `sha384`, or `sha512`; custom +algorithms use the explicit-name constructor. The concurrency contract is in +[section 1](#1-requirements), and publication semantics are in +[section 4](#4-storage-layout-and-publication). + +### 5.3 Entry bundles (optional) + +Entry bundles are application-owned because merklecpp receives precomputed leaf +hashes, not raw entries. Each full bundle stores 256 big-endian `uint16` +length-prefixed entries at `-256w/tile/entries/`. The application +defines the leaf derivation (for example, `leaf_hash = H(entry)`); merklecpp +stores the supplied leaf hash unchanged. + +## 6. Delivery plan + +This PR delivers phases 0 and 1 plus the standalone entry-bundle codec and +storage primitives. Later PRs deliver the remaining independently testable +phases; phases 1-3 need no further core changes, while phase 4 may add one +non-hashing accessor. + +| Phase | Scope | Key tests | +|---|---|---| +| 0. Scaffolding | Headers, PAL, namespace, geometry, aliases, and CMake test wiring | Public-header and build integration | +| 1. Coordinates/store | `TileRef`, index/path encoding, `TileStoreT`, durable atomic I/O, entry-bundle primitives | Encoding vectors; algorithm roots; 256-hash SHA-256/384 tiles; round trips; file/symlink collisions | +| 2. Writers | Incremental `TileWriterT::write_up_to` from `leaf_at`, roll-ups, and `EntryBundleWriterT`; full resources only | Sizes 256 and 70,000 produce the exact tile set; repeated writes preserve immutability | +| 3. Proof engine | `TileHashSource`, `mth_range`, roots, inclusion/consistency proofs, and verification | Tile roots equal tree roots; inclusion equals `path()` / `past_path()` and verifies; consistency reconciles `past_root()` values | +| 4. Combined tree | Optional `subtree_root`; memory/combined sources; `TiledTreeT` append, flush, proof, and compaction APIs | Prove flushed and resident leaves against a non-flushed reference; consistency across a flush boundary | +| 5. Documentation/performance | README usage, design link, and tile-backed benchmarks | Documentation and benchmark coverage | + +Deliverables are `merklecpp_tiles.h`, `merklecpp_pal.h`, `test/tiles_*.cpp`, +CMake wiring, the optional core accessor, and README/design updates. diff --git a/doc/index.rst b/doc/index.rst index fa02e49..1aaf272 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -7,9 +7,9 @@ allows us to create a Merkle tree with a compile-time configurable hash size and function. A default implementation without further dependencies is provided as -:cpp:type:`merkle::Tree`, which uses the SHA256 compression function -(:cpp:func:`merkle::sha256_compress`). merklecpp also provides bindings -for the respective OpenSSL functions (see `Hash functions`_), +:cpp:type:`merkle::Tree`, which uses the built-in SHA256 function +(:cpp:func:`merkle::sha256`). merklecpp also provides bindings for the +respective OpenSSL functions (see `Hash functions`_), which can be specified as a template parameter as illustrated by the following example: @@ -46,13 +46,13 @@ Paths Hash functions -------------- -By default, merklecpp uses the SHA256 compression function -(:cpp:func:`merkle::sha256_compress`) for node hashes. For convenience, -it also provides bindings to the SHA256 implementation in OpenSSL. +By default, merklecpp uses its built-in SHA256 function +(:cpp:func:`merkle::sha256`) for node hashes. For convenience, merklecpp also +provides bindings to the SHA256 implementation in OpenSSL. To enable these bindings, merklecpp requires the compiler macro :code:`HAVE_OPENSSL` to be defined. -.. doxygenfunction:: merkle::sha256_compress +.. doxygenfunction:: merkle::sha256 :project: merklecpp .. doxygenfunction:: merkle::sha256_openssl diff --git a/merklecpp.h b/merklecpp.h index 1170b1c..5709da7 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -3,13 +3,16 @@ #pragma once +#include #include #include #include #include #include #include +#include #include +#include #include #include #include @@ -20,7 +23,6 @@ #ifdef HAVE_OPENSSL # include -# include #endif #ifdef MERKLECPP_TRACE_ENABLED @@ -114,25 +116,28 @@ namespace merkle template struct HashT { + /// Size of the hash in bytes. + static constexpr size_t size_bytes = SIZE; + /// Holds the hash bytes uint8_t bytes[SIZE]; /// @brief Constructs a Hash with all bytes set to zero - HashT() + HashT() { std::fill(bytes, bytes + SIZE, 0); } /// @brief Constructs a Hash from a byte buffer /// @param bytes Buffer with hash value - HashT(const uint8_t* bytes) + HashT(const uint8_t* bytes) { std::copy(bytes, bytes + SIZE, this->bytes); } /// @brief Constructs a Hash from a string /// @param s String to read the hash value from - HashT(const std::string& s) + HashT(const std::string& s) { if (s.length() != 2 * SIZE) { @@ -142,8 +147,9 @@ namespace merkle { uint8_t high = 0; uint8_t low = 0; - if (!decode_hex_digit(s[2 * i], high) || - !decode_hex_digit(s[2 * i + 1], low)) + if ( + !decode_hex_digit(s[2 * i], high) || + !decode_hex_digit(s[2 * i + 1], low)) { throw std::runtime_error("invalid hash string"); } @@ -153,7 +159,7 @@ namespace merkle /// @brief Deserialises a Hash from a vector of bytes /// @param bytes Vector to read the hash value from - HashT(const std::vector& bytes) + HashT(const std::vector& bytes) { if (bytes.size() < SIZE) { @@ -165,9 +171,9 @@ namespace merkle /// @brief Deserialises a Hash from a vector of bytes /// @param bytes Vector to read the hash value from /// @param position Position of the first byte in @p bytes - HashT(const std::vector& bytes, size_t& position) + HashT(const std::vector& bytes, size_t& position) { - if (bytes.size() - position < SIZE) + if (position > bytes.size() || bytes.size() - position < SIZE) { throw std::runtime_error("not enough bytes"); } @@ -176,7 +182,7 @@ namespace merkle /// @brief Deserialises a Hash from an array of bytes /// @param bytes Array to read the hash value from - HashT(const std::array& bytes) + HashT(const std::array& bytes) { std::copy(bytes.data(), bytes.data() + SIZE, this->bytes); } @@ -184,7 +190,7 @@ namespace merkle /// @brief The size of the hash (in number of bytes) [[nodiscard]] size_t size() const { - return SIZE; + return size_bytes; } /// @brief zeros out all bytes in the hash @@ -196,24 +202,35 @@ namespace merkle /// @brief The size of the serialisation of the hash (in number of bytes) [[nodiscard]] size_t serialised_size() const { - return SIZE; + return size_bytes; } /// @brief Convert a hash to a hex-encoded string - /// @param num_bytes The maximum number of bytes to convert + /// @param num_bytes The number of bytes to convert /// @param lower_case Enables lower-case hex characters + /// @throws std::out_of_range if @p num_bytes exceeds the hash size [[nodiscard]] std::string to_string( - size_t num_bytes = SIZE, bool lower_case = true) const + size_t num_bytes = size_bytes, bool lower_case = true) const { + if (num_bytes > size_bytes) + { + throw std::out_of_range("hash string byte count exceeds hash size"); + } size_t const num_chars = 2 * num_bytes; - std::string r(num_chars, '_'); + std::string r; + r.reserve(num_chars); for (size_t i = 0; i < num_bytes; i++) { - snprintf( - const_cast(r.data() + 2 * i), - num_chars + 1 - 2 * i, - lower_case ? "%02x" : "%02X", - bytes[i]); + if (lower_case) + { + std::format_to( + std::back_inserter(r), "{:02x}", static_cast(bytes[i])); + } + else + { + std::format_to( + std::back_inserter(r), "{:02X}", static_cast(bytes[i])); + } } return r; } @@ -254,7 +271,7 @@ namespace merkle void deserialise(const std::vector& buffer, size_t& position) { MERKLECPP_TRACE(MERKLECPP_TOUT << "> HashT::deserialise " << std::endl); - if (buffer.size() - position < SIZE) + if (position > buffer.size() || buffer.size() - position < SIZE) { throw std::runtime_error("not enough bytes"); } @@ -454,14 +471,13 @@ namespace merkle /// @brief The size of the serialised path in number of bytes [[nodiscard]] size_t serialised_size() const { - return sizeof(_leaf) + - sizeof(uint64_t) + // leaf index - sizeof(uint64_t) + // max index - sizeof(uint64_t) + // number of elements - elements.size() * ( - sizeof(Element::hash) + // hash - sizeof(uint8_t) // direction - ); + return sizeof(_leaf) + sizeof(uint64_t) + // leaf index + sizeof(uint64_t) + // max index + sizeof(uint64_t) + // number of elements + elements.size() * + (sizeof(Element::hash) + // hash + sizeof(uint8_t) // direction + ); } /// @brief Index of the leaf of the path @@ -728,6 +744,9 @@ namespace merkle }; public: + /// @brief Hash function used to combine tree nodes. + static constexpr auto hash_function = HASH_FUNCTION; + /// @brief The type of hashes in the tree using Hash = HashT; @@ -798,9 +817,9 @@ namespace merkle /// @param hash Hash to insert void insert(const Hash& hash) { - MERKLECPP_TRACE(MERKLECPP_TOUT << "> insert " - << hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << "> insert " << hash.to_string(TRACE_HASH_SIZE) + << std::endl;); uninserted_leaf_nodes.push_back(Node::make(hash)); statistics.num_insert++; } @@ -842,10 +861,10 @@ namespace merkle walk_to(index, false, [this](Node*& n, bool go_right) { if (go_right && n->left) { - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - conflate " - << n->left->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - conflate " + << n->left->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); if (n->left && n->left->dirty) { hash(n->left); @@ -900,10 +919,10 @@ namespace merkle n->dirty = true; if (go_left && n->right) { - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - eliminate " - << n->right->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - eliminate " + << n->right->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); bool is_root = n == _root; Node* old_left = n->left; @@ -923,18 +942,18 @@ namespace merkle if (is_root) { - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - new root: " - << n->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT + << " - new root: " << n->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); assert(_root == n); } assert(n->invariant()); - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - after elimination: " << std::endl - << to_string(TRACE_HASH_SIZE) << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - after elimination: " << std::endl + << to_string(TRACE_HASH_SIZE) << std::endl;); return false; } return true; @@ -1007,9 +1026,9 @@ namespace merkle statistics.num_root++; compute_root(); assert(_root && !_root->dirty); - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - root: " << _root->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - root: " << _root->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); return _root->hash; } @@ -1079,12 +1098,12 @@ namespace merkle { walk_stack.push_back(cur); } - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - at " << cur->hash.to_string(TRACE_HASH_SIZE) - << " (" << cur->size << "/" << (unsigned)cur->height - << ")" - << " (" << (go_right ? "R" : "L") << ")" - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - at " << cur->hash.to_string(TRACE_HASH_SIZE) + << " (" << cur->size << "/" << (unsigned)cur->height + << ")" + << " (" << (go_right ? "R" : "L") << ")" + << std::endl;); if (cur->height == height) { if (!f(cur, go_right)) @@ -1139,8 +1158,9 @@ namespace merkle /// tree to @p as_of and then extracting the path of @p index. std::shared_ptr past_path(size_t index, size_t as_of) { - MERKLECPP_TRACE(MERKLECPP_TOUT << "> past_path from " << index - << " as of " << as_of << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << "> past_path from " << index << " as of " << as_of + << std::endl;); statistics.num_past_paths++; if ( @@ -1179,24 +1199,24 @@ namespace merkle bool const go_right_i = ((it_i >> (8 * sizeof(it_i) - 1)) & 0x01) != 0U; bool const go_right_a = ((it_a >> (8 * sizeof(it_a) - 1)) & 0x01) != 0U; - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - at " << (unsigned)height << ": " - << cur_i->hash.to_string(TRACE_HASH_SIZE) << " (" - << cur_i->size << "/" << (unsigned)cur_i->height - << "/" << (go_right_i ? "R" : "L") << ")" - << " / " << cur_a->hash.to_string(TRACE_HASH_SIZE) - << " (" << cur_a->size << "/" - << (unsigned)cur_a->height << "/" - << (go_right_a ? "R" : "L") << ")" << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - at " << (unsigned)height << ": " + << cur_i->hash.to_string(TRACE_HASH_SIZE) << " (" + << cur_i->size << "/" << (unsigned)cur_i->height << "/" + << (go_right_i ? "R" : "L") << ")" + << " / " << cur_a->hash.to_string(TRACE_HASH_SIZE) + << " (" << cur_a->size << "/" + << (unsigned)cur_a->height << "/" + << (go_right_a ? "R" : "L") << ")" << std::endl;); if (!fork_node && go_right_i != go_right_a) { assert(cur_i == cur_a); assert(!go_right_i && go_right_a); - MERKLECPP_TRACE(MERKLECPP_TOUT - << " - split at " - << cur_i->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - split at " + << cur_i->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); fork_node = cur_i; } @@ -1340,8 +1360,8 @@ namespace merkle compute_root(); - MERKLECPP_TRACE(MERKLECPP_TOUT << to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << to_string(TRACE_HASH_SIZE) << std::endl;); std::vector extras; walk_to(min_index(), false, [&extras](Node*& n, bool go_right) { @@ -1365,8 +1385,9 @@ namespace merkle /// @param bytes The vector of bytes to serialise to void serialise(size_t from, size_t to, std::vector& bytes) { - MERKLECPP_TRACE(MERKLECPP_TOUT << "> serialise from " << from << " to " - << to << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << "> serialise from " << from << " to " << to + << std::endl;); validate_partial_range(from, to); @@ -1383,8 +1404,8 @@ namespace merkle compute_root(); - MERKLECPP_TRACE(MERKLECPP_TOUT << to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << to_string(TRACE_HASH_SIZE) << std::endl;); std::vector extras; walk_to(from, false, [&extras](Node*& n, bool go_right) { @@ -1448,10 +1469,10 @@ namespace merkle level.insert(level.begin(), n); } - MERKLECPP_TRACE(for (auto& n - : level) MERKLECPP_TOUT - << " " << n->hash.to_string(TRACE_HASH_SIZE); - MERKLECPP_TOUT << std::endl;); + MERKLECPP_TRACE( + for (auto& n : level) MERKLECPP_TOUT + << " " << n->hash.to_string(TRACE_HASH_SIZE); + MERKLECPP_TOUT << std::endl;); // Rebuild the level for (size_t i = 0; i < level.size(); i += 2) @@ -1709,8 +1730,7 @@ namespace merkle protected: void validate_partial_range(size_t from, size_t to) const { - if ( - empty() || !(min_index() <= from && from <= to && to <= max_index())) + if (empty() || !(min_index() <= from && from <= to && to <= max_index())) { throw std::runtime_error("invalid leaf indices"); } @@ -1877,9 +1897,9 @@ namespace merkle { while (true) { - MERKLECPP_TRACE(MERKLECPP_TOUT << " @ " - << n->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " @ " << n->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); assert(n->invariant()); if (n->is_full()) @@ -1916,8 +1936,10 @@ namespace merkle MERKLECPP_TRACE({ std::string nodes; for (size_t i = 0; i < insertion_stack.size(); i++) - nodes += - " " + insertion_stack.at(i).n->hash.to_string(TRACE_HASH_SIZE); + std::format_to( + std::back_inserter(nodes), + " {}", + insertion_stack.at(i).n->hash.to_string(TRACE_HASH_SIZE)); MERKLECPP_TOUT << " X " << (complete ? "complete" : "continue") << ":" << nodes << std::endl; }); @@ -1950,10 +1972,10 @@ namespace merkle if (!complete && !result->is_full()) { - MERKLECPP_TRACE(MERKLECPP_TOUT - << " X save " - << result->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " X save " + << result->hash.to_string(TRACE_HASH_SIZE) + << std::endl;); return result; } } @@ -1968,9 +1990,9 @@ namespace merkle /// @param n New leaf node to insert void insert_leaf(Node*& root, Node* n) { - MERKLECPP_TRACE(MERKLECPP_TOUT << " - insert_leaf " - << n->hash.to_string(TRACE_HASH_SIZE) - << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << " - insert_leaf " + << n->hash.to_string(TRACE_HASH_SIZE) << std::endl;); leaf_nodes.push_back(n); if (insertion_stack.empty() && !root) { @@ -1990,9 +2012,9 @@ namespace merkle { if (!uninserted_leaf_nodes.empty()) { - MERKLECPP_TRACE(MERKLECPP_TOUT - << "* insert_leaves " << leaf_nodes.size() << " +" - << uninserted_leaf_nodes.size() << std::endl;); + MERKLECPP_TRACE( + MERKLECPP_TOUT << "* insert_leaves " << leaf_nodes.size() << " +" + << uninserted_leaf_nodes.size() << std::endl;); // Potential future improvement: make this go fast when there are many // leaves to insert. for (auto& n : uninserted_leaf_nodes) @@ -2008,88 +2030,131 @@ namespace merkle } }; - // clang-format off - /// @brief SHA256 compression function for tree node hashes + namespace detail + { + static inline std::array sha256_initial_state() + { + return { + 0x6a09e667, + 0xbb67ae85, + 0x3c6ef372, + 0xa54ff53a, + 0x510e527f, + 0x9b05688c, + 0x1f83d9ab, + 0x5be0cd19}; + } + + static inline void sha256_transform( + const uint8_t block[64], std::array& state) + { + static constexpr std::array constants = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, + 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, + 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, + 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, + 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + + std::array schedule = {}; + for (size_t i = 0; i < 16; ++i) + { + const size_t offset = i * 4; + schedule[i] = (static_cast(block[offset]) << 24) | + (static_cast(block[offset + 1]) << 16) | + (static_cast(block[offset + 2]) << 8) | + static_cast(block[offset + 3]); + } + + for (size_t i = 16; i < 64; ++i) + { + const uint32_t word15 = schedule[i - 15]; + const uint32_t word2 = schedule[i - 2]; + const uint32_t sigma0 = (word15 >> 7 | word15 << 25) ^ + (word15 >> 18 | word15 << 14) ^ (word15 >> 3); + const uint32_t sigma1 = (word2 >> 17 | word2 << 15) ^ + (word2 >> 19 | word2 << 13) ^ (word2 >> 10); + schedule[i] = schedule[i - 16] + sigma0 + schedule[i - 7] + sigma1; + } + + auto working = state; + for (size_t i = 0; i < 64; ++i) + { + const uint32_t choice = + (working[4] & working[5]) ^ (~working[4] & working[6]); + const uint32_t majority = (working[0] & working[1]) ^ + (working[0] & working[2]) ^ (working[1] & working[2]); + const uint32_t sigma0 = (working[0] >> 2 | working[0] << 30) ^ + (working[0] >> 13 | working[0] << 19) ^ + (working[0] >> 22 | working[0] << 10); + const uint32_t sigma1 = (working[4] >> 6 | working[4] << 26) ^ + (working[4] >> 11 | working[4] << 21) ^ + (working[4] >> 25 | working[4] << 7); + const uint32_t temporary1 = + working[7] + sigma1 + choice + constants[i] + schedule[i]; + const uint32_t temporary2 = sigma0 + majority; + + working[7] = working[6]; + working[6] = working[5]; + working[5] = working[4]; + working[4] = working[3] + temporary1; + working[3] = working[2]; + working[2] = working[1]; + working[1] = working[0]; + working[0] = temporary1 + temporary2; + } + + for (size_t i = 0; i < state.size(); ++i) + { + state[i] += working[i]; + } + } + + static inline void sha256_write_digest( + const std::array& state, HashT<32>& out) + { + for (size_t i = 0; i < state.size(); ++i) + { + out.bytes[i * 4] = static_cast(state[i] >> 24); + out.bytes[i * 4 + 1] = static_cast(state[i] >> 16); + out.bytes[i * 4 + 2] = static_cast(state[i] >> 8); + out.bytes[i * 4 + 3] = static_cast(state[i]); + } + } + } + + /// @brief Built-in SHA256 function for tree node hashes /// @param l Left node hash /// @param r Right node hash /// @param out Output node hash - /// @details This function is the compression function of SHA256, which, for - /// the special case of hashing two hashes, is more efficient than a full - /// SHA256 while providing similar guarantees. - static inline void sha256_compress(const HashT<32> &l, const HashT<32> &r, HashT<32> &out) { - static const uint32_t constants[] = { - 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, - 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, - 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, - 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, - 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, - 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, - 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, - 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 - }; - + /// @details Computes SHA256 over the 64-byte concatenation of @p l and @p r, + /// including standard SHA256 message padding. + static inline void sha256( + const HashT<32>& l, const HashT<32>& r, HashT<32>& out) + { uint8_t block[32 * 2]; memcpy(&block[0], l.bytes, 32); memcpy(&block[32], r.bytes, 32); - static const uint32_t s[8] = { 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, - 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19 }; - - uint32_t cws[64] = {0}; - - for (int i=0; i < 16; i++) { - cws[i] = convert_endianness(reinterpret_cast(block)[i]); -} - - for (int i = 16; i < 64; i++) { - uint32_t const t16 = cws[i - 16]; - uint32_t const t15 = cws[i - 15]; - uint32_t const t7 = cws[i - 7]; - uint32_t const t2 = cws[i - 2]; - uint32_t const s1 = (t2 >> 17 | t2 << 15) ^ ((t2 >> 19 | t2 << 13) ^ t2 >> 10); - uint32_t const s0 = (t15 >> 7 | t15 << 25) ^ ((t15 >> 18 | t15 << 14) ^ t15 >> 3); - cws[i] = (s1 + t7 + s0 + t16); - } - - uint32_t h[8]; - for (int i=0; i < 8; i++) { - h[i] = s[i]; -} - - for (int i=0; i < 64; i++) { - const uint32_t a0 = h[0]; - const uint32_t b0 = h[1]; - const uint32_t c0 = h[2]; - const uint32_t d0 = h[3]; - const uint32_t e0 = h[4]; - const uint32_t f0 = h[5]; - const uint32_t g0 = h[6]; - const uint32_t h03 = h[7]; - uint32_t const w = cws[i]; - uint32_t const t1 = h03 + ((e0 >> 6 | e0 << 26) ^ ((e0 >> 11 | e0 << 21) ^ (e0 >> 25 | e0 << 7))) + ((e0 & f0) ^ (~e0 & g0)) + constants[i] + w; - uint32_t const t2 = ((a0 >> 2 | a0 << 30) ^ ((a0 >> 13 | a0 << 19) ^ (a0 >> 22 | a0 << 10))) + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); - h[0] = t1 + t2; - h[1] = a0; - h[2] = b0; - h[3] = c0; - h[4] = d0 + t1; - h[5] = e0; - h[6] = f0; - h[7] = g0; - } - - for (int i=0; i < 8; i++) { - reinterpret_cast(out.bytes)[i] = convert_endianness(s[i] + h[i]); -} + auto state = detail::sha256_initial_state(); + detail::sha256_transform(block, state); + + uint8_t padding[64] = {0x80}; + padding[62] = 0x02; + detail::sha256_transform(padding, state); + detail::sha256_write_digest(state, out); } - // clang-format on #ifdef HAVE_OPENSSL /// @brief OpenSSL SHA256 /// @param l Left node hash /// @param r Right node hash /// @param out Output node hash - /// @note Some versions of OpenSSL may not provide SHA256_Transform. static inline void sha256_openssl( const merkle::HashT<32>& l, const merkle::HashT<32>& r, @@ -2104,7 +2169,7 @@ namespace merkle EVP_Digest(&block[0], sizeof(block), out.bytes, nullptr, md, nullptr); if (rc != 1) { - throw std::runtime_error("EVP_Digest failed: " + std::to_string(rc)); + throw std::runtime_error(std::format("EVP_Digest failed: {}", rc)); } } @@ -2126,7 +2191,7 @@ namespace merkle EVP_Digest(&block[0], sizeof(block), out.bytes, nullptr, md, nullptr); if (rc != 1) { - throw std::runtime_error("EVP_Digest failed: " + std::to_string(rc)); + throw std::runtime_error(std::format("EVP_Digest failed: {}", rc)); } } @@ -2148,22 +2213,16 @@ namespace merkle EVP_Digest(&block[0], sizeof(block), out.bytes, nullptr, md, nullptr); if (rc != 1) { - throw std::runtime_error("EVP_Digest failed: " + std::to_string(rc)); + throw std::runtime_error(std::format("EVP_Digest failed: {}", rc)); } } - /// @brief Type of hashes in the SHA384 tree type - using Hash384 = HashT<48>; - /// @brief Type of paths in the SHA384 tree type using Path384 = PathT<48, sha384_openssl>; /// @brief SHA384 tree with OpenSSL hash function using Tree384 = TreeT<48, sha384_openssl>; - /// @brief Type of hashes in the SHA512 tree type - using Hash512 = HashT<64>; - /// @brief Type of paths in the SHA512 tree type using Path512 = PathT<64, sha512_openssl>; @@ -2171,12 +2230,18 @@ namespace merkle using Tree512 = TreeT<64, sha512_openssl>; #endif + /// @brief Type of SHA384-sized hashes + using Hash384 = HashT<48>; + + /// @brief Type of SHA512-sized hashes + using Hash512 = HashT<64>; + /// @brief Type of hashes in the default tree type using Hash = HashT<32>; /// @brief Type of paths in the default tree type - using Path = PathT<32, sha256_compress>; + using Path = PathT<32, sha256>; /// @brief Default tree with default hash size and function - using Tree = TreeT<32, sha256_compress>; + using Tree = TreeT<32, sha256>; }; diff --git a/merklecpp_pal.h b/merklecpp_pal.h new file mode 100644 index 0000000..7d688d0 --- /dev/null +++ b/merklecpp_pal.h @@ -0,0 +1,307 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +# ifndef NOMINMAX +# define NOMINMAX +# define MERKLECPP_UNDEF_NOMINMAX +# endif +# include +# ifdef MERKLECPP_UNDEF_NOMINMAX +# undef MERKLECPP_UNDEF_NOMINMAX +# undef NOMINMAX +# endif +#else +# include +# include +#endif + +// Internal platform abstraction for durable file operations. + +namespace merkle // NOLINT(modernize-concat-nested-namespaces) +{ + namespace pal + { +#ifdef _WIN32 + using SystemError = DWORD; +#else + using SystemError = int; +#endif + + static inline SystemError last_system_error() + { +#ifdef _WIN32 + return GetLastError(); +#else + return errno; +#endif + } + + template + std::string system_error_message( + SystemError error, + std::format_string format_string, + Args&&... args) + { + std::string message; + std::format_to( + std::back_inserter(message), + format_string, + std::forward(args)...); +#ifdef _WIN32 + std::format_to(std::back_inserter(message), ": error {}", error); +#else + std::format_to(std::back_inserter(message), ": {}", std::strerror(error)); +#endif + return message; + } + + static inline uint64_t process_id() + { +#ifdef _WIN32 + return static_cast(GetCurrentProcessId()); +#else + return static_cast(::getpid()); +#endif + } + +#ifndef _WIN32 + namespace detail + { + template + static inline int retry_on_eintr(const Operation& operation) + { + int result = 0; + do + { + result = operation(); + } while (result == -1 && errno == EINTR); + return result; + } + + } +#endif + + static inline void require_write_progress( + size_t written, const std::filesystem::path& path) + { + if (written == 0) + { + throw std::runtime_error(std::format("short write: {}", path.string())); + } + } + + static inline void remove_owned_file( + const std::filesystem::path& path) noexcept + { + std::error_code ec; + std::filesystem::remove(path, ec); + } + + /// Writes and syncs a new file, failing if @p path already exists. + /// A file created by this call is removed if a later operation fails. + static inline void write_and_sync_file( + const std::filesystem::path& path, const std::vector& bytes) + { +#ifdef _WIN32 + HANDLE handle = CreateFileW( + path.wstring().c_str(), + GENERIC_WRITE, + 0, + nullptr, + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (handle == INVALID_HANDLE_VALUE) + { + const auto error = last_system_error(); + throw std::runtime_error( + system_error_message(error, "cannot open file {}", path.string())); + } + bool close_handle = true; + try + { + constexpr auto max_write_size = + static_cast(std::numeric_limits::max()); + size_t written = 0; + while (written < bytes.size()) + { + const auto remaining = bytes.size() - written; + const auto chunk = + static_cast(std::min(remaining, max_write_size)); + DWORD done = 0; + if (!WriteFile(handle, bytes.data() + written, chunk, &done, nullptr)) + { + const auto error = last_system_error(); + throw std::runtime_error(system_error_message( + error, "error writing file {}", path.string())); + } + require_write_progress(static_cast(done), path); + written += done; + } + if (!FlushFileBuffers(handle)) + { + const auto error = last_system_error(); + throw std::runtime_error(system_error_message( + error, "error syncing file {}", path.string())); + } + if (!CloseHandle(handle)) + { + const auto error = last_system_error(); + throw std::runtime_error(system_error_message( + error, "error closing file {}", path.string())); + } + close_handle = false; + } + catch (...) + { + if (close_handle) + { + CloseHandle(handle); + } + remove_owned_file(path); + throw; + } +#else + int flags = O_WRONLY | O_CREAT | O_EXCL; +# ifdef O_CLOEXEC + flags |= O_CLOEXEC; +# endif + int fd = ::open(path.c_str(), flags, 0666); + if (fd < 0) + { + const auto error = last_system_error(); + throw std::runtime_error( + system_error_message(error, "cannot open file {}", path.string())); + } + try + { + size_t written = 0; + while (written < bytes.size()) + { + const ssize_t done = + ::write(fd, bytes.data() + written, bytes.size() - written); + if (done < 0) + { + if (errno == EINTR) + { + continue; + } + const auto error = last_system_error(); + throw std::runtime_error(system_error_message( + error, "error writing file {}", path.string())); + } + require_write_progress(static_cast(done), path); + written += static_cast(done); + } + if (detail::retry_on_eintr([fd]() { return ::fsync(fd); }) != 0) + { + const auto error = last_system_error(); + throw std::runtime_error(system_error_message( + error, "error syncing file {}", path.string())); + } + if (::close(fd) != 0) + { + const auto error = last_system_error(); + fd = -1; + throw std::runtime_error(system_error_message( + error, "error closing file {}", path.string())); + } + fd = -1; + } + catch (...) + { + if (fd >= 0) + { + ::close(fd); + } + remove_owned_file(path); + throw; + } +#endif + } + + static inline void replace_file( + const std::filesystem::path& tmp, const std::filesystem::path& path) + { +#ifdef _WIN32 + if (!MoveFileExW( + tmp.wstring().c_str(), + path.wstring().c_str(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) + { + const auto error = last_system_error(); + throw std::runtime_error(system_error_message( + error, + "cannot rename temp file {} to {}", + tmp.string(), + path.string())); + } +#else + std::error_code ec; + std::filesystem::rename(tmp, path, ec); + if (ec) + { + throw std::runtime_error(std::format( + "cannot rename temp file {} to {}: {}", + tmp.string(), + path.string(), + ec.message())); + } +#endif + } + + static inline void sync_directory_on_disk(const std::filesystem::path& path) + { +#ifndef _WIN32 + int flags = O_RDONLY; +# ifdef O_DIRECTORY + flags |= O_DIRECTORY; +# endif +# ifdef O_CLOEXEC + flags |= O_CLOEXEC; +# endif + const int fd = ::open(path.c_str(), flags); + if (fd < 0) + { + const auto error = last_system_error(); + throw std::runtime_error(system_error_message( + error, "cannot open directory {}", path.string())); + } + if (detail::retry_on_eintr([fd]() { return ::fsync(fd); }) != 0) + { + const auto error = last_system_error(); + const std::string message = system_error_message( + error, "error syncing directory {}", path.string()); + ::close(fd); + throw std::runtime_error(message); + } + if (::close(fd) != 0) + { + const auto error = last_system_error(); + throw std::runtime_error(system_error_message( + error, "error closing directory {}", path.string())); + } +#else + (void)path; +#endif + } + } +} diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h new file mode 100644 index 0000000..8485e24 --- /dev/null +++ b/merklecpp_tiles.h @@ -0,0 +1,739 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "merklecpp.h" +#include "merklecpp_pal.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Tiled storage for merklecpp trees, following the full-tile geometry, payload, +// and path encoding of the C2SP tlog-tiles layout +// (https://c2sp.org/tlog-tiles). C2SP defines SHA-256; merklecpp preserves the +// 256-hash tile width for other SHA output sizes and separates each format +// under an algorithm-qualified directory such as sha256-256w or sha384-256w. +// Only complete, immutable tiles are stored. Their hash values are produced by +// the tree's existing HASH_FUNCTION, so tile-derived proofs are byte-identical +// to those produced by merkle::TreeT (see doc/design/tlog-tiles.md). +// +// Thread safety: types in this header do not synchronize access internally. +// Callers must serialize access to each shared object, including const proof +// operations that update the tile cache, and all writers sharing a store +// prefix. Independent store objects may read while the serialized writer +// publishes tiles atomically. + +namespace merkle // NOLINT(modernize-concat-nested-namespaces) +{ + namespace tiles + { + /// @brief Number of tree levels spanned by a single tile. + static constexpr uint16_t TILE_HEIGHT = 8; + + /// @brief Number of hashes in a full tile (2**TILE_HEIGHT). + static constexpr uint16_t TILE_WIDTH = uint16_t{1U << TILE_HEIGHT}; + + /// @brief Highest tile level permitted by the tlog-tiles layout. + static constexpr uint8_t MAX_TILE_LEVEL = 63; + + namespace detail + { + static constexpr std::string_view SHA256_ALGORITHM_SHORT_NAME = "sha256"; + static constexpr std::string_view SHA384_ALGORITHM_SHORT_NAME = "sha384"; + static constexpr std::string_view SHA512_ALGORITHM_SHORT_NAME = "sha512"; + static constexpr size_t ENTRY_LENGTH_PREFIX_SIZE = 2; + static constexpr size_t MAX_ENTRY_SIZE = 0xFFFF; + static constexpr size_t MAX_ENTRY_BUNDLE_SIZE = + TILE_WIDTH * (ENTRY_LENGTH_PREFIX_SIZE + MAX_ENTRY_SIZE); + + template + uint64_t contiguous_prefix_length(uint64_t limit, const Present& present) + { + uint64_t length = 0; + while (length < limit && present(length)) + { + length++; + } + return length; + } + } + + /// @brief Encodes a tile index as tlog-tiles path elements. + /// @param n The tile index + /// @return The index as zero-padded, '/'-separated 3-digit groups, where + /// all but the last group are prefixed with 'x'. For example, 1234067 is + /// encoded as "x001/x234/067" and 5 as "005". + static inline std::string encode_tile_index(uint64_t n) + { + std::vector parts; + do + { + parts.emplace_back(static_cast(n % 1000)); + n /= 1000; + } while (n > 0); + + std::string r; + r.reserve(parts.size() * 5); + for (size_t i = 0; i < parts.size(); i++) + { + const auto part = parts[parts.size() - 1 - i]; + std::format_to( + std::back_inserter(r), + "{}{}{:03}", + i == 0 ? "" : "/", + i + 1 < parts.size() ? "x" : "", + part); + } + return r; + } + + /// @brief Identifies a single (full) tile within a tiled log. + /// @note Only full, TILE_WIDTH-wide tiles are produced and consumed; the + /// incomplete frontier is never tiled (see doc/design/tlog-tiles.md). + struct TileRef + { + /// @brief The level of the tile (0 == leaf hashes, maximum 63). + uint8_t level = 0; + + /// @brief The index of the tile within its level. + uint64_t index = 0; + }; + + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TileWriterT; + + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class EntryBundleWriterT; + + /// @brief Reads and writes tlog-tiles tile files on a local filesystem. + /// @tparam HASH_SIZE Size of each hash in bytes + /// @tparam HASH_FUNCTION The tree's node hash function (carried for use by + /// later components; tile I/O itself does not hash). + /// @warning No internal synchronization is provided. Callers must serialize + /// access to each store object and all writers sharing its prefix. + /// Independent store objects may read while the serialized writer publishes + /// tiles atomically. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TileStoreT + { + friend class TileWriterT; + friend class EntryBundleWriterT; + + public: + /// @brief The type of hashes stored in tiles. + using Hash = HashT; + + /// @brief Constructs a tile store below an algorithm-qualified directory + /// under @p prefix. + /// @param prefix The directory under which the format directory lives + /// @note Built-in SHA functions select sha256, sha384, or sha512. + explicit TileStoreT(std::filesystem::path prefix) : + TileStoreT(std::move(prefix), default_hash_algorithm_short_name()) + {} + + /// @brief Constructs a tile store for an explicitly named hash algorithm. + /// @param prefix The directory under which the format directory lives + /// @param hash_algorithm_short_name Lowercase algorithm short name + TileStoreT( + std::filesystem::path prefix, + const std::string& hash_algorithm_short_name) : + prefix(storage_root(std::move(prefix), hash_algorithm_short_name)) + {} + + /// @brief The algorithm-qualified root directory of the store. + [[nodiscard]] const std::filesystem::path& root() const + { + return prefix; + } + + /// @brief The format directory for @p hash_algorithm_short_name. + /// @note TILE_WIDTH is fixed at 256 for every hash output size. + static std::string storage_directory_name( + const std::string& hash_algorithm_short_name) + { + validate_hash_algorithm_short_name(hash_algorithm_short_name); + return std::format("{}-{}w", hash_algorithm_short_name, TILE_WIDTH); + } + + /// @brief Encodes a tile index (see encode_tile_index). + static std::string encode_index(uint64_t n) + { + return encode_tile_index(n); + } + + /// @brief The filesystem path of a tile. + /// @throws std::runtime_error if the tile level exceeds 63 + [[nodiscard]] std::filesystem::path tile_path(const TileRef& ref) const + { + if (ref.level > MAX_TILE_LEVEL) + { + throw std::runtime_error("tile level out of range"); + } + const auto relative_path = std::format( + "tile/{}/{}", + static_cast(ref.level), + encode_tile_index(ref.index)); + return prefix / relative_path; + } + + /// @brief The filesystem path of an entry bundle. + [[nodiscard]] std::filesystem::path entries_path(uint64_t index) const + { + const auto relative_path = + std::format("tile/entries/{}", encode_tile_index(index)); + return prefix / relative_path; + } + + /// @brief Whether a full tile exists on disk. + [[nodiscard]] bool has_full_tile(uint8_t level, uint64_t index) const + { + if (level > MAX_TILE_LEVEL) + { + return false; + } + std::error_code ec; + const auto path = tile_path(TileRef{level, index}); + if (!std::filesystem::is_regular_file(path, ec) || ec) + { + return false; + } + return std::filesystem::file_size(path, ec) == + (uintmax_t)TILE_WIDTH * HASH_SIZE && + !ec; + } + + /// @brief Writes a tile to disk atomically. + /// @param ref The tile to write + /// @param hashes The tile's hashes (exactly TILE_WIDTH of them) + void write_tile(const TileRef& ref, const std::vector& hashes) + { + if (hashes.size() != TILE_WIDTH) + { + throw std::runtime_error("tile width mismatch"); + } + + std::vector bytes; + bytes.reserve(hashes.size() * HASH_SIZE); + for (const auto& h : hashes) + { + h.serialise(bytes); + } + + write_file_atomically(tile_path(ref), bytes); + } + + /// @brief Reads a tile from disk. + /// @param ref The tile to read + /// @return The tile's hashes (TILE_WIDTH of them) + [[nodiscard]] std::vector read_tile(const TileRef& ref) const + { + const size_t expected = (size_t)TILE_WIDTH * HASH_SIZE; + const auto path = tile_path(ref); + std::vector bytes = read_file(path, expected); + if (bytes.size() != expected) + { + throw std::runtime_error(std::format( + "unexpected tile size for {}: expected {} bytes, got {}", + path.string(), + expected, + bytes.size())); + } + + std::vector hashes; + hashes.reserve(TILE_WIDTH); + size_t position = 0; + for (uint16_t i = 0; i < TILE_WIDTH; i++) + { + hashes.emplace_back(bytes, position); + } + return hashes; + } + + /// @brief Whether a full entry bundle exists on disk. + [[nodiscard]] bool has_entry_bundle(uint64_t index) const + { + std::error_code ec; + const auto path = entries_path(index); + if (!std::filesystem::is_regular_file(path, ec) || ec) + { + return false; + } + try + { + (void)decode_entries( + read_file(path, detail::MAX_ENTRY_BUNDLE_SIZE), TILE_WIDTH); + return true; + } + catch (const std::runtime_error&) + { + return false; + } + } + + /// @brief Writes a full entry bundle to disk atomically. + /// @param index The bundle index + /// @param entries The raw log entries (exactly TILE_WIDTH of them) + /// @note Entries are stored in the tlog-tiles entry-bundle format: a + /// sequence of big-endian uint16 length-prefixed byte strings. + void write_entry_bundle( + uint64_t index, const std::vector>& entries) + { + if (entries.size() != TILE_WIDTH) + { + throw std::runtime_error("entry bundle width mismatch"); + } + write_file_atomically(entries_path(index), encode_entries(entries)); + } + + /// @brief Reads a full entry bundle from disk. + /// @param index The bundle index + /// @return The raw log entries (TILE_WIDTH of them) + [[nodiscard]] std::vector> read_entry_bundle( + uint64_t index) const + { + const auto path = entries_path(index); + const auto bytes = read_file(path, detail::MAX_ENTRY_BUNDLE_SIZE); + try + { + return decode_entries(bytes, TILE_WIDTH); + } + catch (const std::runtime_error& error) + { + throw std::runtime_error(std::format( + "invalid entry bundle {}: {}", path.string(), error.what())); + } + } + + /// @brief Encodes log entries into the tlog-tiles entry-bundle format. + static std::vector encode_entries( + const std::vector>& entries) + { + size_t encoded_size = 0; + for (const auto& e : entries) + { + if (e.size() > detail::MAX_ENTRY_SIZE) + { + throw std::runtime_error( + "entry too large for uint16 length prefix"); + } + encoded_size += detail::ENTRY_LENGTH_PREFIX_SIZE + e.size(); + } + + std::vector bytes; + bytes.reserve(encoded_size); + for (const auto& e : entries) + { + bytes.push_back((uint8_t)((e.size() >> 8) & 0xFF)); + bytes.push_back((uint8_t)(e.size() & 0xFF)); + bytes.insert(bytes.end(), e.begin(), e.end()); + } + return bytes; + } + + /// @brief Decodes @p count entries from the entry-bundle format. + static std::vector> decode_entries( + const std::vector& bytes, size_t count) + { + if (count > bytes.size() / detail::ENTRY_LENGTH_PREFIX_SIZE) + { + throw std::runtime_error("truncated entry bundle"); + } + std::vector> out; + out.reserve(count); + size_t pos = 0; + for (size_t i = 0; i < count; i++) + { + if (bytes.size() - pos < 2) + { + throw std::runtime_error("truncated entry bundle"); + } + const auto len = + (uint16_t)(((uint16_t)bytes[pos] << 8) | bytes[pos + 1]); + pos += 2; + if (len > bytes.size() - pos) + { + throw std::runtime_error("truncated entry bundle"); + } + out.emplace_back( + bytes.begin() + static_cast(pos), + bytes.begin() + static_cast(pos + len)); + pos += len; + } + if (pos != bytes.size()) + { + throw std::runtime_error("trailing bytes in entry bundle"); + } + return out; + } + + /// @cond INTERNAL + protected: + using DirectorySync = std::function; + + TileStoreT(std::filesystem::path prefix, DirectorySync directory_sync) : + TileStoreT( + std::move(prefix), + default_hash_algorithm_short_name(), + std::move(directory_sync)) + {} + + TileStoreT( + std::filesystem::path prefix, + const std::string& hash_algorithm_short_name, + DirectorySync directory_sync) : + prefix(storage_root(std::move(prefix), hash_algorithm_short_name)), + directory_sync(std::move(directory_sync)) + {} + + /// @brief The algorithm-qualified root directory of the store. + std::filesystem::path prefix; + + DirectorySync directory_sync; + std::set durable_directory_entries; + std::set durable_directory_contents; + + static std::string default_hash_algorithm_short_name() + { + if constexpr (HASH_SIZE == merkle::Tree::Hash::size_bytes) + { + if constexpr (HASH_FUNCTION == merkle::Tree::hash_function) + { + return std::string(detail::SHA256_ALGORITHM_SHORT_NAME); + } +#ifdef HAVE_OPENSSL + if constexpr (HASH_FUNCTION == sha256_openssl) + { + return std::string(detail::SHA256_ALGORITHM_SHORT_NAME); + } +#endif + } +#ifdef HAVE_OPENSSL + else if constexpr (HASH_SIZE == merkle::Tree384::Hash::size_bytes) + { + if constexpr (HASH_FUNCTION == merkle::Tree384::hash_function) + { + return std::string(detail::SHA384_ALGORITHM_SHORT_NAME); + } + } + else if constexpr (HASH_SIZE == merkle::Tree512::Hash::size_bytes) + { + if constexpr (HASH_FUNCTION == merkle::Tree512::hash_function) + { + return std::string(detail::SHA512_ALGORITHM_SHORT_NAME); + } + } +#endif + throw std::runtime_error( + "TileStoreT requires a hash algorithm short name"); + } + + static void validate_hash_algorithm_short_name( + const std::string& hash_algorithm_short_name) + { + if ( + hash_algorithm_short_name.empty() || + hash_algorithm_short_name.front() == '-' || + hash_algorithm_short_name.back() == '-') + { + throw std::runtime_error("invalid hash algorithm short name"); + } + for (const char c : hash_algorithm_short_name) + { + if ((c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '-') + { + throw std::runtime_error("invalid hash algorithm short name"); + } + } + } + + static std::filesystem::path storage_root( + std::filesystem::path prefix, + const std::string& hash_algorithm_short_name) + { + if ( + (hash_algorithm_short_name == detail::SHA256_ALGORITHM_SHORT_NAME && + HASH_SIZE != merkle::Hash::size_bytes) || + (hash_algorithm_short_name == detail::SHA384_ALGORITHM_SHORT_NAME && + HASH_SIZE != merkle::Hash384::size_bytes) || + (hash_algorithm_short_name == detail::SHA512_ALGORITHM_SHORT_NAME && + HASH_SIZE != merkle::Hash512::size_bytes)) + { + throw std::runtime_error( + "hash algorithm short name does not match hash size"); + } + prefix /= storage_directory_name(hash_algorithm_short_name); + return prefix; + } + + [[nodiscard]] bool confirm_full_tile(uint8_t level, uint64_t index) + { + const auto path = tile_path(TileRef{level, index}); + if (!has_full_tile(level, index)) + { + return false; + } + confirm_file_durable(path); + return true; + } + + [[nodiscard]] bool confirm_entry_bundle(uint64_t index) + { + const auto path = entries_path(index); + if (!has_entry_bundle(index)) + { + return false; + } + confirm_file_durable(path); + return true; + } + + void begin_write_attempt() + { + durable_directory_contents.clear(); + } + + /// @brief Reads an entire file into a byte vector. + static std::vector read_file( + const std::filesystem::path& path, size_t max_size) + { + std::ifstream f(path, std::ios::binary); + if (!f.good()) + { + throw std::runtime_error( + std::format("cannot open file: {}", path.string())); + } + + std::vector bytes; + std::array buffer{}; + while (f) + { + const size_t remaining = max_size - bytes.size(); + const size_t request = + remaining < buffer.size() ? remaining + 1 : buffer.size(); + f.read( + reinterpret_cast(buffer.data()), + static_cast(request)); + const auto count = static_cast(f.gcount()); + if (count == 0) + { + break; + } + if (count > remaining) + { + throw std::runtime_error( + std::format("file exceeds maximum size: {}", path.string())); + } + bytes.insert(bytes.end(), buffer.begin(), buffer.begin() + count); + } + if (f.bad()) + { + throw std::runtime_error( + std::format("error reading file: {}", path.string())); + } + return bytes; + } + + /// @brief Writes a file atomically via a synced temporary file. + /// @note Uses unique temp names, cleans them up on errors, and syncs the + /// file before publishing it with an atomic replace. POSIX builds also + /// confirm each directory entry in the path and sync the destination + /// directory after rename. Existing files are only reused after these + /// syncs are re-confirmed by the current write attempt. + void write_file_atomically( + const std::filesystem::path& path, const std::vector& bytes) + { + create_directories_durably(path.parent_path()); + + const std::filesystem::path tmp = temp_path(path); + TempFileGuard guard(tmp); + merkle::pal::write_and_sync_file(tmp, bytes); + guard.arm(); + const auto parent = directory_or_dot(path.parent_path()); + durable_directory_contents.erase(parent); + merkle::pal::replace_file(tmp, path); + sync_directory_contents(parent); + guard.dismiss(); + } + + static std::filesystem::path directory_or_dot( + const std::filesystem::path& directory) + { + return directory.empty() ? std::filesystem::path(".") : directory; + } + + void create_directories_durably( + const std::filesystem::path& requested_directory) + { + const auto directory = directory_or_dot(requested_directory); + if (durable_directory_entries.contains(directory)) + { + return; + } + + auto parent = directory_or_dot(directory.parent_path()); + if (parent != directory) + { + create_directories_durably(parent); + } + + std::error_code ec; + const bool exists = std::filesystem::exists(directory, ec); + if (ec) + { + throw std::runtime_error(std::format( + "cannot inspect directory {}: {}", + directory.string(), + ec.message())); + } + if (exists) + { + const bool is_directory = + std::filesystem::is_directory(directory, ec); + if (ec) + { + throw std::runtime_error(std::format( + "cannot inspect directory {}: {}", + directory.string(), + ec.message())); + } + if (!is_directory) + { + throw std::runtime_error(std::format( + "cannot create directory {}: path exists and is not a directory", + directory.string())); + } + } + else + { + const bool created = std::filesystem::create_directory(directory, ec); + if (ec) + { + throw std::runtime_error(std::format( + "cannot create directory {}: {}", + directory.string(), + ec.message())); + } + if (!created && !std::filesystem::is_directory(directory, ec)) + { + throw std::runtime_error( + ec ? + std::format( + "cannot create directory {}: {}", + directory.string(), + ec.message()) : + std::format("cannot create directory {}", directory.string())); + } + } + + if (parent != directory) + { + sync_directory_for_durability(parent); + durable_directory_contents.insert(parent); + } + durable_directory_entries.insert(directory); + } + + void confirm_file_durable(const std::filesystem::path& path) + { + const auto parent = directory_or_dot(path.parent_path()); + create_directories_durably(parent); + sync_directory_contents(parent); + } + + void sync_directory_contents(const std::filesystem::path& directory) + { + const auto path = directory_or_dot(directory); + if (durable_directory_contents.contains(path)) + { + return; + } + sync_directory_for_durability(path); + durable_directory_contents.insert(path); + } + + class TempFileGuard + { + public: + explicit TempFileGuard(std::filesystem::path path) : + path(std::move(path)) + {} + + ~TempFileGuard() + { + if (active) + { + std::error_code ec; + std::filesystem::remove(path, ec); + } + } + + void arm() + { + active = true; + } + + void dismiss() + { + active = false; + } + + private: + std::filesystem::path path; + bool active = false; + }; + + static std::filesystem::path temp_path(const std::filesystem::path& path) + { + static std::atomic counter{0}; + const auto stamp = + std::chrono::steady_clock::now().time_since_epoch().count(); + std::filesystem::path tmp = path; + tmp += std::format( + ".tmp.{}.{}.{}", + merkle::pal::process_id(), + (uint64_t)stamp, + counter.fetch_add(1, std::memory_order_relaxed)); + return tmp; + } + + void sync_directory_for_durability(const std::filesystem::path& path) + { + if (directory_sync) + { + directory_sync(path); + return; + } + merkle::pal::sync_directory_on_disk(path); + } + /// @endcond + }; + + /// @brief Default tile store (SHA256, default hash function). + using TileStore = + TileStoreT; + + } +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6dee12c..e21cadd 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,16 +3,24 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +include(FetchContent) +FetchContent_Declare( + doctest + GIT_REPOSITORY https://github.com/doctest/doctest.git + GIT_TAG 2d0a9359a60c51affe2a9bebb1be1dca47868151 # v2.5.3 +) +set(DOCTEST_NO_INSTALL ON) +set(DOCTEST_WITH_MAIN_IN_STATIC_LIB OFF) +FetchContent_MakeAvailable(doctest) + function(add_merklecpp_test target source) add_unit_test(${MERKLECPP_TEST_PREFIX}${target} ${source}) + target_link_libraries( + ${MERKLECPP_TEST_PREFIX}${target} PRIVATE doctest::doctest + ) set_tests_properties( ${MERKLECPP_TEST_PREFIX}${target} PROPERTIES LABELS "merklecpp" ) - if (DOCTEST_DIR) - target_include_directories( - ${MERKLECPP_TEST_PREFIX}${target} PRIVATE ${DOCTEST_DIR} - ) - endif() if (WIN32) target_compile_definitions(${MERKLECPP_TEST_PREFIX}${target} PRIVATE _CRT_SECURE_NO_WARNINGS) endif() @@ -34,8 +42,5 @@ if(OPENSSL) add_merklecpp_test(compare_hash_functions compare_hash_functions.cpp) endif() -find_path(DOCTEST_DIR doctest.h) - -if(DOCTEST_DIR) - add_merklecpp_test(unit_tests unit_tests.cpp) -endif() +add_merklecpp_test(tiles_store tiles_store.cpp) +add_merklecpp_test(unit_tests unit_tests.cpp) diff --git a/test/compare_hash_functions.cpp b/test/compare_hash_functions.cpp index 2bb7def..bb3a403 100644 --- a/test/compare_hash_functions.cpp +++ b/test/compare_hash_functions.cpp @@ -10,10 +10,93 @@ #include #include +constexpr size_t PRNTSZ = 3; + +#ifdef HAVE_OPENSSL +using OpenSSLTree = merkle::TreeT<32, merkle::sha256_openssl>; +#endif + +template < + void (*HF1)( + const merkle::HashT<32>& l, + const merkle::HashT<32>& r, + merkle::HashT<32>& out), + void (*HF2)( + const merkle::HashT<32>& l, + const merkle::HashT<32>& r, + merkle::HashT<32>& out)> +void compare_roots( + merkle::TreeT<32, HF1>& mt1, merkle::TreeT<32, HF2>& mt2, const char* name) +{ + auto mt1_root = mt1.root(); + auto mt2_root = mt2.root(); + + if (mt1_root != mt2_root) + { + std::cout << mt1.num_leaves() << ": " << mt1_root.to_string() + << " != " << mt2_root.to_string() << '\n'; + std::cout << "mt1: " << '\n'; + std::cout << mt1.to_string(PRNTSZ) << '\n'; + std::cout << name << ": " << '\n'; + std::cout << mt2.to_string(PRNTSZ) << '\n'; + throw std::runtime_error("root hash mismatch"); + } +} + +void compare_sha256_hashes() +{ +#ifndef NDEBUG + const size_t num_trees = 1024; + const size_t root_interval = 31; +#else + const size_t num_trees = 4096; + const size_t root_interval = 128; +#endif + + size_t total_inserts = 0; + size_t total_roots = 0; + + for (size_t k = 0; k < num_trees; k++) + { + merkle::Tree mt; + +#ifdef HAVE_OPENSSL + OpenSSLTree mto; +#endif + + // Build trees with k+1 leaves + int j = 0; + auto hashes = make_hashes(k + 1); + + for (const auto h : hashes) + { + mt.insert(h); + #ifdef HAVE_OPENSSL -using OpenSSLFullTree = merkle::TreeT<32, merkle::sha256_openssl>; + mto.insert(h); #endif + total_inserts++; + + if ((j++ % root_interval) == 0) + { +#ifdef HAVE_OPENSSL + compare_roots(mt, mto, "OpenSSL"); +#endif + + total_roots++; + } + } + +#ifdef HAVE_OPENSSL + compare_roots(mt, mto, "OpenSSL"); +#endif + } + + std::cout << num_trees << " trees, " << total_inserts << " inserts, " + << total_roots << " roots with SHA256: OK" << '\n'; +} + template void bench( const std::vector& hashes, @@ -34,7 +117,9 @@ void bench( mt.root(); auto stop = std::chrono::high_resolution_clock::now(); const double seconds = - static_cast(std::chrono::duration_cast(stop - start).count()) / + static_cast( + std::chrono::duration_cast(stop - start) + .count()) / 1e9; std::cout << std::left << std::setw(10) << name << ": " << mt.statistics.num_insert << " insertions, " @@ -62,7 +147,9 @@ void benchT( mt.root(); auto stop = std::chrono::high_resolution_clock::now(); const double seconds = - static_cast(std::chrono::duration_cast(stop - start).count()) / + static_cast( + std::chrono::duration_cast(stop - start) + .count()) / 1e9; std::cout << std::left << std::setw(10) << name << ": " << mt.statistics.num_insert << " insertions, " @@ -77,6 +164,8 @@ int main() // std::srand(0); std::srand(std::time(nullptr)); + compare_sha256_hashes(); + #ifndef NDEBUG const size_t num_leaves = static_cast(128) * 1024; const size_t root_interval = 128; @@ -87,26 +176,23 @@ int main() auto hashes = make_hashes(num_leaves); - std::cout << "--- merklecpp trees with SHA256 compression function: " - << '\n'; + std::cout << "--- merklecpp trees with SHA256: " << '\n'; bench(hashes, "merklecpp", root_interval); - std::cout << "--- merklecpp trees with full SHA256: " << '\n'; - #ifdef HAVE_OPENSSL - bench(hashes, "OpenSSL", root_interval); + bench(hashes, "OpenSSL", root_interval); #endif #ifdef HAVE_OPENSSL { - std::cout << "--- merklecpp trees with full SHA384: " << '\n'; + std::cout << "--- merklecpp trees with SHA384: " << '\n'; auto hashes384 = make_hashesT<48>(num_leaves); benchT(hashes384, "OpenSSL", root_interval); } { - std::cout << "--- merklecpp trees with full SHA512: " << '\n'; + std::cout << "--- merklecpp trees with SHA512: " << '\n'; auto hashes512 = make_hashesT<64>(num_leaves); benchT(hashes512, "OpenSSL", root_interval); } diff --git a/test/coverage.cpp b/test/coverage.cpp index a930d17..eaa4626 100644 --- a/test/coverage.cpp +++ b/test/coverage.cpp @@ -116,6 +116,9 @@ namespace { require(byte == 0xAA, "valid hash string parsed incorrectly"); } + require_throws( + [&] { (void)valid_hash.to_string(valid_hash.size() + 1); }, + "hash string should reject byte counts beyond the hash size"); std::string mixed_case_hex(64, '0'); mixed_case_hex[0] = 'A'; diff --git a/test/tiles_store.cpp b/test/tiles_store.cpp new file mode 100644 index 0000000..6dbe5d2 --- /dev/null +++ b/test/tiles_store.cpp @@ -0,0 +1,677 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN + +#include + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TileRef; + +static_assert(std::min(1, 2) == 1); +static_assert(std::max(1, 2) == 2); + +class TemporaryDirectory +{ +private: + fs::path path_; + +public: + TemporaryDirectory() + { + static std::atomic sequence = 0; + const auto nonce = + std::chrono::steady_clock::now().time_since_epoch().count(); + path_ = fs::temp_directory_path() / + std::format( + "merklecpp_tiles_{}_{}_{}", + merkle::pal::process_id(), + nonce, + sequence++); + } + + ~TemporaryDirectory() + { + std::error_code ec; + fs::remove_all(path_, ec); + } + + TemporaryDirectory(const TemporaryDirectory&) = delete; + TemporaryDirectory& operator=(const TemporaryDirectory&) = delete; + + [[nodiscard]] const fs::path& path() const + { + return path_; + } +}; + +static void custom_hash(const Hash& lhs, const Hash& rhs, Hash& out) +{ + (void)rhs; + out = lhs; +} + +template +static std::string rel(const Store& store, const fs::path& p) +{ + return p.lexically_relative(store.root()).generic_string(); +} + +static bool any_tmp_files(const fs::path& dir) +{ + if (!fs::exists(dir)) + { + return false; + } + return std::any_of( + fs::recursive_directory_iterator(dir), + fs::recursive_directory_iterator(), + [](const auto& entry) { + return entry.path().filename().string().find(".tmp") != std::string::npos; + }); +} + +struct SyncFault +{ + fs::path fail_path; + size_t failures_remaining = 0; + size_t matching_calls_before_failure = 0; + std::vector calls; + + void sync(const fs::path& path) + { + const auto normalised = path.lexically_normal(); + calls.push_back(normalised); + if (failures_remaining > 0 && normalised == fail_path.lexically_normal()) + { + if (matching_calls_before_failure > 0) + { + matching_calls_before_failure--; + return; + } + failures_remaining--; + throw std::runtime_error("injected directory sync failure"); + } + } + + [[nodiscard]] size_t call_count(const fs::path& path) const + { + const auto normalised = path.lexically_normal(); + return (size_t)std::count(calls.begin(), calls.end(), normalised); + } +}; + +class FaultInjectingTileStore : public merkle::tiles::TileStore +{ +public: + using Store = merkle::tiles::TileStore; + + FaultInjectingTileStore( + fs::path prefix, const std::shared_ptr& fault) : + Store( + std::move(prefix), [fault](const fs::path& path) { fault->sync(path); }) + {} + + void begin_attempt() + { + Store::begin_write_attempt(); + } + + [[nodiscard]] bool confirm_tile(uint8_t level, uint64_t index) + { + return Store::confirm_full_tile(level, index); + } + + [[nodiscard]] bool confirm_bundle(uint64_t index) + { + return Store::confirm_entry_bundle(index); + } +}; + +static void overwrite_file(const fs::path& p, const std::vector& bytes) +{ + std::ofstream f(p, std::ios::binary | std::ios::trunc); + f.write( + reinterpret_cast(bytes.data()), (std::streamsize)bytes.size()); +} + +static std::vector read_file(const fs::path& p) +{ + std::ifstream f(p, std::ios::binary); + if (!f.good()) + { + throw std::runtime_error( + std::format("cannot open test file: {}", p.string())); + } + return {std::istreambuf_iterator(f), std::istreambuf_iterator()}; +} + +static std::vector> make_entries() +{ + std::vector> entries(merkle::tiles::TILE_WIDTH); + entries[1] = {0xA5}; + entries[2] = std::vector(256, 0x5A); + for (size_t i = 3; i < entries.size(); i++) + { + entries[i] = {(uint8_t)i, 0x7F}; + } + return entries; +} + +TEST_CASE("Tile geometry, references, and index encoding") +{ + // 1. Tile geometry, references, and index encoding. + CHECK(merkle::tiles::TILE_WIDTH == (1U << merkle::tiles::TILE_HEIGHT)); + + const TileRef default_ref; + CHECK(default_ref.level == 0); + CHECK(default_ref.index == 0); + + size_t prefix_probes = 0; + const auto prefix_length = + merkle::tiles::detail::contiguous_prefix_length(10, [&](uint64_t index) { + prefix_probes++; + return index < 3; + }); + CHECK(prefix_length == 3); + CHECK(prefix_probes == 4); + + prefix_probes = 0; + const auto empty_prefix_length = + merkle::tiles::detail::contiguous_prefix_length(0, [&](uint64_t) { + prefix_probes++; + return true; + }); + CHECK(empty_prefix_length == 0); + CHECK(prefix_probes == 0); + + CHECK(merkle::tiles::encode_tile_index(0) == "000"); + CHECK(merkle::tiles::encode_tile_index(5) == "005"); + CHECK(merkle::tiles::encode_tile_index(255) == "255"); + CHECK(merkle::tiles::encode_tile_index(999) == "999"); + CHECK(merkle::tiles::encode_tile_index(1000) == "x001/000"); + CHECK(merkle::tiles::encode_tile_index(1234067) == "x001/x234/067"); + CHECK( + merkle::tiles::encode_tile_index(std::numeric_limits::max()) == + "x018/x446/x744/x073/x709/x551/615"); + CHECK(merkle::tiles::TileStore::encode_index(1000) == "x001/000"); +} + +TEST_CASE("Tile store paths and hash namespaces") +{ + // 2. Resource path layout (full tiles and bundles only). + const TemporaryDirectory temporary_directory; + const fs::path& dir = temporary_directory.path(); + const merkle::tiles::TileStore store(dir); + + CHECK(store.root().lexically_relative(dir).generic_string() == "sha256-256w"); + CHECK(rel(store, store.tile_path(TileRef{0, 0})) == "tile/0/000"); + CHECK( + rel(store, store.tile_path(TileRef{1, 1234067})) == "tile/1/x001/x234/067"); + CHECK( + rel(store, store.tile_path(TileRef{merkle::tiles::MAX_TILE_LEVEL, 0})) == + "tile/63/000"); + CHECK_THROWS_AS( + ((void)store.tile_path( + TileRef{static_cast(merkle::tiles::MAX_TILE_LEVEL + 1), 0})), + std::runtime_error); + CHECK_FALSE(store.has_full_tile( + static_cast(merkle::tiles::MAX_TILE_LEVEL + 1), 0)); + CHECK(rel(store, store.entries_path(5)) == "tile/entries/005"); + CHECK( + merkle::tiles::TileStore::storage_directory_name("sha384") == + "sha384-256w"); + CHECK( + merkle::tiles::TileStore::storage_directory_name("sha3-256") == + "sha3-256-256w"); + + for (const std::string& invalid_name : + {"", "-sha256", "sha256-", "SHA256", "sha_256", "sha/256"}) + { + CAPTURE(invalid_name); + CHECK_THROWS_AS( + merkle::tiles::TileStore::storage_directory_name(invalid_name), + std::runtime_error); + } + for (const std::string& mismatched_name : {"sha384", "sha512"}) + { + CAPTURE(mismatched_name); + CHECK_THROWS_AS( + (merkle::tiles::TileStore(dir, mismatched_name)), std::runtime_error); + } + + using CustomStore = + merkle::tiles::TileStoreT; + CHECK_THROWS_AS((void)CustomStore(dir), std::runtime_error); + const CustomStore custom_store(dir, "custom-hash"); + CHECK( + custom_store.root().lexically_relative(dir).generic_string() == + "custom-hash-256w"); + +#ifdef HAVE_OPENSSL + using TileStore256 = + merkle::tiles::TileStoreT; + const TileStore256 store256(dir); + CHECK( + store256.root().lexically_relative(dir).generic_string() == "sha256-256w"); + + using TileStore384 = merkle::tiles::TileStoreT< + merkle::Tree384::Hash::size_bytes, + merkle::Tree384::hash_function>; + for (const std::string& mismatched_name : {"sha256", "sha512"}) + { + CAPTURE(mismatched_name); + CHECK_THROWS_AS((TileStore384(dir, mismatched_name)), std::runtime_error); + } + TileStore384 store384(dir); + const auto full384 = + make_hashesT(merkle::tiles::TILE_WIDTH); + CHECK( + store384.root().lexically_relative(dir).generic_string() == "sha384-256w"); + store384.write_tile(TileRef{0, 0}, full384); + CHECK( + fs::file_size(store384.tile_path(TileRef{0, 0})) == + (uintmax_t)merkle::tiles::TILE_WIDTH * merkle::Tree384::Hash::size_bytes); + CHECK(store384.has_full_tile(0, 0)); + CHECK(store384.read_tile(TileRef{0, 0}) == full384); + + using TileStore512 = merkle::tiles::TileStoreT< + merkle::Tree512::Hash::size_bytes, + merkle::Tree512::hash_function>; + for (const std::string& mismatched_name : {"sha256", "sha384"}) + { + CAPTURE(mismatched_name); + CHECK_THROWS_AS((TileStore512(dir, mismatched_name)), std::runtime_error); + } + TileStore512 store512(dir); + const auto full512 = + make_hashesT(merkle::tiles::TILE_WIDTH); + CHECK( + store512.root().lexically_relative(dir).generic_string() == "sha512-256w"); + store512.write_tile(TileRef{0, 0}, full512); + CHECK( + fs::file_size(store512.tile_path(TileRef{0, 0})) == + (uintmax_t)merkle::tiles::TILE_WIDTH * merkle::Tree512::Hash::size_bytes); + CHECK(store512.has_full_tile(0, 0)); + CHECK(store512.read_tile(TileRef{0, 0}) == full512); +#endif +} + +TEST_CASE("Tile writes sync directory links in order") +{ + // 2b. Production writes sync each directory link and the destination + // directory in order. + const TemporaryDirectory temporary_directory; + const fs::path& dir = temporary_directory.path(); + const fs::path prefix = dir / "durable"; + const auto fault = std::make_shared(); + FaultInjectingTileStore store(prefix, fault); + store.write_tile(TileRef{1, 0}, make_hashes(merkle::tiles::TILE_WIDTH)); + + const std::vector expected = { + dir.parent_path(), + dir, + prefix, + prefix / "sha256-256w", + prefix / "sha256-256w" / "tile", + prefix / "sha256-256w" / "tile" / "1"}; + REQUIRE(fault->calls.size() >= expected.size()); + CHECK(std::equal( + expected.begin(), + expected.end(), + fault->calls.end() - (std::ptrdiff_t)expected.size())); +} + +TEST_CASE("Failed directory-link sync is retried") +{ + // 2c. A failed directory-link sync is retried even when the directory + // created before the failure is already visible. + const TemporaryDirectory temporary_directory; + const fs::path prefix = temporary_directory.path() / "directory_retry"; + const auto fault = std::make_shared(); + fault->fail_path = prefix; + fault->failures_remaining = 1; + FaultInjectingTileStore store(prefix, fault); + const TileRef ref{2, 7}; + const auto full = make_hashes(merkle::tiles::TILE_WIDTH); + + CHECK_THROWS_AS((store.write_tile(ref, full)), std::runtime_error); + CHECK_FALSE(store.has_full_tile(ref.level, ref.index)); + store.write_tile(ref, full); + CHECK(store.read_tile(ref) == full); + CHECK(fault->call_count(prefix) == 2); +} + +TEST_CASE("Visible publication can be confirmed after sync failure") +{ + // 2d. A failure syncing the destination directory leaves a complete, + // visible file that can be re-confirmed on the next write attempt. + const TemporaryDirectory temporary_directory; + const fs::path prefix = temporary_directory.path() / "publication_retry"; + const auto fault = std::make_shared(); + FaultInjectingTileStore store(prefix, fault); + const TileRef ref{3, 9}; + const auto full = make_hashes(merkle::tiles::TILE_WIDTH); + const auto tile_directory = store.tile_path(ref).parent_path(); + fault->fail_path = tile_directory; + fault->failures_remaining = 1; + + CHECK_THROWS_AS((store.write_tile(ref, full)), std::runtime_error); + CHECK(store.has_full_tile(ref.level, ref.index)); + CHECK(store.read_tile(ref) == full); + CHECK_FALSE(any_tmp_files(prefix)); + + const size_t calls_before_confirm = fault->call_count(tile_directory); + store.begin_attempt(); + CHECK(store.confirm_tile(ref.level, ref.index)); + CHECK(fault->call_count(tile_directory) == calls_before_confirm + 1); + CHECK(store.confirm_tile(ref.level, ref.index)); + CHECK(fault->call_count(tile_directory) == calls_before_confirm + 1); + + store.begin_attempt(); + CHECK_FALSE(store.confirm_tile(ref.level, ref.index + 1)); +} + +TEST_CASE("Replacement and directory conflicts clean up temporary files") +{ + // 2e. Failures before atomic replacement clean up temporary files, and a + // non-directory in the destination path is rejected. + const TemporaryDirectory temporary_directory; + const fs::path& dir = temporary_directory.path(); + const auto full = make_hashes(merkle::tiles::TILE_WIDTH); + + const fs::path replacement_prefix = dir / "replacement_failure"; + merkle::tiles::TileStore replacement_store(replacement_prefix); + const TileRef blocked_ref{0, 1}; + fs::create_directories(replacement_store.tile_path(blocked_ref)); + CHECK_THROWS_AS( + (replacement_store.write_tile(blocked_ref, full)), std::runtime_error); + CHECK_FALSE(any_tmp_files(replacement_prefix)); + + const fs::path conflict_prefix = dir / "directory_conflict"; + fs::create_directories(conflict_prefix); + overwrite_file(conflict_prefix / "sha256-256w", {0x00}); + merkle::tiles::TileStore conflict_store(conflict_prefix); + CHECK_THROWS_AS( + (conflict_store.write_tile(TileRef{0, 0}, full)), std::runtime_error); + CHECK_FALSE(any_tmp_files(conflict_prefix)); +} + +TEST_CASE("Exclusive file creation rejects file and symlink collisions") +{ + // 2f. Unique temporary-file creation must fail rather than overwrite an + // existing path, including a pre-created symlink on POSIX. + const TemporaryDirectory temporary_directory; + const fs::path& dir = temporary_directory.path(); + fs::create_directories(dir); + const fs::path existing_path = dir / "preexisting-temp"; + const std::vector original = {0x11, 0x22, 0x33}; + overwrite_file(existing_path, original); + + std::string collision_error; + try + { + merkle::pal::write_and_sync_file(existing_path, {0xAA}); + } + catch (const std::runtime_error& ex) + { + collision_error = ex.what(); + } + REQUIRE_FALSE(collision_error.empty()); + CHECK(collision_error.find(existing_path.string()) != std::string::npos); +#ifdef _WIN32 + CHECK(collision_error.find(": error ") != std::string::npos); +#endif + CHECK(read_file(existing_path) == original); + +#ifndef _WIN32 + const fs::path symlink_path = dir / "preexisting-temp-symlink"; + fs::create_symlink(existing_path, symlink_path); + CHECK_THROWS_AS( + (merkle::pal::write_and_sync_file(symlink_path, {0xBB})), + std::runtime_error); + CHECK(read_file(existing_path) == original); +#endif +} + +TEST_CASE("Successful writes must make progress") +{ + // 2g. A successful write must make progress. This shared guard prevents + // the Windows WriteFile loop from spinning if it reports zero bytes. + CHECK_THROWS_AS( + merkle::pal::require_write_progress(0, "test"), std::runtime_error); + CHECK_NOTHROW(merkle::pal::require_write_progress(1, "test")); +} + +#ifndef _WIN32 +TEST_CASE("Interrupted sync operations are retried") +{ + // 2h. Interrupted sync operations are retried, while other failures are + // returned immediately with errno intact. + size_t attempts = 0; + const int retry_result = merkle::pal::detail::retry_on_eintr([&]() { + attempts++; + if (attempts == 1) + { + errno = EINTR; + return -1; + } + return 0; + }); + CHECK(retry_result == 0); + CHECK(attempts == 2); + + attempts = 0; + const int error_result = merkle::pal::detail::retry_on_eintr([&]() { + attempts++; + errno = EIO; + return -1; + }); + CHECK(error_result == -1); + CHECK(attempts == 1); + CHECK(errno == EIO); +} +#endif + +TEST_CASE("Full tiles use raw bytes and round-trip") +{ + // 3a. Full tile byte round-trip. + const TemporaryDirectory temporary_directory; + merkle::tiles::TileStore store(temporary_directory.path()); + const size_t hash_size = Hash().size(); + const auto full = make_hashes(merkle::tiles::TILE_WIDTH); + const TileRef full_ref{0, 0}; + + store.write_tile(full_ref, full); + CHECK(store.has_full_tile(0, 0)); + CHECK_FALSE(store.has_full_tile(0, 5)); + CHECK_THROWS_AS((void)store.read_tile(TileRef{0, 5}), std::runtime_error); + CHECK( + fs::file_size(store.tile_path(full_ref)) == + (uintmax_t)merkle::tiles::TILE_WIDTH * hash_size); + + std::vector expected_tile_bytes; + expected_tile_bytes.reserve(full.size() * hash_size); + for (const auto& hash : full) + { + hash.serialise(expected_tile_bytes); + } + CHECK(read_file(store.tile_path(full_ref)) == expected_tile_bytes); + + const auto full_roundtrip = store.read_tile(full_ref); + REQUIRE(full_roundtrip.size() == full.size()); + for (size_t i = 0; i < full.size(); i++) + { + CAPTURE(i); + CHECK(full_roundtrip[i] == full[i]); + } + + // 3b. Wrong-width writes are rejected (only 256-wide tiles are valid). + const std::vector three(full.begin(), full.begin() + 3); + CHECK_THROWS_AS((store.write_tile(TileRef{0, 2}, three)), std::runtime_error); +} + +TEST_CASE("Entry encoding enforces bounds and round-trips") +{ + // 4. Entry bundles preserve empty, short, and multi-byte-length entries, + // enforce their width and uint16 length limits, and can be re-confirmed. + const auto entries = make_entries(); + const auto encoded_entries = + merkle::tiles::TileStore::encode_entries(entries); + REQUIRE(encoded_entries.size() > 7); + CHECK(encoded_entries[0] == 0x00); + CHECK(encoded_entries[1] == 0x00); + CHECK(encoded_entries[2] == 0x00); + CHECK(encoded_entries[3] == 0x01); + CHECK(encoded_entries[4] == 0xA5); + CHECK(encoded_entries[5] == 0x01); + CHECK(encoded_entries[6] == 0x00); + CHECK( + merkle::tiles::TileStore::decode_entries( + encoded_entries, merkle::tiles::TILE_WIDTH) == entries); + CHECK(merkle::tiles::TileStore::decode_entries({}, 0).empty()); + CHECK_THROWS_AS( + (merkle::tiles::TileStore::decode_entries( + {}, std::numeric_limits::max())), + std::runtime_error); + + const std::vector maximum_entry(0xFFFF, 0xC3); + const auto maximum_encoded = + merkle::tiles::TileStore::encode_entries({maximum_entry}); + const auto maximum_decoded = + merkle::tiles::TileStore::decode_entries(maximum_encoded, 1); + CHECK(maximum_encoded.size() == maximum_entry.size() + 2); + CHECK(maximum_encoded[0] == 0xFF); + CHECK(maximum_encoded[1] == 0xFF); + REQUIRE(maximum_decoded.size() == 1); + CHECK(maximum_decoded[0] == maximum_entry); + CHECK_THROWS_AS( + (merkle::tiles::TileStore::encode_entries({std::vector(0x10000)})), + std::runtime_error); +} + +TEST_CASE("Entry bundle storage validates and confirms publications") +{ + const TemporaryDirectory temporary_directory; + const fs::path& dir = temporary_directory.path(); + merkle::tiles::TileStore store(dir); + const auto entries = make_entries(); + const auto encoded_entries = + merkle::tiles::TileStore::encode_entries(entries); + + CHECK_FALSE(store.has_entry_bundle(17)); + CHECK_THROWS_AS((void)store.read_entry_bundle(17), std::runtime_error); + const std::vector> short_bundle( + entries.begin(), entries.end() - 1); + CHECK_THROWS_AS( + (store.write_entry_bundle(1, short_bundle)), std::runtime_error); + auto oversized_bundle = entries; + oversized_bundle[0].resize(0x10000); + CHECK_THROWS_AS( + (store.write_entry_bundle(2, oversized_bundle)), std::runtime_error); + CHECK_FALSE(fs::exists(store.entries_path(1))); + CHECK_FALSE(fs::exists(store.entries_path(2))); + + store.write_entry_bundle(0, entries); + CHECK(store.has_entry_bundle(0)); + CHECK(store.read_entry_bundle(0) == entries); + CHECK(fs::file_size(store.entries_path(0)) == encoded_entries.size()); + CHECK(read_file(store.entries_path(0)) == encoded_entries); + + const fs::path prefix = dir / "bundle_confirmation"; + const auto fault = std::make_shared(); + FaultInjectingTileStore confirming_store(prefix, fault); + confirming_store.write_entry_bundle(4, entries); + const auto entries_directory = confirming_store.entries_path(4).parent_path(); + const size_t calls_before_confirm = fault->call_count(entries_directory); + confirming_store.begin_attempt(); + CHECK(confirming_store.confirm_bundle(4)); + CHECK(fault->call_count(entries_directory) == calls_before_confirm + 1); + CHECK(confirming_store.confirm_bundle(4)); + CHECK(fault->call_count(entries_directory) == calls_before_confirm + 1); + confirming_store.begin_attempt(); + CHECK_FALSE(confirming_store.confirm_bundle(5)); +} + +TEST_CASE("Corrupt tiles and entry bundles are rejected") +{ + // 5. Corrupt / truncated files are rejected on read (integrity check), so + // a torn write can never be served as a valid tile or bundle. + const TemporaryDirectory temporary_directory; + merkle::tiles::TileStore store(temporary_directory.path()); + const size_t hash_size = Hash().size(); + const auto full = make_hashes(merkle::tiles::TILE_WIDTH); + const TileRef full_ref{0, 0}; + const auto entries = make_entries(); + store.write_tile(full_ref, full); + + overwrite_file(store.tile_path(full_ref), std::vector(hash_size, 0)); + CHECK_FALSE(store.has_full_tile(0, 0)); + const auto short_tile_error = std::format( + "unexpected tile size for {}: expected {} bytes, got {}", + store.tile_path(full_ref).string(), + merkle::tiles::TILE_WIDTH * hash_size, + hash_size); + CHECK_THROWS_WITH_AS( + (void)store.read_tile(full_ref), + short_tile_error.c_str(), + std::runtime_error); + store.write_tile(full_ref, full); + CHECK(store.has_full_tile(0, 0)); + CHECK(store.read_tile(full_ref) == full); + + overwrite_file( + store.tile_path(full_ref), + std::vector((merkle::tiles::TILE_WIDTH + 1) * hash_size, 0)); + CHECK_FALSE(store.has_full_tile(0, 0)); + CHECK_THROWS_AS((void)store.read_tile(full_ref), std::runtime_error); + + store.write_entry_bundle(0, entries); + CHECK(store.has_entry_bundle(0)); + CHECK(store.read_entry_bundle(0) == entries); + overwrite_file(store.entries_path(0), {0x00, 0x05, 0x01}); + CHECK_FALSE(store.has_entry_bundle(0)); + const auto invalid_bundle_error = std::format( + "invalid entry bundle {}: truncated entry bundle", + store.entries_path(0).string()); + CHECK_THROWS_WITH_AS( + (void)store.read_entry_bundle(0), + invalid_bundle_error.c_str(), + std::runtime_error); + + auto encoded = merkle::tiles::TileStore::encode_entries(entries); + encoded.push_back(0xFF); + overwrite_file(store.entries_path(0), encoded); + CHECK_FALSE(store.has_entry_bundle(0)); + CHECK_THROWS_AS((void)store.read_entry_bundle(0), std::runtime_error); + + CHECK_THROWS_AS( + (merkle::tiles::TileStore::decode_entries({0xFF, 0xFF, 0x00}, 1)), + std::runtime_error); + CHECK_THROWS_AS( + (merkle::tiles::TileStore::decode_entries({}, 1)), std::runtime_error); + CHECK_THROWS_AS( + (merkle::tiles::TileStore::decode_entries({0x00, 0x00, 0xFF}, 1)), + std::runtime_error); +} diff --git a/test/unit_tests.cpp b/test/unit_tests.cpp index e6f6967..ce7ce8d 100644 --- a/test/unit_tests.cpp +++ b/test/unit_tests.cpp @@ -4,9 +4,25 @@ #include "util.h" #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN -#include +#include #include +TEST_CASE("Built-in SHA256 hashes complete messages") +{ + const merkle::Hash zero; + merkle::Hash digest; + merkle::sha256(zero, zero, digest); + + REQUIRE( + digest.to_string() == + "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b"); + + merkle::Tree tree; + tree.insert(zero); + tree.insert(zero); + REQUIRE(tree.root() == digest); +} + TEST_CASE("HashT constructors and error paths") { // Default constructor: all bytes zero @@ -72,6 +88,11 @@ TEST_CASE("HashT constructors and error paths") size_t pos2 = 10; const std::vector too_short(41, 0); // only 31 bytes from position 10 REQUIRE_THROWS(merkle::Hash(too_short, pos2)); + + // Vector+position constructor: position beyond the buffer throws + size_t beyond_end = too_short.size() + 1; + REQUIRE_THROWS(merkle::Hash(too_short, beyond_end)); + REQUIRE(beyond_end == too_short.size() + 1); } TEST_CASE("HashT methods") @@ -86,6 +107,12 @@ TEST_CASE("HashT methods") REQUIRE(b == 0); } + // deserialise: position beyond the buffer throws without advancing + const std::vector empty; + size_t beyond_end = 1; + REQUIRE_THROWS(h.deserialise(empty, beyond_end)); + REQUIRE(beyond_end == 1); + // to_string (lower case) merkle::Hash h2; h2.bytes[0] = 0xAB; @@ -102,9 +129,10 @@ TEST_CASE("HashT methods") const std::string s2 = h2.to_string(2); REQUIRE(s2.size() == 4); REQUIRE(s2 == "abcd"); + REQUIRE_THROWS_AS((void)h2.to_string(33), std::out_of_range); // operator== and operator!= - merkle::Hash ha; + const merkle::Hash ha; merkle::Hash hb; REQUIRE(ha == hb); REQUIRE_FALSE(ha != hb); @@ -149,7 +177,7 @@ TEST_CASE("PathT equality") const auto path0a = tree.path(0); const auto path0b = tree.path(0); // same path extracted twice - const auto path1 = tree.path(1); // path to a different leaf + const auto path1 = tree.path(1); // path to a different leaf // Two paths to the same leaf should be equal REQUIRE(*path0a == *path0b); @@ -166,7 +194,7 @@ TEST_CASE("PathT equality") h3.bytes[0] = 3; tree_diff.insert(h3); - const auto path_orig = tree.path(0); // h0 leaf, element has h1 + const auto path_orig = tree.path(0); // h0 leaf, element has h1 const auto path_diff = tree_diff.path(0); // h0 leaf, element has h3 REQUIRE_FALSE(*path_orig == *path_diff); REQUIRE(*path_orig != *path_diff); @@ -175,7 +203,7 @@ TEST_CASE("PathT equality") TEST_CASE("TreeT to_string") { // Empty tree produces "" - merkle::Tree empty_tree; + const merkle::Tree empty_tree; const std::string empty_str = empty_tree.to_string(); REQUIRE(empty_str.find("") != std::string::npos); @@ -192,7 +220,7 @@ TEST_CASE("TreeT to_string") TEST_CASE("TreeT leaf bounds and uninserted leaves") { // leaf() out of bounds on empty tree - merkle::Tree empty_tree; + const merkle::Tree empty_tree; REQUIRE_THROWS(empty_tree.leaf(0)); // Access leaf before insertion is flushed (uninserted_leaf_nodes path) @@ -216,7 +244,7 @@ TEST_CASE("TreeT size with uninserted leaves") { merkle::Tree tree; // size() when tree has uninserted leaves triggers insert_leaves() - merkle::Tree::Hash h; + const merkle::Tree::Hash h; tree.insert(h); // size() forces lazy insertion const size_t sz = tree.size(); diff --git a/test/util.h b/test/util.h index 2abfc77..196f8b9 100644 --- a/test/util.h +++ b/test/util.h @@ -38,9 +38,9 @@ inline std::vector> make_hashesT( inline size_t random_index(merkle::Tree& mt) { - return (size_t)( - mt.min_index() + - (std::rand() / (double)RAND_MAX) * (mt.max_index() - mt.min_index())); + return (size_t)(mt.min_index() + + (std::rand() / (double)RAND_MAX) * + (mt.max_index() - mt.min_index())); } inline double get_timeout()