diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index f264f8e..5219ef2 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -253,19 +253,55 @@ algorithms use the explicit-name constructor. The concurrency contract is in [section 1](#1-requirements), and publication semantics are in [section 4](#4-storage-layout-and-publication). -### 5.3 Entry bundles (optional) +### 5.3 `TileWriterT` + +`TileWriterT` incrementally persists every newly complete tile at every level: + +```cpp +class TileWriterT { +public: + explicit TileWriterT(TileStoreT& store); + + struct Stats { uint64_t full_written; }; + Stats write_up_to( + uint64_t size, + const std::function& leaf_at); +}; +``` + +For each level `L`, `write_up_to` computes the number of complete level entries +as `size >> (8 * L)`, then writes each complete group of 256 entries. Level 0 +reads leaf hashes from `leaf_at`; higher levels roll up the complete child tiles +already on disk. The incomplete frontier is never written. + +A writer caches the next full-tile index at each level. A fresh writer rebuilds +each cursor with a bounded scan of the confirmed contiguous prefix, then checks +later indices individually so malformed or missing interior tiles are repaired. +Existing valid tiles are immutable. If publication succeeds but a directory +sync fails, a retry confirms the visible tile's durability instead of rewriting +it. + +The resulting tile counts provide useful test vectors: + +- Size 256 produces one level-0 tile and no level-1 tile. +- Size 70,000 produces 273 level-0 tiles and one level-1 tile. +- Size 65,536 completes both level-0 tile 255 and level-1 tile 0. + +### 5.4 Entry bundles (optional) Entry bundles are application-owned because merklecpp receives precomputed leaf hashes, not raw entries. Each full bundle stores 256 big-endian `uint16` length-prefixed entries at `-256w/tile/entries/`. The application defines the leaf derivation (for example, `leaf_hash = H(entry)`); merklecpp -stores the supplied leaf hash unchanged. +stores the supplied leaf hash unchanged. `EntryBundleWriterT` mirrors +`TileWriterT`: it writes only complete 256-entry bundles, confirms existing +bundles before reusing them, and leaves the incomplete tail with the application. ## 6. Delivery plan -This PR delivers phases 0 and 1 plus the standalone entry-bundle codec and -storage primitives. Later PRs deliver the remaining independently testable -phases; phases 1-3 need no further core changes, while phase 4 may add one +Phases 0-2 now deliver the storage primitives plus incremental tile and +entry-bundle writers. Later PRs deliver the remaining independently testable +phases; phase 3 needs no further core changes, while phase 4 may add one non-hashing accessor. | Phase | Scope | Key tests | diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 8485e24..5169d14 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -731,9 +731,318 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @endcond }; + /// @brief Computes the Merkle Tree Hash of a perfect (balanced) subtree. + /// @param leaves The subtree's leaves; the count MUST be a power of two. + /// @return The subtree root, computed with the tree's HASH_FUNCTION. + /// @note This is exactly a merkle::TreeT full-node hash, which is why tile + /// entries (such roots) are immutable: an unbalanced subtree would still + /// change as leaves are added and must therefore never be tiled. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + inline HashT perfect_root( + const std::vector>& leaves) + { + if (leaves.empty()) + { + throw std::runtime_error("perfect_root requires at least one leaf"); + } + if ((leaves.size() & (leaves.size() - 1)) != 0) + { + throw std::runtime_error( + "perfect_root requires a power-of-two number of leaves"); + } + + std::vector> level = leaves; + while (level.size() > 1) + { + std::vector> next; + next.reserve(level.size() / 2); + for (size_t i = 0; i + 1 < level.size(); i += 2) + { + HashT h; + HASH_FUNCTION(level[i], level[i + 1], h); + next.push_back(h); + } + level.swap(next); + } + return level.front(); + } + + /// @brief Computes and persists tlog-tiles tiles for a growing tree. + /// @tparam HASH_SIZE Size of each hash in bytes + /// @tparam HASH_FUNCTION The tree's node hash function + /// @note Only balanced subtrees are tiled: a level-L entry is the root of a + /// complete 2**(8L)-leaf subtree. Only full tiles (256 such entries) are + /// written; they are therefore immutable and written exactly once. Entries + /// beyond the last full-tile boundary remain in memory until a later flush + /// completes the next tile. + /// @warning No internal synchronization is provided. Callers must serialize + /// access to a writer and its store. + /// @warning A writer trusts existing full tiles as output from the same + /// tree and hash function. Callers resuming a store must establish that + /// ownership and restore the matching tree state. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TileWriterT + { + public: + /// @brief The type of hashes stored in tiles. + using Hash = HashT; + + /// @brief The associated tile store type. + using Store = TileStoreT; + + /// @brief Supplies the level-0 leaf hash for a given leaf index. + using LeafFn = std::function; + + /// @brief Counts of work performed by a write_up_to call. + struct Stats + { + /// @brief Number of full tiles written. + uint64_t full_written = 0; + }; + + /// @brief Constructs a writer over @p store. + explicit TileWriterT(Store& store) : store(store) {} + + /// @brief Writes all newly-complete full tiles for a tree of @p size + /// leaves. + /// @param size The current tree size + /// @param leaf_at Returns the level-0 leaf hash for a leaf index in + /// [0, size); only ever queried for leaves of complete subtrees. + /// @return Counts of tiles written + /// @note Incremental: full tiles already on disk are immutable and are + /// never rewritten once validated and confirmed durable. Malformed files + /// are replaced. Entries that do not complete a tile are not written. + /// Tiles are always rolled up through MAX_TILE_LEVEL, so the on-disk set + /// always contains the higher-level roll-ups that proof generation relies + /// on. + Stats write_up_to(uint64_t size, const LeafFn& leaf_at) + { + Stats stats; + store.begin_write_attempt(); + + // The loop stops early once a level has no complete entries (see the + // entries == 0 break below). + for (uint8_t level = 0; level <= MAX_TILE_LEVEL; level++) + { + // Number of complete (balanced) level-L entries available; this + // deliberately excludes the incomplete frontier subtree. + const uint64_t entries = entries_at_level(size, level); + if (entries == 0) + { + break; + } + ensure_level(level); + + const uint64_t full_tiles = entries / TILE_WIDTH; + + if (cursor_inited[level] == 0) + { + next_full[level] = full_prefix_length(level, full_tiles); + cursor_inited[level] = 1; + } + + for (uint64_t n = next_full[level]; n < full_tiles; n++) + { + if (store.confirm_full_tile(level, n)) + { + continue; // immutable: never rewrite an existing full tile + } + store.write_tile( + TileRef{level, n}, + collect(level, n * TILE_WIDTH, TILE_WIDTH, leaf_at)); + stats.full_written++; + } + if (full_tiles > next_full[level]) + { + next_full[level] = full_tiles; + } + } + + return stats; + } + + protected: + /// @brief The tile store written to. + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + Store& store; + + /// @brief Per-level index of the next full tile to write. + std::vector next_full; + + /// @brief Per-level flag indicating next_full has been initialised. + std::vector cursor_inited; + + /// @brief Number of complete level-@p level entries for a tree of @p + /// size. + static uint64_t entries_at_level(uint64_t size, uint8_t level) + { + const unsigned shift = + static_cast(TILE_HEIGHT) * static_cast(level); + return shift >= 64 ? 0 : (size >> shift); + } + + /// @brief Ensures per-level bookkeeping vectors cover @p level. + void ensure_level(uint8_t level) + { + const size_t needed = (size_t)level + 1; + if (next_full.size() < needed) + { + next_full.resize(needed, 0); + cursor_inited.resize(needed, 0); + } + } + + /// @brief Length of the confirmed contiguous prefix, bounded by @p limit. + [[nodiscard]] uint64_t full_prefix_length(uint8_t level, uint64_t limit) + { + return detail::contiguous_prefix_length(limit, [&](uint64_t index) { + return store.confirm_full_tile(level, index); + }); + } + + /// @brief Collects @p count consecutive level-@p level entries, each the + /// root of a complete (balanced) subtree. + std::vector collect( + uint8_t level, + uint64_t first_entry, + uint64_t count, + const LeafFn& leaf_at) + { + std::vector out; + out.reserve(count); + for (uint64_t i = 0; i < count; i++) + { + const uint64_t g = first_entry + i; + if (level == 0) + { + out.push_back(leaf_at(g)); + } + else + { + // Roll up the complete child full tile (256 complete entries). + out.push_back( + perfect_root( + store.read_tile(TileRef{(uint8_t)(level - 1), g}))); + } + } + return out; + } + }; + + /// @brief Writes tlog-tiles entry bundles (raw log entries) for a growing + /// log. + /// @note Entry bundles are level-0 only and application-owned: merklecpp + /// stores leaf hashes, while the raw entries (and the leaf-hash derivation + /// linking each entry to its level-0 tile hash) are the application's + /// responsibility. Only full bundles (TILE_WIDTH entries) are written; they + /// are immutable and written exactly once. The incomplete tail stays with + /// the application until it grows into a full bundle, mirroring the + /// un-tiled Merkle frontier. + /// @warning No internal synchronization is provided. Callers must serialize + /// access to a writer and its store. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class EntryBundleWriterT + { + public: + using Store = TileStoreT; + + /// @brief Supplies the raw bytes of the log entry at a given index. + using EntryFn = std::function(uint64_t)>; + + /// @brief Counts of work performed by a write_up_to call. + struct Stats + { + /// @brief Number of full bundles written. + uint64_t full_written = 0; + }; + + explicit EntryBundleWriterT(Store& store) : store(store) {} + + /// @brief Writes all newly-complete full bundles for a log of @p size + /// entries. + /// @param size The current number of entries + /// @param entry_at Returns the raw bytes of the entry at an index in + /// [0, size); only ever queried for entries of complete bundles. + /// @return Counts of bundles written + /// @note Incremental: full bundles already on disk are immutable and are + /// never rewritten once validated and confirmed durable. Malformed files + /// are replaced. The incomplete tail is never bundled. + Stats write_up_to(uint64_t size, const EntryFn& entry_at) + { + Stats stats; + store.begin_write_attempt(); + const uint64_t full = size / TILE_WIDTH; + + if (!cursor_inited) + { + next_full = full_prefix_length(full); + cursor_inited = true; + } + + for (uint64_t n = next_full; n < full; n++) + { + if (store.confirm_entry_bundle(n)) + { + continue; // immutable: never rewrite an existing full bundle + } + store.write_entry_bundle( + n, collect(n * TILE_WIDTH, TILE_WIDTH, entry_at)); + stats.full_written++; + } + if (full > next_full) + { + next_full = full; + } + return stats; + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + Store& store; + uint64_t next_full = 0; + bool cursor_inited = false; + + std::vector> collect( + uint64_t first, uint64_t count, const EntryFn& entry_at) + { + std::vector> out; + out.reserve(count); + for (uint64_t i = 0; i < count; i++) + { + out.push_back(entry_at(first + i)); + } + return out; + } + + [[nodiscard]] uint64_t full_prefix_length(uint64_t limit) + { + return detail::contiguous_prefix_length(limit, [&](uint64_t index) { + return store.confirm_entry_bundle(index); + }); + } + }; + /// @brief Default tile store (SHA256, default hash function). using TileStore = TileStoreT; + /// @brief Default tile writer (SHA256, default hash function). + using TileWriter = + TileWriterT; + + /// @brief Default entry-bundle writer (SHA256, default hash function). + using EntryBundleWriter = EntryBundleWriterT< + merkle::Tree::Hash::size_bytes, + merkle::Tree::hash_function>; + } } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e21cadd..9e61e03 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -43,4 +43,6 @@ if(OPENSSL) endif() add_merklecpp_test(tiles_store tiles_store.cpp) +add_merklecpp_test(tiles_writer tiles_writer.cpp) +add_merklecpp_test(tiles_entries tiles_entries.cpp) add_merklecpp_test(unit_tests unit_tests.cpp) diff --git a/test/tiles_entries.cpp b/test/tiles_entries.cpp new file mode 100644 index 0000000..8d0efb9 --- /dev/null +++ b/test/tiles_entries.cpp @@ -0,0 +1,212 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "tiles_test_util.h" +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::EntryBundleWriter; +using merkle::tiles::TileRef; +using merkle::tiles::TileStore; +using merkle::tiles::TileWriter; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +static void overwrite_file(const fs::path& p, const std::vector& bytes) +{ + std::ofstream f(p, std::ios::binary | std::ios::trunc); + f.write( + reinterpret_cast(bytes.data()), (std::streamsize)bytes.size()); +} + +int main() +{ + const TemporaryDirectory temporary_directory("merklecpp_tiles_entries"); + const fs::path& base = temporary_directory.path(); + + try + { + // 1. Encode/decode round-trip with variable-length entries. + { + const std::vector> entries = { + {}, {0x01}, {0x02, 0x03, 0x04}, std::vector(300, 0xAB)}; + const auto enc = TileStore::encode_entries(entries); + // 4 length prefixes (8 bytes) + 0 + 1 + 3 + 300 payload bytes. + expect(enc.size() == 8 + 0 + 1 + 3 + 300, "encoded size"); + const auto dec = TileStore::decode_entries(enc, entries.size()); + expect(dec == entries, "encode/decode round-trip"); + + auto trailing = enc; + trailing.push_back(0xFF); + bool trailing_threw = false; + try + { + (void)TileStore::decode_entries(trailing, entries.size()); + } + catch (const std::exception&) + { + trailing_threw = true; + } + expect(trailing_threw, "decode rejects trailing bytes"); + } + + const auto hashes = make_hashes(70000); + const auto entry_at = [&](uint64_t i) -> std::vector { + return hashes[i]; // HashT -> vector + }; + + // 2. size 256: a single full bundle, no tail. + { + TileStore store(base / "e256"); + EntryBundleWriter writer(store); + const auto s = writer.write_up_to(256, entry_at); + expect(s.full_written == 1, "256 counts"); + expect(store.has_entry_bundle(0), "256 full bundle"); + expect(!store.has_entry_bundle(1), "256 no second bundle"); + + const auto b0 = store.read_entry_bundle(0); + expect(b0.size() == 256, "256 bundle width"); + for (size_t i = 0; i < 256; i++) + { + expect(Hash(b0[i]) == hashes[i], "256 entry round-trip"); + } + } + + // 3. size 70000: 273 full bundles (the 112-entry tail is not bundled), + // plus tile linkage. + { + TileStore store(base / "e70k"); + EntryBundleWriter writer(store); + const auto s = writer.write_up_to(70000, entry_at); + expect(s.full_written == 273, "70000 counts"); + expect(store.has_entry_bundle(0), "70000 bundle 0"); + expect(store.has_entry_bundle(272), "70000 bundle 272"); + expect(!store.has_entry_bundle(273), "70000 no bundle 273"); + + // Level-0 tile entries (leaf hashes) correspond to the bundle entries + // under the identity leaf-hash used here. + TileWriter tw(store); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + tw.write_up_to(70000, leaf_at); + const auto tile0 = store.read_tile(TileRef{0, 0}); + const auto bundle0 = store.read_entry_bundle(0); + for (size_t i = 0; i < 256; i++) + { + expect(Hash(bundle0[i]) == tile0[i], "bundle<->tile linkage"); + } + } + + // 4. Incremental writes: full bundles are immutable, the tail is not + // bundled. + { + TileStore store(base / "einc"); + EntryBundleWriter writer(store); + expect(writer.write_up_to(256, entry_at).full_written == 1, "inc s1"); + const auto s2 = writer.write_up_to(256, entry_at); + expect(s2.full_written == 0, "inc immutable rerun"); + overwrite_file(store.entries_path(0), {0x00, 0x05, 0x01}); + expect(!store.has_entry_bundle(0), "inc corrupt bundle not durable"); + EntryBundleWriter resumed(store); + expect( + resumed.write_up_to(256, entry_at).full_written == 1, + "inc rewrite corrupt"); + expect(store.has_entry_bundle(0), "inc rewritten bundle durable"); + const auto s3 = writer.write_up_to(600, entry_at); + expect(s3.full_written == 1, "inc s3 full"); + expect(store.has_entry_bundle(1), "inc bundle 1"); + expect(!store.has_entry_bundle(2), "inc no tail bundle"); + } + + // 5. Resume repairs an interior malformed bundle even when later files are + // valid. + { + const fs::path dir = base / "eholes"; + { + TileStore store(dir); + EntryBundleWriter writer(store); + expect( + writer.write_up_to(2048, entry_at).full_written == 8, + "holes initial bundles"); + overwrite_file(store.entries_path(3), {0x00, 0x05, 0x01}); + expect(!store.has_entry_bundle(3), "holes interior bundle malformed"); + expect(store.has_entry_bundle(7), "holes later bundle remains valid"); + } + + TileStore store(dir); + EntryBundleWriter writer(store); + expect( + writer.write_up_to(2048, entry_at).full_written == 1, + "holes rewrites interior bundle"); + std::vector> expected; + expected.reserve(merkle::tiles::TILE_WIDTH); + constexpr uint64_t hole_begin = (uint64_t)merkle::tiles::TILE_WIDTH * 3; + constexpr uint64_t hole_end = (uint64_t)merkle::tiles::TILE_WIDTH * 4; + for (uint64_t i = hole_begin; i < hole_end; i++) + { + expected.push_back(entry_at(i)); + } + expect( + store.read_entry_bundle(3) == expected, + "holes repaired bundle contents"); + } + + // 6. Recovery examines only bundle indices relevant to the requested size. + { + const fs::path dir = base / "esparse"; + const std::vector> bundle( + merkle::tiles::TILE_WIDTH, {0x42}); + { + TileStore store(dir); + store.write_entry_bundle(0, bundle); + for (uint64_t index = 1;; index <<= 1) + { + store.write_entry_bundle(index, bundle); + if (index == (uint64_t{1} << 63)) + { + break; + } + } + } + + TileStore store(dir); + EntryBundleWriter writer(store); + expect( + writer.write_up_to(merkle::tiles::TILE_WIDTH, entry_at).full_written == + 0, + "sparse bounded recovery"); + expect( + store.has_entry_bundle(0), "sparse requested bundle remains valid"); + } + + std::cout << "tiles_entries: OK" << '\n'; + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + return 1; + } + + return 0; +} diff --git a/test/tiles_store.cpp b/test/tiles_store.cpp index 6dbe5d2..07e2495 100644 --- a/test/tiles_store.cpp +++ b/test/tiles_store.cpp @@ -5,12 +5,11 @@ #include +#include "tiles_test_util.h" #include "util.h" #include -#include #include -#include #include #include #include @@ -32,40 +31,6 @@ using merkle::tiles::TileRef; static_assert(std::min(1, 2) == 1); static_assert(std::max(1, 2) == 2); -class TemporaryDirectory -{ -private: - fs::path path_; - -public: - TemporaryDirectory() - { - static std::atomic sequence = 0; - const auto nonce = - std::chrono::steady_clock::now().time_since_epoch().count(); - path_ = fs::temp_directory_path() / - std::format( - "merklecpp_tiles_{}_{}_{}", - merkle::pal::process_id(), - nonce, - sequence++); - } - - ~TemporaryDirectory() - { - std::error_code ec; - fs::remove_all(path_, ec); - } - - TemporaryDirectory(const TemporaryDirectory&) = delete; - TemporaryDirectory& operator=(const TemporaryDirectory&) = delete; - - [[nodiscard]] const fs::path& path() const - { - return path_; - } -}; - static void custom_hash(const Hash& lhs, const Hash& rhs, Hash& out) { (void)rhs; @@ -393,6 +358,72 @@ TEST_CASE("Visible publication can be confirmed after sync failure") CHECK_FALSE(store.confirm_tile(ref.level, ref.index + 1)); } +TEST_CASE("Tile writer confirms visible tiles after sync failure") +{ + const TemporaryDirectory temporary_directory; + const fs::path prefix = + temporary_directory.path() / "writer_publication_retry"; + const auto fault = std::make_shared(); + constexpr uint64_t growing_size = + (uint64_t)merkle::tiles::TILE_WIDTH * 2; + const auto growing = make_hashes((size_t)growing_size); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return growing[i]; }; + + FaultInjectingTileStore retry_store(prefix, fault); + merkle::tiles::TileWriter retry_writer(retry_store); + const fs::path destination = + retry_store.tile_path(TileRef{0, 0}).parent_path(); + fault->fail_path = destination; + REQUIRE( + retry_writer.write_up_to(merkle::tiles::TILE_WIDTH, leaf_at).full_written == + 1); + + fault->matching_calls_before_failure = 1; + fault->failures_remaining = 1; + { + FaultInjectingTileStore failed_store(prefix, fault); + merkle::tiles::TileWriter writer(failed_store); + CHECK_THROWS_AS( + (writer.write_up_to(growing_size, leaf_at)), std::runtime_error); + CHECK(failed_store.has_full_tile(0, 1)); + } + + CHECK(retry_writer.write_up_to(growing_size, leaf_at).full_written == 0); + CHECK(fault->call_count(destination) == 4); + const std::vector second( + growing.begin() + (std::ptrdiff_t)merkle::tiles::TILE_WIDTH, growing.end()); + CHECK(retry_store.read_tile(TileRef{0, 1}) == second); +} + +TEST_CASE("Tile writer retries ancestor sync before publication") +{ + const TemporaryDirectory temporary_directory; + const fs::path prefix = temporary_directory.path() / "writer_ancestor_retry"; + const auto fault = std::make_shared(); + fault->fail_path = prefix; + fault->failures_remaining = 1; + const auto full = make_hashes(merkle::tiles::TILE_WIDTH); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return full[i]; }; + + { + FaultInjectingTileStore failed_store(prefix, fault); + merkle::tiles::TileWriter writer(failed_store); + CHECK_THROWS_AS( + (writer.write_up_to(merkle::tiles::TILE_WIDTH, leaf_at)), + std::runtime_error); + CHECK(fs::is_directory(prefix / "sha256-256w")); + CHECK_FALSE(failed_store.has_full_tile(0, 0)); + } + + FaultInjectingTileStore retry_store(prefix, fault); + merkle::tiles::TileWriter retry_writer(retry_store); + CHECK( + retry_writer.write_up_to(merkle::tiles::TILE_WIDTH, leaf_at).full_written == + 1); + CHECK(fault->call_count(prefix) == 2); + CHECK(retry_store.has_full_tile(0, 0)); +} + TEST_CASE("Replacement and directory conflicts clean up temporary files") { // 2e. Failures before atomic replacement clean up temporary files, and a diff --git a/test/tiles_test_util.h b/test/tiles_test_util.h new file mode 100644 index 0000000..b775838 --- /dev/null +++ b/test/tiles_test_util.h @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +class TemporaryDirectory +{ +private: + std::filesystem::path path_; + +public: + explicit TemporaryDirectory( + std::string_view prefix = "merklecpp_tiles") + { + static std::atomic sequence = 0; + const auto nonce = + std::chrono::steady_clock::now().time_since_epoch().count(); + path_ = std::filesystem::temp_directory_path() / + std::format( + "{}_{}_{}_{}", + prefix, + merkle::pal::process_id(), + nonce, + sequence++); + } + + ~TemporaryDirectory() + { + std::error_code ec; + std::filesystem::remove_all(path_, ec); + } + + TemporaryDirectory(const TemporaryDirectory&) = delete; + TemporaryDirectory& operator=(const TemporaryDirectory&) = delete; + + [[nodiscard]] const std::filesystem::path& path() const + { + return path_; + } +}; diff --git a/test/tiles_writer.cpp b/test/tiles_writer.cpp new file mode 100644 index 0000000..235346f --- /dev/null +++ b/test/tiles_writer.cpp @@ -0,0 +1,435 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "tiles_test_util.h" +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TileRef; +using merkle::tiles::TileStore; +using merkle::tiles::TileWriter; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +template +static size_t tile_file_count(const Store& store) +{ + const fs::path tiles = store.root() / "tile"; + if (!fs::exists(tiles)) + { + return 0; + } + const fs::recursive_directory_iterator it(tiles); + return (size_t)std::count_if( + begin(it), end(it), [](const fs::directory_entry& e) { + return e.is_regular_file(); + }); +} + +// Roll up a full level-0 tile and compare with a level-1 tile entry. +static Hash rollup(const std::vector& leaves) +{ + return merkle::tiles::perfect_root< + merkle::Tree::Hash::size_bytes, + merkle::Tree::hash_function>(leaves); +} + +class TileWriterProbe : public TileWriter +{ +public: + using TileWriter::TileWriter; + + void mark_level_complete(uint8_t level, uint64_t full_tiles) + { + ensure_level(level); + next_full[level] = full_tiles; + cursor_inited[level] = 1; + } +}; + +static void overwrite_file( + const fs::path& path, const std::vector& bytes) +{ + std::ofstream file(path, std::ios::binary | std::ios::trunc); + file.write( + reinterpret_cast(bytes.data()), (std::streamsize)bytes.size()); +} + +int main() +{ + const TemporaryDirectory temporary_directory("merklecpp_tiles_writer"); + const fs::path& base = temporary_directory.path(); + + try + { + // ---- A. size 256: exactly one full L0 tile and no L1 tile (the L1 root is + // not yet a full tile, so it stays in memory). The level-1 entry it + // would hold equals the real merkle::Tree root. + { + const auto hashes = make_hashes(256); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + + TileStore store(base / "a"); + TileWriter writer(store); + const auto s = writer.write_up_to(256, leaf_at); + + expect(s.full_written == 1, "A full_written"); + + expect(store.has_full_tile(0, 0), "A L0 full tile"); + expect( + fs::file_size(store.tile_path(TileRef{0, 0})) == 256U * Hash().size(), + "A L0 full tile size"); + expect(!store.has_full_tile(1, 0), "A no L1 full tile"); + expect(tile_file_count(store) == 1, "A exact tile file count"); + + // Level-0 tile is the leaf hashes verbatim. + const auto l0 = store.read_tile(TileRef{0, 0}); + for (size_t i = 0; i < hashes.size(); i++) + { + expect(l0[i] == hashes[i], "A L0 entry == leaf"); + } + + // The (un-tiled) level-1 entry == root of the equivalent merkle::Tree. + merkle::Tree tree; + for (const auto& h : hashes) + { + tree.insert(h); + } + expect(rollup(l0) == tree.root(), "A rollup(L0) == tree root"); + + std::cout << "A (size 256): OK" << '\n'; + } + + // ---- B. size 70000: full tiles only (273 full L0 + 1 full L1 = 274); the + // incomplete frontier at every level is left untiled. + { + const auto hashes = make_hashes(70000); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + + TileStore store(base / "b"); + TileWriter writer(store); + const auto s = writer.write_up_to(70000, leaf_at); + + expect(s.full_written == 274, "B full_written"); + + expect(store.has_full_tile(0, 0), "B L0 tile 0"); + expect(store.has_full_tile(0, 272), "B L0 tile 272"); + expect(!store.has_full_tile(0, 273), "B no L0 tile 273"); + + expect(store.has_full_tile(1, 0), "B L1 tile 0"); + expect(!store.has_full_tile(1, 1), "B no L1 tile 1"); + + expect(!fs::exists(store.root() / "tile" / "2"), "B no level 2"); + expect(!fs::exists(store.root() / "tile" / "3"), "B no level 3"); + expect(tile_file_count(store) == 274, "B exact tile file count"); + + // Higher-level entries are roll-ups of the complete child tiles. + const auto l1 = store.read_tile(TileRef{1, 0}); + expect(l1.size() == 256, "B L1 full width"); + expect(l1[0] == rollup(store.read_tile(TileRef{0, 0})), "B L1[0]"); + expect(l1[255] == rollup(store.read_tile(TileRef{0, 255})), "B L1[255]"); + + std::cout << "B (size 70000): OK" << '\n'; + } + + // ---- C. incremental writes preserve immutability and idempotency. + { + const auto hashes = make_hashes(1024); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + + TileStore store(base / "c"); + TileWriter writer(store); + + const auto s1 = writer.write_up_to(256, leaf_at); + expect(s1.full_written == 1, "C s1 full"); + + // Re-running at the same size writes nothing (full tiles are immutable). + const auto s2 = writer.write_up_to(256, leaf_at); + expect(s2.full_written == 0, "C s2 full immutable"); + + // Grow to 600: one new full L0 tile (covering 512), nothing else. + const auto s3 = writer.write_up_to(600, leaf_at); + expect(s3.full_written == 1, "C s3 full"); + + expect(store.has_full_tile(0, 0), "C L0 tile 0"); + expect(store.has_full_tile(0, 1), "C L0 tile 1"); + expect(!store.has_full_tile(0, 2), "C no L0 tile 2"); + expect(!store.has_full_tile(1, 0), "C no L1 full tile"); + expect(tile_file_count(store) == 2, "C exact tile file count"); + + std::cout << "C (incremental): OK" << '\n'; + } + + // ---- D. crossing into a full level-1 tile: the roll-up appears as a full + // tile and prior full tiles are never rewritten. + { + const auto hashes = make_hashes(65536); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + + TileStore store(base / "d"); + TileWriter writer(store); + + // One leaf short of a full L1 tile: 255 full L0 tiles, no L1 yet. + const auto s1 = writer.write_up_to(65535, leaf_at); + expect(s1.full_written == 255, "D s1 255 full L0"); + expect(!store.has_full_tile(1, 0), "D no L1 before completion"); + expect(tile_file_count(store) == 255, "D initial tile file count"); + + // Completing the 256th L0 tile yields one new L0 tile and one full L1. + const auto s2 = writer.write_up_to(65536, leaf_at); + expect(s2.full_written == 2, "D s2 new L0 + new L1"); + expect(store.has_full_tile(0, 255), "D L0 tile 255"); + expect(store.has_full_tile(1, 0), "D L1 tile 0"); + expect(!store.has_full_tile(1, 1), "D no L1 tile 1"); + expect(!store.has_full_tile(2, 0), "D no L2 tile"); + expect(tile_file_count(store) == 257, "D final tile file count"); + + const auto l1 = store.read_tile(TileRef{1, 0}); + expect(l1[0] == rollup(store.read_tile(TileRef{0, 0})), "D L1[0]"); + + // Re-running writes nothing (everything already full and immutable). + const auto s3 = writer.write_up_to(65536, leaf_at); + expect(s3.full_written == 0, "D s3 immutable rerun"); + + std::cout << "D (full L1 tile): OK" << '\n'; + } + + // ---- E. resume: a fresh writer over an existing store rebuilds its cursor + // from disk via full_prefix_length, never rewriting full tiles and + // writing only the newly-complete ones. + { + const auto hashes = make_hashes(70000); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + + const fs::path dir = base / "e"; + { + TileStore store(dir); + TileWriter writer(store); + expect( + writer.write_up_to(600, leaf_at).full_written == 2, + "E first writer 2 L0 tiles"); // 600 / 256 == 2 full L0 tiles + } + + // A brand-new store + writer over the same directory: its cursor must + // resume from what is already on disk. + TileStore store(dir); + TileWriter writer(store); + expect( + writer.write_up_to(600, leaf_at).full_written == 0, + "E resume rewrites nothing"); + + // Extending to 70000 writes only the missing tiles: 273 - 2 == 271 new L0 + // plus 1 new L1 (which rolls up the L0 tiles the first writer wrote). + expect( + writer.write_up_to(70000, leaf_at).full_written == 272, + "E resume writes only new tiles"); + expect(store.has_full_tile(0, 0), "E L0 tile 0 still present"); + expect(store.has_full_tile(0, 272), "E L0 tile 272"); + expect(store.has_full_tile(1, 0), "E L1 tile 0"); + expect( + store.read_tile(TileRef{1, 0})[0] == + rollup(store.read_tile(TileRef{0, 0})), + "E L1[0] rolled up from resumed L0"); + expect(tile_file_count(store) == 274, "E exact tile file count"); + + // A third fresh writer confirms full idempotence after a resume. + TileStore store3(dir); + TileWriter writer3(store3); + expect( + writer3.write_up_to(70000, leaf_at).full_written == 0, + "E second resume idempotent"); + + std::cout << "E (writer resume): OK" << '\n'; + } + + // ---- F. Resume discovers the first hole rather than trusting later valid + // files as proof that the prefix is contiguous. + { + const auto hashes = make_hashes(2048); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + const fs::path dir = base / "f"; + + { + TileStore store(dir); + TileWriter writer(store); + expect( + writer.write_up_to(2048, leaf_at).full_written == 8, + "F initial tiles"); + overwrite_file(store.tile_path(TileRef{0, 3}), {0}); + expect(!store.has_full_tile(0, 3), "F interior tile corrupt"); + expect(store.has_full_tile(0, 7), "F later tile remains valid"); + } + + TileStore store(dir); + TileWriter writer(store); + expect( + writer.write_up_to(2048, leaf_at).full_written == 1, + "F rewrites interior hole"); + const std::vector expected( + hashes.begin() + (std::ptrdiff_t)merkle::tiles::TILE_WIDTH * 3, + hashes.begin() + (std::ptrdiff_t)merkle::tiles::TILE_WIDTH * 4); + expect( + store.read_tile(TileRef{0, 3}) == expected, "F repaired tile contents"); + expect(tile_file_count(store) == 8, "F exact tile file count"); + + std::cout << "F (interior recovery): OK" << '\n'; + } + + // ---- G. Recovery is bounded by the requested tree size, so sparse files + // at geometrically increasing indices cannot overflow its search. + { + const fs::path dir = base / "g"; + const auto hashes = make_hashes(merkle::tiles::TILE_WIDTH); + { + TileStore store(dir); + store.write_tile(TileRef{0, 0}, hashes); + for (uint64_t index = 1;; index <<= 1) + { + store.write_tile(TileRef{0, index}, hashes); + if (index == (uint64_t{1} << 63)) + { + break; + } + } + } + + TileStore store(dir); + TileWriter writer(store); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + expect( + writer.write_up_to(merkle::tiles::TILE_WIDTH, leaf_at).full_written == + 0, + "G bounded sparse recovery"); + expect(store.has_full_tile(0, 0), "G requested tile remains valid"); + + std::cout << "G (bounded sparse recovery): OK" << '\n'; + } + + // ---- H. A complete level-2 tile rolls up 256 level-1 child tiles. Prime + // the lower-level cursors to model a long-lived writer without allocating + // all 2^24 source leaves. + { + const fs::path dir = base / "h"; + TileStore store(dir); + const auto child_template = make_hashes(merkle::tiles::TILE_WIDTH); + std::vector expected; + expected.reserve(merkle::tiles::TILE_WIDTH); + + for (uint64_t index = 0; index < merkle::tiles::TILE_WIDTH; index++) + { + auto child = child_template; + child[0].bytes[0] = static_cast(index); + store.write_tile(TileRef{1, index}, child); + expected.push_back(rollup(child)); + } + + TileWriterProbe writer(store); + writer.mark_level_complete( + 0, + (uint64_t)merkle::tiles::TILE_WIDTH * merkle::tiles::TILE_WIDTH); + writer.mark_level_complete(1, merkle::tiles::TILE_WIDTH); + + bool leaf_requested = false; + const Hash unused; + const auto leaf_at = [&](uint64_t) -> const Hash& { + leaf_requested = true; + return unused; + }; + constexpr uint64_t size = (uint64_t)merkle::tiles::TILE_WIDTH * + merkle::tiles::TILE_WIDTH * merkle::tiles::TILE_WIDTH; + const auto stats = writer.write_up_to(size, leaf_at); + + expect(stats.full_written == 1, "H writes one L2 tile"); + expect(!leaf_requested, "H does not revisit completed lower levels"); + expect(store.has_full_tile(2, 0), "H full L2 tile"); + expect(store.read_tile(TileRef{2, 0}) == expected, "H L2 roll-up"); + expect(!store.has_full_tile(3, 0), "H no L3 tile"); + expect(tile_file_count(store) == 257, "H exact tile file count"); + + std::cout << "H (full L2 tile): OK" << '\n'; + } + +#ifdef HAVE_OPENSSL + // ---- I. The templated writer uses the selected hash size and function + // when producing higher-level tiles. + { + using Hash384 = merkle::Tree384::Hash; + using TileStore384 = merkle::tiles::TileStoreT< + Hash384::size_bytes, + merkle::Tree384::hash_function>; + using TileWriter384 = merkle::tiles::TileWriterT< + Hash384::size_bytes, + merkle::Tree384::hash_function>; + + constexpr uint64_t size = (uint64_t)merkle::tiles::TILE_WIDTH * + merkle::tiles::TILE_WIDTH; + const auto hashes = make_hashesT((size_t)size); + const auto leaf_at = [&](uint64_t index) -> const Hash384& { + return hashes[index]; + }; + + TileStore384 store(base / "i"); + TileWriter384 writer(store); + const auto stats = writer.write_up_to(size, leaf_at); + + expect(stats.full_written == 257, "I writes L0 and L1 SHA-384 tiles"); + expect(store.has_full_tile(1, 0), "I full SHA-384 L1 tile"); + const auto level1 = store.read_tile(TileRef{1, 0}); + const std::vector first_tile( + hashes.begin(), hashes.begin() + merkle::tiles::TILE_WIDTH); + expect( + level1[0] == + merkle::tiles::perfect_root< + Hash384::size_bytes, + merkle::Tree384::hash_function>(first_tile), + "I first SHA-384 roll-up"); + + merkle::Tree384 tree; + for (const auto& hash : hashes) + { + tree.insert(hash); + } + expect( + merkle::tiles::perfect_root< + Hash384::size_bytes, + merkle::Tree384::hash_function>(level1) == tree.root(), + "I SHA-384 tiled root"); + expect(tile_file_count(store) == 257, "I exact tile file count"); + + std::cout << "I (SHA-384 writer): OK" << '\n'; + } +#endif + + std::cout << "tiles_writer: OK" << '\n'; + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + return 1; + } + + return 0; +}