From 9fa6fddf93f5f61686b27851275a88a4a8285215 Mon Sep 17 00:00:00 2001 From: Sanya239 Date: Mon, 20 Jul 2026 02:00:23 +0300 Subject: [PATCH 1/2] Naive huffman implementation --- CMakeLists.txt | 21 +- include/pixie/huffman.h | 89 +++++ include/pixie/huffman/implementations.h | 12 + include/pixie/huffman/pivco_huffman.h | 343 ++++++++++++++++++++ src/benchmarks/pivco_huffman_benchmarks.cpp | 189 +++++++++++ src/tests/pivco_huffman_tests.cpp | 108 ++++++ 6 files changed, 761 insertions(+), 1 deletion(-) create mode 100644 include/pixie/huffman.h create mode 100644 include/pixie/huffman/implementations.h create mode 100644 include/pixie/huffman/pivco_huffman.h create mode 100644 src/benchmarks/pivco_huffman_benchmarks.cpp create mode 100644 src/tests/pivco_huffman_tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6101a98..6828f92 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -256,6 +256,15 @@ if (PIXIE_TESTS) PRIVATE ${sdsl_lite_SOURCE_DIR}/include) endif () + add_executable(pivco_huffman_tests + src/tests/pivco_huffman_tests.cpp) + target_include_directories(pivco_huffman_tests + PUBLIC include) + target_link_libraries(pivco_huffman_tests + gtest + gtest_main + ${PIXIE_DIAGNOSTICS_LIBS}) + set(PIXIE_TEST_TARGETS bit_algorithms_unittests rank_select_unittests @@ -267,7 +276,8 @@ if (PIXIE_TESTS) storage_tests excess_positions_tests excess_record_lows_tests - rmq_tests) + rmq_tests + pivco_huffman_tests) foreach (test_target IN LISTS PIXIE_TEST_TARGETS) gtest_discover_tests(${test_target} DISCOVERY_MODE PRE_TEST @@ -399,6 +409,15 @@ if (PIXIE_BENCHMARKS) benchmark benchmark_main ${PIXIE_DIAGNOSTICS_LIBS}) + + add_executable(pivco_huffman_benchmarks + src/benchmarks/pivco_huffman_benchmarks.cpp) + target_include_directories(pivco_huffman_benchmarks + PUBLIC include) + target_link_libraries(pivco_huffman_benchmarks + benchmark + benchmark_main + ${PIXIE_DIAGNOSTICS_LIBS}) endif () # --------------------------------------------------------------------------- diff --git a/include/pixie/huffman.h b/include/pixie/huffman.h new file mode 100644 index 0000000..68acab5 --- /dev/null +++ b/include/pixie/huffman.h @@ -0,0 +1,89 @@ +#pragma once + +/** + * @file huffman.h + * @brief Common CRTP contract for Huffman entropy codecs. + * + * PivCo-Huffman ("Pivot-Coded Huffman") reuses the wavelet-tree "tree of + * bitmaps" layout to turn sequential, bit-by-bit Huffman-tree traversals into + * vectorizable operations. A codec encodes a byte sequence into a compressed + * stream by building a Huffman-shaped tree of per-node bitmaps, and decodes it + * back by traversing that tree. Concrete implementations live under + * ``. + * + * @see Marcin Zukowski, "PivCo-Huffman", v1.0 (2026). + */ + +#include +#include +#include +#include + +namespace pixie { + +/** + * @brief CRTP facade for Huffman entropy codecs. + * + * The contract is the source of truth for observable codec semantics. Each + * concrete implementation inherits `HuffmanBase` and supplies the + * required `*_impl()` extension points. There is no virtual dispatch: the + * facade delegates statically through CRTP, mirroring the other Pixie families. + * + * Range and ownership conventions: + * - Symbol sequences are zero-based byte streams (`symbol_type`). + * - Compressed streams are byte-oriented views (`std::byte`). + * - The caller keeps any non-owning view alive for the codec lifetime. + * + * @tparam Impl Concrete codec type implementing the `*_impl()` contract. + * + * @see `` for the available concrete + * implementations. + */ +template +class HuffmanBase { + public: + /** @brief Symbol type handled by the codec: one byte per symbol. */ + using symbol_type = std::uint8_t; + + /** + * @brief Number of symbols in the original (uncompressed) stream. + * @return Logical uncompressed symbol count. + */ + std::size_t uncompressed_size() const { + return impl().uncompressed_size_impl(); + } + + /** + * @brief Number of bytes in the compressed representation. + * @return Compressed stream size in bytes. + */ + std::size_t compressed_size() const { return impl().compressed_size_impl(); } + + /** + * @brief Check whether the codec holds no data. + * @return `true` when `uncompressed_size() == 0`. + */ + bool empty() const { return uncompressed_size() == 0; } + + /** + * @brief Read-only view of the compressed byte stream. + * @return Span over the serialized representation. + * + * @note The returned view is invalidated by codec destruction. + */ + std::span compressed_data() const { + return impl().compressed_data_impl(); + } + + /** + * @brief Reconstruct the original symbol sequence. + * @return Decoded symbols of length `uncompressed_size()`. + */ + std::vector decode() const { return impl().decode_impl(); } + + private: + /** @brief Return this facade as its concrete CRTP implementation. */ + const Impl& impl() const { return static_cast(*this); } +}; + +} // namespace pixie diff --git a/include/pixie/huffman/implementations.h b/include/pixie/huffman/implementations.h new file mode 100644 index 0000000..e5455cf --- /dev/null +++ b/include/pixie/huffman/implementations.h @@ -0,0 +1,12 @@ +#pragma once + +/** + * @file implementations.h + * @brief All Huffman codec implementations provided by Pixie. + * + * - `PivCoHuffman`: simple scalar reference codec storing a Huffman-shaped + * tree of per-node routing bitmaps as packed 64-bit words. + */ + +#include +#include diff --git a/include/pixie/huffman/pivco_huffman.h b/include/pixie/huffman/pivco_huffman.h new file mode 100644 index 0000000..c675be7 --- /dev/null +++ b/include/pixie/huffman/pivco_huffman.h @@ -0,0 +1,343 @@ +#pragma once + +/** + * @file pivco_huffman.h + * @brief Simple scalar PivCo-Huffman codec. + * + * Stores a Huffman-shaped tree of per-node routing bitmaps (the "tree of + * bitmaps" layout shared with wavelet trees). Encoding walks each input symbol + * from the root, appending one direction bit to every internal node on its + * path. Decoding walks each output position from the root down to a leaf, + * consuming one bit per internal node. + * + * This is a deliberately simple, unoptimized reference implementation: node + * bitmaps are stored as packed `std::uint64_t` words, traversal is scalar, + * and there are no flat-subtree, non-canonical, SIMD, or selective-ANS + * optimizations. It is intended as a correct baseline for the PivCo-Huffman + * layout and for future optimized variants. + * + * Range and ownership conventions follow `HuffmanBase`: symbols are bytes, + * the compressed stream is a byte view, and the codec owns its serialized form. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie { + +/** + * @brief Simple scalar PivCo-Huffman codec. + * @details Implements `HuffmanBase` by building a Huffman tree over the + * byte alphabet, storing one routing bitmap per internal node as + * packed 64-bit words, and decoding by top-down per-position + * traversal. The serialized form is a header followed by the tree + * structure and packed node bitmaps. + */ +class PivCoHuffman : public HuffmanBase { + public: + /** @brief Symbol type handled by the codec: one byte per symbol. */ + using symbol_type = std::uint8_t; + + /** + * @brief Build a codec by encoding @p input. + * @param input Symbol sequence to compress. + */ + explicit PivCoHuffman(std::span input) { + build(input); + serialize(); + } + + /** + * @brief Load a codec from a previously serialized compressed stream. + * @param compressed Compressed byte view produced by another instance. + */ + explicit PivCoHuffman(std::span compressed) { + deserialize(compressed); + } + + // --- CRTP extension points ---------------------------------------------- + + /** @brief Size of the original symbol stream. */ + std::size_t uncompressed_size_impl() const { return uncompressed_size_; } + + /** @brief Size of the compressed byte stream. */ + std::size_t compressed_size_impl() const { return compressed_.size(); } + + /** @brief Read-only view of the compressed byte stream. */ + std::span compressed_data_impl() const { + return compressed_; + } + + /** @brief Reconstruct the original symbol sequence. */ + std::vector decode_impl() const { return decode_from_tree(); } + + private: + /// @brief Sentinel node index meaning "no node". + static constexpr std::size_t kNpos = static_cast(-1); + + /// @brief Byte-alphabet size (256 symbols). + static constexpr std::size_t kAlphabet = 256; + + /// @brief Bits per word used by the packed bitmap. + static constexpr std::size_t kWordBits = 64; + + /** + * @brief Packed routing bitmap stored as 64-bit words. + * @details Bit `i` lives in word `i / 64` at position `i % 64`. This avoids + * the slow bit-proxy access of `std::vector` and allows + * bulk serialization via direct word `memcpy`. + */ + struct Bitmap { + std::vector words; + std::size_t count = 0; + + /** @brief Append one bit to the end of the bitmap. */ + void push_back(bool bit) { + if (count % kWordBits == 0) { + words.push_back(0); + } + if (bit) { + words.back() |= (1ull << (count % kWordBits)); + } + ++count; + } + + /** @brief Read the bit at position @p i. */ + bool get(std::size_t i) const { + return ((words[i / kWordBits] >> (i % kWordBits)) & 1ull) != 0; + } + + /** @brief Number of stored bits. */ + std::size_t size() const { return count; } + + /** @brief Number of 64-bit words backing the bitmap. */ + std::size_t word_count() const { + return (count + kWordBits - 1) / kWordBits; + } + }; + + /** + * @brief A node in the Huffman tree of bitmaps. + * @details Internal nodes carry a routing bitmap with one bit per symbol + * that passes through the node (0 = left, 1 = right), in input + * order. Leaves carry only their symbol. + */ + struct Node { + std::size_t left = kNpos; + std::size_t right = kNpos; + std::uint8_t symbol = 0; + bool is_leaf = false; + Bitmap bits; + }; + + std::size_t uncompressed_size_ = 0; + std::size_t root_ = kNpos; + std::vector nodes_; + std::vector compressed_; + + // --- construction -------------------------------------------------------- + + /** @brief Build the Huffman tree and per-node bitmaps from @p input. */ + void build(std::span input) { + uncompressed_size_ = input.size(); + if (uncompressed_size_ == 0) { + root_ = kNpos; + return; + } + + std::array freq{}; + for (auto s : input) { + freq[s]++; + } + + struct HeapItem { + std::size_t weight; + std::size_t idx; + }; + auto cmp = [](const HeapItem& a, const HeapItem& b) { + return a.weight > b.weight; + }; + std::priority_queue, decltype(cmp)> heap( + cmp); + + for (std::size_t s = 0; s < kAlphabet; s++) { + if (freq[s] > 0) { + Node n; + n.is_leaf = true; + n.symbol = static_cast(s); + nodes_.push_back(std::move(n)); + heap.push({freq[s], nodes_.size() - 1}); + } + } + + while (heap.size() > 1) { + HeapItem a = heap.top(); + heap.pop(); + HeapItem b = heap.top(); + heap.pop(); + Node n; + n.is_leaf = false; + n.left = a.idx; + n.right = b.idx; + nodes_.push_back(std::move(n)); + heap.push({a.weight + b.weight, nodes_.size() - 1}); + } + root_ = heap.top().idx; + + // Record, for each symbol, the path of (node, direction-bit) from root. + std::array>, kAlphabet> paths; + std::vector> path; + assign_paths(root_, path, paths); + + // Append one direction bit per visited node, in input order. + for (auto s : input) { + for (const auto& [idx, bit] : paths[s]) { + nodes_[idx].bits.push_back(bit); + } + } + } + + /** @brief Depth-first assignment of root-to-leaf paths. */ + void assign_paths( + std::size_t idx, + std::vector>& path, + std::array>, kAlphabet>& paths) { + if (nodes_[idx].is_leaf) { + paths[nodes_[idx].symbol] = path; + return; + } + path.emplace_back(idx, false); + assign_paths(nodes_[idx].left, path, paths); + path.pop_back(); + path.emplace_back(idx, true); + assign_paths(nodes_[idx].right, path, paths); + path.pop_back(); + } + + // --- decode -------------------------------------------------------------- + + /** @brief Reconstruct the original sequence by top-down traversal. */ + std::vector decode_from_tree() const { + std::vector out(uncompressed_size_); + if (root_ == kNpos) { + return out; + } + if (nodes_[root_].is_leaf) { + const std::uint8_t sym = nodes_[root_].symbol; + for (auto& s : out) { + s = sym; + } + return out; + } + std::vector cursor(nodes_.size(), 0); + for (std::size_t i = 0; i < uncompressed_size_; i++) { + std::size_t idx = root_; + while (!nodes_[idx].is_leaf) { + const bool bit = nodes_[idx].bits.get(cursor[idx]++); + idx = bit ? nodes_[idx].right : nodes_[idx].left; + } + out[i] = nodes_[idx].symbol; + } + return out; + } + + // --- serialization ------------------------------------------------------- + + /** @brief Write the in-memory tree into the serialized byte buffer. */ + void serialize() { + compressed_.clear(); + write(uncompressed_size_); + if (uncompressed_size_ == 0) { + return; + } + write(root_); + write(nodes_.size()); + for (const auto& n : nodes_) { + write(static_cast(n.is_leaf ? 1 : 0)); + write(n.symbol); + write(n.left); + write(n.right); + write(n.bits.count); + write_words(n.bits.words); + } + } + + /** @brief Rebuild the in-memory tree from a serialized byte buffer. */ + void deserialize(std::span data) { + compressed_.assign(data.begin(), data.end()); + nodes_.clear(); + if (data.empty()) { + uncompressed_size_ = 0; + root_ = kNpos; + return; + } + std::size_t pos = 0; + uncompressed_size_ = read(data, pos); + if (uncompressed_size_ == 0) { + root_ = kNpos; + return; + } + root_ = read(data, pos); + const std::size_t count = read(data, pos); + nodes_.resize(count); + for (std::size_t i = 0; i < count; i++) { + const std::uint8_t flags = read(data, pos); + nodes_[i].is_leaf = (flags & 1) != 0; + nodes_[i].symbol = read(data, pos); + nodes_[i].left = read(data, pos); + nodes_[i].right = read(data, pos); + nodes_[i].bits.count = read(data, pos); + nodes_[i].bits.words = read_words(data, pos, nodes_[i].bits.word_count()); + } + } + + /** @brief Append a little-endian, fixed-width value to the byte buffer. */ + template + void write(const T& value) { + const std::size_t old = compressed_.size(); + compressed_.resize(old + sizeof(T)); + std::memcpy(compressed_.data() + old, &value, sizeof(T)); + } + + /** @brief Append a packed word array directly to the byte buffer. */ + void write_words(const std::vector& words) { + const std::size_t bytes = words.size() * sizeof(std::uint64_t); + const std::size_t old = compressed_.size(); + compressed_.resize(old + bytes); + if (bytes > 0) { + std::memcpy(compressed_.data() + old, words.data(), bytes); + } + } + + /** @brief Read a fixed-width value from @p data at @p pos. */ + template + static T read(std::span data, std::size_t& pos) { + T value; + std::memcpy(&value, data.data() + pos, sizeof(T)); + pos += sizeof(T); + return value; + } + + /** @brief Read a packed word array of @p word_count words from @p data. */ + static std::vector read_words(std::span data, + std::size_t& pos, + std::size_t word_count) { + std::vector words(word_count); + const std::size_t bytes = word_count * sizeof(std::uint64_t); + if (bytes > 0) { + std::memcpy(words.data(), data.data() + pos, bytes); + pos += bytes; + } + return words; + } +}; + +} // namespace pixie diff --git a/src/benchmarks/pivco_huffman_benchmarks.cpp b/src/benchmarks/pivco_huffman_benchmarks.cpp new file mode 100644 index 0000000..5812bf4 --- /dev/null +++ b/src/benchmarks/pivco_huffman_benchmarks.cpp @@ -0,0 +1,189 @@ +#include +#include + +#include +#include +#include +#include +#include + +using pixie::PivCoHuffman; + +// --------------------------------------------------------------------------- +// Configuration via environment variables (follows the Pixie convention used +// by EXCESS_POS_CASES / RECORD_LOWS_CASES etc.): +// PIVCO_BENCH_SIZE - input size in bytes (default 1<<16 = 64 KiB) +// PIVCO_BENCH_SEED - random seed (default 42) +// +// To run a subset of datasets, use Google Benchmark's built-in filter flag, +// e.g. `--benchmark_filter=".*Skewed.*|.*Text.*"`. No code change needed. +// --------------------------------------------------------------------------- + +namespace { + +/// @brief Input size in bytes, overridable via `PIVCO_BENCH_SIZE`. +std::size_t bench_size() { + if (const char* env = std::getenv("PIVCO_BENCH_SIZE")) { + try { + const unsigned long long v = std::stoull(env); + if (v != 0) { + return static_cast(v); + } + } catch (...) { + } + } + return 1ull << 16; +} + +/// @brief RNG seed, overridable via `PIVCO_BENCH_SEED`. +std::uint64_t bench_seed() { + if (const char* env = std::getenv("PIVCO_BENCH_SEED")) { + try { + return std::stoull(env); + } catch (...) { + } + } + return 42; +} + +/// @brief Signature of a dataset generator: produces @p n bytes from @p rng. +using DatasetMaker = std::vector (*)(std::size_t, + std::mt19937_64&); + +// --- built-in datasets ----------------------------------------------------- + +/// @brief Uniformly distributed bytes over the full 256-symbol alphabet. +std::vector make_uniform256(std::size_t n, std::mt19937_64& rng) { + std::vector data(n); + for (auto& b : data) { + b = static_cast(rng()); + } + return data; +} + +/// @brief Binary alphabet {0, 1}: shortest possible codes (depth 1). +std::vector make_binary(std::size_t n, std::mt19937_64& rng) { + std::vector data(n); + for (auto& b : data) { + b = static_cast(rng() & 1u); + } + return data; +} + +/// @brief Four-symbol alphabet {0,1,2,3}: code depth 2. +std::vector make_low_entropy_4(std::size_t n, + std::mt19937_64& rng) { + std::vector data(n); + for (auto& b : data) { + b = static_cast(rng() & 3u); + } + return data; +} + +/// @brief English-text-like distribution: ~17% space, ~80% lowercase letters, +/// ~2% digits, ~1% punctuation. +std::vector make_text_like(std::size_t n, std::mt19937_64& rng) { + std::vector data(n); + std::uniform_int_distribution pick(0, 99); + std::uniform_int_distribution letter('a', 'z'); + std::uniform_int_distribution digit('0', '9'); + std::uniform_int_distribution punct('!', '/'); + for (auto& b : data) { + const int r = pick(rng); + if (r < 17) { + b = ' '; + } else if (r < 97) { + b = static_cast(letter(rng)); + } else if (r < 99) { + b = static_cast(digit(rng)); + } else { + b = static_cast(punct(rng)); + } + } + return data; +} + +/// @brief Highly skewed: 99% one symbol, 1% random over 256. Exercises a deep, +/// lopsided Huffman tree. +std::vector make_skewed99(std::size_t n, std::mt19937_64& rng) { + std::vector data(n, 0); + std::uniform_int_distribution coin(0, 99); + for (auto& b : data) { + if (coin(rng) < 1) { + b = static_cast(rng()); + } + } + return data; +} + +/// @brief Degenerate single-symbol input (root is a leaf): best-case path. +std::vector make_single_symbol(std::size_t n, + std::mt19937_64& rng) { + (void)rng; + return std::vector(n, 123); +} + +/// @brief Compression efficiency in bits per input byte (8 = no compression). +void report_ratio(benchmark::State& state, + std::size_t compressed_bytes, + std::size_t input_bytes) { + state.counters["bpb"] = static_cast(compressed_bytes) * 8.0 / + static_cast(input_bytes); +} + +// --- benchmarks ------------------------------------------------------------ + +/// @brief Measure full encoding: build Huffman tree, fill node bitmaps, and +/// serialize the compressed stream. +void BM_Encode(benchmark::State& state, DatasetMaker make) { + const std::size_t size = bench_size(); + std::mt19937_64 rng(bench_seed()); + const std::vector data = make(size, rng); + + // Encode once outside timing to report compression ratio. + const PivCoHuffman probe(data); + report_ratio(state, probe.compressed_size(), size); + + for (auto _ : state) { + PivCoHuffman codec(data); + benchmark::DoNotOptimize(codec.compressed_data().data()); + benchmark::ClobberMemory(); + } + state.SetBytesProcessed(static_cast(state.iterations() * size)); +} + +/// @brief Measure decoding: reconstruct the original byte stream from the +/// in-memory tree. Encoding is done once, outside the timed loop. +void BM_Decode(benchmark::State& state, DatasetMaker make) { + const std::size_t size = bench_size(); + std::mt19937_64 rng(bench_seed()); + const std::vector data = make(size, rng); + + const PivCoHuffman codec(data); + report_ratio(state, codec.compressed_size(), size); + + for (auto _ : state) { + std::vector out = codec.decode(); + benchmark::DoNotOptimize(out.data()); + benchmark::ClobberMemory(); + } + state.SetBytesProcessed(static_cast(state.iterations() * size)); +} + +} // namespace + +// Register encode + decode for every built-in dataset. Each registration is +// named `BM_/`, so `--benchmark_filter` selects subsets freely. +BENCHMARK_CAPTURE(BM_Encode, Uniform256, make_uniform256); +BENCHMARK_CAPTURE(BM_Encode, Binary, make_binary); +BENCHMARK_CAPTURE(BM_Encode, LowEntropy4, make_low_entropy_4); +BENCHMARK_CAPTURE(BM_Encode, TextLike, make_text_like); +BENCHMARK_CAPTURE(BM_Encode, Skewed99, make_skewed99); +BENCHMARK_CAPTURE(BM_Encode, SingleSymbol, make_single_symbol); + +BENCHMARK_CAPTURE(BM_Decode, Uniform256, make_uniform256); +BENCHMARK_CAPTURE(BM_Decode, Binary, make_binary); +BENCHMARK_CAPTURE(BM_Decode, LowEntropy4, make_low_entropy_4); +BENCHMARK_CAPTURE(BM_Decode, TextLike, make_text_like); +BENCHMARK_CAPTURE(BM_Decode, Skewed99, make_skewed99); +BENCHMARK_CAPTURE(BM_Decode, SingleSymbol, make_single_symbol); diff --git a/src/tests/pivco_huffman_tests.cpp b/src/tests/pivco_huffman_tests.cpp new file mode 100644 index 0000000..fcc656b --- /dev/null +++ b/src/tests/pivco_huffman_tests.cpp @@ -0,0 +1,108 @@ +#include +#include + +#include +#include +#include +#include +#include + +using pixie::PivCoHuffman; + +static_assert( + std::is_base_of_v, PivCoHuffman>); +static_assert(std::is_same_v); + +namespace { +// Round-trip through the in-memory tree of the same codec instance. +std::vector decode_same(const PivCoHuffman& codec) { + return codec.decode(); +} + +// Round-trip through the serialized byte stream: simulate writing the +// compressed form to disk and loading it into a fresh codec instance. +std::vector decode_via_bytes(const PivCoHuffman& codec) { + const auto bytes = codec.compressed_data(); + PivCoHuffman copy(bytes); + return copy.decode(); +} +} // namespace + +TEST(PivCoHuffmanSmoke, EmptyInputRoundTrips) { + const std::vector data; + const PivCoHuffman codec(data); + EXPECT_TRUE(codec.empty()); + EXPECT_EQ(codec.uncompressed_size(), 0u); + EXPECT_EQ(decode_same(codec), data); + EXPECT_EQ(decode_via_bytes(codec), data); +} + +TEST(PivCoHuffmanSmoke, SingleSymbolRoundTrips) { + const std::vector data(1000, 42); + const PivCoHuffman codec(data); + EXPECT_EQ(codec.uncompressed_size(), data.size()); + EXPECT_EQ(decode_same(codec), data); + EXPECT_EQ(decode_via_bytes(codec), data); +} + +TEST(PivCoHuffmanSmoke, TwoSymbolRoundTrips) { + std::vector data; + for (std::size_t i = 0; i < 500; i++) { + data.push_back(static_cast(i % 2)); + } + const PivCoHuffman codec(data); + EXPECT_EQ(decode_same(codec), data); + EXPECT_EQ(decode_via_bytes(codec), data); +} + +TEST(PivCoHuffmanSmoke, KnownInputRoundTrips) { + const std::vector data{3, 2, 0, 3, 1, 1, 2, 0, 3, 1}; + const PivCoHuffman codec(data); + EXPECT_EQ(decode_same(codec), data); + EXPECT_EQ(decode_via_bytes(codec), data); +} + +TEST(PivCoHuffmanSmoke, CompressedDataIsNonEmpty) { + const std::vector data{3, 2, 0, 3, 1, 1, 2, 0, 3, 1}; + const PivCoHuffman codec(data); + EXPECT_GT(codec.compressed_size(), 0u); + EXPECT_EQ(codec.compressed_data().size(), codec.compressed_size()); +} + +TEST(PivCoHuffmanSmoke, FullAlphabetRoundTrips) { + std::vector data(256); + for (std::size_t i = 0; i < 256; i++) { + data[i] = static_cast(i); + } + const PivCoHuffman codec(data); + EXPECT_EQ(decode_same(codec), data); + EXPECT_EQ(decode_via_bytes(codec), data); +} + +TEST(PivCoHuffmanSmoke, RandomUniformRoundTrips) { + std::mt19937 rng(239); + std::uniform_int_distribution dist(0, 255); + std::vector data(10000); + for (auto& b : data) { + b = static_cast(dist(rng)); + } + const PivCoHuffman codec(data); + EXPECT_EQ(decode_same(codec), data); + EXPECT_EQ(decode_via_bytes(codec), data); +} + +TEST(PivCoHuffmanSmoke, SkewedInputRoundTrips) { + // Most symbols are one value, a few are outliers: exercises a deep, lopsided + // Huffman tree without any single-symbol shortcut. + std::mt19937 rng(7); + std::uniform_int_distribution coin(0, 99); + std::vector data(10000, 7); + for (std::size_t i = 0; i < data.size(); i++) { + if (coin(rng) < 5) { + data[i] = static_cast(rng() % 256); + } + } + const PivCoHuffman codec(data); + EXPECT_EQ(decode_same(codec), data); + EXPECT_EQ(decode_via_bytes(codec), data); +} From 9f80561440c38b3a089198db6ad9e97ffec7d9d2 Mon Sep 17 00:00:00 2001 From: Sanya239 Date: Tue, 21 Jul 2026 18:20:00 +0300 Subject: [PATCH 2/2] Bottom-up implementation with naive primitives --- include/pixie/huffman/pivco_huffman.h | 159 ++++++++++++++------ src/benchmarks/pivco_huffman_benchmarks.cpp | 92 +++++------ 2 files changed, 162 insertions(+), 89 deletions(-) diff --git a/include/pixie/huffman/pivco_huffman.h b/include/pixie/huffman/pivco_huffman.h index c675be7..48f40be 100644 --- a/include/pixie/huffman/pivco_huffman.h +++ b/include/pixie/huffman/pivco_huffman.h @@ -7,8 +7,8 @@ * Stores a Huffman-shaped tree of per-node routing bitmaps (the "tree of * bitmaps" layout shared with wavelet trees). Encoding walks each input symbol * from the root, appending one direction bit to every internal node on its - * path. Decoding walks each output position from the root down to a leaf, - * consuming one bit per internal node. + * path. Decoding merges child symbol streams bottom-up from leaves to root, + * using the node bitmap as a selector. * * This is a deliberately simple, unoptimized reference implementation: node * bitmaps are stored as packed `std::uint64_t` words, traversal is scalar, @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -37,9 +38,9 @@ namespace pixie { * @brief Simple scalar PivCo-Huffman codec. * @details Implements `HuffmanBase` by building a Huffman tree over the * byte alphabet, storing one routing bitmap per internal node as - * packed 64-bit words, and decoding by top-down per-position - * traversal. The serialized form is a header followed by the tree - * structure and packed node bitmaps. + * packed 64-bit words, and decoding by bottom-up merging of child + * symbol streams. The serialized form is a header followed by the + * tree structure and packed node bitmaps. */ class PivCoHuffman : public HuffmanBase { public: @@ -99,29 +100,19 @@ class PivCoHuffman : public HuffmanBase { std::vector words; std::size_t count = 0; - /** @brief Append one bit to the end of the bitmap. */ - void push_back(bool bit) { - if (count % kWordBits == 0) { - words.push_back(0); - } - if (bit) { - words.back() |= (1ull << (count % kWordBits)); - } - ++count; - } - - /** @brief Read the bit at position @p i. */ - bool get(std::size_t i) const { - return ((words[i / kWordBits] >> (i % kWordBits)) & 1ull) != 0; - } - - /** @brief Number of stored bits. */ - std::size_t size() const { return count; } - /** @brief Number of 64-bit words backing the bitmap. */ std::size_t word_count() const { return (count + kWordBits - 1) / kWordBits; } + + /** @brief Number of set (1) bits in the bitmap. */ + std::size_t popcount() const { + std::size_t total = 0; + for (std::uint64_t w : words) { + total += std::popcount(w); + } + return total; + } }; /** @@ -168,6 +159,10 @@ class PivCoHuffman : public HuffmanBase { std::priority_queue, decltype(cmp)> heap( cmp); + // At most 2*kAlphabet-1 nodes (leaves + internal). Pre-allocate to avoid + // reallocation during tree construction. + nodes_.reserve(2 * kAlphabet - 1); + for (std::size_t s = 0; s < kAlphabet; s++) { if (freq[s] > 0) { Node n; @@ -187,8 +182,11 @@ class PivCoHuffman : public HuffmanBase { n.is_leaf = false; n.left = a.idx; n.right = b.idx; + std::size_t w = a.weight + b.weight; + n.bits.count = w; + n.bits.words.assign((w + kWordBits - 1) / kWordBits, 0); nodes_.push_back(std::move(n)); - heap.push({a.weight + b.weight, nodes_.size() - 1}); + heap.push({w, nodes_.size() - 1}); } root_ = heap.top().idx; @@ -197,12 +195,9 @@ class PivCoHuffman : public HuffmanBase { std::vector> path; assign_paths(root_, path, paths); - // Append one direction bit per visited node, in input order. - for (auto s : input) { - for (const auto& [idx, bit] : paths[s]) { - nodes_[idx].bits.push_back(bit); - } - } + // Top-down partitioning: recursively split the symbol stream and fill + // pre-allocated bitmaps. Leaves are skipped (zero overhead). + encode_node(root_, 0, input, paths); } /** @brief Depth-first assignment of root-to-leaf paths. */ @@ -222,29 +217,99 @@ class PivCoHuffman : public HuffmanBase { path.pop_back(); } + /** @brief Top-down encoding: recursively partition symbols and fill bitmaps. + * @details At each internal node, fill the pre-allocated bitmap with the + * direction bit of each symbol's code at this depth, and split the + * symbol stream into left (0) and right (1) child vectors. Leaves + * return immediately — zero overhead. + * @param idx Current node index. + * @param depth Current tree depth (indexes into each symbol's code path). + * @param symbols Symbol stream arriving at this node, in order. + * @param paths Per-symbol code paths from `assign_paths`. */ + void encode_node(std::size_t idx, + std::size_t depth, + std::span symbols, + const std::array>, + kAlphabet>& paths) { + if (nodes_[idx].is_leaf) { + return; + } + + auto& n = nodes_[idx]; + std::vector left_symbols, right_symbols; + std::size_t left_size = nodes_[n.left].bits.count; + std::size_t right_size = nodes_[n.right].bits.count; + left_symbols.reserve(left_size); + right_symbols.reserve(right_size); + + // Fill bitmap via incremental word/bit tracking — avoids per-symbol + // division and modulo. + std::size_t word_idx = 0; + std::size_t bit_pos = 0; + for (std::size_t i = 0; i < symbols.size(); i++) { + bool bit = paths[symbols[i]][depth].second; + if (bit) { + n.bits.words[word_idx] |= (1ull << bit_pos); + right_symbols.push_back(symbols[i]); + } else { + left_symbols.push_back(symbols[i]); + } + if (++bit_pos == kWordBits) { + bit_pos = 0; + ++word_idx; + } + } + + encode_node(n.left, depth + 1, left_symbols, paths); + encode_node(n.right, depth + 1, right_symbols, paths); + } + // --- decode -------------------------------------------------------------- - /** @brief Reconstruct the original sequence by top-down traversal. */ + /** @brief Reconstruct the original sequence by bottom-up merging. */ std::vector decode_from_tree() const { - std::vector out(uncompressed_size_); if (root_ == kNpos) { - return out; + return std::vector(); } - if (nodes_[root_].is_leaf) { - const std::uint8_t sym = nodes_[root_].symbol; - for (auto& s : out) { - s = sym; - } - return out; + return decode_node(root_, uncompressed_size_); + } + + /** @brief Recursively decode a node's output stream bottom-up. + * @param weight Number of symbols this node must produce. For internal + * nodes this equals `bits.count`; for leaves it is passed down from + * the parent (no bitmap to read it from). */ + std::vector decode_node(std::size_t idx, + std::size_t weight) const { + const auto& n = nodes_[idx]; + if (n.is_leaf) { + return std::vector(weight, n.symbol); } - std::vector cursor(nodes_.size(), 0); - for (std::size_t i = 0; i < uncompressed_size_; i++) { - std::size_t idx = root_; - while (!nodes_[idx].is_leaf) { - const bool bit = nodes_[idx].bits.get(cursor[idx]++); - idx = bit ? nodes_[idx].right : nodes_[idx].left; + + std::size_t right_weight = n.bits.popcount(); + std::size_t left_weight = n.bits.count - right_weight; + std::vector left_symbols = decode_node(n.left, left_weight); + std::vector right_symbols = decode_node(n.right, right_weight); + + std::vector out(n.bits.count); + std::size_t c_left = 0; + std::size_t c_right = 0; + // Word-wise merge: read each 64-bit word once and shift through its bits, + // avoiding per-bit division/modulo. + for (std::size_t w = 0; w < n.bits.words.size(); w++) { + std::uint64_t word = n.bits.words[w]; + std::size_t base = w * kWordBits; + std::size_t limit = base + kWordBits; + if (limit > n.bits.count) { + limit = n.bits.count; + } + for (std::size_t i = base; i < limit; i++) { + if (word & 1ull) { + out[i] = right_symbols[c_right++]; + } else { + out[i] = left_symbols[c_left++]; + } + word >>= 1; } - out[i] = nodes_[idx].symbol; } return out; } diff --git a/src/benchmarks/pivco_huffman_benchmarks.cpp b/src/benchmarks/pivco_huffman_benchmarks.cpp index 5812bf4..cebb0ed 100644 --- a/src/benchmarks/pivco_huffman_benchmarks.cpp +++ b/src/benchmarks/pivco_huffman_benchmarks.cpp @@ -4,37 +4,24 @@ #include #include #include -#include #include using pixie::PivCoHuffman; // --------------------------------------------------------------------------- -// Configuration via environment variables (follows the Pixie convention used -// by EXCESS_POS_CASES / RECORD_LOWS_CASES etc.): -// PIVCO_BENCH_SIZE - input size in bytes (default 1<<16 = 64 KiB) +// Configuration: // PIVCO_BENCH_SEED - random seed (default 42) // -// To run a subset of datasets, use Google Benchmark's built-in filter flag, -// e.g. `--benchmark_filter=".*Skewed.*|.*Text.*"`. No code change needed. +// Input sizes are swept by Google Benchmark's range mechanism, not an env +// var: 1 KiB -> 1 MiB -> 1 GiB (multiplier 1024). This exercises the codec +// from cache-resident to gigabyte scale in a single run. +// +// To run a subset of datasets or sizes, use Google Benchmark's filter flag, +// e.g. `--benchmark_filter="BM_Encode/Uniform256"`. // --------------------------------------------------------------------------- namespace { -/// @brief Input size in bytes, overridable via `PIVCO_BENCH_SIZE`. -std::size_t bench_size() { - if (const char* env = std::getenv("PIVCO_BENCH_SIZE")) { - try { - const unsigned long long v = std::stoull(env); - if (v != 0) { - return static_cast(v); - } - } catch (...) { - } - } - return 1ull << 16; -} - /// @brief RNG seed, overridable via `PIVCO_BENCH_SEED`. std::uint64_t bench_seed() { if (const char* env = std::getenv("PIVCO_BENCH_SEED")) { @@ -131,31 +118,42 @@ void report_ratio(benchmark::State& state, static_cast(input_bytes); } +/// @brief Common range of input sizes: 1 KiB, 1 MiB, 1 GiB. +/// @details Applied to every registered benchmark. The 1 GiB tier exercises +/// gigabyte-scale inputs; at that size Google Benchmark runs a single +/// iteration per case. +constexpr std::int64_t kSizeLo = 1ull << 10; // 1 KiB +constexpr std::int64_t kSizeHi = 1ull << 30; // 1 GiB +constexpr std::int64_t kSizeMult = 1024; + // --- benchmarks ------------------------------------------------------------ /// @brief Measure full encoding: build Huffman tree, fill node bitmaps, and -/// serialize the compressed stream. +/// serialize the compressed stream. The compressed size is read from +/// the first timed iteration (no separate probe build), which keeps +/// gigabyte-scale setup cost to a single encode. void BM_Encode(benchmark::State& state, DatasetMaker make) { - const std::size_t size = bench_size(); + const std::size_t size = static_cast(state.range(0)); std::mt19937_64 rng(bench_seed()); const std::vector data = make(size, rng); - // Encode once outside timing to report compression ratio. - const PivCoHuffman probe(data); - report_ratio(state, probe.compressed_size(), size); - + std::size_t compressed = 0; for (auto _ : state) { PivCoHuffman codec(data); + compressed = codec.compressed_size(); benchmark::DoNotOptimize(codec.compressed_data().data()); benchmark::ClobberMemory(); } + if (compressed > 0) { + report_ratio(state, compressed, size); + } state.SetBytesProcessed(static_cast(state.iterations() * size)); } /// @brief Measure decoding: reconstruct the original byte stream from the /// in-memory tree. Encoding is done once, outside the timed loop. void BM_Decode(benchmark::State& state, DatasetMaker make) { - const std::size_t size = bench_size(); + const std::size_t size = static_cast(state.range(0)); std::mt19937_64 rng(bench_seed()); const std::vector data = make(size, rng); @@ -172,18 +170,28 @@ void BM_Decode(benchmark::State& state, DatasetMaker make) { } // namespace -// Register encode + decode for every built-in dataset. Each registration is -// named `BM_/`, so `--benchmark_filter` selects subsets freely. -BENCHMARK_CAPTURE(BM_Encode, Uniform256, make_uniform256); -BENCHMARK_CAPTURE(BM_Encode, Binary, make_binary); -BENCHMARK_CAPTURE(BM_Encode, LowEntropy4, make_low_entropy_4); -BENCHMARK_CAPTURE(BM_Encode, TextLike, make_text_like); -BENCHMARK_CAPTURE(BM_Encode, Skewed99, make_skewed99); -BENCHMARK_CAPTURE(BM_Encode, SingleSymbol, make_single_symbol); - -BENCHMARK_CAPTURE(BM_Decode, Uniform256, make_uniform256); -BENCHMARK_CAPTURE(BM_Decode, Binary, make_binary); -BENCHMARK_CAPTURE(BM_Decode, LowEntropy4, make_low_entropy_4); -BENCHMARK_CAPTURE(BM_Decode, TextLike, make_text_like); -BENCHMARK_CAPTURE(BM_Decode, Skewed99, make_skewed99); -BENCHMARK_CAPTURE(BM_Decode, SingleSymbol, make_single_symbol); +// Register encode + decode for every built-in dataset, each swept over the +// full size range. Each registration is named `BM_//`, so +// `--benchmark_filter` selects subsets freely. +#define PIXIE_PIVCO_REGISTER(Op, Dataset, Maker) \ + BENCHMARK_CAPTURE(BM_##Op, Dataset, Maker) \ + ->Unit(benchmark::kMillisecond) \ + ->UseRealTime() \ + ->Repetitions(5) \ + ->ReportAggregatesOnly(true) \ + ->RangeMultiplier(kSizeMult) \ + ->Range(kSizeLo, kSizeHi); + +PIXIE_PIVCO_REGISTER(Encode, Uniform256, make_uniform256) +PIXIE_PIVCO_REGISTER(Encode, Binary, make_binary) +PIXIE_PIVCO_REGISTER(Encode, LowEntropy4, make_low_entropy_4) +PIXIE_PIVCO_REGISTER(Encode, TextLike, make_text_like) +PIXIE_PIVCO_REGISTER(Encode, Skewed99, make_skewed99) +PIXIE_PIVCO_REGISTER(Encode, SingleSymbol, make_single_symbol) + +PIXIE_PIVCO_REGISTER(Decode, Uniform256, make_uniform256) +PIXIE_PIVCO_REGISTER(Decode, Binary, make_binary) +PIXIE_PIVCO_REGISTER(Decode, LowEntropy4, make_low_entropy_4) +PIXIE_PIVCO_REGISTER(Decode, TextLike, make_text_like) +PIXIE_PIVCO_REGISTER(Decode, Skewed99, make_skewed99) +PIXIE_PIVCO_REGISTER(Decode, SingleSymbol, make_single_symbol)