From 2adb200e67aba4d793860f75afa7e65e15fb0112 Mon Sep 17 00:00:00 2001 From: Teo Lemane Date: Mon, 23 Mar 2026 11:54:29 +0100 Subject: [PATCH 1/6] add: static repartition --- include/kmtricks/cmd.hpp | 2 +- include/kmtricks/cmd/all.hpp | 3 ++- include/kmtricks/cmd/repart.hpp | 5 ++++- include/kmtricks/repartition.hpp | 34 ++++++++++++++++++++++++++++- include/kmtricks/task.hpp | 31 +++++++++++++++++--------- include/kmtricks/task_scheduler.hpp | 11 ++++++++-- src/cli.cpp | 8 +++++++ 7 files changed, 78 insertions(+), 16 deletions(-) diff --git a/include/kmtricks/cmd.hpp b/include/kmtricks/cmd.hpp index ce7ba82f..4101a95a 100644 --- a/include/kmtricks/cmd.hpp +++ b/include/kmtricks/cmd.hpp @@ -88,7 +88,7 @@ struct main_repart opt->nb_parts); ConfigTask config_task(opt->fof, props, opt->bloom_size, opt->nb_parts); config_task.exec(); - RepartTask repart_task(opt->fof); repart_task.exec(); repart_task.postprocess(); + RepartTask repart_task(opt->fof, "", opt->static_repart); repart_task.exec(); repart_task.postprocess(); Storage* config_storage = StorageFactory(STORAGE_FILE).load(KmDir::get().m_config_storage); LOCAL(config_storage); diff --git a/include/kmtricks/cmd/all.hpp b/include/kmtricks/cmd/all.hpp index 44551162..b0d74f9a 100644 --- a/include/kmtricks/cmd/all.hpp +++ b/include/kmtricks/cmd/all.hpp @@ -54,7 +54,7 @@ struct all_options : km_options bool lz4 {false}; bool kff {false}; bool hist {false}; - + bool static_repart {false}; uint32_t bwidth {0}; uint32_t max_memory {8000}; @@ -100,6 +100,7 @@ struct all_options : km_options RECORD(ss, lz4); RECORD(ss, kff); RECORD(ss, hist); + RECORD(ss, static_repart); RECORD(ss, focus); RECORD(ss, restrict_to); RECORD(ss, bwidth); diff --git a/include/kmtricks/cmd/repart.hpp b/include/kmtricks/cmd/repart.hpp index 4ccd8093..e3160114 100644 --- a/include/kmtricks/cmd/repart.hpp +++ b/include/kmtricks/cmd/repart.hpp @@ -39,6 +39,8 @@ struct repart_options : km_options uint32_t nb_parts; uint64_t bloom_size; + bool static_repart {false}; + std::string display() { std::stringstream ss; @@ -49,6 +51,7 @@ struct repart_options : km_options RECORD(ss, minim_type); RECORD(ss, repart_type); RECORD(ss, nb_parts); + RECORD(ss, static_repart); std::string ret = ss.str(); ret.pop_back(); ret.pop_back(); return ret; } @@ -56,4 +59,4 @@ struct repart_options : km_options using repart_options_t = std::shared_ptr; -}; \ No newline at end of file +}; diff --git a/include/kmtricks/repartition.hpp b/include/kmtricks/repartition.hpp index 108e0532..7b208b0c 100644 --- a/include/kmtricks/repartition.hpp +++ b/include/kmtricks/repartition.hpp @@ -22,11 +22,19 @@ #include #include +#include + namespace km { class Repartition { - static const uint32_t s_gatb_magic = 0x12345678; + inline static const uint32_t s_gatb_magic = 0x12345678; + Repartition(std::size_t nb_parts, std::size_t nb_minims) + : m_nb_part(nb_parts), m_nb_minims(nb_minims), m_nb_pass(1), m_has_freq(false) + { + m_repart_table.resize(m_nb_minims); + } + public: Repartition(const std::string& path, const std::string& fpath = "") : m_path(path), m_fpath(fpath) @@ -34,6 +42,30 @@ class Repartition load(); } + static Repartition from_xxh(std::size_t nb_partitions, std::size_t minim_size) + { + std::size_t nb_minims = std::pow(4, minim_size); + Repartition repart(nb_partitions, nb_minims); + + for (std::uint32_t m = 0; m < nb_minims; ++m) + { + repart.m_repart_table[m] = XXH64(&m, sizeof(m), 0) % nb_partitions; + } + + return repart; + } + + void save(const std::string& path) const + { + std::ofstream out(path, std::ios::binary | std::ios::out); check_fstream_good(path, out); + out.write((const char*)&m_nb_part, sizeof(m_nb_part)); + out.write((const char*)&m_nb_minims, sizeof(m_nb_minims)); + out.write((const char*)&m_nb_pass, sizeof(m_nb_pass)); + out.write((const char*)m_repart_table.data(), sizeof(uint16_t)*m_nb_minims); + out.write((const char*)&m_has_freq, sizeof(m_has_freq)); + out.write((const char*)&s_gatb_magic, sizeof(s_gatb_magic)); + } + void load() { std::ifstream in(m_path, std::ios::binary | std::ios::in); check_fstream_good(m_path, in); diff --git a/include/kmtricks/task.hpp b/include/kmtricks/task.hpp index b97fa446..b32f36eb 100644 --- a/include/kmtricks/task.hpp +++ b/include/kmtricks/task.hpp @@ -109,8 +109,8 @@ template class RepartTask : public ITask { public: - RepartTask(const std::string& path, const std::string& from = "") - : ITask(1), m_path(path), m_from(from) {} + RepartTask(const std::string& path, const std::string& from = "", bool static_repart = false) + : ITask(1), m_path(path), m_from(from), m_static_repart(static_repart) {} void preprocess() {} void postprocess() @@ -138,16 +138,26 @@ class RepartTask : public ITask if (m_from.empty()) { - Fof fof(m_path); - IBank* bank = Bank::open(fof.get_all()); LOCAL(bank); - Storage* rep_store = - StorageFactory(STORAGE_FILE).create(KmDir::get().m_repart_storage, true, false); + if (!m_static_repart) + { + Fof fof(m_path); + IBank* bank = Bank::open(fof.get_all()); LOCAL(bank); + Storage* rep_store = + StorageFactory(STORAGE_FILE).create(KmDir::get().m_repart_storage, true, false); - LOCAL(rep_store); + LOCAL(rep_store); - RepartitorAlgorithm repartition( - bank, rep_store->getGroup("repartition"), config, 1); - repartition.execute(); + RepartitorAlgorithm repartition( + bank, rep_store->getGroup("repartition"), config, 1); + repartition.execute(); + } + else + { + auto repart = Repartition::from_xxh(m_nb_parts, m_minim_size); + auto repart_directory = fmt::format("{}/repartition_gatb", KmDir::get().m_root); + fs::create_directories(repart_directory); + repart.save(fmt::format("{}/repartition.minimRepart", repart_directory)); + } } else { @@ -174,6 +184,7 @@ class RepartTask : public ITask int m_cores; uint32_t m_nb_parts {0}; uint32_t m_minim_size {0}; + bool m_static_repart {false}; }; template diff --git a/include/kmtricks/task_scheduler.hpp b/include/kmtricks/task_scheduler.hpp index 3d14cbde..991f328d 100644 --- a/include/kmtricks/task_scheduler.hpp +++ b/include/kmtricks/task_scheduler.hpp @@ -104,8 +104,15 @@ class TaskScheduler void exec_repart() { - spdlog::info("Compute minimizer repartition..."); - RepartTask repart_task(m_opt->fof, m_opt->from); + if (m_opt->static_repart) + { + spdlog::info("Use static repartition"); + } + else + { + spdlog::info("Compute minimizer repartition..."); + } + RepartTask repart_task(m_opt->fof, m_opt->from, m_opt->static_repart); repart_task.exec(); repart_task.postprocess(); m_opt->m_ab_min_vec.resize(KmDir::get().m_fof.size()); m_hw = HashWindow(KmDir::get().m_hash_win); diff --git a/src/cli.cpp b/src/cli.cpp index cf43d517..08c5ef62 100644 --- a/src/cli.cpp +++ b/src/cli.cpp @@ -304,6 +304,10 @@ km_options_t all_cli(std::shared_ptr> cli, all_options_t options) ->checker(bc::check::f::range(0.05, 1.0)) ->setter(options->restrict_to); + all_cmd->add_param("--static-repart", "Do not estimate repartition by sampling") + ->as_flag() + ->setter(options->static_repart); + auto rtl_setter = [options](const std::string& v) { auto partitions = bc::utils::split(v, ','); for (auto& p : partitions) @@ -422,6 +426,10 @@ km_options_t repart_cli(std::shared_ptr> cli, repart_options_t opt ->checker(bc::check::is_number) ->setter(options->nb_parts); + repart_cmd->add_param("--static-repart", "Do not estimate repartition by sampling") + ->as_flag() + ->setter(options->static_repart); + repart_cmd->add_param("--bloom-size", "bloom filter size") ->meta("INT") ->def("10000000") From e6dea46ec85d01f24cec8622872c8744a29b678d Mon Sep 17 00:00:00 2001 From: Teo Lemane Date: Tue, 24 Mar 2026 09:27:24 +0100 Subject: [PATCH 2/6] gcc 14 support --- include/kmtricks/cli.hpp | 1 + thirdparty/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/include/kmtricks/cli.hpp b/include/kmtricks/cli.hpp index 3a3a470d..e682c514 100644 --- a/include/kmtricks/cli.hpp +++ b/include/kmtricks/cli.hpp @@ -22,6 +22,7 @@ #include #include +#include // ext #include diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index d5bea695..ae55b102 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -85,7 +85,7 @@ ExternalProject_Add(TURBOP BUILD_IN_SOURCE 1 CONFIGURE_COMMAND "" BUILD_COMMAND ${CMAKE_COMMAND} -E env - make libic.a + make OPT="-fstrict-aliasing -fPIC -Wno-incompatible-pointer-types" libic.a COMMAND ${CMAKE_COMMAND} -E copy "/libic.a" ${PROJECT_BINARY_DIR}/thirdparty/TURBOP INSTALL_COMMAND make clean LOG_CONFIGURE ON From 137e00e7b8866021a260c68ec22ae85eaff73e5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=A9o=20Lemane?= <33129766+tlemane@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:36:13 +0200 Subject: [PATCH 3/6] fix RepartTask ctor --- include/kmtricks/task_scheduler.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/kmtricks/task_scheduler.hpp b/include/kmtricks/task_scheduler.hpp index 5ebdb529..10681d5f 100644 --- a/include/kmtricks/task_scheduler.hpp +++ b/include/kmtricks/task_scheduler.hpp @@ -113,7 +113,7 @@ class TaskScheduler { spdlog::info("Compute minimizer repartition..."); } - RepartTask repart_task(m_opt->fof, m_opt->from, m_opt->static_repart); + RepartTask repart_task(m_opt->fof, "", 0, 0, const std::string& from = "" m_opt->from, m_opt->static_repart); repart_task.exec(); repart_task.postprocess(); m_opt->m_ab_min_vec.resize(KmDir::get().m_fof.size()); m_hw = HashWindow(KmDir::get().m_hash_win); From 395c764abbc02cdfca8b782923eac9f897bc0d5f Mon Sep 17 00:00:00 2001 From: Teo Lemane Date: Fri, 26 Jun 2026 13:42:48 +0200 Subject: [PATCH 4/6] typo --- include/kmtricks/task_scheduler.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/kmtricks/task_scheduler.hpp b/include/kmtricks/task_scheduler.hpp index 10681d5f..d177fa3c 100644 --- a/include/kmtricks/task_scheduler.hpp +++ b/include/kmtricks/task_scheduler.hpp @@ -113,7 +113,7 @@ class TaskScheduler { spdlog::info("Compute minimizer repartition..."); } - RepartTask repart_task(m_opt->fof, "", 0, 0, const std::string& from = "" m_opt->from, m_opt->static_repart); + RepartTask repart_task(m_opt->fof, "", 0, 0, m_opt->from, m_opt->static_repart); repart_task.exec(); repart_task.postprocess(); m_opt->m_ab_min_vec.resize(KmDir::get().m_fof.size()); m_hw = HashWindow(KmDir::get().m_hash_win); From 8d35993a398092f6648637e9b683bcf6d71d193d Mon Sep 17 00:00:00 2001 From: Teo Lemane Date: Fri, 26 Jun 2026 13:44:08 +0200 Subject: [PATCH 5/6] turbop --- thirdparty/CMakeLists.txt | 3 ++- thirdparty/TurboPFor-Integer-Compression | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index ae55b102..f852d45d 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -85,7 +85,8 @@ ExternalProject_Add(TURBOP BUILD_IN_SOURCE 1 CONFIGURE_COMMAND "" BUILD_COMMAND ${CMAKE_COMMAND} -E env - make OPT="-fstrict-aliasing -fPIC -Wno-incompatible-pointer-types" libic.a + make libic.a + #make OPT="-fstrict-aliasing -fPIC -Wno-incompatible-pointer-types" libic.a COMMAND ${CMAKE_COMMAND} -E copy "/libic.a" ${PROJECT_BINARY_DIR}/thirdparty/TURBOP INSTALL_COMMAND make clean LOG_CONFIGURE ON diff --git a/thirdparty/TurboPFor-Integer-Compression b/thirdparty/TurboPFor-Integer-Compression index 06d6aad9..2b4c6bd5 160000 --- a/thirdparty/TurboPFor-Integer-Compression +++ b/thirdparty/TurboPFor-Integer-Compression @@ -1 +1 @@ -Subproject commit 06d6aad98b4be5471289f35d5d04fac4469cf6df +Subproject commit 2b4c6bd588ce560decea11ea44b0a1fe23ab634b From 9c27ba5a1ba91cf3387e561207ad771bfe3ac3cc Mon Sep 17 00:00:00 2001 From: Teo Lemane Date: Fri, 26 Jun 2026 14:02:26 +0200 Subject: [PATCH 6/6] turbop --- thirdparty/TurboPFor-Integer-Compression | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thirdparty/TurboPFor-Integer-Compression b/thirdparty/TurboPFor-Integer-Compression index 2b4c6bd5..06d6aad9 160000 --- a/thirdparty/TurboPFor-Integer-Compression +++ b/thirdparty/TurboPFor-Integer-Compression @@ -1 +1 @@ -Subproject commit 2b4c6bd588ce560decea11ea44b0a1fe23ab634b +Subproject commit 06d6aad98b4be5471289f35d5d04fac4469cf6df