diff --git a/include/kmtricks/cli.hpp b/include/kmtricks/cli.hpp index 3a3a470..e682c51 100644 --- a/include/kmtricks/cli.hpp +++ b/include/kmtricks/cli.hpp @@ -22,6 +22,7 @@ #include #include +#include // ext #include diff --git a/include/kmtricks/cmd.hpp b/include/kmtricks/cmd.hpp index f5c795f..09fe300 100644 --- a/include/kmtricks/cmd.hpp +++ b/include/kmtricks/cmd.hpp @@ -89,7 +89,8 @@ struct main_repart ConfigTask config_task(opt->fof, props, opt->bloom_size, opt->nb_parts, opt->bam_exclude_refs, opt->bam_include_flags, opt->bam_exclude_flags); config_task.exec(); - RepartTask repart_task(opt->fof, opt->bam_exclude_refs, opt->bam_include_flags, opt->bam_exclude_flags); + + RepartTask repart_task(opt->fof, opt->bam_exclude_refs, opt->bam_include_flags, opt->bam_exclude_flags, "", opt->static_repart); repart_task.exec(); repart_task.postprocess(); Storage* config_storage = StorageFactory(STORAGE_FILE).load(KmDir::get().m_config_storage); diff --git a/include/kmtricks/cmd/all.hpp b/include/kmtricks/cmd/all.hpp index 4a4bccf..568b8cf 100644 --- a/include/kmtricks/cmd/all.hpp +++ b/include/kmtricks/cmd/all.hpp @@ -54,7 +54,7 @@ struct all_options : km_options bool lz4 {false}; bool kff {false}; bool hist {false}; - + bool static_repart {false}; uint32_t bwidth {0}; uint32_t max_memory {8000}; @@ -104,6 +104,7 @@ struct all_options : km_options RECORD(ss, lz4); RECORD(ss, kff); RECORD(ss, hist); + RECORD(ss, static_repart); RECORD(ss, focus); RECORD(ss, restrict_to); RECORD(ss, bwidth); diff --git a/include/kmtricks/cmd/repart.hpp b/include/kmtricks/cmd/repart.hpp index 406099c..47dfc41 100644 --- a/include/kmtricks/cmd/repart.hpp +++ b/include/kmtricks/cmd/repart.hpp @@ -42,6 +42,8 @@ struct repart_options : km_options uint32_t bam_include_flags {0}; uint32_t bam_exclude_flags {0}; + bool static_repart {false}; + std::string display() { std::stringstream ss; @@ -52,6 +54,7 @@ struct repart_options : km_options RECORD(ss, minim_type); RECORD(ss, repart_type); RECORD(ss, nb_parts); + RECORD(ss, static_repart); RECORD(ss, bam_exclude_refs); RECORD(ss, bam_include_flags); RECORD(ss, bam_exclude_flags); @@ -62,4 +65,4 @@ struct repart_options : km_options using repart_options_t = std::shared_ptr; -}; \ No newline at end of file +}; diff --git a/include/kmtricks/repartition.hpp b/include/kmtricks/repartition.hpp index 108e053..7b208b0 100644 --- a/include/kmtricks/repartition.hpp +++ b/include/kmtricks/repartition.hpp @@ -22,11 +22,19 @@ #include #include +#include + namespace km { class Repartition { - static const uint32_t s_gatb_magic = 0x12345678; + inline static const uint32_t s_gatb_magic = 0x12345678; + Repartition(std::size_t nb_parts, std::size_t nb_minims) + : m_nb_part(nb_parts), m_nb_minims(nb_minims), m_nb_pass(1), m_has_freq(false) + { + m_repart_table.resize(m_nb_minims); + } + public: Repartition(const std::string& path, const std::string& fpath = "") : m_path(path), m_fpath(fpath) @@ -34,6 +42,30 @@ class Repartition load(); } + static Repartition from_xxh(std::size_t nb_partitions, std::size_t minim_size) + { + std::size_t nb_minims = std::pow(4, minim_size); + Repartition repart(nb_partitions, nb_minims); + + for (std::uint32_t m = 0; m < nb_minims; ++m) + { + repart.m_repart_table[m] = XXH64(&m, sizeof(m), 0) % nb_partitions; + } + + return repart; + } + + void save(const std::string& path) const + { + std::ofstream out(path, std::ios::binary | std::ios::out); check_fstream_good(path, out); + out.write((const char*)&m_nb_part, sizeof(m_nb_part)); + out.write((const char*)&m_nb_minims, sizeof(m_nb_minims)); + out.write((const char*)&m_nb_pass, sizeof(m_nb_pass)); + out.write((const char*)m_repart_table.data(), sizeof(uint16_t)*m_nb_minims); + out.write((const char*)&m_has_freq, sizeof(m_has_freq)); + out.write((const char*)&s_gatb_magic, sizeof(s_gatb_magic)); + } + void load() { std::ifstream in(m_path, std::ios::binary | std::ios::in); check_fstream_good(m_path, in); diff --git a/include/kmtricks/task.hpp b/include/kmtricks/task.hpp index 5bab7ec..ee3544e 100644 --- a/include/kmtricks/task.hpp +++ b/include/kmtricks/task.hpp @@ -150,10 +150,11 @@ template class RepartTask : public ITask { public: + RepartTask(const std::string& path, const std::string& bam_exclude_refs = "", - uint32_t bam_include_flags = 0, uint32_t bam_exclude_flags = 0, const std::string& from = "") + uint32_t bam_include_flags = 0, uint32_t bam_exclude_flags = 0, const std::string& from = "", bool static_repart = false) : ITask(1), m_path(path), m_bam_exclude_refs(bam_exclude_refs), - m_bam_include_flags(bam_include_flags), m_bam_exclude_flags(bam_exclude_flags), m_from(from) {} + m_bam_include_flags(bam_include_flags), m_bam_exclude_flags(bam_exclude_flags), m_from(from), m_static_repart(static_repart) {} void preprocess() {} void postprocess() @@ -181,17 +182,28 @@ class RepartTask : public ITask if (m_from.empty()) { - Fof fof(m_path); - IBank* bank = Bank::open(fof.get_all()); LOCAL(bank); - apply_bam_filtering(bank, m_bam_exclude_refs, m_bam_include_flags, m_bam_exclude_flags); - Storage* rep_store = - StorageFactory(STORAGE_FILE).create(KmDir::get().m_repart_storage, true, false); + if (!m_static_repart) + { + Fof fof(m_path); + IBank* bank = Bank::open(fof.get_all()); LOCAL(bank); + apply_bam_filtering(bank, m_bam_exclude_refs, m_bam_include_flags, m_bam_exclude_flags); + Storage* rep_store = + StorageFactory(STORAGE_FILE).create(KmDir::get().m_repart_storage, true, false); + - LOCAL(rep_store); + LOCAL(rep_store); - RepartitorAlgorithm repartition( - bank, rep_store->getGroup("repartition"), config, 1); - repartition.execute(); + RepartitorAlgorithm repartition( + bank, rep_store->getGroup("repartition"), config, 1); + repartition.execute(); + } + else + { + auto repart = Repartition::from_xxh(m_nb_parts, m_minim_size); + auto repart_directory = fmt::format("{}/repartition_gatb", KmDir::get().m_root); + fs::create_directories(repart_directory); + repart.save(fmt::format("{}/repartition.minimRepart", repart_directory)); + } } else { @@ -221,6 +233,7 @@ class RepartTask : public ITask int m_cores; uint32_t m_nb_parts {0}; uint32_t m_minim_size {0}; + bool m_static_repart {false}; }; template diff --git a/include/kmtricks/task_scheduler.hpp b/include/kmtricks/task_scheduler.hpp index 6b93b12..d177fa3 100644 --- a/include/kmtricks/task_scheduler.hpp +++ b/include/kmtricks/task_scheduler.hpp @@ -105,8 +105,15 @@ class TaskScheduler void exec_repart() { - spdlog::info("Compute minimizer repartition..."); - RepartTask repart_task(m_opt->fof, m_opt->from); + if (m_opt->static_repart) + { + spdlog::info("Use static repartition"); + } + else + { + spdlog::info("Compute minimizer repartition..."); + } + RepartTask repart_task(m_opt->fof, "", 0, 0, m_opt->from, m_opt->static_repart); repart_task.exec(); repart_task.postprocess(); m_opt->m_ab_min_vec.resize(KmDir::get().m_fof.size()); m_hw = HashWindow(KmDir::get().m_hash_win); diff --git a/src/cli.cpp b/src/cli.cpp index 2bfcdc8..2047228 100644 --- a/src/cli.cpp +++ b/src/cli.cpp @@ -304,6 +304,10 @@ km_options_t all_cli(std::shared_ptr> cli, all_options_t options) ->checker(bc::check::f::range(0.05, 1.0)) ->setter(options->restrict_to); + all_cmd->add_param("--static-repart", "Do not estimate repartition by sampling") + ->as_flag() + ->setter(options->static_repart); + auto rtl_setter = [options](const std::string& v) { auto partitions = bc::utils::split(v, ','); for (auto& p : partitions) @@ -424,6 +428,10 @@ km_options_t repart_cli(std::shared_ptr> cli, repart_options_t opt ->checker(bc::check::is_number) ->setter(options->nb_parts); + repart_cmd->add_param("--static-repart", "Do not estimate repartition by sampling") + ->as_flag() + ->setter(options->static_repart); + repart_cmd->add_param("--bloom-size", "bloom filter size") ->meta("INT") ->def("10000000") diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index d5bea69..f852d45 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -86,6 +86,7 @@ ExternalProject_Add(TURBOP CONFIGURE_COMMAND "" BUILD_COMMAND ${CMAKE_COMMAND} -E env make libic.a + #make OPT="-fstrict-aliasing -fPIC -Wno-incompatible-pointer-types" libic.a COMMAND ${CMAKE_COMMAND} -E copy "/libic.a" ${PROJECT_BINARY_DIR}/thirdparty/TURBOP INSTALL_COMMAND make clean LOG_CONFIGURE ON