Static repartition

include/kmtricks/cli.hpp

@@ -22,6 +22,7 @@
 #include <memory>
 #include <string>
 
+#include <algorithm>
 // ext
 #include <bcli/bcli.hpp>
 

include/kmtricks/cmd.hpp

@@ -89,7 +89,8 @@ struct main_repart
     ConfigTask<MAX_K> config_task(opt->fof, props, opt->bloom_size, opt->nb_parts,
                                    opt->bam_exclude_refs, opt->bam_include_flags, opt->bam_exclude_flags);
     config_task.exec();
-    RepartTask<MAX_K> repart_task(opt->fof, opt->bam_exclude_refs, opt->bam_include_flags, opt->bam_exclude_flags);
+
+    RepartTask<MAX_K> repart_task(opt->fof, opt->bam_exclude_refs, opt->bam_include_flags, opt->bam_exclude_flags, "", opt->static_repart);
     repart_task.exec(); repart_task.postprocess();
 
     Storage* config_storage = StorageFactory(STORAGE_FILE).load(KmDir::get().m_config_storage);

include/kmtricks/cmd/all.hpp

@@ -54,7 +54,7 @@ struct all_options : km_options
   bool lz4 {false};
   bool kff {false};
   bool hist {false};
-
+  bool static_repart {false};
   uint32_t bwidth {0};
 
   uint32_t max_memory {8000};
@@ -104,6 +104,7 @@ struct all_options : km_options
     RECORD(ss, lz4);
     RECORD(ss, kff);
     RECORD(ss, hist);
+    RECORD(ss, static_repart);
     RECORD(ss, focus);
     RECORD(ss, restrict_to);
     RECORD(ss, bwidth);

include/kmtricks/cmd/repart.hpp

@@ -42,6 +42,8 @@ struct repart_options : km_options
   uint32_t bam_include_flags {0};
   uint32_t bam_exclude_flags {0};
 
+  bool static_repart {false};
+
   std::string display()
   {
     std::stringstream ss;
@@ -52,6 +54,7 @@ struct repart_options : km_options
     RECORD(ss, minim_type);
     RECORD(ss, repart_type);
     RECORD(ss, nb_parts);
+    RECORD(ss, static_repart);
     RECORD(ss, bam_exclude_refs);
     RECORD(ss, bam_include_flags);
     RECORD(ss, bam_exclude_flags);
@@ -62,4 +65,4 @@ struct repart_options : km_options
 
 using repart_options_t = std::shared_ptr<struct repart_options>;
 
-};
+};

include/kmtricks/repartition.hpp

@@ -22,18 +22,50 @@
 #include <kmtricks/minimizer.hpp>
 #include <kmtricks/utils.hpp>
 
+#include <xxhash.h>
+
 namespace km {
 
 class Repartition
 {
-  static const uint32_t s_gatb_magic = 0x12345678;
+  inline static const uint32_t s_gatb_magic = 0x12345678;
+  Repartition(std::size_t nb_parts, std::size_t nb_minims)
+    : m_nb_part(nb_parts), m_nb_minims(nb_minims), m_nb_pass(1), m_has_freq(false)
+  {
+    m_repart_table.resize(m_nb_minims);
+  }
+
 public:
   Repartition(const std::string& path, const std::string& fpath = "")
     : m_path(path), m_fpath(fpath)
   {
     load();
   }
 
+  static Repartition from_xxh(std::size_t nb_partitions, std::size_t minim_size)
+  {
+    std::size_t nb_minims = std::pow(4, minim_size);
+    Repartition repart(nb_partitions, nb_minims);
+
+    for (std::uint32_t m = 0; m < nb_minims; ++m)
+    {
+      repart.m_repart_table[m] = XXH64(&m, sizeof(m), 0) % nb_partitions;
+    }
+
+    return repart;
+  }
+
+  void save(const std::string& path) const
+  {
+    std::ofstream out(path, std::ios::binary | std::ios::out); check_fstream_good(path, out);
+    out.write((const char*)&m_nb_part, sizeof(m_nb_part));
+    out.write((const char*)&m_nb_minims, sizeof(m_nb_minims));
+    out.write((const char*)&m_nb_pass, sizeof(m_nb_pass));
+    out.write((const char*)m_repart_table.data(), sizeof(uint16_t)*m_nb_minims);
+    out.write((const char*)&m_has_freq, sizeof(m_has_freq));
+    out.write((const char*)&s_gatb_magic, sizeof(s_gatb_magic));
+  }
+
   void load()
   {
     std::ifstream in(m_path, std::ios::binary | std::ios::in); check_fstream_good(m_path, in);

include/kmtricks/task.hpp

@@ -150,10 +150,11 @@ template<size_t span>
 class RepartTask : public ITask
 {
 public:
+
   RepartTask(const std::string& path, const std::string& bam_exclude_refs = "",
-             uint32_t bam_include_flags = 0, uint32_t bam_exclude_flags = 0, const std::string& from = "")
+             uint32_t bam_include_flags = 0, uint32_t bam_exclude_flags = 0, const std::string& from = "", bool static_repart = false)
     : ITask(1), m_path(path), m_bam_exclude_refs(bam_exclude_refs),
-      m_bam_include_flags(bam_include_flags), m_bam_exclude_flags(bam_exclude_flags), m_from(from) {}
+      m_bam_include_flags(bam_include_flags), m_bam_exclude_flags(bam_exclude_flags), m_from(from), m_static_repart(static_repart) {}
 
   void preprocess() {}
   void postprocess()
@@ -181,17 +182,28 @@ class RepartTask : public ITask
 
     if (m_from.empty())
     {
-      Fof fof(m_path);
-      IBank* bank = Bank::open(fof.get_all()); LOCAL(bank);
-      apply_bam_filtering(bank, m_bam_exclude_refs, m_bam_include_flags, m_bam_exclude_flags);
-      Storage* rep_store =
-        StorageFactory(STORAGE_FILE).create(KmDir::get().m_repart_storage, true, false);
+      if (!m_static_repart)
+      {
+        Fof fof(m_path);
+        IBank* bank = Bank::open(fof.get_all()); LOCAL(bank);
+        apply_bam_filtering(bank, m_bam_exclude_refs, m_bam_include_flags, m_bam_exclude_flags);
+        Storage* rep_store =
+          StorageFactory(STORAGE_FILE).create(KmDir::get().m_repart_storage, true, false);
+
 
-      LOCAL(rep_store);
+        LOCAL(rep_store);
 
-      RepartitorAlgorithm<span> repartition(
-        bank, rep_store->getGroup("repartition"), config, 1);
-      repartition.execute();
+        RepartitorAlgorithm<span> repartition(
+          bank, rep_store->getGroup("repartition"), config, 1);
+        repartition.execute();
+      }
+      else
+      {
+        auto repart = Repartition::from_xxh(m_nb_parts, m_minim_size);
+        auto repart_directory = fmt::format("{}/repartition_gatb", KmDir::get().m_root);
+        fs::create_directories(repart_directory);
+        repart.save(fmt::format("{}/repartition.minimRepart", repart_directory));
+      }
     }
     else
     {
@@ -221,6 +233,7 @@ class RepartTask : public ITask
   int m_cores;
   uint32_t m_nb_parts {0};
   uint32_t m_minim_size {0};
+  bool m_static_repart {false};
 };
 
 template<size_t span>

include/kmtricks/task_scheduler.hpp

@@ -105,8 +105,15 @@ class TaskScheduler
 
   void exec_repart()
   {
-    spdlog::info("Compute minimizer repartition...");
-    RepartTask<MAX_K> repart_task(m_opt->fof, m_opt->from);
+    if (m_opt->static_repart)
+    {
+      spdlog::info("Use static repartition");
+    }
+    else
+    {
+      spdlog::info("Compute minimizer repartition...");
+    }
+    RepartTask<MAX_K> repart_task(m_opt->fof, "", 0, 0, m_opt->from, m_opt->static_repart);
     repart_task.exec(); repart_task.postprocess();
     m_opt->m_ab_min_vec.resize(KmDir::get().m_fof.size());
     m_hw = HashWindow(KmDir::get().m_hash_win);

src/cli.cpp

@@ -304,6 +304,10 @@ km_options_t all_cli(std::shared_ptr<bc::Parser<1>> cli, all_options_t options)
     ->checker(bc::check::f::range(0.05, 1.0))
     ->setter(options->restrict_to);
 
+  all_cmd->add_param("--static-repart", "Do not estimate repartition by sampling")
+    ->as_flag()
+    ->setter(options->static_repart);
+
   auto rtl_setter = [options](const std::string& v) {
     auto partitions = bc::utils::split(v, ',');
     for (auto& p : partitions)
@@ -424,6 +428,10 @@ km_options_t repart_cli(std::shared_ptr<bc::Parser<1>> cli, repart_options_t opt
     ->checker(bc::check::is_number)
     ->setter(options->nb_parts);
 
+  repart_cmd->add_param("--static-repart", "Do not estimate repartition by sampling")
+    ->as_flag()
+    ->setter(options->static_repart);
+
   repart_cmd->add_param("--bloom-size", "bloom filter size")
     ->meta("INT")
     ->def("10000000")

thirdparty/CMakeLists.txt

@@ -86,6 +86,7 @@ ExternalProject_Add(TURBOP
   CONFIGURE_COMMAND ""
   BUILD_COMMAND ${CMAKE_COMMAND} -E env
   make libic.a
+  #make OPT="-fstrict-aliasing -fPIC -Wno-incompatible-pointer-types" libic.a
   COMMAND ${CMAKE_COMMAND} -E copy "<SOURCE_DIR>/libic.a" ${PROJECT_BINARY_DIR}/thirdparty/TURBOP
   INSTALL_COMMAND make clean
   LOG_CONFIGURE ON