Merge remote-tracking branch 'origin/master' into altpaths

Author: glennhickey

.github/workflows/testmac.yml

@@ -61,6 +61,7 @@ jobs:
           brew update && \
           brew install protobuf && \
           brew install libomp && \
+          brew install bwa graphviz && \
           npm install -g [email protected] && \
           brew config && \
           (brew doctor || echo "brew doctor is unhappy")

.gitignore

@@ -23,6 +23,9 @@ test/.*
 test/*.lcp
 test/*/*.lcp
 test/**/*.index/
+test/wiki-data/mock-hs37d5/graphs/
+test/wiki-data/mock-hs37d5/indexes/
+test/wiki-data/mock-hs37d5/logs/
 trash
 src/*.gch
 .vscode

.gitlab-ci.yml

@@ -15,7 +15,7 @@ before_script:
   - sudo apt-get -q -y update
   # Make sure we have some curl stuff for pycurl which we need for some Python stuff
   # And the CI report upload needs uuidgen from uuid-runtime
-  - sudo apt-get -q -y install --no-upgrade docker.io python3-pip python3-virtualenv libcurl4-gnutls-dev python-dev-is-python3 npm nodejs node-gyp uuid-runtime libgnutls28-dev doxygen libzstd-dev bcftools
+  - sudo apt-get -q -y install --no-upgrade docker.io python3-pip python3-virtualenv libcurl4-gnutls-dev python-dev-is-python3 npm nodejs node-gyp uuid-runtime libgnutls28-dev doxygen libzstd-dev bcftools bwa graphviz
   - which junit-merge || sudo npm install -g junit-merge
   # Configure Docker to use a mirror for Docker Hub and restart the daemon
   - |
@@ -85,7 +85,7 @@ local-build-test-job:
       - obj/
   before_script:
     - sudo apt-get -q -y update
-    - sudo apt-get install -y gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION}
+    - sudo apt-get install -y gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} bwa graphviz
     - sudo rm -f /usr/bin/gcc && sudo ln -s /usr/bin/gcc-${COMPILER_VERSION} /usr/bin/gcc
     - sudo rm -f /usr/bin/g++ && sudo ln -s /usr/bin/g++-${COMPILER_VERSION} /usr/bin/g++
     # We need to make sure we get the right submodule files for this version

deps/gbwt

@@ -210,20 +210,20 @@ void set_score(GaplessExtension& extension, const Aligner* aligner) {
 
 // Match the initial node, assuming that read_offset or node_offset is 0.
 // Updates internal_score and old_score; use set_score() to compute score.
-void match_initial(GaplessExtension& match, const std::string& seq, gbwtgraph::view_type target) {
+void match_initial(GaplessExtension& match, const std::string& seq, std::string_view target) {
     size_t node_offset = match.offset;
-    size_t left = std::min(seq.length() - match.read_interval.second, target.second - node_offset);
+    size_t left = std::min(seq.length() - match.read_interval.second, target.size() - node_offset);
     while (left > 0) {
         size_t len = std::min(left, sizeof(std::uint64_t));
         std::uint64_t a = 0, b = 0;
         std::memcpy(&a, seq.data() + match.read_interval.second, len);
-        std::memcpy(&b, target.first + node_offset, len);
+        std::memcpy(&b, target.data() + node_offset, len);
         if (a == b) {
             match.read_interval.second += len;
             node_offset += len;
         } else {
             for (size_t i = 0; i < len; i++) {
-                if (seq[match.read_interval.second] != target.first[node_offset]) {
+                if (seq[match.read_interval.second] != target[node_offset]) {
                     match.internal_score++;
                 }
                 match.read_interval.second++;
@@ -238,20 +238,20 @@ void match_initial(GaplessExtension& match, const std::string& seq, gbwtgraph::v
 // Match forward but stop before the mismatch count reaches the limit.
 // Updates internal_score; use set_score() to recompute score.
 // Returns the tail offset (the number of characters matched).
-size_t match_forward(GaplessExtension& match, const std::string& seq, gbwtgraph::view_type target, uint32_t mismatch_limit) {
+size_t match_forward(GaplessExtension& match, const std::string& seq, std::string_view target, uint32_t mismatch_limit) {
     size_t node_offset = 0;
-    size_t left = std::min(seq.length() - match.read_interval.second, target.second - node_offset);
+    size_t left = std::min(seq.length() - match.read_interval.second, target.size() - node_offset);
     while (left > 0) {
         size_t len = std::min(left, sizeof(std::uint64_t));
         std::uint64_t a = 0, b = 0;
         std::memcpy(&a, seq.data() + match.read_interval.second, len);
-        std::memcpy(&b, target.first + node_offset, len);
+        std::memcpy(&b, target.data() + node_offset, len);
         if (a == b) {
             match.read_interval.second += len;
             node_offset += len;
         } else {
             for (size_t i = 0; i < len; i++) {
-                if (seq[match.read_interval.second] != target.first[node_offset]) {
+                if (seq[match.read_interval.second] != target[node_offset]) {
                     if (match.internal_score + 1 >= mismatch_limit) {
                         return node_offset;
                     }
@@ -269,19 +269,19 @@ size_t match_forward(GaplessExtension& match, const std::string& seq, gbwtgraph:
 // Match forward but stop before the mismatch count reaches the limit.
 // Starts from the offset in the match and updates it.
 // Updates internal_score; use set_score() to recompute score.
-void match_backward(GaplessExtension& match, const std::string& seq, gbwtgraph::view_type target, uint32_t mismatch_limit) {
+void match_backward(GaplessExtension& match, const std::string& seq, std::string_view target, uint32_t mismatch_limit) {
     size_t left = std::min(match.read_interval.first, match.offset);
     while (left > 0) {
         size_t len = std::min(left, sizeof(std::uint64_t));
         std::uint64_t a = 0, b = 0;
         std::memcpy(&a, seq.data() + match.read_interval.first - len, len);
-        std::memcpy(&b, target.first + match.offset - len, len);
+        std::memcpy(&b, target.data() + match.offset - len, len);
         if (a == b) {
             match.read_interval.first -= len;
             match.offset -= len;
         } else {
             for (size_t i = 0; i < len; i++) {
-                if (seq[match.read_interval.first - 1] != target.first[match.offset - 1]) {
+                if (seq[match.read_interval.first - 1] != target[match.offset - 1]) {
                     if (match.internal_score + 1 >= mismatch_limit) {
                         return;
                     }
@@ -373,9 +373,9 @@ void find_mismatches(const std::string& seq, const gbwtgraph::CachedGBWTGraph& g
         extension.mismatch_positions.reserve(extension.internal_score);
         size_t node_offset = extension.offset, read_offset = extension.read_interval.first;
         for (const handle_t& handle : extension.path) {
-            gbwtgraph::view_type target = graph.get_sequence_view(handle);
-            while (node_offset < target.second && read_offset < extension.read_interval.second) {
-                if (target.first[node_offset] != seq[read_offset]) {
+            std::string_view target = graph.get_sequence_view(handle);
+            while (node_offset < target.size() && read_offset < extension.read_interval.second) {
+                if (target[node_offset] != seq[read_offset]) {
                     extension.mismatch_positions.push_back(read_offset);
                 }
                 node_offset++;
@@ -1546,10 +1546,10 @@ struct WFANode {
     bool append_node(const gbwtgraph::CachedGBWTGraph& graph, gbwt::SearchState next, pos_t target) {
         this->state = next;
         this->path.push_back(this->state.node);
-        gbwtgraph::view_type view = graph.get_sequence_view(gbwtgraph::GBWTGraph::node_to_handle(this->state.node));
-        this->node_sequence.append(view.first, view.second);
+        std::string_view view = graph.get_sequence_view(gbwtgraph::GBWTGraph::node_to_handle(this->state.node));
+        this->node_sequence.append(view.data(), view.size());
         if (gbwt::Node::encode(id(target), is_rev(target)) == this->state.node) {
-            this->target_offset = this->node_sequence.length() - (view.second - offset(target));
+            this->target_offset = this->node_sequence.length() - (view.size() - offset(target));
             return true;
         }
         return false;

deps/gbwtgraph

@@ -4011,7 +4011,7 @@ IndexRegistry VGIndexes::get_vg_index_registry() {
 
         // jointly generate the GBWT and record sequences
         unique_ptr<gbwt::GBWT> gbwt_index;
-        unique_ptr<gbwtgraph::SequenceSource> seq_source;
+        unique_ptr<gbwtgraph::NaiveGraph> seq_source;
         tie(gbwt_index, seq_source) = gbwtgraph::gfa_to_gbwt(gfa_filename, params);
 
         // convert sequences into GBZ and save

doc/wiki

@@ -242,9 +242,9 @@ static string get_haplotype_sequence(gbwt::size_type sequence_visit_offset, hand
     if (curr == end_handle) {
         return haplotype;
     }
-    gbwtgraph::view_type view = gbwt_graph.get_sequence_view(curr);
-    size_t offset = (view.second > start_max ? view.second - start_max : 0);
-    haplotype.append(view.first + offset, view.second - offset);
+    std::string_view view = gbwt_graph.get_sequence_view(curr);
+    size_t offset = (view.size() > start_max ? view.size() - start_max : 0);
+    haplotype.append(view.data() + offset, view.size() - offset);
 
     while (true) {
         pos = gbwt_graph.index->LF(pos);
@@ -254,10 +254,10 @@ static string get_haplotype_sequence(gbwt::size_type sequence_visit_offset, hand
         curr = gbwtgraph::GBWTGraph::node_to_handle(pos.first);
         view = gbwt_graph.get_sequence_view(curr);
         if (curr == end_handle) {
-            haplotype.append(view.first, std::min(view.second, end_max));
+            haplotype.append(view.data(), std::min(view.size(), end_max));
             break;
         } else {
-            haplotype.append(view.first, view.second);
+            haplotype.append(view.data(), view.size());
         }
     }
     return haplotype;

src/gbwt_extender.cpp

@@ -921,10 +921,10 @@ std::vector<std::string> generate_haplotype(
     if (curr == end) {
         return haplotype;
     }
-    gbwtgraph::view_type view = partitioner.gbz.graph.get_sequence_view(curr);
-    size_t offset = (view.second > start_max ? view.second - start_max : 0);
+    std::string_view view = partitioner.gbz.graph.get_sequence_view(curr);
+    size_t offset = (view.size() > start_max ? view.size() - start_max : 0);
     haplotype.emplace_back();
-    haplotype.back().append(view.first + offset, view.second - offset);
+    haplotype.back().append(view.data() + offset, view.size() - offset);
 
     while (true) {
         pos = partitioner.gbz.index.LF(pos);
@@ -948,10 +948,10 @@ std::vector<std::string> generate_haplotype(
         curr = gbwtgraph::GBWTGraph::node_to_handle(pos.first);
         view = partitioner.gbz.graph.get_sequence_view(curr);
         if (curr == end) {
-            haplotype.back().append(view.first, std::min(view.second, end_max));
+            haplotype.back().append(view.data(), std::min(view.size(), end_max));
             break;
         } else {
-            haplotype.back().append(view.first, view.second);
+            haplotype.back().append(view.data(), view.size());
         }
     }