csv to gl conversion in python, .gl file is now 64-byte aligned for AVX-512.

Author: KrishSingaria

.gitignore

@@ -2,5 +2,4 @@
 *.gl
 graph.gl
 venv/
-genGraph-10T
-genGraph-100T
+*.exe

genGraph-100T

@@ -1,7 +1,18 @@
-#include "csrFilegen.hpp"
+#include "src/csrFilegen.hpp"
 
-int main(int argc, char const *argv[])
-{
-    generateLargeGraph(100000,0.05f,"graph-100T.gl");
+// int main(int argc, char const *argv[])
+// {
+//     // generateLargeGraph(100000,0.05f,"graph-100T.gl");
+//     return 0;
+// }
+
+// csv_to_gl
+int main(int argc, char** argv) {
+    if (argc < 3) {
+        std::cerr << "Usage: ./converter <input.csv> <output.gl> [directed=0]" << std::endl;
+        return 1;
+    }
+    bool directed = (argc > 3 && std::string(argv[3]) == "1");
+    convert_csv(argv[1], argv[2], directed);
     return 0;
-}
+}

genGraph-10T

@@ -3,7 +3,6 @@
 #include "MemoryMap.hpp"
 #include <span>
 #include <cstddef>
-#include <sys/mman.h>
 
 class CSR{
     // size_t is the type used for every data!!
@@ -13,9 +12,13 @@ class CSR{
     // defined as nnzRow[i] = nnzRow[i-1] + no of non zero row entries of ith row
     size_t* nnzRow; 
     size_t  sizeofnnzRow; // size in bytes
+    size_t  N; // num of nodes
 
     size_t* colPtr;
     size_t  sizeofcolPtr; // size in bytes
+    size_t  M; // num of edges
+
+    size_t flags;
 public:
     CSR(const char* graphPath);
     ~CSR();
@@ -42,6 +45,9 @@ inline CSR::CSR(const char* graphPath){
     }
     this->sizeofnnzRow = header.sizeofnnzRow;
     this->sizeofcolPtr = header.sizeofcolPtr;
+    this->N = header.num_nodes;
+    this->M = header.num_edges;
+    this->flags = header.flags;
 
     this->nnzRow = reinterpret_cast<size_t*>(this->graphMap->get_data()) + header.offset_nnz/sizeof(size_t);
     this->colPtr = reinterpret_cast<size_t*>(this->graphMap->get_data()) + header.offset_col/sizeof(size_t);

generateGraph.cpp

@@ -165,23 +165,35 @@ inline std::vector<size_t> Graphzero::batchRandomWalk(const std::vector<size_t>&
     std::vector<size_t> results;
     results.reserve(walkLength*startNodes.size());
 
+    // set only for random walks 
+    storage->set_access_pattern(true);
+
     #pragma omp parallel for
     for(size_t startNode: startNodes){
         std::vector<size_t> walk = randomWalk(startNode,walkLength,p,q);
         results.insert(results.end(),walk.begin(),walk.end()); // extend the results 
     }
+
+    // reset
+    storage->set_access_pattern(false);
     return results;
 }
 
 inline std::vector<size_t> Graphzero::batchRandomUniformWalk(const std::vector<size_t>& startNodes, size_t walkLength){
     std::vector<size_t> results;
     results.reserve(walkLength*startNodes.size());
+    
+    // set only for random walks 
+    storage->set_access_pattern(true);
 
     #pragma omp parallel for
     for(size_t startNode: startNodes){
         std::vector<size_t> walk = ReservoirSampling(startNode,walkLength);
         results.insert(results.end(),walk.begin(),walk.end()); // extend the results 
     }
+
+    // reset
+    storage->set_access_pattern(false);
     return results;
 }
 #endif

src/CSR.hpp

@@ -9,7 +9,7 @@
 #include <stdexcept>
 
 // only here
-const size_t MAGIC_NUM = 8388354976772092519; // 'graphlit' converted in size_t
+const uint64_t MAGIC_NUM = 8388354976772092519; // 'graphlit' converted in size_t
 
 struct GraphHeader {
     uint64_t MAGIC_NUM 
@@ -18,7 +18,10 @@ struct GraphHeader {
     uint64_t sizeofcolPtr;      // Needed to know size of col_indices (M)
     uint64_t offset_nnz;        // Byte offset where nnzRow start
     uint64_t offset_col;        // Byte offset where colPtr start
-};
+    uint64_t num_nodes;         // Explicit count (N)
+    uint64_t num_edges;         // Explicit count (M)
+    uint64_t flags;     // flags later user
+}; // 64 byte header perfect AVX-512 alignment
 
 class MemoryMap
 {
@@ -55,6 +58,9 @@ inline MemoryMap::MemoryMap(const char* path){
         close(fd); // Clean up the fd we just opened
         throw std::runtime_error("mmap failed");
     }
+
+    // memory advise to use huge pages
+    madvise(mappedptr,length, MADV_HUGEPAGE);
 }
 
 inline MemoryMap::~MemoryMap(){
@@ -69,7 +75,7 @@ inline MemoryMap::~MemoryMap(){
     }
 
     fd = -1;
-    length = 0;   
+    length = 0;
 }
 inline void* MemoryMap::get_data(){
     // get data pointer 

src/Graphzero.hpp

@@ -4,6 +4,7 @@
 #include <nanobind/stl/tuple.h>
 #include <nanobind/stl/string.h>
 #include "Graphzero.hpp"
+#include "csrFilegen.hpp"
 #include <vector>
 namespace nb = nanobind;
 
@@ -90,5 +91,12 @@ NB_MODULE(graphzero,m) {
             return new Graphzero(filename.c_str());
         })
         ;
-
+    
+    // convert csv to gl 
+    m.def("convert_csv_to_gl", &convert_csv,
+        "Convert a CSV edge list to GraphZero binary format (.gl)",
+        nb::arg("csv_path"), 
+        nb::arg("out_path"), 
+        nb::arg("directed") = false,
+        nb::call_guard<nb::gil_scoped_release>());
 }