wip

include/scea/algo/algo_interface.hpp

@@ -3,13 +3,22 @@
 
 #pragma once
 
+#include <optional>
+#include <ostream>
+
 #include "scea/graph/mutable_graph_interface.hpp"
 
 namespace scea::algo {
 
 class Algo {
 public:
   virtual void operator()(scea::graph::MutableGraph& g) = 0;
+
+  /* Run the algorithm and output the result to the given stream. */
+  virtual void operator()(scea::graph::MutableGraph& g, std::ostream& output) {
+    operator()(g);
+    output << "done" << std::endl;
+  }
 };
 
 } // namespace scea::algo

include/scea/algo/bc.hpp

@@ -8,6 +8,8 @@
 #include <string>
 #include <limits>
 #include <unordered_set>
+#include <algorithm>
+#include <random>
 
 #include "galois/AtomicHelpers.h"
 #include "galois/LargeArray.h"
@@ -23,23 +25,31 @@ class BetweennessCentrality : public Algo {
       std::numeric_limits<uint64_t>::max() / 4;
   constexpr static const unsigned LEVEL_CHUNK_SIZE = 256u;
 
-  std::vector<uint64_t> sources; // empty if all vertices are sources
+  unsigned int const rseed;
+  uint64_t const num_src;
 
 public:
-  BetweennessCentrality(uint64_t num_vertices, unsigned int rseed = 0,
-                        uint64_t num_src = 0) {
-    sources.resize(num_vertices);
-    std::iota(sources.begin(), sources.end(), 0);
+  BetweennessCentrality(unsigned int rseed = 0, uint64_t num_src = 0)
+      : rseed(rseed), num_src(num_src) {}
 
-    if (num_src > 0 && num_src != num_vertices) {
+  static galois::LargeArray<double>
+  compute(scea::graph::MutableGraph& g, unsigned int rseed, uint64_t num_src) {
+    std::vector<uint64_t> sources;
+
+    // Select source vertices.
+    sources.resize(g.size());
+    galois::do_all(
+        galois::iterate(0ul, g.size()),       //
+        [&](size_t i) { sources[i] = i; },    //
+        galois::no_stats(),                   //
+        galois::loopname("InitializeSources") //
+    );
+    if (num_src > 0) {
       std::mt19937 gen(rseed);
       std::shuffle(sources.begin(), sources.end(), gen);
-      sources.resize(num_src);
+      sources.resize(std::min(g.size(), num_src));
     }
-  }
 
-  static galois::LargeArray<double>
-  compute(scea::graph::MutableGraph& g, std::vector<uint64_t> const& sources) {
     galois::LargeArray<uint64_t> distance;
     galois::LargeArray<std::atomic<uint64_t>> shortestPathCount;
     galois::LargeArray<double> dependency;
@@ -155,7 +165,7 @@ class BetweennessCentrality : public Algo {
   }
 
   void operator()(scea::graph::MutableGraph& g) override {
-    compute(g, sources);
+    compute(g, rseed, num_src);
   }
 };
 

include/scea/algo/bfs.hpp

@@ -30,6 +30,10 @@ class SSSP_BFS : public Algo {
     galois::LargeArray<uint64_t> shortest_path;
     shortest_path.create(g.size(), UNVISITED);
 
+    if (src >= g.size()) {
+      std::cout << "warn: source vertex out of bounds" << std::endl;
+      return shortest_path;
+    }
     nextSt->push(src);
 
     uint64_t level = 0U;

include/scea/graph/adj.hpp

@@ -12,27 +12,39 @@
 namespace scea::graph {
 
 class AdjGraph : public MutableGraph {
-private:
-  std::vector<std::vector<uint64_t>> edges;
+protected:
+  std::vector<std::vector<uint64_t>> m_edges;
 
 public:
-  explicit AdjGraph(uint64_t num_vertices) : edges(num_vertices) {}
-
   virtual ~AdjGraph() {}
 
-  uint64_t size() noexcept override { return edges.size(); }
+  uint64_t size() noexcept override { return m_edges.size(); }
 
-  uint64_t get_out_degree(uint64_t src) override { return edges[src].size(); }
+  uint64_t get_out_degree(uint64_t src) override { return m_edges[src].size(); }
 
   void ingest(
       std::vector<std::pair<uint64_t, std::vector<uint64_t>>> edges) override {
-    // todo: account for new vertex ids
+    galois::GReduceMax<uint64_t> max_vertex_id;
+    max_vertex_id.reset();
+    galois::do_all(
+        galois::iterate(edges), //
+        [&](auto const& pair) {
+          auto const& [src, dsts] = pair;
+          max_vertex_id.update(src);
+          // update with max element of dsts
+          if (!dsts.empty())
+            max_vertex_id.update(*std::max_element(dsts.begin(), dsts.end()));
+        },
+        galois::steal(),                  //
+        galois::loopname("FindMaxVertex") //
+    );
+    m_edges.resize(std::max(m_edges.size(), max_vertex_id.reduce() + 1));
+
     galois::do_all(
         galois::iterate(edges),
         [&](auto const& pair) {
           auto const& [src, dsts] = pair;
-          std::copy(dsts.begin(), dsts.end(),
-                    std::back_inserter(this->edges[src]));
+          std::copy(dsts.begin(), dsts.end(), std::back_inserter(m_edges[src]));
         },
         galois::steal(), galois::loopname("Ingest"));
   }
@@ -41,16 +53,16 @@ class AdjGraph : public MutableGraph {
 
   void for_each_edge(uint64_t src,
                      std::function<void(uint64_t const&)> callback) override {
-    for (auto const& dst : edges[src])
+    for (auto const& dst : m_edges[src])
       callback(dst);
   }
 
   void sort_edges(uint64_t src) {
-    std::sort(edges[src].begin(), edges[src].end());
+    std::sort(m_edges[src].begin(), m_edges[src].end());
   }
 
   bool find_edge_sorted(uint64_t src, uint64_t dst) {
-    return std::binary_search(edges[src].begin(), edges[src].end(), dst);
+    return std::binary_search(m_edges[src].begin(), m_edges[src].end(), dst);
   }
 };
 

include/scea/graph/csr.hpp

@@ -9,16 +9,16 @@
 #include <algorithm>
 
 #include "galois/Galois.h"
-#include "scea/graph/mutable_graph_interface.hpp"
+#include "scea/graph/adj.hpp"
 
 namespace scea::graph {
 
-class CSR : public MutableGraph {
+class CSR : public AdjGraph {
 private:
-  std::vector<std::vector<uint64_t>> m_adj;
+  static constexpr uint64_t PARALLEL_PREFIX_SUM_VERTEX_THRESHOLD = 1ul << 25;
 
   std::vector<uint64_t> m_vertices;
-  std::vector<uint64_t> m_edges;
+  std::vector<uint64_t> m_edges_compact;
 
   static uint64_t transmute(std::vector<uint64_t> const& p) { return p.size(); }
   static uint64_t scan_op(std::vector<uint64_t> const& p, const uint64_t& l) {
@@ -32,62 +32,60 @@ class CSR : public MutableGraph {
       m_pfx_sum;
 
 public:
-  explicit CSR(uint64_t num_vertices)
-      : m_adj(num_vertices),
-        m_vertices(num_vertices + 1, 0ul), m_pfx_sum{&m_adj[0],
-                                                     &m_vertices[1]} {}
+  CSR() : m_vertices(1, 0ul), m_pfx_sum(&m_edges[0], &m_vertices[1]) {}
+  virtual ~CSR(){};
 
-  virtual ~CSR() {}
-
-  uint64_t size() noexcept override { return m_adj.size(); }
+  uint64_t size() noexcept override { return m_vertices.size() - 1; }
 
   uint64_t get_out_degree(uint64_t src) override {
     return m_vertices[src + 1] - m_vertices[src];
   }
 
-  void ingest(
-      std::vector<std::pair<uint64_t, std::vector<uint64_t>>> edges) override {
-    // todo: account for new vertex ids
-    galois::do_all(
-        galois::iterate(edges),
-        [&](auto const& pair) {
-          auto const& [src, dsts] = pair;
-          for (auto const& dst : dsts) {
-            m_adj[src].emplace_back(dst);
-          }
-          // std::copy(dsts.begin(), dsts.end(),
-          // std::back_inserter(m_adj[src]));
-        },
-        galois::steal(), galois::loopname("Ingest"));
-  }
-
   void post_ingest() override {
-    m_pfx_sum.computePrefixSum(m_adj.size());
-    m_edges.resize(m_vertices[m_adj.size()]);
+    // from the adjacency list:
+    const size_t num_vertices = m_edges.size();
+    if (num_vertices == 0)
+      return;
+
+    // compute prefix sum on the vertices
+    m_vertices.resize(num_vertices + 1);
+    m_pfx_sum.src = &m_edges[0];
+    // note: this prefix sum is inclusive!
+    m_pfx_sum.dst = &m_vertices[1];
+    if (num_vertices < PARALLEL_PREFIX_SUM_VERTEX_THRESHOLD) {
+      m_pfx_sum.computePrefixSumSerially(num_vertices);
+    } else {
+      m_pfx_sum.computePrefixSum(num_vertices);
+    }
+    GALOIS_ASSERT(m_vertices[0] == 0,
+                  "First entry of CSR vertex array should always be 0!");
+
+    // Compact edges.
+    m_edges_compact.resize(m_vertices.back());
     galois::do_all(
-        galois::iterate(0ul, m_adj.size()),
+        galois::iterate(0ul, num_vertices),
         [&](uint64_t const& i) {
-          std::copy(m_adj[i].begin(), m_adj[i].end(),
-                    m_edges.begin() + m_vertices[i]);
+          std::copy(m_edges[i].begin(), m_edges[i].end(),
+                    m_edges_compact.begin() + m_vertices[i]);
         },
-        galois::steal(), galois::loopname("PostIngest"));
+        galois::steal(), galois::loopname("CreateCSR"));
   }
 
   void for_each_edge(uint64_t src,
                      std::function<void(uint64_t const&)> callback) override {
-    for (auto i = m_vertices[src]; i < m_vertices[src + 1]; ++i) {
-      callback(m_edges[i]);
-    }
+    for (auto i = m_vertices[src]; i < m_vertices[src + 1]; ++i)
+      callback(m_edges_compact[i]);
   }
 
   void sort_edges(uint64_t src) override {
-    std::sort(m_edges.begin() + m_vertices[src],
-              m_edges.begin() + m_vertices[src]);
+    std::sort(m_edges_compact.begin() + m_vertices[src],
+              m_edges_compact.begin() + m_vertices[src]);
   }
 
   bool find_edge_sorted(uint64_t src, uint64_t dst) override {
-    return std::binary_search(m_edges.begin() + m_vertices[src],
-                              m_edges.begin() + m_vertices[src + 1], dst);
+    return std::binary_search(m_edges_compact.begin() + m_vertices[src],
+                              m_edges_compact.begin() + m_vertices[src + 1],
+                              dst);
   }
 };
 

include/scea/graph/lccsr.hpp

@@ -3,56 +3,46 @@
 
 #pragma once
 
-#include <vector>
 #include <memory>
 #include <atomic>
 #include <algorithm>
 
 #include "galois/graphs/LC_CSR_64_Graph.h"
-#include "scea/graph/mutable_graph_interface.hpp"
+#include "scea/graph/adj.hpp"
 
 namespace scea::graph {
 
-class LC_CSR : public MutableGraph {
+class LC_CSR : public AdjGraph {
 private:
   using GraphTy =
       galois::graphs::LC_CSR_64_Graph<void, void>::with_no_lockable<true>::type;
-  std::vector<std::vector<uint64_t>> m_adj;
+
   std::unique_ptr<GraphTy> m_graph;
   std::atomic<size_t> num_edges = ATOMIC_VAR_INIT(0);
 
 public:
-  explicit LC_CSR(uint64_t num_vertices) : m_adj(num_vertices) {}
-
   virtual ~LC_CSR() {}
 
-  uint64_t size() noexcept override { return m_adj.size(); }
-
   uint64_t get_out_degree(uint64_t src) override {
     return m_graph->getDegree(src);
   }
 
-  void ingest(
-      std::vector<std::pair<uint64_t, std::vector<uint64_t>>> edges) override {
-    // todo: account for new vertex ids
-    galois::GAccumulator<uint64_t> num_new_edges_accum;
+  void post_ingest() override {
+    galois::GAccumulator<uint64_t> num_edges_accum;
+    num_edges_accum.reset();
     galois::do_all(
-        galois::iterate(edges),
-        [&](auto const& pair) {
-          auto const& [src, dsts] = pair;
-          num_new_edges_accum += dsts.size();
-          std::copy(dsts.begin(), dsts.end(), std::back_inserter(m_adj[src]));
-        },
-        galois::steal(), galois::loopname("Ingest"));
-    num_edges += num_new_edges_accum.reduce();
-  }
+        galois::iterate(m_edges),                                  //
+        [&](auto const& dsts) { num_edges_accum += dsts.size(); }, //
+        galois::loopname("CountEdges"));
 
-  void post_ingest() override {
     m_graph = std::make_unique<GraphTy>(
-        m_adj.size(), num_edges.load(std::memory_order_acquire),
-        [&](size_t const& src) { return m_adj[src].size(); },
-        [&](size_t const& src, size_t const& idx) { return m_adj[src][idx]; },
-        [&](size_t const& src, size_t const& idx) { return 0; });
+        m_edges.size(), num_edges_accum.reduce(),
+        [&](size_t const& src) { return m_edges[src].size(); },
+        [&](size_t const& src, size_t const& idx) { return m_edges[src][idx]; },
+        [&](size_t const& /*src*/, size_t const& /*idx*/) {
+          /* no edge data*/
+          return 0;
+        });
   }
 
   void for_each_edge(uint64_t src,

include/scea/graph/lscsr.hpp

@@ -5,6 +5,7 @@
 
 #include <vector>
 
+#include "galois/ParallelSTL.h"
 #include "galois/graphs/LS_LC_CSR_Graph.h"
 #include "scea/graph/mutable_graph_interface.hpp"
 
@@ -16,8 +17,8 @@ class LS_CSR : public MutableGraph {
   float const compact_threshold;
 
 public:
-  LS_CSR(uint64_t num_vertices, float compact_threshold)
-      : graph(num_vertices), compact_threshold(compact_threshold) {}
+  LS_CSR(float compact_threshold)
+      : graph(0), compact_threshold(compact_threshold) {}
 
   virtual ~LS_CSR() {}
 
@@ -29,6 +30,10 @@ class LS_CSR : public MutableGraph {
 
   void ingest(
       std::vector<std::pair<uint64_t, std::vector<uint64_t>>> edges) override {
+    // galois::ParallelSTL::sort(
+    //     edges.begin(), edges.end(),
+    //     [](auto const& a, auto const& b) { return a.first < b.first; });
+
     graph.addBatchTopologyOnly<false>(edges);
   }
 
@@ -40,7 +45,8 @@ class LS_CSR : public MutableGraph {
                 << ", memory_usage = " << memory_usage << ")" << std::endl;
       graph.compact();
       std::cout << "compacted (csr_usage = " << graph.getCSRMemoryUsageBytes()
-                << ", memory_usage = " << graph.getLogMemoryUsageBytes() << ")" << std::endl;
+                << ", memory_usage = " << graph.getLogMemoryUsageBytes() << ")"
+                << std::endl;
     }
   }