improving performance of lscsr, plus generation of Kronecker graphs

include/scea/graph/lscsr.hpp

@@ -14,6 +14,7 @@ class LS_CSR : public MutableGraph {
 private:
   galois::graphs::LS_LC_CSR_Graph<void, void> graph;
   float const compact_threshold;
+  bool first_ingest = true;
 
 public:
   LS_CSR(uint64_t num_vertices, float compact_threshold)
@@ -33,16 +34,19 @@ class LS_CSR : public MutableGraph {
 
   void post_ingest() override {
     float holes_usage  = graph.getLogHolesMemoryUsageBytes();
-    float memory_usage = graph.getMemoryUsageBytes();
-    if (memory_usage > 0.0 && holes_usage / memory_usage > compact_threshold) {
+    float memory_usage = graph.getLogMemoryUsageBytes();
+    if (first_ingest || ((memory_usage > 0.0) &&
+                         ((holes_usage / memory_usage) > compact_threshold))) {
+      std::cout << "compact (holes_usage = " << holes_usage
+                << ", memory_usage = " << memory_usage << ")" << std::endl;
       graph.compact();
     }
+    first_ingest = false;
   }
 
   void for_each_edge(uint64_t src,
                      std::function<void(uint64_t const&)> callback) override {
-    for (auto const& edge : graph.edges(src))
-      callback(graph.getEdgeDst(edge));
+    graph.for_each_edge(src, callback);
   }
 
   void sort_edges(uint64_t src) override { graph.sortEdges(src); }

include/scea/stats.hpp

@@ -49,8 +49,8 @@ class PerfEvent {
     pe.size           = sizeof(struct perf_event_attr);
     pe.config         = config;
     pe.disabled       = 1;
-    pe.exclude_kernel = 1; // Exclude kernel events
-    pe.exclude_hv     = 1; // Exclude hypervisor events
+    pe.exclude_kernel = 0; // Exclude kernel events
+    pe.exclude_hv     = 0; // Exclude hypervisor events
 
     fd = perf_event_open(&pe, 0, -1, -1, 0);
     if (fd == -1) {
@@ -114,12 +114,12 @@ class ScopeBenchmarker {
         minorPageFaultsEvent(PERF_TYPE_SOFTWARE, PERF_COUNT_SW_PAGE_FAULTS_MIN),
         majorPageFaultsEvent(PERF_TYPE_SOFTWARE,
                              PERF_COUNT_SW_PAGE_FAULTS_MAJ) {
-    timer.start();
     cacheMissesEvent.start();
     cacheReferencesEvent.start();
     instructionsEvent.start();
     minorPageFaultsEvent.start();
     majorPageFaultsEvent.start();
+    timer.start();
   }
 
   ~ScopeBenchmarker() {

scripts/edit_scalability.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright (c) 2023. University of Texas at Austin. All rights reserved.
+
+BUILD="${BUILD:-/home/$(whoami)/scea/graph-log-sketch/build}"
+echo "BUILD=$BUILD"
+
+SCRATCH="${SCRATCH:-/var/local/mzinn}"
+echo "SCRATCH=$SCRATCH"
+
+echo "WORKLOAD=$WORKLOAD"
+WORKLOAD_FILE="${WORKLOAD_FILE:-$SCRATCH/graphs/$WORKLOAD.txt}"
+# check that WORKLOAD_FILE exists
+if [ ! -f $WORKLOAD_FILE ]; then
+	echo "WORKLOAD_FILE does not exist: $WORKLOAD_FILE"
+	exit 1
+fi
+echo "WORKLOAD_FILE=$WORKLOAD_FILE"
+
+RESULTS_DIR="${RESULTS_DIR:-$/scea/graph-log-sketch/results/$WORKLOAD}"
+
+# Count the number of vertices in the workload, if not already set.
+if [ -z "$WORKLOAD_NUM_VERTICES" ]; then
+	WORKLOAD_NUM_VERTICES="$($BUILD/scripts/count-batched -q -v $WORKLOAD_FILE)"
+fi
+echo "WORKLOAD_NUM_VERTICES=$WORKLOAD_NUM_VERTICES"
+
+GRAPHS="${GRAPHS:-lscsr adj csr}"
+echo "GRAPHS=$GRAPHS"
+
+THREADS="${THREADS:-32}"
+echo "THREADS=$THREADS"
+
+ALGOS="${ALGOS:-bfs pr bc tc}"
+echo "ALGOS=$ALGOS"
+
+COMPACT_THRESHOLDS="${COMPACT_THRESHOLDS:-0.6}"
+echo "COMPACT_THRESHOLDS=$COMPACT_THRESHOLDS"
+
+NOW=$(date "+%s")
+echo "NOW=$NOW"
+
+CORES="${CORES:-0-31}"
+echo "CORES=$CORES"
+
+BFS_SRC="${BFS_SRC:-95}"
+echo "BFS_SRC=$BFS_SRC"
+
+BC_NUM_SRC="${BC_NUM_SRC:-64}"
+echo "BC_NUM_SRC=$BC_NUM_SRC"
+
+for threads in $THREADS; do
+	for algo in $ALGOS; do
+		for graph in $GRAPHS; do
+			for compact_threshold in $COMPACT_THRESHOLDS; do
+				TASKSET_CMD="taskset --cpu-list $CORES"
+				EDIT_SCALABILITY_CMD="${BUILD}/microbench/edit-scalability --algo $algo --bfs-src $BFS_SRC --bc-num-src $BC_NUM_SRC --graph $graph --lscsr-compact-threshold $compact_threshold --ingest-threads $threads --algo-threads $threads --input-file $WORKLOAD_FILE --num-vertices $WORKLOAD_NUM_VERTICES"
+				JOBN="$SCRATCH/data/$WORKLOAD/$algo/edit-scalability_t=${threads}_g=${graph}_c=${compact_threshold}_${NOW}"
+				mkdir -p $(dirname $JOBN)
+
+				CMD="$TASKSET_CMD $EDIT_SCALABILITY_CMD"
+				echo "$JOBN: $CMD"
+				$CMD 2>$JOBN.err 1>$JOBN.out
+			done
+		done
+	done
+done

scripts/krongen/go.mod

@@ -0,0 +1,9 @@
+module graph500/AdityaAtulTewari/input_gen
+
+go 1.18
+
+require (
+	github.com/MichaelTJones/pcg v0.0.0-20180122055547-df440c6ed7ed
+	github.com/emer/empi v1.0.22
+	github.com/vmunoz82/shuffle v1.0.2
+)

scripts/krongen/go.sum

@@ -0,0 +1,6 @@
+github.com/MichaelTJones/pcg v0.0.0-20180122055547-df440c6ed7ed h1:hQC4FSwvsLH6rOLJTndsHnANARF9RwW4PbrDTjks/0A=
+github.com/MichaelTJones/pcg v0.0.0-20180122055547-df440c6ed7ed/go.mod h1:NQ4UMHqyfXyYVmZopcfwPRWJa0rw2aH16eDIltReVUo=
+github.com/emer/empi v1.0.22 h1:FPrM68LNQYydODqgs6ZddVEGRBOdFOrbUrLhIsn3eL8=
+github.com/emer/empi v1.0.22/go.mod h1:wr0eFV/S2/sAyowA7/8jGmK5AjiWW4saT5QMQuw87GY=
+github.com/vmunoz82/shuffle v1.0.2 h1:9QxvJN9F1OdsnRVrPu1+N1F+IU14PPtHATtvlxTSDVI=
+github.com/vmunoz82/shuffle v1.0.2/go.mod h1:MvogPEkxTyK+LU0v+nYDCzWiD8Jlu/rocUCJOvs8mcA=