tmp

ldbc · Aug 23, 2024 · 6c7f94b · 6c7f94b
1 parent f26d723
commit 6c7f94b
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 4 deletions.
diff --git a/scripts/run_cluster.sh b/scripts/run_cluster.sh
@@ -18,18 +18,18 @@ echo "start: " `date`
 time spark-submit --master spark://finbench-large-00:7077 \
     --class ldbc.finbench.datagen.LdbcDatagen \
     --num-executors 2 \
-    --conf "spark.default.parallelism=640" \
+    --conf "spark.default.parallelism=400" \
     --conf "spark.network.timeout=100000" \
     --conf "spark.shuffle.compress=true" \
     --conf "spark.shuffle.spill.compress=true" \
     --conf "spark.serializer=org.apache.spark.serializer.KryoSerializer" \
     --conf "spark.driver.memory=200g" \
-    --conf "spark.driver.maxResultSize=5g" \
+    --conf "spark.driver.maxResultSize=10g" \
     --conf "spark.executor.memory=300g" \
     --conf "spark.executor.memoryOverheadFactor=0.2" \
     --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC" \
     ${LDBC_FINBENCH_DATAGEN_JAR} \
-    --scale-factor 30 \
+    --scale-factor 10 \
     --output-dir ${OUTPUT_DIR}
 
 echo "End: " `date`
diff --git a/src/main/java/ldbc/finbench/datagen/generation/DatagenParams.java b/src/main/java/ldbc/finbench/datagen/generation/DatagenParams.java
@@ -142,6 +142,7 @@ public static void readConf(DatagenConfiguration conf) {
 
             System.out.println(" ... Num Persons " + numPersons);
             System.out.println(" ... Num Companies " + numCompanies);
+            System.out.println(" ... Num Mediums " + numMediums);
             System.out.println(" ... Start Year " + startYear);
             System.out.println(" ... Num Years " + numYears);
             System.out.println(" ... Max degree of Powerlaw " + transferMaxDegree);

diff --git a/src/main/resources/scale_factors.xml b/src/main/resources/scale_factors.xml
@@ -161,4 +161,27 @@
         </property>
     </scale_factor>
 
+    <scale_factor name="100">
+        <property>
+            <name>generator.numPersons</name>
+            <value>2000000</value>
+        </property>
+        <property>
+            <name>generator.numCompanies</name>
+            <value>2000000</value>
+        </property>
+        <property>
+            <name>generator.numMediums</name>
+            <value>3500000</value>
+        </property>
+        <property>
+            <name>transfer.minNumDegree</name>
+            <value>1</value>
+        </property>
+        <property>
+            <name>transfer.maxNumDegree</name>
+            <value>100000</value>
+        </property>
+    </scale_factor>
+
 </scale_factors>
diff --git a/src/main/scala/ldbc/finbench/datagen/LdbcDatagen.scala b/src/main/scala/ldbc/finbench/datagen/LdbcDatagen.scala
@@ -136,7 +136,7 @@ object LdbcDatagen extends SparkApp with Logging {
         format = args.factorFormat
       )
       log.info("[Main] Starting factoring stage")
-      // FactorGenerationStage.run(factorArgs)
+      FactorGenerationStage.run(factorArgs)
     }
   }
 }