Checking the memory issue

google · May 14, 2024 · 8b73086 · 8b73086
1 parent 88f34e9
commit 8b73086
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/e2e-tests/pipeline_validation.sh b/e2e-tests/pipeline_validation.sh
@@ -206,7 +206,11 @@ function test_parquet_sink() {
   | awk '{print $3}')
   print_message "Total encounters synced to parquet ---> ${total_encounters_streamed}"
 
-  local total_obs_streamed=$(java -Xms16g -Xmx16g -jar ./controller-spark/parquet-tools-1.11.1.jar \
+  print_message "Java Heap with no memory set ----> $(java -XX:+PrintFlagsFinal -version | grep 'HeapSize')"
+  print_message "Java Heap with 4g memory set ----> $(java -Xmx4g -Xms4g -XX:+PrintFlagsFinal -version | grep 'HeapSize')"
+  local files=$(ls -lh "${HOME_PATH}/${PARQUET_SUBDIR}/Observation/")
+  print_message "Files in the obs directory ----> ${files}"
+  local total_obs_streamed=$(java -Xms4g -Xmx4g -jar ./controller-spark/parquet-tools-1.11.1.jar \
   rowcount "${HOME_PATH}/${PARQUET_SUBDIR}/Observation/" | awk '{print $3}')
   print_message "Total obs synced to parquet ---> ${total_obs_streamed}"