Merge branch 'master' into issue-144

google · Apr 13, 2021 · 54073e5 · 54073e5
2 parents 4a5d2f1 + ae598d7
commit 54073e5
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 4 deletions.
diff --git a/batch/Dockerfile b/batch/Dockerfile
@@ -36,7 +36,7 @@ ENV JDBC_FETCH_SIZE=10000
 ENV JDBC_MAX_POOL_SIZE=50
 ENV JDBC_INITIAL_POOL_SIZE=10
 ENV JDBC_URL="jdbc:mysql://localhost:3306/openmrs"
-ENV NUM_PARQUET_SHARDS=3
+ENV NUM_FILE_SHARDS=3
 
 
 RUN echo 'Batch Job started'
@@ -48,4 +48,4 @@ ENTRYPOINT java -cp app.jar org.openmrs.analytics.FhirEtl \
            --fhirSinkPath=${SINK_PATH}  --sinkUserName=${SINK_USERNAME} --sinkPassword=${SINK_PASSWORD} \
            --outputParquetPath=${PARQUET_PATH} --jdbcModeEnabled=${JDBC_MODE_ENABLED} --jdbcDriverClass=${JDBC_DRIVER_CLASS} \
            --dbUser=${DB_USER} --dbPassword=${DB_PASSWORD} --jdbcMaxPoolSize=${JDBC_MAX_POOL_SIZE} --jdbcUrl=${JDBC_URL} \
-           --jdbcInitialPoolSize=${JDBC_INITIAL_POOL_SIZE} --jdbcFetchSize=${JDBC_FETCH_SIZE} --numParquetShards=${NUM_PARQUET_SHARDS}
+           --jdbcInitialPoolSize=${JDBC_INITIAL_POOL_SIZE} --jdbcFetchSize=${JDBC_FETCH_SIZE} --numFileShards=${NUM_FILE_SHARDS}
diff --git a/batch/src/main/java/org/openmrs/analytics/FhirEtl.java b/batch/src/main/java/org/openmrs/analytics/FhirEtl.java
@@ -139,7 +139,7 @@ private static void fetchSegments(PCollection<SearchSegmentDescriptor> inputSegm
 			        .withNumShards(options.getNumFileShards()));
 			// TODO add Avro output option
 			// apply("WriteToAvro", AvroIO.writeGenericRecords(schema).to(outputFile).withSuffix(".avro")
-			//        .withNumShards(options.getNumParquetShards()));
+			//        .withNumShards(options.getnumFileShards()));
 		}
 		if (!options.getOutputJsonPath().isEmpty()) {
 			PCollection<String> windowedRecords = addWindow(records.get(fetchSearchPageFn.jsonTag),

diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
@@ -45,7 +45,7 @@ services:
      - JDBC_MAX_POOL_SIZE=50
      - JDBC_INITIAL_POOL_SIZE=10
      - JDBC_URL=jdbc:mysql://localhost:3306/openmrs
-     - NUM_PARQUET_SHARDS=3
+     - NUM_FILE_SHARDS=3
      # the 2 variable below should be same as volume mappings above
      - PARQUET_PATH=/tmp/
      - FHIR_DEBEZIUM_CONFIG_PATH=/usr/src/Main/dbz_event_to_fhir_config.json