diff --git a/batch/Dockerfile b/batch/Dockerfile index 6a1ab44cb..3ca7340f2 100644 --- a/batch/Dockerfile +++ b/batch/Dockerfile @@ -36,7 +36,7 @@ ENV JDBC_FETCH_SIZE=10000 ENV JDBC_MAX_POOL_SIZE=50 ENV JDBC_INITIAL_POOL_SIZE=10 ENV JDBC_URL="jdbc:mysql://localhost:3306/openmrs" -ENV NUM_PARQUET_SHARDS=3 +ENV NUM_FILE_SHARDS=3 RUN echo 'Batch Job started' @@ -48,4 +48,4 @@ ENTRYPOINT java -cp app.jar org.openmrs.analytics.FhirEtl \ --fhirSinkPath=${SINK_PATH} --sinkUserName=${SINK_USERNAME} --sinkPassword=${SINK_PASSWORD} \ --outputParquetPath=${PARQUET_PATH} --jdbcModeEnabled=${JDBC_MODE_ENABLED} --jdbcDriverClass=${JDBC_DRIVER_CLASS} \ --dbUser=${DB_USER} --dbPassword=${DB_PASSWORD} --jdbcMaxPoolSize=${JDBC_MAX_POOL_SIZE} --jdbcUrl=${JDBC_URL} \ - --jdbcInitialPoolSize=${JDBC_INITIAL_POOL_SIZE} --jdbcFetchSize=${JDBC_FETCH_SIZE} --numParquetShards=${NUM_PARQUET_SHARDS} + --jdbcInitialPoolSize=${JDBC_INITIAL_POOL_SIZE} --jdbcFetchSize=${JDBC_FETCH_SIZE} --numFileShards=${NUM_FILE_SHARDS} diff --git a/batch/src/main/java/org/openmrs/analytics/FhirEtl.java b/batch/src/main/java/org/openmrs/analytics/FhirEtl.java index 8a8118267..87f0e0edc 100644 --- a/batch/src/main/java/org/openmrs/analytics/FhirEtl.java +++ b/batch/src/main/java/org/openmrs/analytics/FhirEtl.java @@ -139,7 +139,7 @@ private static void fetchSegments(PCollection inputSegm .withNumShards(options.getNumFileShards())); // TODO add Avro output option // apply("WriteToAvro", AvroIO.writeGenericRecords(schema).to(outputFile).withSuffix(".avro") - // .withNumShards(options.getNumParquetShards())); + // .withNumShards(options.getnumFileShards())); } if (!options.getOutputJsonPath().isEmpty()) { PCollection windowedRecords = addWindow(records.get(fetchSearchPageFn.jsonTag), diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index a0fbdcb24..814a0dbe7 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -45,7 +45,7 @@ services: - JDBC_MAX_POOL_SIZE=50 - JDBC_INITIAL_POOL_SIZE=10 - JDBC_URL=jdbc:mysql://localhost:3306/openmrs - - NUM_PARQUET_SHARDS=3 + - NUM_FILE_SHARDS=3 # the 2 variable below should be same as volume mappings above - PARQUET_PATH=/tmp/ - FHIR_DEBEZIUM_CONFIG_PATH=/usr/src/Main/dbz_event_to_fhir_config.json