Debug

aoe-data-analytics/.env

@@ -1,23 +1,18 @@
 spring.datasource.primary.username=aoeuser
 spring.datasource.primary.password=aoepassword
 
-spring.datasource.secondary.username=aoeuser
-spring.datasource.secondary.password=aoepassword
-
 kafka.bootstrap-servers=kafka:9092,kafka2:9092
 spring.kafka.bootstrap-servers=kafka:9092,kafka2:9092
 spring.kafka.consumer.bootstrap-servers=kafka:9092,kafka2:9092
 spring.kafka.producer.bootstrap-servers=kafka:9092,kafka2:9092
 
-
 mongodb.primary.host=aoe-mongodb
 mongodb.primary.port=27017
 mongodb.primary.database=aoe
 mongodb.primary.username=aoeuser
 mongodb.primary.password=aoepassword
 
-mongodb.secondary.host=aoe-mongodb
-mongodb.secondary.port=27017
-mongodb.secondary.database=aoe
-mongodb.secondary.username=aoeuser
-mongodb.secondary.password=aoepassword
+mongodb.primary.enable.ssl=false
+
+kafka.enabled=false
+spring.kafka.consumer.auto.startup=false

aoe-data-analytics/service-etl-processor/Dockerfile

@@ -24,4 +24,4 @@ COPY --from=build /certs /certs
 
 WORKDIR /app
 COPY --from=build /app/service-etl-processor/target/service-etl-processor-0.0.1-exec.jar service-etl-processor.jar
-ENTRYPOINT ["java", "-Xms512m", "-Xmx512m", "-Djavax.net.ssl.trustStore=/certs/rds-truststore.jks", "-Djava.security.egd=file:/dev/./urandom", "-jar", "service-etl-processor.jar"]
+ENTRYPOINT ["sh", "-c", "java -Xms512m -Xmx512m -Djavax.net.ssl.trustStorePassword=$TRUST_STORE_PASS -Djavax.net.ssl.trustStore=/certs/rds-truststore.jks -Djava.security.egd=file:/dev/./urandom -jar service-etl-processor.jar"]

aoe-data-analytics/service-etl-processor/import_rds_certs.sh

@@ -11,9 +11,9 @@ fi
 mydir=/certs
 truststore=${mydir}/rds-truststore.jks
 storepassword="$TRUST_STORE_PASSWORD"
+curl -sS "https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem" > ${mydir}/global-bundle.pem
 
-curl -sS "https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem" > ${mydir}/rds-combined-ca-bundle.pem
-awk 'split_after == 1 {n++;split_after=0} /-----END CERTIFICATE-----/ {split_after=1}{print > "rds-ca-" n ".pem"}' < ${mydir}/rds-combined-ca-bundle.pem
+awk 'split_after == 1 {n++;split_after=0} /-----END CERTIFICATE-----/ {split_after=1}{print > "rds-ca-" n ".pem"}' < ${mydir}/global-bundle.pem
 
 for CERT in rds-ca-*; do
   alias=$(openssl x509 -noout -text -in $CERT | perl -ne 'next unless /Subject:/; s/.*(CN=|CN = )//; print')
@@ -22,4 +22,4 @@ for CERT in rds-ca-*; do
   rm $CERT
 done
 
-rm ${mydir}/rds-combined-ca-bundle.pem
+rm ${mydir}/global-bundle.pem

aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/ServiceEtlProcessorApplication.java

@@ -1,15 +1,20 @@
 package fi.csc.processor;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.boot.Banner;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 
 @SpringBootApplication
 public class ServiceEtlProcessorApplication {
 
+    private static final Logger LOG = LoggerFactory.getLogger(ServiceEtlProcessorApplication.class.getSimpleName());
+
     public static void main(String[] args) {
         SpringApplication app = new SpringApplication(ServiceEtlProcessorApplication.class);
         app.setBannerMode(Banner.Mode.OFF);
         app.run(args);
     }
+
 }

aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/configuration/MongoPrimaryConfiguration.java

@@ -43,12 +43,19 @@ public MongoProperties primaryProperties() {
 
     @Bean(name = "primaryMongoClient")
     public MongoClient mongoClient(@Qualifier("primaryProperties") MongoProperties mongoProperties) {
-        return MongoClients.create(MongoClientSettings.builder()
+        boolean enableSsl = Boolean.parseBoolean(System.getenv().getOrDefault("mongodb.primary.enable.ssl", "true"));
+
+        MongoClientSettings.Builder builder = MongoClientSettings.builder();
+
+        if (enableSsl) {
+            builder.applyToSslSettings(b -> b.enabled(true).invalidHostNameAllowed(true));
+        }
+
+        return MongoClients.create(builder
             .credential(MongoCredential.createCredential(
                 mongoProperties.getUsername(),
                 mongoProperties.getDatabase(),
                 mongoProperties.getPassword()))
-            .applyToSslSettings(builder -> builder.enabled(true).invalidHostNameAllowed(true))
             .applyToClusterSettings(settings -> settings.hosts(singletonList(new ServerAddress(
                 mongoProperties.getHost(),
                 mongoProperties.getPort()

aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/consumer/KafkaConsumer.java

@@ -9,6 +9,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.kafka.annotation.KafkaListener;
 import org.springframework.kafka.listener.ConsumerSeekAware;
 import org.springframework.kafka.support.KafkaHeaders;
@@ -21,6 +22,7 @@
 import java.time.format.DateTimeFormatter;
 
 @Service
+@ConditionalOnProperty(value = "kafka.enabled", matchIfMissing = true)
 public class KafkaConsumer implements ConsumerSeekAware {
     private final Logger LOG = LoggerFactory.getLogger(KafkaConsumer.class.getSimpleName());
     private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")

docker-compose.local-dev.yml

@@ -35,6 +35,7 @@ services:
     command: sh -c "yarn serve"
 
   aoe-data-analytics:
+    entrypoint: ["java", "-Xms512m", "-Xmx512m", "-Djava.security.egd=file:/dev/./urandom", "-jar", "service-etl-processor.jar"]
     ports:
       - 8080:8080
     env_file:

start-local-env.sh

@@ -33,6 +33,8 @@ check_env_files() {
 
 check_env_files
 
+export TRUST_STORE_PASSWORD=myPassword
+
 export REVISION=${revision}
 compose="docker compose -f ./docker-compose.yml"
 compose="$compose -f ./docker-compose.local-dev.yml"