From ff57fae52051c5e5f5da1cca07ec81f4bba92dcf Mon Sep 17 00:00:00 2001 From: Mikko Kauhanen Date: Wed, 4 Dec 2024 14:48:04 +0200 Subject: [PATCH] Debug --- aoe-data-analytics/.env | 13 ++---- .../service-etl-processor/Dockerfile | 2 +- .../service-etl-processor/import_rds_certs.sh | 6 +-- .../ServiceEtlProcessorApplication.java | 5 +++ .../MongoPrimaryConfiguration.java | 11 ++++- .../csc/processor/consumer/KafkaConsumer.java | 2 + .../controller/StatusController.java | 2 +- .../processor/producer/JksFileChecker.java | 44 ------------------- docker-compose.local-dev.yml | 1 + start-local-env.sh | 2 + 10 files changed, 28 insertions(+), 60 deletions(-) delete mode 100644 aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/producer/JksFileChecker.java diff --git a/aoe-data-analytics/.env b/aoe-data-analytics/.env index e75aa0fef..e8664634c 100644 --- a/aoe-data-analytics/.env +++ b/aoe-data-analytics/.env @@ -1,23 +1,18 @@ spring.datasource.primary.username=aoeuser spring.datasource.primary.password=aoepassword -spring.datasource.secondary.username=aoeuser -spring.datasource.secondary.password=aoepassword - kafka.bootstrap-servers=kafka:9092,kafka2:9092 spring.kafka.bootstrap-servers=kafka:9092,kafka2:9092 spring.kafka.consumer.bootstrap-servers=kafka:9092,kafka2:9092 spring.kafka.producer.bootstrap-servers=kafka:9092,kafka2:9092 - mongodb.primary.host=aoe-mongodb mongodb.primary.port=27017 mongodb.primary.database=aoe mongodb.primary.username=aoeuser mongodb.primary.password=aoepassword -mongodb.secondary.host=aoe-mongodb -mongodb.secondary.port=27017 -mongodb.secondary.database=aoe -mongodb.secondary.username=aoeuser -mongodb.secondary.password=aoepassword \ No newline at end of file +mongodb.primary.enable.ssl=false + +kafka.enabled=false +spring.kafka.consumer.auto.startup=false diff --git a/aoe-data-analytics/service-etl-processor/Dockerfile b/aoe-data-analytics/service-etl-processor/Dockerfile index 52163c53c..371c5b75c 100644 --- a/aoe-data-analytics/service-etl-processor/Dockerfile +++ b/aoe-data-analytics/service-etl-processor/Dockerfile @@ -24,4 +24,4 @@ COPY --from=build /certs /certs WORKDIR /app COPY --from=build /app/service-etl-processor/target/service-etl-processor-0.0.1-exec.jar service-etl-processor.jar -ENTRYPOINT ["java", "-Xms512m", "-Xmx512m", "-Djavax.net.ssl.trustStore=/certs/rds-truststore.jks", "-Djava.security.egd=file:/dev/./urandom", "-jar", "service-etl-processor.jar"] +ENTRYPOINT ["sh", "-c", "java -Xms512m -Xmx512m -Djavax.net.ssl.trustStorePassword=$TRUST_STORE_PASS -Djavax.net.ssl.trustStore=/certs/rds-truststore.jks -Djava.security.egd=file:/dev/./urandom -jar service-etl-processor.jar"] diff --git a/aoe-data-analytics/service-etl-processor/import_rds_certs.sh b/aoe-data-analytics/service-etl-processor/import_rds_certs.sh index bf307b97d..78af94487 100755 --- a/aoe-data-analytics/service-etl-processor/import_rds_certs.sh +++ b/aoe-data-analytics/service-etl-processor/import_rds_certs.sh @@ -11,9 +11,9 @@ fi mydir=/certs truststore=${mydir}/rds-truststore.jks storepassword="$TRUST_STORE_PASSWORD" +curl -sS "https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem" > ${mydir}/global-bundle.pem -curl -sS "https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem" > ${mydir}/rds-combined-ca-bundle.pem -awk 'split_after == 1 {n++;split_after=0} /-----END CERTIFICATE-----/ {split_after=1}{print > "rds-ca-" n ".pem"}' < ${mydir}/rds-combined-ca-bundle.pem +awk 'split_after == 1 {n++;split_after=0} /-----END CERTIFICATE-----/ {split_after=1}{print > "rds-ca-" n ".pem"}' < ${mydir}/global-bundle.pem for CERT in rds-ca-*; do alias=$(openssl x509 -noout -text -in $CERT | perl -ne 'next unless /Subject:/; s/.*(CN=|CN = )//; print') @@ -22,4 +22,4 @@ for CERT in rds-ca-*; do rm $CERT done -rm ${mydir}/rds-combined-ca-bundle.pem +rm ${mydir}/global-bundle.pem diff --git a/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/ServiceEtlProcessorApplication.java b/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/ServiceEtlProcessorApplication.java index 45580d41d..375fb1fdb 100644 --- a/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/ServiceEtlProcessorApplication.java +++ b/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/ServiceEtlProcessorApplication.java @@ -1,5 +1,7 @@ package fi.csc.processor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.boot.Banner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -7,9 +9,12 @@ @SpringBootApplication public class ServiceEtlProcessorApplication { + private static final Logger LOG = LoggerFactory.getLogger(ServiceEtlProcessorApplication.class.getSimpleName()); + public static void main(String[] args) { SpringApplication app = new SpringApplication(ServiceEtlProcessorApplication.class); app.setBannerMode(Banner.Mode.OFF); app.run(args); } + } diff --git a/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/configuration/MongoPrimaryConfiguration.java b/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/configuration/MongoPrimaryConfiguration.java index 82a7e8b8f..ebc0fab18 100644 --- a/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/configuration/MongoPrimaryConfiguration.java +++ b/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/configuration/MongoPrimaryConfiguration.java @@ -43,12 +43,19 @@ public MongoProperties primaryProperties() { @Bean(name = "primaryMongoClient") public MongoClient mongoClient(@Qualifier("primaryProperties") MongoProperties mongoProperties) { - return MongoClients.create(MongoClientSettings.builder() + boolean enableSsl = Boolean.parseBoolean(System.getenv().getOrDefault("mongodb.primary.enable.ssl", "true")); + + MongoClientSettings.Builder builder = MongoClientSettings.builder(); + + if (enableSsl) { + builder.applyToSslSettings(b -> b.enabled(true).invalidHostNameAllowed(true)); + } + + return MongoClients.create(builder .credential(MongoCredential.createCredential( mongoProperties.getUsername(), mongoProperties.getDatabase(), mongoProperties.getPassword())) - .applyToSslSettings(builder -> builder.enabled(true).invalidHostNameAllowed(true)) .applyToClusterSettings(settings -> settings.hosts(singletonList(new ServerAddress( mongoProperties.getHost(), mongoProperties.getPort() diff --git a/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/consumer/KafkaConsumer.java b/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/consumer/KafkaConsumer.java index 998c806a0..7880b5596 100644 --- a/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/consumer/KafkaConsumer.java +++ b/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/consumer/KafkaConsumer.java @@ -9,6 +9,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.listener.ConsumerSeekAware; import org.springframework.kafka.support.KafkaHeaders; @@ -21,6 +22,7 @@ import java.time.format.DateTimeFormatter; @Service +@ConditionalOnProperty(value = "kafka.enabled", matchIfMissing = true) public class KafkaConsumer implements ConsumerSeekAware { private final Logger LOG = LoggerFactory.getLogger(KafkaConsumer.class.getSimpleName()); private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") diff --git a/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/controller/StatusController.java b/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/controller/StatusController.java index 0f299ec03..5529163de 100644 --- a/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/controller/StatusController.java +++ b/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/controller/StatusController.java @@ -13,7 +13,7 @@ @RestController public class StatusController { - @GetMapping(path = "/status", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.TEXT_PLAIN_VALUE) + @GetMapping(path = "/status", produces = MediaType.TEXT_PLAIN_VALUE) public CompletableFuture> getStatus() { return async(() -> new ResponseEntity<>("Service operable: true", HttpStatus.OK)); } diff --git a/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/producer/JksFileChecker.java b/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/producer/JksFileChecker.java deleted file mode 100644 index a100502e7..000000000 --- a/aoe-data-analytics/service-etl-processor/src/main/java/fi/csc/processor/producer/JksFileChecker.java +++ /dev/null @@ -1,44 +0,0 @@ -package fi.csc.processor.producer; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.CommandLineRunner; -import org.springframework.stereotype.Component; - -import java.io.FileInputStream; -import java.security.KeyStore; -import java.util.Enumeration; - -@Component -public class JksFileChecker implements CommandLineRunner { - - private static final Logger LOG = LoggerFactory.getLogger(JksFileChecker.class.getSimpleName()); - - - @Override - public void run(String... args) throws Exception { - String trustStorePath = System.getProperty("javax.net.ssl.trustStore"); - String trustStorePassword = System.getProperty("javax.net.ssl.trustStorePassword"); - - if (trustStorePath == null || trustStorePassword == null) { - LOG.error("TrustStore properties are not set!"); - return; - } - - try (FileInputStream trustStoreStream = new FileInputStream(trustStorePath)) { - KeyStore trustStore = KeyStore.getInstance("JKS"); - trustStore.load(trustStoreStream, trustStorePassword.toCharArray()); - - LOG.info("JKS file loaded successfully!"); - LOG.info("Aliases in the JKS:"); - - Enumeration aliases = trustStore.aliases(); - while (aliases.hasMoreElements()) { - System.out.println(" - " + aliases.nextElement()); - } - } catch (Exception e) { - LOG.error("Failed to load JKS file: " + e.getMessage()); - e.printStackTrace(); - } - } -} \ No newline at end of file diff --git a/docker-compose.local-dev.yml b/docker-compose.local-dev.yml index 3a99d594b..e63a825de 100644 --- a/docker-compose.local-dev.yml +++ b/docker-compose.local-dev.yml @@ -35,6 +35,7 @@ services: command: sh -c "yarn serve" aoe-data-analytics: + entrypoint: ["java", "-Xms512m", "-Xmx512m", "-Djava.security.egd=file:/dev/./urandom", "-jar", "service-etl-processor.jar"] ports: - 8080:8080 env_file: diff --git a/start-local-env.sh b/start-local-env.sh index 2f6af0912..212d7fb22 100755 --- a/start-local-env.sh +++ b/start-local-env.sh @@ -33,6 +33,8 @@ check_env_files() { check_env_files +export TRUST_STORE_PASSWORD=myPassword + export REVISION=${revision} compose="docker compose -f ./docker-compose.yml" compose="$compose -f ./docker-compose.local-dev.yml"