diff --git a/etl/debezium-jdbc/Dockerfile b/etl/debezium-jdbc/Dockerfile index 997ff2ac..14ddecf2 100644 --- a/etl/debezium-jdbc/Dockerfile +++ b/etl/debezium-jdbc/Dockerfile @@ -34,6 +34,8 @@ COPY --from=builder /app/target/*.jar $KAFKA_CONNECT_JDBC_DIR/ COPY --from=builder /app/lib/*.jar $KAFKA_CONNECT_JDBC_DIR/ COPY InstallCert.class . COPY "InstallCert\$SavingTrustManager.class" . +COPY connect-log4j.properties /kafka/config/ +COPY log4j.properties /kafka/config/ RUN java InstallCert --quiet "${ORACLE_DB_HOST}:${ORACLE_DB_PORT}" RUN keytool -exportcert -alias "$ORACLE_DB_HOST-1" -keystore jssecacerts -storepass changeit -file oracle.cer diff --git a/etl/debezium-jdbc/connect-log4j.properties b/etl/debezium-jdbc/connect-log4j.properties new file mode 100644 index 00000000..f17e2bb2 --- /dev/null +++ b/etl/debezium-jdbc/connect-log4j.properties @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=ERROR, stdout, connectAppender + +# Send the logs to the console. +# +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + +# Send the logs to a file, rolling the file at midnight local time. For example, the `File` option specifies the +# location of the log files (e.g. ${kafka.logs.dir}/connect.log), and at midnight local time the file is closed +# and copied in the same directory but with a filename that ends in the `DatePattern` option. +# +log4j.appender.connectAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.connectAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.connectAppender.File=${kafka.logs.dir}/connect.log +log4j.appender.connectAppender.layout=org.apache.log4j.PatternLayout + +# The `%X{connector.context}` parameter in the layout includes connector-specific and task-specific information +# in the log messages, where appropriate. This makes it easier to identify those log messages that apply to a +# specific connector. +# +connect.log.pattern=[%d] %p %X{connector.context}%m (%c:%L)%n + +log4j.appender.stdout.layout.ConversionPattern=${connect.log.pattern} +log4j.appender.connectAppender.layout.ConversionPattern=${connect.log.pattern} + +log4j.logger.org.apache.zookeeper=ERROR +log4j.logger.org.reflections=ERROR diff --git a/etl/debezium-jdbc/log4j.properties b/etl/debezium-jdbc/log4j.properties new file mode 100644 index 00000000..272d67e7 --- /dev/null +++ b/etl/debezium-jdbc/log4j.properties @@ -0,0 +1,23 @@ +kafka.logs.dir=logs + +log4j.rootLogger=INFO, stdout, appender + +# Disable excessive reflection warnings - KAFKA-5229 +log4j.logger.org.reflections=ERROR + +# Disable excessive logging by consumers and producers. +log4j.logger.org.apache.kafka=ERROR +log4j.logger.org.apache.kafka.clients.consumer=WARN + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.threshold=INFO +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n + + +log4j.appender.appender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.appender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.appender.File=${kafka.logs.dir}/connect-service.log +log4j.appender.appender.layout=org.apache.log4j.PatternLayout +log4j.appender.appender.layout.ConversionPattern=%d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n + diff --git a/etl/docker-compose.yaml b/etl/docker-compose.yaml index 841eeff0..579f5c39 100644 --- a/etl/docker-compose.yaml +++ b/etl/docker-compose.yaml @@ -19,7 +19,7 @@ services: build: context: debezium-postgis/15-3.3 ports: - - "5432:5432" + - "7432:5432" environment: - POSTGRES_USER=postgres - POSTGRES_PASSWORD=postgres diff --git a/openshift/templates/etl/debezium-jdbc.deploy.yaml b/openshift/templates/etl/debezium-jdbc.deploy.yaml index 7c632661..b3121d70 100644 --- a/openshift/templates/etl/debezium-jdbc.deploy.yaml +++ b/openshift/templates/etl/debezium-jdbc.deploy.yaml @@ -68,7 +68,7 @@ objects: - name: STATUS_STORAGE_TOPIC value: "epd_connect_statuses" - name: KAFKA_DEBUG - value: "true" + value: "false" - name: DEBUG_SUSPEND_FLAG value: "n" resources: