Merge pull request #571 from bcgov/feat/srs-26

Feat/srs 26

etl/debezium-jdbc/Dockerfile

@@ -5,9 +5,9 @@ WORKDIR /app/
 COPY ./ ./
 RUN mvn clean install
 
+RUN echo "DEBEZIUM_VERSION is $DEBEZIUM_VERSION"
 
-
-FROM debezium/connect:$DEBEZIUM_VERSION
+FROM debezium/connect:1.9
 
 ARG ORACLE_JDBC_VERSION
 ARG ORACLE_JDBC_MD5
@@ -34,6 +34,8 @@ COPY --from=builder /app/target/*.jar $KAFKA_CONNECT_JDBC_DIR/
 COPY --from=builder /app/lib/*.jar $KAFKA_CONNECT_JDBC_DIR/
 COPY InstallCert.class .
 COPY "InstallCert\$SavingTrustManager.class" .
+COPY connect-log4j.properties /kafka/config/
+COPY log4j.properties /kafka/config/
 
 RUN java InstallCert --quiet "${ORACLE_DB_HOST}:${ORACLE_DB_PORT}"
 RUN keytool -exportcert -alias "$ORACLE_DB_HOST-1" -keystore jssecacerts -storepass changeit -file oracle.cer

etl/debezium-jdbc/connect-log4j.properties

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=ERROR, stdout, connectAppender
+
+# Send the logs to the console.
+#
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+
+# Send the logs to a file, rolling the file at midnight local time. For example, the `File` option specifies the
+# location of the log files (e.g. ${kafka.logs.dir}/connect.log), and at midnight local time the file is closed
+# and copied in the same directory but with a filename that ends in the `DatePattern` option.
+#
+log4j.appender.connectAppender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.connectAppender.DatePattern='.'yyyy-MM-dd-HH
+log4j.appender.connectAppender.File=${kafka.logs.dir}/connect.log
+log4j.appender.connectAppender.layout=org.apache.log4j.PatternLayout
+
+# The `%X{connector.context}` parameter in the layout includes connector-specific and task-specific information
+# in the log messages, where appropriate. This makes it easier to identify those log messages that apply to a
+# specific connector.
+#
+connect.log.pattern=[%d] %p %X{connector.context}%m (%c:%L)%n
+
+log4j.appender.stdout.layout.ConversionPattern=${connect.log.pattern}
+log4j.appender.connectAppender.layout.ConversionPattern=${connect.log.pattern}
+
+log4j.logger.org.apache.zookeeper=ERROR
+log4j.logger.org.reflections=ERROR

etl/debezium-jdbc/log4j.properties

@@ -0,0 +1,23 @@
+kafka.logs.dir=logs
+
+log4j.rootLogger=INFO, stdout, appender
+
+# Disable excessive reflection warnings - KAFKA-5229
+log4j.logger.org.reflections=ERROR
+
+# Disable excessive logging by consumers and producers.
+log4j.logger.org.apache.kafka=ERROR
+log4j.logger.org.apache.kafka.clients.consumer=WARN
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.threshold=INFO
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n
+
+
+log4j.appender.appender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.appender.DatePattern='.'yyyy-MM-dd-HH
+log4j.appender.appender.File=${kafka.logs.dir}/connect-service.log
+log4j.appender.appender.layout=org.apache.log4j.PatternLayout
+log4j.appender.appender.layout.ConversionPattern=%d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n
+

etl/debezium-postgis/15-3.3/Dockerfile

@@ -44,4 +44,6 @@ COPY --from=build /usr/share/postgresql/$PG_MAJOR/extension/decoderbufs.control
 COPY postgresql.conf.sample /usr/share/postgresql/postgresql.conf.sample
 
 # Copy the script which will initialize the replication permissions
-COPY /docker-entrypoint-initdb.d /docker-entrypoint-initdb.d
+COPY /docker-entrypoint-initdb.d/init-permissions.sh /docker-entrypoint-initdb.d/init-permissions.sh
+
+RUN chmod +x /docker-entrypoint-initdb.d/init-permissions.sh

etl/docker-compose.yaml

@@ -19,7 +19,7 @@ services:
  build:
  context: debezium-postgis/15-3.3
  ports:
- - "5432:5432"
+ - "7432:5432"
  environment:
  - POSTGRES_USER=postgres
  - POSTGRES_PASSWORD=postgres

openshift/templates/etl/debezium-jdbc.build.yaml

@@ -0,0 +1,101 @@
+---
+kind: Template
+apiVersion: template.openshift.io/v1
+metadata:
+ name: "${API_NAME}-build-template"
+ creationTimestamp: 
+objects:
+- kind: ImageStream
+ apiVersion: v1
+ metadata:
+ name: "${API_NAME}"
+- kind: BuildConfig
+ apiVersion: v1
+ metadata:
+ name: "${API_NAME}-build"
+ labels:
+ app: "${API_NAME}-build"
+ spec:
+ runPolicy: Serial
+ source:
+ type: Git
+ git:
+ uri: "${GIT_REPO_URL}"
+ ref: "${GIT_REF}"
+ contextDir: "${SOURCE_CONTEXT_DIR}"
+ strategy:
+ type: Docker
+ dockerStrategy:
+ dockerfilePath: "${DOCKER_FILE_PATH}"
+ env:
+ - name: "DEBEZIUM_VERSION"
+ value: "${DEBEZIUM_VERSION}"
+ - name: "ORACLE_JDBC_VERSION"
+ value: "${ORACLE_JDBC_VERSION}"
+ - name: "ORACLE_JDBC_MD5"
+ value: "${ORACLE_JDBC_MD5}"
+ - name: "KAFKA_JDBC_VERSION"
+ value: "${KAFKA_JDBC_VERSION}"
+ - name: "ORACLE_DB_HOST"
+ value: "${ORACLE_DB_HOST}"
+ - name: "ORACLE_DB_PORT"
+ value: "${ORACLE_DB_PORT}" 
+
+ output:
+ to:
+ kind: ImageStreamTag
+ name: "${API_NAME}:latest"
+parameters:
+- name: API_NAME
+ displayName: Name
+ description: The name assigned to all of the resources defined in this template.
+ required: true
+ value: debezium-jdbc
+- name: GIT_REPO_URL
+ displayName: Git Repo URL
+ description: The URL to your GIT repo.
+ required: true
+ value: https://github.com/bcgov/nr-epd-digital-services
+- name: GIT_REF
+ displayName: Git Reference
+ description: The git reference or branch.
+ required: true
+ value: feat/srs-26
+- name: SOURCE_CONTEXT_DIR
+ displayName: Source Context Directory
+ description: The source context directory.
+ required: false
+ value: etl/debezium-jdbc
+- name: DOCKER_FILE_PATH
+ displayName: Docker File Path
+ description: The path to the docker file defining the build.
+ required: false
+ value: "Dockerfile"
+- name: DEBEZIUM_VERSION
+ displayName: Debezium Version
+ description: Debezium image version.
+ required: false
+ value: "1.9"
+- name: ORACLE_JDBC_VERSION
+ displayName: Oracle JDBC Version
+ description: Oracle JDBC Version.
+ required: false
+ value: "21.6.0.0"
+- name: ORACLE_JDBC_MD5
+ displayName: Oracle JDBC Checksum
+ description: Oracle JDBC File Checksum.
+ required: false
+ value: "312e6f4ec9932bbf74a4461669970c4b"
+- name: KAFKA_JDBC_VERSION
+ displayName: Kafka JDBC Version
+ description: Kafka Jdbc Version.
+ required: false
+ value: "5.3.1" 
+- name: ORACLE_DB_HOST
+ displayName: Oracle DB Host
+ description: Oracle Database Host Name
+ required: true
+- name: ORACLE_DB_PORT
+ displayName: Oracle DB Port
+ description: Oracle Database Port.
+ required: true