build and deployment file changes

etl/ReadMe.md

@@ -37,18 +37,60 @@ jdbc:oracle:thin:@(description=(address=(protocol=tcps)(host=172.18.215.225.nip.
 
 # Create sample data in postgresql using init.sql
 
+# OpenShift Build and Deploy process
+
+## 1. Build custom postgis
+
+```
+ oc process -f debezium-postgis.build.yaml |oc apply -f - 
+```
+
+## 2. Build kafka jdbc connect component
+
+```
+ oc process -f debezium-jdbc.build.yaml --param-file=.env |oc apply -f - 
+```
+
+## 3. Build kafka broker.
+```
+ oc process -f debezium-kafka.build.yaml |oc apply -f - 
+```
+
+## 4. Deploy postgis (switch to correct project for env before this step)
+```
+ oc process -f debezium-postgis.deploy.yaml |oc apply -f - 
+```
+## 5. Deploy zookeeper
+```
+ oc process -f debezium-zookeeper.deploy.yaml |oc apply -f - 
+```
+## 6. Deploy kafka
+```
+ oc process -f debezium-kafka.deploy.yaml |oc apply -f - 
+```
+## 7. Deploy kafka-jdbc connect.
+```
+ oc process -f debezium-jdbc.deploy.yaml |oc apply -f - 
+```
 
 # Register postgresql connector
 ```
 curl -H "Content-Type: application/json" -d @register-postgres-source-connector.json http://localhost:8083/connectors/ 
 
+curl -H "Content-Type: application/json" -d @register-postgres-source-connector.json https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/
+
+
+
 ```
 
 # Register Oracle jdbc sink connector
 
 ```
 curl -H "Content-Type: application/json" -d @register-oracle-jdbc-sink-connector.json http://localhost:8083/connectors/
 
+curl -H "Content-Type: application/json" -d @register-oracle-jdbc-sink-connector.json https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/
+
+
 ```
 
 # Test with a kafka console consumer.
@@ -67,6 +109,16 @@ curl -X DELETE localhost:8083/connectors/<connector-name>
 
 curl -X DELETE localhost:8083/connectors/oracle-jdbc-sink-connector
 curl -X DELETE localhost:8083/connectors/postgres-source-connector
-``` 
+
+curl -X DELETE https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/oracle-jdbc-sink-connector
+curl -X DELETE https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/postgres-source-connector
+
+``` 
+
+# Get all connectors registered.
+```
+curl localhost:8083/connectors/
+curl https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors
+```
 
 

etl/debezium-kafka/Dockerfile

@@ -1,5 +1,3 @@
-ARG DEBEZIUM_VERSION
-
 
 FROM debezium/kafka:1.9
 

etl/register-oracle-jdbc-sink-connector.json

@@ -6,7 +6,7 @@
  "topics.regex": "dbserver1.public.(.*)",
  "connection.url": "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCPS)(PORT=1543)(HOST=nrcdb01.bcgov))(CONNECT_DATA=(SERVICE_NAME=SD57387.NRS.BCGOV))(SECURITY=(ssl_server_cert_dn=\"CN=nrcdb01.bcgov\")))",
  "connection.user": "JAITHOMA",
- "connection.password": "jaithoma#1020",
+ "connection.password": "xxxxx",
  "dialect.name": "OracleDatabaseDialect",
  "security.protocol":"SSL",
  "ssl.enabled.protocols": "TLSv1.2,TLSv1.1",
@@ -16,7 +16,9 @@
  "table.name.format" : "SIS.${topic}", 
  "delete.enabled": "true",
  "pk.mode": "record_key",
- "insert.mode": "upsert", 
+ "insert.mode": "upsert",
+ "max.retries": "30",
+ "retry.backoff.ms": "10000",
  "quote.sql.identifiers": "never",
  "transforms":"route,ConvertCreatedTimestamp,ConvertUpdatedTimestamp,topicCase",
  "transforms.route.type": "org.apache.kafka.connect.transforms.RegexRouter",

openshift/templates/etl/debezium-kafka.build.yaml

@@ -0,0 +1,59 @@
+---
+kind: Template
+apiVersion: template.openshift.io/v1
+metadata:
+ name: "${API_NAME}-build-template"
+ creationTimestamp: 
+objects:
+- kind: ImageStream
+ apiVersion: v1
+ metadata:
+ name: "${API_NAME}"
+- kind: BuildConfig
+ apiVersion: v1
+ metadata:
+ name: "${API_NAME}-build"
+ labels:
+ app: "${API_NAME}-build"
+ spec:
+ runPolicy: Serial
+ source:
+ type: Git
+ git:
+ uri: "${GIT_REPO_URL}"
+ ref: "${GIT_REF}"
+ contextDir: "${SOURCE_CONTEXT_DIR}"
+ strategy:
+ type: Docker
+ dockerStrategy:
+ dockerfilePath: "${DOCKER_FILE_PATH}"
+ output:
+ to:
+ kind: ImageStreamTag
+ name: "${API_NAME}:latest"
+parameters:
+- name: API_NAME
+ displayName: Name
+ description: The name assigned to all of the resources defined in this template.
+ required: true
+ value: debezium-kafka
+- name: GIT_REPO_URL
+ displayName: Git Repo URL
+ description: The URL to your GIT repo.
+ required: true
+ value: https://github.com/bcgov/nr-epd-digital-services
+- name: GIT_REF
+ displayName: Git Reference
+ description: The git reference or branch.
+ required: true
+ value: fix/task-164
+- name: SOURCE_CONTEXT_DIR
+ displayName: Source Context Directory
+ description: The source context directory.
+ required: false
+ value: etl/debezium-kafka
+- name: DOCKER_FILE_PATH
+ displayName: Docker File Path
+ description: The path to the docker file defining the build.
+ required: false
+ value: "Dockerfile"

openshift/templates/etl/debezium-kafka.deploy.yaml

@@ -26,6 +26,15 @@ objects:
  maxSurge: 25%
  triggers:
  - type: ConfigChange
+ - type: ImageChange
+ imageChangeParams:
+ automatic: true
+ containerNames:
+ - "${API_NAME}"
+ from:
+ kind: ImageStreamTag
+ namespace: "${IMAGE_NAMESPACE}"
+ name: "${API_NAME}:${TAG_NAME}"
  replicas: 1
  test: false
  selector:
@@ -42,7 +51,7 @@ objects:
  spec:
  containers:
  - name: "${API_NAME}" 
- image: debezium/kafka:1.9
+ image: "${API_NAME}"
  imagePullPolicy: Always
  ports:
  - containerPort: 9092
@@ -108,4 +117,4 @@ parameters:
  displayName: Environment TAG name
  description: The TAG name for this environment, e.g., dev, test, prod
  required: true
- value: dev
+ value: latest