From 3983ee56bcf4fcb0d5e7205a501349d401f01053 Mon Sep 17 00:00:00 2001 From: jaise-aot Date: Thu, 7 Dec 2023 13:41:54 -0800 Subject: [PATCH] build and deployment file changes --- etl/ReadMe.md | 54 ++++++++++++++++- etl/debezium-kafka/Dockerfile | 2 - etl/register-oracle-jdbc-sink-connector.json | 6 +- .../templates/etl/debezium-kafka.build.yaml | 59 +++++++++++++++++++ .../templates/etl/debezium-kafka.deploy.yaml | 13 +++- 5 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 openshift/templates/etl/debezium-kafka.build.yaml diff --git a/etl/ReadMe.md b/etl/ReadMe.md index 6808de55..7bc95c96 100644 --- a/etl/ReadMe.md +++ b/etl/ReadMe.md @@ -37,11 +37,50 @@ jdbc:oracle:thin:@(description=(address=(protocol=tcps)(host=172.18.215.225.nip. # Create sample data in postgresql using init.sql +# OpenShift Build and Deploy process + +## 1. Build custom postgis + +``` + oc process -f debezium-postgis.build.yaml |oc apply -f - +``` + +## 2. Build kafka jdbc connect component + +``` + oc process -f debezium-jdbc.build.yaml --param-file=.env |oc apply -f - +``` + +## 3. Build kafka broker. +``` + oc process -f debezium-kafka.build.yaml |oc apply -f - +``` + +## 4. Deploy postgis (switch to correct project for env before this step) +``` + oc process -f debezium-postgis.deploy.yaml |oc apply -f - +``` +## 5. Deploy zookeeper +``` + oc process -f debezium-zookeeper.deploy.yaml |oc apply -f - +``` +## 6. Deploy kafka +``` + oc process -f debezium-kafka.deploy.yaml |oc apply -f - +``` +## 7. Deploy kafka-jdbc connect. +``` + oc process -f debezium-jdbc.deploy.yaml |oc apply -f - +``` # Register postgresql connector ``` curl -H "Content-Type: application/json" -d @register-postgres-source-connector.json http://localhost:8083/connectors/ +curl -H "Content-Type: application/json" -d @register-postgres-source-connector.json https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/ + + + ``` # Register Oracle jdbc sink connector @@ -49,6 +88,9 @@ curl -H "Content-Type: application/json" -d @register-postgres-source-connector. ``` curl -H "Content-Type: application/json" -d @register-oracle-jdbc-sink-connector.json http://localhost:8083/connectors/ +curl -H "Content-Type: application/json" -d @register-oracle-jdbc-sink-connector.json https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/ + + ``` # Test with a kafka console consumer. @@ -67,6 +109,16 @@ curl -X DELETE localhost:8083/connectors/ curl -X DELETE localhost:8083/connectors/oracle-jdbc-sink-connector curl -X DELETE localhost:8083/connectors/postgres-source-connector -``` + +curl -X DELETE https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/oracle-jdbc-sink-connector +curl -X DELETE https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/postgres-source-connector + +``` + +# Get all connectors registered. +``` +curl localhost:8083/connectors/ +curl https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors +``` diff --git a/etl/debezium-kafka/Dockerfile b/etl/debezium-kafka/Dockerfile index 68f6e4a4..29d7ae26 100644 --- a/etl/debezium-kafka/Dockerfile +++ b/etl/debezium-kafka/Dockerfile @@ -1,5 +1,3 @@ -ARG DEBEZIUM_VERSION - FROM debezium/kafka:1.9 diff --git a/etl/register-oracle-jdbc-sink-connector.json b/etl/register-oracle-jdbc-sink-connector.json index b72a9285..0addcae8 100644 --- a/etl/register-oracle-jdbc-sink-connector.json +++ b/etl/register-oracle-jdbc-sink-connector.json @@ -6,7 +6,7 @@ "topics.regex": "dbserver1.public.(.*)", "connection.url": "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCPS)(PORT=1543)(HOST=nrcdb01.bcgov))(CONNECT_DATA=(SERVICE_NAME=SD57387.NRS.BCGOV))(SECURITY=(ssl_server_cert_dn=\"CN=nrcdb01.bcgov\")))", "connection.user": "JAITHOMA", - "connection.password": "jaithoma#1020", + "connection.password": "xxxxx", "dialect.name": "OracleDatabaseDialect", "security.protocol":"SSL", "ssl.enabled.protocols": "TLSv1.2,TLSv1.1", @@ -16,7 +16,9 @@ "table.name.format" : "SIS.${topic}", "delete.enabled": "true", "pk.mode": "record_key", - "insert.mode": "upsert", + "insert.mode": "upsert", + "max.retries": "30", + "retry.backoff.ms": "10000", "quote.sql.identifiers": "never", "transforms":"route,ConvertCreatedTimestamp,ConvertUpdatedTimestamp,topicCase", "transforms.route.type": "org.apache.kafka.connect.transforms.RegexRouter", diff --git a/openshift/templates/etl/debezium-kafka.build.yaml b/openshift/templates/etl/debezium-kafka.build.yaml new file mode 100644 index 00000000..b77e417b --- /dev/null +++ b/openshift/templates/etl/debezium-kafka.build.yaml @@ -0,0 +1,59 @@ +--- +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: "${API_NAME}-build-template" + creationTimestamp: +objects: +- kind: ImageStream + apiVersion: v1 + metadata: + name: "${API_NAME}" +- kind: BuildConfig + apiVersion: v1 + metadata: + name: "${API_NAME}-build" + labels: + app: "${API_NAME}-build" + spec: + runPolicy: Serial + source: + type: Git + git: + uri: "${GIT_REPO_URL}" + ref: "${GIT_REF}" + contextDir: "${SOURCE_CONTEXT_DIR}" + strategy: + type: Docker + dockerStrategy: + dockerfilePath: "${DOCKER_FILE_PATH}" + output: + to: + kind: ImageStreamTag + name: "${API_NAME}:latest" +parameters: +- name: API_NAME + displayName: Name + description: The name assigned to all of the resources defined in this template. + required: true + value: debezium-kafka +- name: GIT_REPO_URL + displayName: Git Repo URL + description: The URL to your GIT repo. + required: true + value: https://github.com/bcgov/nr-epd-digital-services +- name: GIT_REF + displayName: Git Reference + description: The git reference or branch. + required: true + value: fix/task-164 +- name: SOURCE_CONTEXT_DIR + displayName: Source Context Directory + description: The source context directory. + required: false + value: etl/debezium-kafka +- name: DOCKER_FILE_PATH + displayName: Docker File Path + description: The path to the docker file defining the build. + required: false + value: "Dockerfile" \ No newline at end of file diff --git a/openshift/templates/etl/debezium-kafka.deploy.yaml b/openshift/templates/etl/debezium-kafka.deploy.yaml index 5fc8c37b..4d2717c2 100644 --- a/openshift/templates/etl/debezium-kafka.deploy.yaml +++ b/openshift/templates/etl/debezium-kafka.deploy.yaml @@ -26,6 +26,15 @@ objects: maxSurge: 25% triggers: - type: ConfigChange + - type: ImageChange + imageChangeParams: + automatic: true + containerNames: + - "${API_NAME}" + from: + kind: ImageStreamTag + namespace: "${IMAGE_NAMESPACE}" + name: "${API_NAME}:${TAG_NAME}" replicas: 1 test: false selector: @@ -42,7 +51,7 @@ objects: spec: containers: - name: "${API_NAME}" - image: debezium/kafka:1.9 + image: "${API_NAME}" imagePullPolicy: Always ports: - containerPort: 9092 @@ -108,4 +117,4 @@ parameters: displayName: Environment TAG name description: The TAG name for this environment, e.g., dev, test, prod required: true - value: dev \ No newline at end of file + value: latest \ No newline at end of file