From 4a7917a3014a44125385d460b01606a0551d738b Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 06:05:11 +0000
Subject: [PATCH 001/220] Add .gitlab-ci.yml

---
 .gitlab-ci.yml | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 .gitlab-ci.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 000000000..3074a761a
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,41 @@
+# You can copy and paste this template into a new `.gitlab-ci.yml` file.
+# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword.
+#
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Scala.gitlab-ci.yml
+
+# Official OpenJDK Java image. Look for the different tagged releases at
+# https://hub.docker.com/_/openjdk/ . A Java image is not required
+# but an image with a JVM speeds up the build a bit.
+image: openjdk:8
+
+before_script:
+  # Enable the usage of sources over https
+  - apt-get update -yqq
+  - apt-get install apt-transport-https -yqq
+  # Add keyserver for SBT
+  - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
+  - mkdir -p /root/.gnupg
+  - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
+  - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
+  # Install SBT
+  - apt-get update -yqq
+  - apt-get install sbt -yqq
+  # Log the sbt version
+  - sbt sbtVersion
+
+test:
+  script:
+    # Execute your project's tests
+    - sbt clean test
+
+build:
+  script:
+    # Execute your project's tests
+    - sbt clean compile test
+
+deploy:
+  stage: deploy
+  script: echo "Define your deployment script!"

From b1d84592ec5267f96eb23d918f9f786ef3d29b86 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 06:14:59 +0000
Subject: [PATCH 002/220] Updated .gitlab-ci.yml file

---
 .gitlab-ci.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3074a761a..e467986e0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,11 +26,6 @@ before_script:
   # Log the sbt version
   - sbt sbtVersion
 
-test:
-  script:
-    # Execute your project's tests
-    - sbt clean test
-
 build:
   script:
     # Execute your project's tests

From 1205e60b5d3c406d08c43ba1de1a10dbc661c9b4 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 06:41:13 +0000
Subject: [PATCH 003/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e467986e0..f0c9864ed 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -30,6 +30,25 @@ build:
   script:
     # Execute your project's tests
     - sbt clean compile test
+    - sbt universal:packageBin
+
+package:
+  script:
+    # Execute your project's tests
+   - ls
+   - unzip ingest/target/universal/*.zip -d ps-publish
+   - ls
+   - mv ps-publish/hydra-ingest*/* ps-publish
+   - rm -rf ps-publish/hydra-ingest*
+   - cd ps-publish/bin
+   - ls -la
+   - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic-agent-4.4.0.jar
+   - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic.yml
+   - cd ../../
+   - echo %build.number% > VERSION
+   - tar czf %env.ARTIFACT_NAME%-%build.number%.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
+   - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-%build.number%.tgz']"
+
 
 deploy:
   stage: deploy

From 726312a664f0838eb210701825a4c0bdea8c112f Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 06:49:08 +0000
Subject: [PATCH 004/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f0c9864ed..fe70b6487 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -11,6 +11,12 @@
 # but an image with a JVM speeds up the build a bit.
 image: openjdk:8
 
+stages:
+  - build
+  - package
+  - deploy
+
+
 before_script:
   # Enable the usage of sources over https
   - apt-get update -yqq
@@ -26,13 +32,16 @@ before_script:
   # Log the sbt version
   - sbt sbtVersion
 
+
 build:
+  stage: build
   script:
     # Execute your project's tests
     - sbt clean compile test
     - sbt universal:packageBin
 
 package:
+  stage: package
   script:
     # Execute your project's tests
    - ls
@@ -49,7 +58,6 @@ package:
    - tar czf %env.ARTIFACT_NAME%-%build.number%.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-%build.number%.tgz']"
 
-
 deploy:
   stage: deploy
   script: echo "Define your deployment script!"

From 72e38ecf76e8b18dba285d778fc6cec6c5659c17 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 06:58:03 +0000
Subject: [PATCH 005/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 41 +++++++++++++----------------------------
 1 file changed, 13 insertions(+), 28 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fe70b6487..0e8b92e55 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -11,12 +11,6 @@
 # but an image with a JVM speeds up the build a bit.
 image: openjdk:8
 
-stages:
-  - build
-  - package
-  - deploy
-
-
 before_script:
   # Enable the usage of sources over https
   - apt-get update -yqq
@@ -39,25 +33,16 @@ build:
     # Execute your project's tests
     - sbt clean compile test
     - sbt universal:packageBin
-
-package:
-  stage: package
-  script:
-    # Execute your project's tests
-   - ls
-   - unzip ingest/target/universal/*.zip -d ps-publish
-   - ls
-   - mv ps-publish/hydra-ingest*/* ps-publish
-   - rm -rf ps-publish/hydra-ingest*
-   - cd ps-publish/bin
-   - ls -la
-   - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic-agent-4.4.0.jar
-   - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic.yml
-   - cd ../../
-   - echo %build.number% > VERSION
-   - tar czf %env.ARTIFACT_NAME%-%build.number%.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
-   - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-%build.number%.tgz']"
-
-deploy:
-  stage: deploy
-  script: echo "Define your deployment script!"
+    - ls
+    - unzip ingest/target/universal/*.zip -d ps-publish
+    - ls
+    - mv ps-publish/hydra-ingest*/* ps-publish
+    - rm -rf ps-publish/hydra-ingest*
+    - cd ps-publish/bin
+    - ls -la
+    - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic-agent-4.4.0.jar
+    - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic.yml
+    - cd ../../
+    - echo %build.number% > VERSION
+    - tar czf %env.ARTIFACT_NAME%-%build.number%.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
+    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-%build.number%.tgz']"

From 8f3e17cd75ef5d87a09d04f2254212749ac4d801 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 07:13:42 +0000
Subject: [PATCH 006/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0e8b92e55..b0aa46cef 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -31,8 +31,11 @@ build:
   stage: build
   script:
     # Execute your project's tests
+    # Clean compile Test
     - sbt clean compile test
+    #Package
     - sbt universal:packageBin
+    # Zip Artifact with Dependency Libs
     - ls
     - unzip ingest/target/universal/*.zip -d ps-publish
     - ls
@@ -46,3 +49,18 @@ build:
     - echo %build.number% > VERSION
     - tar czf %env.ARTIFACT_NAME%-%build.number%.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-%build.number%.tgz']"
+    # Md5sum and Publish Md5Sum Artifact
+    - md5sum %env.ARTIFACT_NAME%-%build.number%.tgz > %env.ARTIFACT_NAME%-%build.number%.tgz.md5
+    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-%build.number%.tgz.md5']"
+    # Docker Build and Docker Push
+    - pwd
+    - ls -lh
+    - docker build -t %image.name% -f Dockerfile.new .
+    - docker tag %image.name% harbor.vnerd.com/library/%image.name%:%build.number%
+    - docker push harbor.vnerd.com/library/%image.name%:%build.number%
+    - docker tag %image.name% harbor.vnerd.com/library/%image.name%:latest
+    - docker push harbor.vnerd.com/library/%image.name%:latest
+    # Update VERSION File
+    - CURYEAR=$(date +%Y)
+    - CURMONTH=$(date +%-m)
+    - echo -n "${CURYEAR}.${CURMONTH}.%env.BUILD_NUMBER%.%teamcity.build.id%" > VERSION
\ No newline at end of file

From 4ad580870ecb9828d9e3d92c4cffd860ca16d87b Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 07:33:01 +0000
Subject: [PATCH 007/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b0aa46cef..0f490ac6d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,6 +25,8 @@ before_script:
   - apt-get install sbt -yqq
   # Log the sbt version
   - sbt sbtVersion
+  # Install docker 
+  - sudo apt install docker-ce
 
 
 build:

From bb0a0290df26c6b94a7e012298ea47dd69113225 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 07:35:40 +0000
Subject: [PATCH 008/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0f490ac6d..df435a9c9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,7 +26,7 @@ before_script:
   # Log the sbt version
   - sbt sbtVersion
   # Install docker 
-  - sudo apt install docker-ce
+  - apt install docker-ce
 
 
 build:

From c024b8425df48a1756e4b6a326558fd67fa847c2 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 07:38:26 +0000
Subject: [PATCH 009/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index df435a9c9..45774e732 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,7 +26,7 @@ before_script:
   # Log the sbt version
   - sbt sbtVersion
   # Install docker 
-  - apt install docker-ce
+  - apt install docker
 
 
 build:

From 44f2004fc11690e46b44ff4c0b90a5ca23137612 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 07:39:29 +0000
Subject: [PATCH 010/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 45774e732..cf5e70db8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,7 +26,7 @@ before_script:
   # Log the sbt version
   - sbt sbtVersion
   # Install docker 
-  - apt install docker
+  - apt-get install docker -yqq
 
 
 build:

From a68ff9e9627b2c146d7e7286eb7f3c30764bcdfe Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 08:24:02 +0000
Subject: [PATCH 011/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cf5e70db8..04f0e40df 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -11,6 +11,11 @@
 # but an image with a JVM speeds up the build a bit.
 image: openjdk:8
 
+variables:
+  ARTIFACT_NAME: hydra-publish
+  IMAGE_NAME: hydra
+
+
 before_script:
   # Enable the usage of sources over https
   - apt-get update -yqq
@@ -48,21 +53,21 @@ build:
     - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic-agent-4.4.0.jar
     - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic.yml
     - cd ../../
-    - echo %build.number% > VERSION
-    - tar czf %env.ARTIFACT_NAME%-%build.number%.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
-    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-%build.number%.tgz']"
+    - echo $CI_PIPELINE_IID > VERSION
+    - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
+    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz']"
     # Md5sum and Publish Md5Sum Artifact
-    - md5sum %env.ARTIFACT_NAME%-%build.number%.tgz > %env.ARTIFACT_NAME%-%build.number%.tgz.md5
-    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-%build.number%.tgz.md5']"
+    - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
+    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
     # Docker Build and Docker Push
     - pwd
     - ls -lh
-    - docker build -t %image.name% -f Dockerfile.new .
-    - docker tag %image.name% harbor.vnerd.com/library/%image.name%:%build.number%
-    - docker push harbor.vnerd.com/library/%image.name%:%build.number%
-    - docker tag %image.name% harbor.vnerd.com/library/%image.name%:latest
-    - docker push harbor.vnerd.com/library/%image.name%:latest
+    - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
+    - docker tag ${IMAGE_NAME} harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
+    - docker push harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
+    - docker tag ${IMAGE_NAME} harbor.vnerd.com/library/${IMAGE_NAME}:latest
+    - docker push harbor.vnerd.com/library/${IMAGE_NAME}:latest
     # Update VERSION File
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)
-    - echo -n "${CURYEAR}.${CURMONTH}.%env.BUILD_NUMBER%.%teamcity.build.id%" > VERSION
\ No newline at end of file
+    - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID} > VERSION
\ No newline at end of file

From c4b666559701ee504fc7be313c3fbc31bd6af6f5 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 08:51:23 +0000
Subject: [PATCH 012/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 04f0e40df..6df50668e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,6 +32,7 @@ before_script:
   - sbt sbtVersion
   # Install docker 
   - apt-get install docker -yqq
+  - docker --version
 
 
 build:

From e89d9f01b1e606d9b4ca41de30fbf20aef906eb3 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 11:09:06 +0000
Subject: [PATCH 013/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6df50668e..470a0839f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,9 +32,9 @@ before_script:
   - sbt sbtVersion
   # Install docker 
   - apt-get install docker -yqq
+  - apt-get install apt-utils
   - docker --version
 
-
 build:
   stage: build
   script:

From a5b9e45c0209a957650f56bdc2a7f342ab66141d Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 11:11:38 +0000
Subject: [PATCH 014/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 470a0839f..7e0909b0c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -31,8 +31,8 @@ before_script:
   # Log the sbt version
   - sbt sbtVersion
   # Install docker 
+  - apt-get install apt-utils 
   - apt-get install docker -yqq
-  - apt-get install apt-utils
   - docker --version
 
 build:

From 4c2a657cbb3a0f5ddd603af37dcb66718109a387 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 23 Jan 2024 08:19:35 +0000
Subject: [PATCH 015/220] Updated .gitlab-ci.yml file

---
 .gitlab-ci.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7e0909b0c..a228b437e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,7 +32,10 @@ before_script:
   - sbt sbtVersion
   # Install docker 
   - apt-get install apt-utils 
-  - apt-get install docker -yqq
+  - apt-get update -yqq
+  - apt-get install docker.io -yqq
+  - curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+  - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
   - docker --version
 
 build:

From 5b092d1530f729e287e41ec31ce66fa157391596 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 23 Jan 2024 09:04:42 +0000
Subject: [PATCH 016/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a228b437e..cbc16ab20 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -37,6 +37,7 @@ before_script:
   - curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
   - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
   - docker --version
+  - sudo service docker start
 
 build:
   stage: build

From e0852fb980841fbbdfe32cf83e9364266cfa0184 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 23 Jan 2024 10:03:43 +0000
Subject: [PATCH 017/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cbc16ab20..7c430421e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -37,7 +37,7 @@ before_script:
   - curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
   - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
   - docker --version
-  - sudo service docker start
+  - service docker start
 
 build:
   stage: build

From 78c93859de046a8a9e65f6065d782d398ecea24f Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 23 Jan 2024 14:33:35 +0000
Subject: [PATCH 018/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7c430421e..e5251fb73 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -69,6 +69,7 @@ build:
     - ls -lh
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
     - docker tag ${IMAGE_NAME} harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
+    - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD harbor.vnerd.com
     - docker push harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
     - docker tag ${IMAGE_NAME} harbor.vnerd.com/library/${IMAGE_NAME}:latest
     - docker push harbor.vnerd.com/library/${IMAGE_NAME}:latest

From af5ab4cb6bc8ddd792735c26b7fc1554bbf333a3 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 07:31:16 +0000
Subject: [PATCH 019/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e5251fb73..fc5bc5acf 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -69,10 +69,10 @@ build:
     - ls -lh
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
     - docker tag ${IMAGE_NAME} harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
-    - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD harbor.vnerd.com
-    - docker push harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
+    #- docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD harbor.vnerd.com
+    - docker push $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
     - docker tag ${IMAGE_NAME} harbor.vnerd.com/library/${IMAGE_NAME}:latest
-    - docker push harbor.vnerd.com/library/${IMAGE_NAME}:latest
+    - docker push $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD harbor.vnerd.com/library/${IMAGE_NAME}:latest
     # Update VERSION File
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)

From f2d796178fe96cad2313ae14f1e01ff66d997e1b Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 11:00:40 +0000
Subject: [PATCH 020/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 73 +++++++++++++++++++-------------------------------
 1 file changed, 27 insertions(+), 46 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fc5bc5acf..f2ee03e49 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,56 +1,34 @@
-# You can copy and paste this template into a new `.gitlab-ci.yml` file.
-# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword.
-#
-# To contribute improvements to CI/CD templates, please follow the Development guide at:
-# https://docs.gitlab.com/ee/development/cicd/templates.html
-# This specific template is located at:
-# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Scala.gitlab-ci.yml
-
-# Official OpenJDK Java image. Look for the different tagged releases at
-# https://hub.docker.com/_/openjdk/ . A Java image is not required
-# but an image with a JVM speeds up the build a bit.
 image: openjdk:8
 
 variables:
   ARTIFACT_NAME: hydra-publish
   IMAGE_NAME: hydra
-
+  DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
 
 before_script:
-  # Enable the usage of sources over https
-  - apt-get update -yqq
-  - apt-get install apt-transport-https -yqq
-  # Add keyserver for SBT
-  - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
-  - mkdir -p /root/.gnupg
-  - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
-  - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
-  # Install SBT
   - apt-get update -yqq
-  - apt-get install sbt -yqq
-  # Log the sbt version
-  - sbt sbtVersion
-  # Install docker 
-  - apt-get install apt-utils 
-  - apt-get update -yqq
-  - apt-get install docker.io -yqq
+  - apt-get install -yqq apt-transport-https apt-utils sbt docker.io
   - curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
   - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
   - docker --version
   - service docker start
 
+stages:
+  - build
+  - package
+  - publish
+  - finalize
+
 build:
   stage: build
   script:
-    # Execute your project's tests
-    # Clean compile Test
     - sbt clean compile test
-    #Package
+
+package:
+  stage: package
+  script:
     - sbt universal:packageBin
-    # Zip Artifact with Dependency Libs
-    - ls
-    - unzip ingest/target/universal/*.zip -d ps-publish
-    - ls
+    - ls ingest/target/universal/*.zip | xargs -I {} unzip {} -d ps-publish
     - mv ps-publish/hydra-ingest*/* ps-publish
     - rm -rf ps-publish/hydra-ingest*
     - cd ps-publish/bin
@@ -61,19 +39,22 @@ build:
     - echo $CI_PIPELINE_IID > VERSION
     - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz']"
-    # Md5sum and Publish Md5Sum Artifact
+
+publish:
+  stage: publish
+  script:
     - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
-    # Docker Build and Docker Push
-    - pwd
-    - ls -lh
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
-    - docker tag ${IMAGE_NAME} harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
-    #- docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD harbor.vnerd.com
-    - docker push $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
-    - docker tag ${IMAGE_NAME} harbor.vnerd.com/library/${IMAGE_NAME}:latest
-    - docker push $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD harbor.vnerd.com/library/${IMAGE_NAME}:latest
-    # Update VERSION File
+    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
+
+finalize:
+  stage: finalize
+  script:
+    - docker push ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
+    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest
+    - docker push ${DOCKER_REGISTRY_IMAGE}:latest
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)
-    - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID} > VERSION
\ No newline at end of file
+    - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID}" > VERSION
+

From 54f86b8d7abe46a96c93585237bebac3c9260cc8 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 11:07:36 +0000
Subject: [PATCH 021/220] Updated .gitlab-ci.yml file

---
 .gitlab-ci.yml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f2ee03e49..d2cc1765a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,11 +7,13 @@ variables:
 
 before_script:
   - apt-get update -yqq
-  - apt-get install -yqq apt-transport-https apt-utils sbt docker.io
-  - curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-  - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
-  - docker --version
-  - service docker start
+  - apt-get install -yqq apt-transport-https apt-utils
+  - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
+  - mkdir -p /root/.gnupg
+  - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
+  - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
+  - apt-get update -yqq
+  - apt-get install -yqq sbt docker.io
 
 stages:
   - build

From 02e5e280e52e91448c010a6667ed9e798cacbb62 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 11:19:02 +0000
Subject: [PATCH 022/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d2cc1765a..f517ac825 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,7 +7,7 @@ variables:
 
 before_script:
   - apt-get update -yqq
-  - apt-get install -yqq apt-transport-https apt-utils
+  - apt-get install -yqq apt-transport-https apt-utils sbt docker.io
   - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
   - mkdir -p /root/.gnupg
   - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
@@ -59,4 +59,3 @@ finalize:
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)
     - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID}" > VERSION
-

From ddd7b38ebdb2ff18acd844b6261b1cbd0bc0cf40 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 11:32:32 +0000
Subject: [PATCH 023/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f517ac825..f0b8a6030 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,7 +7,7 @@ variables:
 
 before_script:
   - apt-get update -yqq
-  - apt-get install -yqq apt-transport-https apt-utils sbt docker.io
+  - apt-get install -yqq apt-transport-https apt-utils
   - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
   - mkdir -p /root/.gnupg
   - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823

From c40f5005222157585e1aaeb59296bb958f771001 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 14:37:36 +0000
Subject: [PATCH 024/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f0b8a6030..042b5a9c2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,6 +14,7 @@ before_script:
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   - apt-get update -yqq
   - apt-get install -yqq sbt docker.io
+  - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
 
 stages:
   - build
@@ -53,6 +54,7 @@ publish:
 finalize:
   stage: finalize
   script:
+    - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
     - docker push ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest
     - docker push ${DOCKER_REGISTRY_IMAGE}:latest

From 928174c1ed7098ea6a0e78a8ec0509d9c59e39b2 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 14:41:15 +0000
Subject: [PATCH 025/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 042b5a9c2..d98edb6b9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,8 @@ before_script:
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   - apt-get update -yqq
   - apt-get install -yqq sbt docker.io
-  - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
+  - docker login harbor.vnerd.com -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD}
+  - echo "${DOCKER_REGISTRY_USERNAME}"
 
 stages:
   - build

From cfc413abba508a4cb632936bf258e1d183fba39e Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 14:50:58 +0000
Subject: [PATCH 026/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d98edb6b9..65074d62a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,8 +14,7 @@ before_script:
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   - apt-get update -yqq
   - apt-get install -yqq sbt docker.io
-  - docker login harbor.vnerd.com -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD}
-  - echo "${DOCKER_REGISTRY_USERNAME}"
+  - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com
 
 stages:
   - build

From de327d43d80b95b771b73dec92cd843e118d46cc Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 14:56:57 +0000
Subject: [PATCH 027/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 65074d62a..14aa1b5c3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,7 @@ before_script:
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   - apt-get update -yqq
   - apt-get install -yqq sbt docker.io
-  - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com
+
 
 stages:
   - build
@@ -23,9 +23,17 @@ stages:
   - finalize
 
 build:
+  #stage: build
+ #script:
+   # - sbt clean compile test
+
   stage: build
+  image: harbor.vnerd.com/proxy/library/docker:19.03.0
+  services:
+    - harbor.vnerd.com/proxy/library/docker:19.03.5-dind
   script:
-    - sbt clean compile test
+    - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com
+    - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
 
 package:
   stage: package
@@ -53,7 +61,11 @@ publish:
 
 finalize:
   stage: finalize
+  image: harbor.vnerd.com/proxy/library/docker:19.03.0
+  services:
+    - harbor.vnerd.com/proxy/library/docker:19.03.5-dind
   script:
+    - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com
     - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
     - docker push ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest

From acd6027c6c1564ff244ce145ef3f2958bb8be6e5 Mon Sep 17 00:00:00 2001
From: Yuvi0710 <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 11:31:46 +0530
Subject: [PATCH 028/220] Updated

---
 .gitlab-ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 14aa1b5c3..d3661702f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,14 +6,14 @@ variables:
   DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
 
 before_script:
-  - apt-get update -yqq
-  - apt-get install -yqq apt-transport-https apt-utils
+  - apk update update -yqq
+  - apk-get install -yqq apt-transport-https apt-utils
   - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
   - mkdir -p /root/.gnupg
   - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
-  - apt-get update -yqq
-  - apt-get install -yqq sbt docker.io
+  - apk-get update -yqq
+  - apk-get install -yqq sbt docker.io
 
 
 stages:

From 9f8b99309f4fbab1b73add225daf21662c1fa250 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 06:05:09 +0000
Subject: [PATCH 029/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 31 ++++++++++++-------------------
 1 file changed, 12 insertions(+), 19 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d3661702f..c3a87a4bc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,15 +6,16 @@ variables:
   DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
 
 before_script:
-  - apk update update -yqq
-  - apk-get install -yqq apt-transport-https apt-utils
+  - apt-get update -yqq
+  - apt-get install -yqq apt-transport-https apt-utils sbt docker.io
   - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
   - mkdir -p /root/.gnupg
   - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
-  - apk-get update -yqq
-  - apk-get install -yqq sbt docker.io
-
+  - apt-get update -yqq
+  - apt-get install -yqq sbt docker.io
+  - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com
+  - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
 
 stages:
   - build
@@ -23,17 +24,9 @@ stages:
   - finalize
 
 build:
-  #stage: build
- #script:
-   # - sbt clean compile test
-
   stage: build
-  image: harbor.vnerd.com/proxy/library/docker:19.03.0
-  services:
-    - harbor.vnerd.com/proxy/library/docker:19.03.5-dind
   script:
-    - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com
-    - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
+    - sbt clean compile test
 
 package:
   stage: package
@@ -61,15 +54,15 @@ publish:
 
 finalize:
   stage: finalize
-  image: harbor.vnerd.com/proxy/library/docker:19.03.0
-  services:
-    - harbor.vnerd.com/proxy/library/docker:19.03.5-dind
   script:
-    - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com
-    - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
     - docker push ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest
     - docker push ${DOCKER_REGISTRY_IMAGE}:latest
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)
     - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID}" > VERSION
+
+
+
+
+

From c9b5076459a649fa1c4cd554daf1a7c2bd0856b7 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 06:06:48 +0000
Subject: [PATCH 030/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c3a87a4bc..ce984e3f1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,7 +7,7 @@ variables:
 
 before_script:
   - apt-get update -yqq
-  - apt-get install -yqq apt-transport-https apt-utils sbt docker.io
+  - apt-get install -yqq apt-transport-https apt-utils
   - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
   - mkdir -p /root/.gnupg
   - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823

From 203abdceea48ede48591b70401df33a34a9107b0 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 06:09:38 +0000
Subject: [PATCH 031/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ce984e3f1..c77f4c3ef 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,7 @@ before_script:
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   - apt-get update -yqq
   - apt-get install -yqq sbt docker.io
-  - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com
+  - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com || exit 1
   - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
 
 stages:

From efe3932cae8b7538dbeefc8591ee8112e3a97ef3 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 06:23:42 +0000
Subject: [PATCH 032/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c77f4c3ef..b7a431358 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,6 +4,9 @@ variables:
   ARTIFACT_NAME: hydra-publish
   IMAGE_NAME: hydra
   DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
+  DOCKER_REGISTRY_URL: "harbor.vnerd.com"
+  DOCKER_REGISTRY_USERNAME: $DOCKER_REGISTRY_USERNAME
+  DOCKER_REGISTRY_PASSWORD: $DOCKER_REGISTRY_PASSWORD
 
 before_script:
   - apt-get update -yqq
@@ -14,9 +17,8 @@ before_script:
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   - apt-get update -yqq
   - apt-get install -yqq sbt docker.io
-  - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com || exit 1
-  - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
-
+  - echo "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin $DOCKER_REGISTRY_URL
+  
 stages:
   - build
   - package

From 2c51475111e884417a84af1853b97a1cd3ac3235 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 06:27:56 +0000
Subject: [PATCH 033/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b7a431358..a4a48da26 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,6 +17,8 @@ before_script:
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   - apt-get update -yqq
   - apt-get install -yqq sbt docker.io
+  - echo "$DOCKER_REGISTRY_PASSWORD"
+  - echo "$DOCKER_REGISTRY_USERNAME"
   - echo "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin $DOCKER_REGISTRY_URL
   
 stages:

From f3776ca64ac5d742544f9f1098603ad543754ee0 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 06:44:32 +0000
Subject: [PATCH 034/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a4a48da26..120f1dea4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,7 +19,7 @@ before_script:
   - apt-get install -yqq sbt docker.io
   - echo "$DOCKER_REGISTRY_PASSWORD"
   - echo "$DOCKER_REGISTRY_USERNAME"
-  - echo "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin $DOCKER_REGISTRY_URL
+  - echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USER} --password-stdin
   
 stages:
   - build

From e926e4a3988eda224307a98ed18757cf8f7d6641 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 06:56:25 +0000
Subject: [PATCH 035/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 120f1dea4..be83fb501 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,7 +19,8 @@ before_script:
   - apt-get install -yqq sbt docker.io
   - echo "$DOCKER_REGISTRY_PASSWORD"
   - echo "$DOCKER_REGISTRY_USERNAME"
-  - echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USER} --password-stdin
+  #- echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
+  #- echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USER} --password-stdin
   
 stages:
   - build
@@ -48,8 +49,12 @@ package:
     - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz']"
 
+
 publish:
   stage: publish
+  image: harbor.vnerd.com/proxy/library/docker:cli
+  before_script:
+    - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
     - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"

From 14f4bed1c708261aab69d25a2b134f0392bebedb Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 07:12:55 +0000
Subject: [PATCH 036/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index be83fb501..78a69a68c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -23,11 +23,22 @@ before_script:
   #- echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USER} --password-stdin
   
 stages:
+  - prestep
   - build
   - package
   - publish
   - finalize
 
+prestep:
+  stage: prestep
+  image: harbor.vnerd.com/proxy/library/docker:cli
+  before_script:
+    - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
+  script:
+    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
+    - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
+    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
+
 build:
   stage: build
   script:

From b3b9c7425e5f4eab9972507338241815f3674ab4 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 07:15:14 +0000
Subject: [PATCH 037/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 78a69a68c..da8c69d12 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,8 +5,6 @@ variables:
   IMAGE_NAME: hydra
   DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
   DOCKER_REGISTRY_URL: "harbor.vnerd.com"
-  DOCKER_REGISTRY_USERNAME: $DOCKER_REGISTRY_USERNAME
-  DOCKER_REGISTRY_PASSWORD: $DOCKER_REGISTRY_PASSWORD
 
 before_script:
   - apt-get update -yqq

From e018baeb106d96b5a3ba7f3afdbef1d9db0c1415 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 08:05:47 +0000
Subject: [PATCH 038/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index da8c69d12..2bd362989 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,7 +4,7 @@ variables:
   ARTIFACT_NAME: hydra-publish
   IMAGE_NAME: hydra
   DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
-  DOCKER_REGISTRY_URL: "harbor.vnerd.com"
+  DOCKER_REGISTRY_URL: "harbor.vnerd.com/library"
 
 before_script:
   - apt-get update -yqq
@@ -31,7 +31,8 @@ prestep:
   stage: prestep
   image: harbor.vnerd.com/proxy/library/docker:cli
   before_script:
-    - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
+    - echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USERNAME} --password-stdin
+    #- echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .

From c1a6d8a8fe9fd043e6fcc6fb36e71f0591abbe3d Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 09:20:43 +0000
Subject: [PATCH 039/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2bd362989..5dda7b8e8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -30,6 +30,11 @@ stages:
 prestep:
   stage: prestep
   image: harbor.vnerd.com/proxy/library/docker:cli
+  variables:
+    DOCKER_HOST: tcp://localhost:2376
+    DOCKER_TLS_CERTDIR: "/certs"
+    DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
+    DOCKER_TLS_VERIFY: 1
   before_script:
     - echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USERNAME} --password-stdin
     #- echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"

From 60fe3c60113e0f4e7708cb9b1ca9f651c59c0fd7 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 09:33:20 +0000
Subject: [PATCH 040/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5dda7b8e8..ba82bf0f6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,6 +5,11 @@ variables:
   IMAGE_NAME: hydra
   DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
   DOCKER_REGISTRY_URL: "harbor.vnerd.com/library"
+  DOCKER_HOST: tcp://localhost:2375
+  DOCKER_TLS_CERTDIR: ''
+  DOCKER_DRIVER: overlay2
+  DOCKER_TLS_VERIFY: 1
+
 
 before_script:
   - apt-get update -yqq
@@ -29,12 +34,7 @@ stages:
 
 prestep:
   stage: prestep
-  image: harbor.vnerd.com/proxy/library/docker:cli
-  variables:
-    DOCKER_HOST: tcp://localhost:2376
-    DOCKER_TLS_CERTDIR: "/certs"
-    DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
-    DOCKER_TLS_VERIFY: 1
+  image: harbor.vnerd.com/proxy/library/docker:19.03.0
   before_script:
     - echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USERNAME} --password-stdin
     #- echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"

From 2750215cc97fb5ffddd56b2c12a02a2f8e9b9ad2 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 09:39:13 +0000
Subject: [PATCH 041/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ba82bf0f6..bef0e3e61 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,12 +5,11 @@ variables:
   IMAGE_NAME: hydra
   DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
   DOCKER_REGISTRY_URL: "harbor.vnerd.com/library"
-  DOCKER_HOST: tcp://localhost:2375
-  DOCKER_TLS_CERTDIR: ''
-  DOCKER_DRIVER: overlay2
+  DOCKER_HOST: tcp://localhost:2376
+  DOCKER_TLS_CERTDIR: "/certs"
+  DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
   DOCKER_TLS_VERIFY: 1
 
-
 before_script:
   - apt-get update -yqq
   - apt-get install -yqq apt-transport-https apt-utils
@@ -34,7 +33,10 @@ stages:
 
 prestep:
   stage: prestep
-  image: harbor.vnerd.com/proxy/library/docker:19.03.0
+  image: harbor.vnerd.com/proxy/library/docker:cli
+  tags:
+    - ps
+    - docker
   before_script:
     - echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USERNAME} --password-stdin
     #- echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"

From e968dfe55b66a0b3e6f87b433e14d2844b0d6837 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 09:42:49 +0000
Subject: [PATCH 042/220] Update .gitlab-ci.yml

---
 .gitlab-ci.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bef0e3e61..7601c9e18 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -70,6 +70,9 @@ package:
 publish:
   stage: publish
   image: harbor.vnerd.com/proxy/library/docker:cli
+  tags:
+    - ps
+    - docker
   before_script:
     - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
@@ -80,6 +83,12 @@ publish:
 
 finalize:
   stage: finalize
+  image: harbor.vnerd.com/proxy/library/docker:cli
+  tags:
+    - ps
+    - docker
+  before_script:
+    - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
     - docker push ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest

From ad4ceb80b9de3b23d4b06459ddc964c9be6b3578 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 09:55:49 +0000
Subject: [PATCH 043/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7601c9e18..c541f89ff 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,26 +25,11 @@ before_script:
   #- echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USER} --password-stdin
   
 stages:
-  - prestep
   - build
   - package
   - publish
   - finalize
 
-prestep:
-  stage: prestep
-  image: harbor.vnerd.com/proxy/library/docker:cli
-  tags:
-    - ps
-    - docker
-  before_script:
-    - echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USERNAME} --password-stdin
-    #- echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
-  script:
-    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
-    - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
-    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
-
 build:
   stage: build
   script:
@@ -66,7 +51,6 @@ package:
     - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz']"
 
-
 publish:
   stage: publish
   image: harbor.vnerd.com/proxy/library/docker:cli

From 84733396fc232c87683da33941f224cfb516da33 Mon Sep 17 00:00:00 2001
From: Aman Minz <aman-minz@pluralsight.com>
Date: Tue, 23 Jan 2024 21:03:06 +0530
Subject: [PATCH 044/220] ADAPT-1823: Add skipValidations query parameter in
 PUT /v2/metadata endpoint

---
 .../endpoints/TopicMetadataEndpoint.scala     |  12 +-
 .../hydra/kafka/model/SkipValidation.scala    |  41 ++
 .../kafka/programs/CreateTopicProgram.scala   |   7 +-
 .../KeyAndValueSchemaV2Validator.scala        | 227 +++++---
 .../serializers/TopicMetadataV2Parser.scala   |  28 +
 .../kafka/programs/SkipValidationSpec.scala   | 544 ++++++++++++++++++
 6 files changed, 786 insertions(+), 73 deletions(-)
 create mode 100644 ingestors/kafka/src/main/scala/hydra/kafka/model/SkipValidation.scala
 create mode 100644 ingestors/kafka/src/test/scala/hydra/kafka/programs/SkipValidationSpec.scala

diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/endpoints/TopicMetadataEndpoint.scala b/ingestors/kafka/src/main/scala/hydra/kafka/endpoints/TopicMetadataEndpoint.scala
index 1a8b3b469..5ffd73356 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/endpoints/TopicMetadataEndpoint.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/endpoints/TopicMetadataEndpoint.scala
@@ -16,7 +16,7 @@ import hydra.kafka.algebras.{MetadataAlgebra, TagsAlgebra}
 import hydra.kafka.consumer.KafkaConsumerProxy.{GetPartitionInfo, ListTopics, ListTopicsResponse, PartitionInfoResponse}
 import hydra.kafka.marshallers.HydraKafkaJsonSupport
 import hydra.kafka.model.TopicMetadataV2Request.Subject
-import hydra.kafka.model.{ContactMethod, DataClassification, MetadataOnlyRequest, Schemas, StreamTypeV2, TopicMetadataV2, TopicMetadataV2Key, TopicMetadataV2Request, TopicMetadataV2Response}
+import hydra.kafka.model.{MetadataOnlyRequest, Schemas, SkipValidation, TopicMetadataV2Request, TopicMetadataV2Response}
 import hydra.kafka.serializers.TopicMetadataV2Parser._
 import hydra.kafka.util.KafkaUtils
 import hydra.kafka.util.KafkaUtils.TopicDetails
@@ -29,7 +29,7 @@ import scala.collection.immutable.Map
 import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Failure, Success, Try}
-import akka.http.scaladsl.server.Directives.onComplete
+import akka.http.scaladsl.server.Directives.{onComplete, parameter}
 import akka.http.scaladsl.server.directives.Credentials
 import cats.data.NonEmptyList
 import hydra.avro.registry.SchemaRegistry
@@ -138,7 +138,9 @@ class TopicMetadataEndpoint[F[_]: Futurable](consumerProxy:ActorSelection,
           handleExceptions(exceptionHandler(startTime, method.value, topic)) {
             put {
               auth.mskAuth(KafkaAction.CreateTopic) { roleName =>
-                putV2Metadata(startTime, topic, roleName)
+                parameter(Symbol("skipValidations").as[List[SkipValidation]].?) { maybeSkipValidations =>
+                  putV2Metadata(startTime, topic, roleName, maybeSkipValidations)
+                }
               }
             }
           }
@@ -155,7 +157,7 @@ class TopicMetadataEndpoint[F[_]: Futurable](consumerProxy:ActorSelection,
       valueSchema.getOrElse(throw new SchemaParseException("Unable to get Value Schema, please create Value Schema in Schema Registry and try again")))
   }
 
-  private def putV2Metadata(startTime: Instant, topic: String, userRoleName: Option[RoleName]): Route = {
+  private def putV2Metadata(startTime: Instant, topic: String, userRoleName: Option[RoleName], maybeSkipValidations: Option[List[SkipValidation]]): Route = {
     extractMethod { method =>
       Subject.createValidated(topic) match {
         case Some(t) => {
@@ -179,7 +181,7 @@ class TopicMetadataEndpoint[F[_]: Futurable](consumerProxy:ActorSelection,
                       val req = TopicMetadataV2Request.fromMetadataOnlyRequest(schemas, mor)
                       onComplete(
                         Futurable[F].unsafeToFuture(createTopicProgram
-                          .createTopicFromMetadataOnly(t, req, withRequiredFields = true))
+                          .createTopicFromMetadataOnly(t, req, withRequiredFields = true, maybeSkipValidations))
                       ) {
                         case Failure(exception) => exception match {
                           case e @ (_:IncompatibleSchemaException | _:ValidationError) =>
diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/model/SkipValidation.scala b/ingestors/kafka/src/main/scala/hydra/kafka/model/SkipValidation.scala
new file mode 100644
index 000000000..8dbf7c204
--- /dev/null
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/model/SkipValidation.scala
@@ -0,0 +1,41 @@
+package hydra.kafka.model
+
+import akka.http.scaladsl.unmarshalling.Unmarshaller
+import enumeratum.{Enum, EnumEntry}
+import spray.json.DeserializationException
+
+import scala.collection.immutable
+
+sealed trait SkipValidation extends EnumEntry
+
+object SkipValidation extends Enum[SkipValidation] {
+
+  case object all extends SkipValidation
+  case object emptyKeyFields extends SkipValidation
+  case object keySchemaEvolution extends SkipValidation
+  case object valueSchemaEvolution extends SkipValidation
+  case object requiredDocField extends SkipValidation
+  case object requiredCreatedAtField extends SkipValidation
+  case object requiredUpdatedAtField extends SkipValidation
+  case object sameFieldsTypeMismatchInKeyValueSchemas extends SkipValidation
+  case object nullableFieldsInKeySchema extends SkipValidation
+  case object missingDefaultInNullableFieldsOfValueSchema extends SkipValidation
+  case object unsupportedLogicalTypeFieldsInKeySchema extends SkipValidation
+  case object unsupportedLogicalTypeFieldsInValueSchema extends SkipValidation
+
+  override val values: immutable.IndexedSeq[SkipValidation] = findValues
+
+  implicit val skipValidationUnmarshaller: Unmarshaller[String, List[SkipValidation]] =
+    Unmarshaller.strict[String, List[SkipValidation]] { commaSeparatedValidations =>
+      val stringValues = values.map(_.entryName)
+      val invalidValues = commaSeparatedValidations.split(",").filterNot(s => stringValues.contains(s) )
+      val validValues = commaSeparatedValidations.split(",").flatMap(s => values.find(v => v.entryName == s.trim)).toList
+
+      if (invalidValues.nonEmpty) {
+        throw DeserializationException(
+          s"Expected single or comma-separated values from enum[${values.mkString(", ")}] but received invalid value(s): ${invalidValues.mkString(",")}")
+      }
+
+      validValues
+    }
+}
diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/programs/CreateTopicProgram.scala b/ingestors/kafka/src/main/scala/hydra/kafka/programs/CreateTopicProgram.scala
index f7fe85b2b..8d656fd1d 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/programs/CreateTopicProgram.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/programs/CreateTopicProgram.scala
@@ -4,7 +4,7 @@ import cats.effect.{Bracket, ExitCase, Resource, Sync}
 import hydra.avro.registry.SchemaRegistry
 import hydra.avro.registry.SchemaRegistry.SchemaVersion
 import hydra.kafka.algebras.{KafkaAdminAlgebra, KafkaClientAlgebra, MetadataAlgebra}
-import hydra.kafka.model.{AdditionalValidation, StreamTypeV2, TopicMetadataV2, TopicMetadataV2Key, TopicMetadataV2Request}
+import hydra.kafka.model.{AdditionalValidation, SkipValidation, StreamTypeV2, TopicMetadataV2, TopicMetadataV2Key, TopicMetadataV2Request}
 import hydra.kafka.programs.CreateTopicProgram._
 import hydra.kafka.util.KafkaUtils.TopicDetails
 import org.typelevel.log4cats.Logger
@@ -137,10 +137,11 @@ final class CreateTopicProgram[F[_]: Bracket[*[_], Throwable]: Sleep: Logger] pr
     } yield ()
 
   //todo: workaround for https://pluralsight.atlassian.net/browse/ADAPT-929, should be removed in the future
-  def createTopicFromMetadataOnly(topicName: Subject, createTopicRequest: TopicMetadataV2Request, withRequiredFields: Boolean = false): F[Unit] =
+  def createTopicFromMetadataOnly(topicName: Subject, createTopicRequest: TopicMetadataV2Request, withRequiredFields: Boolean = false,
+                                  maybeSkipValidations: Option[List[SkipValidation]] = None): F[Unit] =
     for {
       _ <- checkThatTopicExists(topicName.value)
-      _ <- schemaValidator.validate(createTopicRequest, topicName, withRequiredFields)
+      _ <- schemaValidator.validate(createTopicRequest, topicName, withRequiredFields, maybeSkipValidations)
       _ <- publishMetadata(topicName, createTopicRequest)
     } yield ()
 
diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/programs/KeyAndValueSchemaV2Validator.scala b/ingestors/kafka/src/main/scala/hydra/kafka/programs/KeyAndValueSchemaV2Validator.scala
index 9e8b21292..a69e863fe 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/programs/KeyAndValueSchemaV2Validator.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/programs/KeyAndValueSchemaV2Validator.scala
@@ -5,35 +5,46 @@ import cats.effect.Sync
 import cats.syntax.all._
 import hydra.avro.convert.IsoDate
 import hydra.avro.registry.SchemaRegistry
-import hydra.kafka.model.{AdditionalValidation, RequiredField, SchemaAdditionalValidation, Schemas, StreamTypeV2, TopicMetadataV2Request}
+import hydra.kafka.model.{AdditionalValidation, RequiredField, SchemaAdditionalValidation, Schemas, SkipValidation, StreamTypeV2, TopicMetadataV2Request}
 import hydra.kafka.model.TopicMetadataV2Request.Subject
 import hydra.kafka.programs.TopicSchemaError._
 import org.apache.avro.{Schema, SchemaBuilder}
 import RequiredFieldStructures._
 import hydra.common.validation.Validator
-import hydra.common.validation.Validator.ValidationChain
+import hydra.common.validation.Validator.{ValidationChain, valid}
 import hydra.kafka.algebras.MetadataAlgebra
+import hydra.kafka.model.RequiredField.RequiredField
 
 import scala.jdk.CollectionConverters.collectionAsScalaIterableConverter
 import scala.language.higherKinds
 
 class KeyAndValueSchemaV2Validator[F[_]: Sync] private (schemaRegistry: SchemaRegistry[F],
                                                         metadataAlgebra: MetadataAlgebra[F]) extends Validator {
-  def validate(request: TopicMetadataV2Request, subject: Subject, withRequiredFields: Boolean): F[Unit] =
+
+  def validate(request: TopicMetadataV2Request, subject: Subject, withRequiredFields: Boolean,
+               maybeSkipValidations: Option[List[SkipValidation]] = None): F[Unit] = {
+
+    val skipValidations = maybeSkipValidations.getOrElse(List.empty)
+    if (skipValidations.contains(SkipValidation.all)) {
+      return ().pure
+    }
+
     for {
       metadata <- metadataAlgebra.getMetadataFor(subject)
       validateDefaultInRequiredField = AdditionalValidation.isPresent(metadata, SchemaAdditionalValidation.defaultInRequiredField)
       schemas = request.schemas
-      _ <- (schemas.key.getType, schemas.value.getType) match {
+      _ <- (Option(schemas.key).map(_.getType).getOrElse(Schema.Type.NULL), Option(schemas.value).map(_.getType).getOrElse(Schema.Type.NULL)) match {
         case (Schema.Type.RECORD, Schema.Type.RECORD) =>
-          validateRecordRecordTypeSchemas(schemas, subject, request.streamType, withRequiredFields, validateDefaultInRequiredField)
+          validateRecordRecordTypeSchemas(schemas, subject, request.streamType, withRequiredFields, validateDefaultInRequiredField, skipValidations)
         case (Schema.Type.STRING, Schema.Type.RECORD) if request.tags.contains("KSQL") =>
-          validateKSQLSchemas(schemas, subject, request.streamType)
+          validateKSQLSchemas(schemas, subject, request.streamType, skipValidations)
         case _ => resultOf(Validated.Invalid(NonEmptyChain.one(InvalidSchemaTypeError)))
       }
     } yield()
+  }
 
-  private def validateKSQLSchemas(schemas: Schemas, subject: Subject, streamType: StreamTypeV2): F[Unit] = {
+  private def validateKSQLSchemas(schemas: Schemas, subject: Subject, streamType: StreamTypeV2,
+                                  skipValidations: List[SkipValidation]): F[Unit] = {
     val concoctedKeyFields = SchemaBuilder
       .record("uselessRecord") //This is a useless record whose only purpose is to transform the string key into a list of fields.
       .fields()
@@ -44,13 +55,21 @@ class KeyAndValueSchemaV2Validator[F[_]: Sync] private (schemaRegistry: SchemaRe
       .endRecord().getFields.asScala.toList
 
     val valueFields = schemas.value.getFields.asScala.toList
+
+    val doKeySchemaEvolutionValidation = !skipValidations.contains(SkipValidation.keySchemaEvolution)
+    val doValueSchemaEvolutionValidation = !skipValidations.contains(SkipValidation.valueSchemaEvolution)
+    val doSameFieldsTypeMismatchInKeyValueSchemasValidation = !skipValidations.contains(SkipValidation.sameFieldsTypeMismatchInKeyValueSchemas)
+    val doMissingDefaultInNullableFieldsOfValueSchemaValidation = !skipValidations.contains(SkipValidation.missingDefaultInNullableFieldsOfValueSchema)
+    val doUnsupportedLogicalTypeFieldsInKeySchemaValidation = !skipValidations.contains(SkipValidation.unsupportedLogicalTypeFieldsInKeySchema)
+    val doUnsupportedLogicalTypeFieldsInValueSchemaValidation = !skipValidations.contains(SkipValidation.unsupportedLogicalTypeFieldsInValueSchema)
+
     val validators = for {
-        keySchemaEvolutionValidationResult         <- validateKeySchemaEvolution(schemas, subject)
-        valueSchemaEvolutionValidationResult       <- validateValueSchemaEvolution(schemas, subject)
-        defaultNullableValueFieldsValidationResult <- checkForDefaultNullableValueFields(valueFields, streamType)
-        unsupportedLogicalTypesKey                 <- checkForUnsupportedLogicalType(concoctedKeyFields)
-        unsupportedLogicalTypesValues              <- checkForUnsupportedLogicalType(valueFields)
-        mismatchesValidationResult                 <- checkForMismatches(concoctedKeyFields, valueFields)
+        keySchemaEvolutionValidationResult         <- validateKeySchemaEvolution(schemas, subject, doKeySchemaEvolutionValidation)
+        valueSchemaEvolutionValidationResult       <- validateValueSchemaEvolution(schemas, subject, doValueSchemaEvolutionValidation)
+        defaultNullableValueFieldsValidationResult <- checkForDefaultNullableValueFields(valueFields, doMissingDefaultInNullableFieldsOfValueSchemaValidation)
+        unsupportedLogicalTypesKey                 <- checkForUnsupportedLogicalType(concoctedKeyFields, doUnsupportedLogicalTypeFieldsInKeySchemaValidation)
+        unsupportedLogicalTypesValues              <- checkForUnsupportedLogicalType(valueFields, doUnsupportedLogicalTypeFieldsInValueSchemaValidation)
+        mismatchesValidationResult                 <- checkForMismatches(concoctedKeyFields, valueFields, doSameFieldsTypeMismatchInKeyValueSchemasValidation)
       } yield {
         keySchemaEvolutionValidationResult ++
           valueSchemaEvolutionValidationResult ++
@@ -67,23 +86,39 @@ class KeyAndValueSchemaV2Validator[F[_]: Sync] private (schemaRegistry: SchemaRe
                                               subject: Subject,
                                               streamType: StreamTypeV2,
                                               withRequiredFields: Boolean,
-                                              validateDefaultInRequiredField: Boolean): F[Unit] = {
+                                              validateDefaultInRequiredField: Boolean,
+                                              skipValidations: List[SkipValidation]): F[Unit] = {
     val keyFields   = schemas.key.getFields.asScala.toList
     val valueFields = schemas.value.getFields.asScala.toList
 
+    val doEmptyKeyFieldsValidation = !skipValidations.contains(SkipValidation.emptyKeyFields)
+    val doKeySchemaEvolutionValidation = !skipValidations.contains(SkipValidation.keySchemaEvolution)
+    val doValueSchemaEvolutionValidation = !skipValidations.contains(SkipValidation.valueSchemaEvolution)
+    val doSameFieldsTypeMismatchInKeyValueSchemasValidation = !skipValidations.contains(SkipValidation.sameFieldsTypeMismatchInKeyValueSchemas)
+    val doNullableFieldsInKeySchemaValidation = !skipValidations.contains(SkipValidation.nullableFieldsInKeySchema)
+    val doMissingDefaultInNullableFieldsOfValueSchemaValidation = !skipValidations.contains(SkipValidation.missingDefaultInNullableFieldsOfValueSchema)
+    val doUnsupportedLogicalTypeFieldsInKeySchemaValidation = !skipValidations.contains(SkipValidation.unsupportedLogicalTypeFieldsInKeySchema)
+    val doUnsupportedLogicalTypeFieldsInValueSchemaValidation = !skipValidations.contains(SkipValidation.unsupportedLogicalTypeFieldsInValueSchema)
+
     val validators = for {
-      keyFieldsValidationResult                  <- validateKeyFields(keyFields)
-      keySchemaEvolutionValidationResult         <- validateKeySchemaEvolution(schemas, subject)
-      valueSchemaEvolutionValidationResult       <- validateValueSchemaEvolution(schemas, subject)
-      validateRequiredKeyFieldsResult            <-
-        if (withRequiredFields) validateRequiredKeyFields(schemas.key, streamType, validateDefaultInRequiredField) else Nil.pure
-      validateRequiredValueFieldsResult          <-
-        if (withRequiredFields) validateRequiredValueFields(schemas.value, streamType, validateDefaultInRequiredField) else Nil.pure
-      mismatchesValidationResult                 <- checkForMismatches(keyFields, valueFields)
-      nullableKeyFieldsValidationResult          <- checkForNullableKeyFields(keyFields, streamType)
-      defaultNullableValueFieldsValidationResult <- checkForDefaultNullableValueFields(valueFields, streamType)
-      unsupportedLogicalTypesKey                 <- checkForUnsupportedLogicalType(keyFields)
-      unsupportedLogicalTypesValues              <- checkForUnsupportedLogicalType(valueFields)
+      keyFieldsValidationResult                  <- validateKeyFields(keyFields, doEmptyKeyFieldsValidation)
+      keySchemaEvolutionValidationResult         <- validateKeySchemaEvolution(schemas, subject, doKeySchemaEvolutionValidation)
+      valueSchemaEvolutionValidationResult       <- validateValueSchemaEvolution(schemas, subject, doValueSchemaEvolutionValidation)
+      validateRequiredKeyFieldsResult            <- if (withRequiredFields) {
+          validateRequiredKeyFields(schemas.key, streamType, validateDefaultInRequiredField, skipValidations)
+        } else {
+          Nil.pure
+        }
+      validateRequiredValueFieldsResult          <- if (withRequiredFields) {
+          validateRequiredValueFields(schemas.value, streamType, validateDefaultInRequiredField, skipValidations)
+        } else {
+          Nil.pure
+        }
+      mismatchesValidationResult                 <- checkForMismatches(keyFields, valueFields, doSameFieldsTypeMismatchInKeyValueSchemasValidation)
+      nullableKeyFieldsValidationResult          <- checkForNullableKeyFields(keyFields, streamType, doNullableFieldsInKeySchemaValidation)
+      defaultNullableValueFieldsValidationResult <- checkForDefaultNullableValueFields(valueFields, doMissingDefaultInNullableFieldsOfValueSchemaValidation)
+      unsupportedLogicalTypesKey                 <- checkForUnsupportedLogicalType(keyFields, doUnsupportedLogicalTypeFieldsInKeySchemaValidation)
+      unsupportedLogicalTypesValues              <- checkForUnsupportedLogicalType(valueFields, doUnsupportedLogicalTypeFieldsInValueSchemaValidation)
     } yield {
         (keyFieldsValidationResult +: keySchemaEvolutionValidationResult) ++
         valueSchemaEvolutionValidationResult ++
@@ -99,19 +134,36 @@ class KeyAndValueSchemaV2Validator[F[_]: Sync] private (schemaRegistry: SchemaRe
     resultOf(validators)
   }
 
-  private def checkForUnsupportedLogicalType(fields: List[Schema.Field]): F[List[ValidationChain]] =
-    fields.map { field =>
-      validate(!getLogicalType(field.schema()).contains(IsoDate.IsoDateLogicalTypeName), UnsupportedLogicalType(field, getLogicalType(field.schema()).getOrElse("")))
-    }.pure
+  private def checkForUnsupportedLogicalType(fields: List[Schema.Field], doValidation: Boolean): F[List[ValidationChain]] =
+    if (doValidation) {
+      fields.map { field =>
+        validate(!getLogicalType(field.schema()).contains(IsoDate.IsoDateLogicalTypeName),
+          UnsupportedLogicalType(field, getLogicalType(field.schema()).getOrElse("")))
+      }.pure
+    } else {
+      List(valid).pure
+    }
 
-  private def validateKeyFields(keyFields: List[Schema.Field]): F[ValidationChain] =
-    validate(keyFields.nonEmpty, TopicSchemaError.KeyIsEmptyError).pure
+  private def validateKeyFields(keyFields: List[Schema.Field], doValidation: Boolean): F[ValidationChain] =
+    if (doValidation) {
+      validate(keyFields.nonEmpty, TopicSchemaError.KeyIsEmptyError).pure
+    } else {
+      valid.pure
+    }
 
-  private def validateKeySchemaEvolution(schemas: Schemas, subject: Subject): F[List[ValidationChain]] =
-    validateSchemaEvolution(schemas, subject + "-key")
+  private def validateKeySchemaEvolution(schemas: Schemas, subject: Subject, doValidation: Boolean): F[List[ValidationChain]] =
+    if (doValidation) {
+      validateSchemaEvolution(schemas, subject + "-key")
+    } else {
+      List(valid).pure
+    }
 
-  private def validateValueSchemaEvolution(schemas: Schemas, subject: Subject): F[List[ValidationChain]] =
-    validateSchemaEvolution(schemas, subject + "-value")
+  private def validateValueSchemaEvolution(schemas: Schemas, subject: Subject, doValidation: Boolean): F[List[ValidationChain]] =
+    if (doValidation) {
+      validateSchemaEvolution(schemas, subject + "-value")
+    } else {
+      List(valid).pure
+    }
 
   private def validateSchemaEvolution(schemas: Schemas, subject: String): F[List[ValidationChain]] =
     schemaRegistry.getLatestSchemaBySubject(subject).map {
@@ -121,31 +173,62 @@ class KeyAndValueSchemaV2Validator[F[_]: Sync] private (schemaRegistry: SchemaRe
       case _ => List(Validator.valid)
     }
 
-  private def validateRequiredKeyFields(keySchema: Schema, streamType: StreamTypeV2, validateDefaultInRequiredField: Boolean): F[List[ValidationChain]] =
-    validateRequiredFields(isKey = true, keySchema, streamType, validateDefaultInRequiredField)
+  private def validateRequiredKeyFields(keySchema: Schema, streamType: StreamTypeV2, validateDefaultInRequiredField: Boolean,
+                                        skipValidations: List[SkipValidation]): F[List[ValidationChain]] =
+    validateRequiredFields(isKey = true, keySchema, streamType, validateDefaultInRequiredField, skipValidations)
 
-  private def validateRequiredValueFields(valueSchema: Schema, streamType: StreamTypeV2, validateDefaultInRequiredField: Boolean): F[List[ValidationChain]] =
-    validateRequiredFields(isKey = false, valueSchema, streamType, validateDefaultInRequiredField)
+  private def validateRequiredValueFields(valueSchema: Schema, streamType: StreamTypeV2, validateDefaultInRequiredField: Boolean,
+                                          skipValidations: List[SkipValidation]): F[List[ValidationChain]] =
+    validateRequiredFields(isKey = false, valueSchema, streamType, validateDefaultInRequiredField, skipValidations)
 
   private def validateRequiredFields(isKey: Boolean, schema: Schema, streamType: StreamTypeV2,
-                                     validateDefaultInRequiredField: Boolean): F[List[ValidationChain]] =
+                                     validateDefaultInRequiredField: Boolean, skipValidations: List[SkipValidation]): F[List[ValidationChain]] = {
+    val doRequiredDocFieldValidation = !skipValidations.contains(SkipValidation.requiredDocField)
+    val doRequiredCreatedAtFieldValidation = !skipValidations.contains(SkipValidation.requiredCreatedAtField)
+    val doRequiredUpdatedAtFieldValidation = !skipValidations.contains(SkipValidation.requiredUpdatedAtField)
+
     streamType match {
       case (StreamTypeV2.Entity | StreamTypeV2.Event) =>
         if (isKey) {
-          List(validate(docFieldValidator(schema), getFieldMissingError(isKey, RequiredField.DOC, schema, streamType.toString))).pure
+          List(validateRequiredField(doRequiredDocFieldValidation, RequiredField.DOC, isKey, schema, streamType.toString)).pure
         } else {
           List(
-            validate(docFieldValidator(schema), getFieldMissingError(isKey, RequiredField.DOC, schema, streamType.toString)),
-            validate(createdAtFieldValidator(schema), getFieldMissingError(isKey, RequiredField.CREATED_AT, schema, streamType.toString)),
-            validate(updatedAtFieldValidator(schema), getFieldMissingError(isKey, RequiredField.UPDATED_AT, schema, streamType.toString)),
-            validate(defaultFieldOfRequiredFieldValidator(schema, RequiredField.CREATED_AT, validateDefaultInRequiredField),
-              RequiredSchemaValueFieldWithDefaultValueError(RequiredField.CREATED_AT, schema, streamType.toString)),
-            validate(defaultFieldOfRequiredFieldValidator(schema, RequiredField.UPDATED_AT, validateDefaultInRequiredField),
-              RequiredSchemaValueFieldWithDefaultValueError(RequiredField.UPDATED_AT, schema, streamType.toString))
+            validateRequiredField(doRequiredDocFieldValidation, RequiredField.DOC, isKey, schema, streamType.toString),
+            validateRequiredField(doRequiredCreatedAtFieldValidation, RequiredField.CREATED_AT, isKey, schema, streamType.toString),
+            validateRequiredField(doRequiredUpdatedAtFieldValidation, RequiredField.UPDATED_AT, isKey, schema, streamType.toString),
+            validateDefaultFieldInRequiredField(doRequiredCreatedAtFieldValidation, RequiredField.CREATED_AT,
+              validateDefaultInRequiredField, schema, streamType.toString),
+            validateDefaultFieldInRequiredField(doRequiredUpdatedAtFieldValidation, RequiredField.UPDATED_AT,
+              validateDefaultInRequiredField, schema, streamType.toString)
           ).pure
         }
       case _ =>
-        List(validate(docFieldValidator(schema), getFieldMissingError(isKey, RequiredField.DOC, schema, streamType.toString))).pure
+        List(validateRequiredField(doRequiredDocFieldValidation, RequiredField.DOC, isKey, schema, streamType.toString)).pure
+    }
+  }
+
+  private def validateRequiredField(doValidation: Boolean, requiredField: RequiredField, isKey: Boolean, schema: Schema, streamType: String): ValidationChain =
+    if (doValidation) {
+      val result = requiredField match {
+        case RequiredField.DOC         => docFieldValidator(schema)
+        case RequiredField.CREATED_AT  => createdAtFieldValidator(schema)
+        case RequiredField.UPDATED_AT  => updatedAtFieldValidator(schema)
+      }
+      validate(result, getFieldMissingError(isKey, requiredField, schema, streamType))
+    } else {
+      valid
+    }
+
+  private def validateDefaultFieldInRequiredField(doValidation: Boolean,
+                                                  requiredField: RequiredField,
+                                                  validateDefaultInRequiredField: Boolean,
+                                                  schema: Schema,
+                                                  streamType: String): ValidationChain =
+    if (doValidation) {
+      validate(defaultFieldOfRequiredFieldValidator(schema, requiredField, validateDefaultInRequiredField),
+        RequiredSchemaValueFieldWithDefaultValueError(requiredField, schema, streamType))
+    } else {
+      valid
     }
 
   private def checkForIllegalLogicalTypeEvolutions(existingSchema: Schema, newSchema: Schema, fieldName: String): List[ValidationChain] = {
@@ -180,7 +263,8 @@ class KeyAndValueSchemaV2Validator[F[_]: Sync] private (schemaRegistry: SchemaRe
     Option(schema.getLogicalType)
       .fold(Option(schema.getProp("logicalType")))(_.getName.some)
 
-  private def checkForNullableKeyFields(keyFields: List[Schema.Field], streamType: StreamTypeV2): F[List[ValidationChain]] = {
+  private def checkForNullableKeyFields(keyFields: List[Schema.Field], streamType: StreamTypeV2,
+                                        doValidation: Boolean): F[List[ValidationChain]] = {
     def fieldIsNotNull(field: Schema.Field): Boolean =
       field.schema().getType match {
         case Schema.Type.UNION if field.schema.getTypes.asScala.toList.exists(_.isNullable) => false
@@ -188,30 +272,43 @@ class KeyAndValueSchemaV2Validator[F[_]: Sync] private (schemaRegistry: SchemaRe
         case _                                                                              => true
       }
 
-    keyFields.map(field =>
-      validate(streamType == StreamTypeV2.Event || fieldIsNotNull(field), KeyHasNullableFieldError(field.name(), field.schema()))
-    ).pure
+    if (doValidation) {
+      keyFields.map(field =>
+        validate(streamType == StreamTypeV2.Event || fieldIsNotNull(field), KeyHasNullableFieldError(field.name(), field.schema()))
+      ).pure
+    } else {
+      List(valid).pure
+    }
   }
 
-  private def checkForDefaultNullableValueFields(valueFields: List[Schema.Field], streamType: StreamTypeV2): F[List[ValidationChain]] = {
+  private def checkForDefaultNullableValueFields(valueFields: List[Schema.Field], doValidation: Boolean): F[List[ValidationChain]] = {
     def validateIfFieldIsNullable(field: Schema.Field): Boolean =
       field.schema().getType match {
         case Schema.Type.UNION if field.schema().getTypes.asScala.toList.exists(_.isNullable) && Option(field.defaultVal()).isEmpty => false
         case _ => true
       }
 
-    valueFields.map(field =>
-      validate(validateIfFieldIsNullable(field), NullableFieldWithoutDefaultValueError(field.name(), field.schema()))
-    ).pure
+    if (doValidation) {
+      valueFields.map(field =>
+        validate(validateIfFieldIsNullable(field), NullableFieldWithoutDefaultValueError(field.name(), field.schema()))
+      ).pure
+    } else {
+      List(valid).pure
+    }
   }
 
-  private def checkForMismatches(keyFields: List[Schema.Field], valueFields: List[Schema.Field]): F[List[ValidationChain]] =
-    keyFields.flatMap { keyField =>
-      valueFields.map { valueField =>
-        validate(keyField.name() != valueField.name() || keyField.schema().equals(valueField.schema()),
-          IncompatibleKeyAndValueFieldNamesError(keyField.name(), keyField.schema(), valueField.schema()))
-      }
-    }.pure
+  private def checkForMismatches(keyFields: List[Schema.Field], valueFields: List[Schema.Field],
+                                 doValidation: Boolean): F[List[ValidationChain]] =
+    if (doValidation) {
+      keyFields.flatMap { keyField =>
+        valueFields.map { valueField =>
+          validate(keyField.name() != valueField.name() || keyField.schema().equals(valueField.schema()),
+            IncompatibleKeyAndValueFieldNamesError(keyField.name(), keyField.schema(), valueField.schema()))
+        }
+      }.pure
+    } else {
+      List(valid).pure
+    }
 }
 
 object KeyAndValueSchemaV2Validator {
diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/serializers/TopicMetadataV2Parser.scala b/ingestors/kafka/src/main/scala/hydra/kafka/serializers/TopicMetadataV2Parser.scala
index af8226af0..4f8ffb89c 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/serializers/TopicMetadataV2Parser.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/serializers/TopicMetadataV2Parser.scala
@@ -269,9 +269,37 @@ sealed trait TopicMetadataV2Parser
     }
   }
 
+  class ListEnumEntryJsonFormat[E <: List[EnumEntry]](values: E) extends RootJsonFormat[E] {
+
+    override def write(obj: E): JsValue = JsString(obj.map(_.entryName).mkString(","))
+
+    override def read(json: JsValue): E = json match {
+      case commaSeparatedStrings: JsString => {
+        val invalidValues = commaSeparatedStrings.value.split(",").flatMap(s => values.find(v => v.entryName != s))
+        val validValues = commaSeparatedStrings.value.split(",").flatMap(s => values.find(v => v.entryName == s))
+
+        if (invalidValues.isEmpty && validValues.nonEmpty) {
+          values
+        } else {
+          deserializationListError(JsString(invalidValues.mkString(",")))
+        }
+      }
+      case x => deserializationListError(x)
+    }
+
+    private def deserializationListError(value: JsValue) = {
+      val className = values.headOption.map(_.getClass.getEnclosingClass.getSimpleName).getOrElse("")
+      throw DeserializationException(
+        s"For '$className': Expected single or comma-separated values from enum $values but received invalid $value")
+    }
+  }
+
   implicit val additionalValidationFormat: EnumEntryJsonFormat[AdditionalValidation] =
     new EnumEntryJsonFormat[AdditionalValidation](Seq.empty)
 
+  implicit val skipValidationFormat: ListEnumEntryJsonFormat[List[SkipValidation]] =
+    new ListEnumEntryJsonFormat[List[SkipValidation]](SkipValidation.values.toList)
+
   implicit object TopicMetadataV2Format
       extends RootJsonFormat[TopicMetadataV2Request] {
 
diff --git a/ingestors/kafka/src/test/scala/hydra/kafka/programs/SkipValidationSpec.scala b/ingestors/kafka/src/test/scala/hydra/kafka/programs/SkipValidationSpec.scala
new file mode 100644
index 000000000..4f26c5191
--- /dev/null
+++ b/ingestors/kafka/src/test/scala/hydra/kafka/programs/SkipValidationSpec.scala
@@ -0,0 +1,544 @@
+package hydra.kafka.programs
+
+import cats.effect._
+import cats.syntax.all._
+import hydra.common.validation.ValidationError.ValidationCombinedErrors
+import hydra.kafka.IOSuite
+import hydra.kafka.model.SkipValidation
+import hydra.kafka.programs.TopicSchemaError._
+import org.apache.avro.{Schema, SchemaBuilder}
+import org.scalatest.freespec.AsyncFreeSpec
+import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper
+
+class SkipValidationSpec extends AsyncFreeSpec with IOSuite {
+  import CreateTopicProgramSpec._
+
+  "SkipValidationSpec" - {
+
+    "skip validation: empty key schema fields" in {
+      val emptyFieldsSchema = SchemaBuilder.record("EmptyFieldsSchema").fields().endRecord()
+
+      val result = createTopicMetadataV2Request(emptyFieldsSchema)
+      result.attempt.map(_ shouldBe KeyIsEmptyError.asLeft)
+
+      val resultWithSkipValidation = createTopicMetadataV2Request(emptyFieldsSchema, skipValidations = Some(List(SkipValidation.emptyKeyFields)))
+      resultWithSkipValidation.map(_ shouldBe ())
+    }
+
+    "skip validation: key schema evolution" in {
+      val schemaString =
+        """
+          |{
+          |  "type": "record",
+          |  "name": "Date",
+          |  "fields": [
+          |    {
+          |      "name": "keyThing",
+          |      "doc": "text",
+          |      "type":{
+          |        "type": "string",
+          |        "logicalType":"uuid"
+          |      }
+          |    }
+          |  ]
+          |}
+      """.stripMargin
+      val evolvedSchemaString =
+        """
+          |{
+          |  "type": "record",
+          |  "name": "Date",
+          |  "fields": [
+          |    {
+          |      "name": "keyThing",
+          |      "doc": "text",
+          |      "type":{
+          |        "type": "string",
+          |        "logicalType":"date"
+          |      }
+          |    }
+          |  ]
+          |}
+      """.stripMargin
+
+      val schema = new Schema.Parser().parse(schemaString)
+      val evolvedSchema = new Schema.Parser().parse(evolvedSchemaString)
+
+      val result = evolveTopicMetadataV2Request(keySchema = schema, evolvedKeySchema = evolvedSchema)
+      result.attempt.map(_ shouldBe IllegalLogicalTypeChangeError("uuid", "date", "keyThing").asLeft)
+
+      val resultWithSkipValidation = evolveTopicMetadataV2Request(keySchema = schema, evolvedKeySchema = evolvedSchema,
+        skipValidations = Some(List(SkipValidation.keySchemaEvolution)))
+      resultWithSkipValidation.attempt.map(_ shouldBe ().asRight)
+    }
+
+    "skip validation: value schema evolution" in {
+      val schemaString =
+        """
+          |{
+          |  "type": "record",
+          |  "name": "test",
+          |  "fields": [
+          |     {
+          |       "name": "id",
+          |       "default": "abc",
+          |       "type": ["string", "null"],
+          |       "doc": "text"
+          |     },
+          |    {
+          |      "name": "createdAt",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      },
+          |      "doc": "text"
+          |    },
+          |    {
+          |      "name": "updatedAt",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      },
+          |      "doc": "text"
+          |    }
+          |  ]
+          |}
+        """.stripMargin
+      val evolvedSchemaString =
+        """
+          |{
+          |  "type": "record",
+          |  "name": "test",
+          |  "fields": [
+          |     {
+          |        "name":"id",
+          |        "default": "abc",
+          |        "doc": "text",
+          |        "type":[
+          |                 {
+          |                   "type": "string",
+          |                   "logicalType": "uuid"
+          |                 },
+          |                 "null"
+          |               ]
+          |     },
+          |    {
+          |      "name": "createdAt",
+          |      "doc": "text",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      }
+          |    },
+          |    {
+          |      "name": "updatedAt",
+          |      "doc": "text",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      }
+          |    }
+          |  ]
+          |}
+        """.stripMargin
+
+      val schema = new Schema.Parser().parse(schemaString)
+      val evolvedSchema = new Schema.Parser().parse(evolvedSchemaString)
+
+      val result = evolveTopicMetadataV2Request(valueSchema = schema, evolvedValueSchema = evolvedSchema)
+      result.attempt.map(_ shouldBe IllegalLogicalTypeChangeError("null", "uuid", "id").asLeft)
+
+      val resultWithSkipValidation = evolveTopicMetadataV2Request(valueSchema = schema, evolvedValueSchema = evolvedSchema,
+        skipValidations = Some(List(SkipValidation.valueSchemaEvolution)))
+      resultWithSkipValidation.attempt.map(_ shouldBe ().asRight)
+    }
+
+    "skip validation: required doc field in key schema" in {
+      val schemaWithoutDocField =
+        """
+          |{
+          |  "type": "record",
+          |  "name": "Date",
+          |  "fields": [
+          |    {
+          |      "name": "keyThing",
+          |      "type":{
+          |        "type": "string",
+          |        "logicalType":"uuid"
+          |      }
+          |    }
+          |  ]
+          |}
+      """.stripMargin
+
+      val schema = new Schema.Parser().parse(schemaWithoutDocField)
+
+      val result = createTopicMetadataV2Request(keySchema = schema)
+      result.attempt.map(_ shouldBe RequiredSchemaKeyFieldMissingError("doc", schema, "Entity").asLeft)
+
+      val resultWithSkipValidation = createTopicMetadataV2Request(keySchema = schema, skipValidations = Some(List(SkipValidation.requiredDocField)))
+      resultWithSkipValidation.attempt.map(_ shouldBe ().asRight)
+    }
+
+    "skip validation: required doc field in value schema" in {
+      val schemaWithoutDocField =
+        """
+          |{
+          |  "type": "record",
+          |  "name": "test",
+          |  "fields": [
+          |    {
+          |      "name": "createdAt",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      }
+          |    },
+          |    {
+          |      "name": "updatedAt",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      },
+          |      "doc": "text"
+          |    }
+          |  ]
+          |}
+        """.stripMargin
+
+      val schema = new Schema.Parser().parse(schemaWithoutDocField)
+
+      val result = createTopicMetadataV2Request(valueSchema = schema)
+      result.attempt.map(_ shouldBe RequiredSchemaValueFieldMissingError("doc", schema, "Entity").asLeft)
+
+      val resultWithSkipValidation = createTopicMetadataV2Request(valueSchema = schema, skipValidations = Some(List(SkipValidation.requiredDocField)))
+      resultWithSkipValidation.attempt.map(_ shouldBe ().asRight)
+    }
+
+    "skip validation: required createdAt field in value schema" in {
+      val schemaWithoutDocField =
+        """
+          |{
+          |  "type": "record",
+          |  "name": "test",
+          |  "fields": [
+          |    {
+          |      "name": "updatedAt",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      },
+          |      "doc": "text"
+          |    }
+          |  ]
+          |}
+        """.stripMargin
+
+      val schema = new Schema.Parser().parse(schemaWithoutDocField)
+
+      val result = createTopicMetadataV2Request(valueSchema = schema)
+      result.attempt.map(_ shouldBe RequiredSchemaValueFieldMissingError("createdAt", schema, "Entity").asLeft)
+
+      val resultWithSkipValidation = createTopicMetadataV2Request(valueSchema = schema, skipValidations = Some(List(SkipValidation.requiredCreatedAtField)))
+      resultWithSkipValidation.attempt.map(_ shouldBe ().asRight)
+    }
+
+    "skip validation: required updatedAt field in value schema" in {
+      val schemaWithoutDocField =
+        """
+          |{
+          |  "type": "record",
+          |  "name": "test",
+          |  "fields": [
+          |    {
+          |      "name": "createdAt",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      },
+          |      "doc": "text"
+          |    }
+          |  ]
+          |}
+        """.stripMargin
+
+      val schema = new Schema.Parser().parse(schemaWithoutDocField)
+
+      val result = createTopicMetadataV2Request(valueSchema = schema)
+      result.attempt.map(_ shouldBe RequiredSchemaValueFieldMissingError("updatedAt", schema, "Entity").asLeft)
+
+      val resultWithSkipValidation = createTopicMetadataV2Request(valueSchema = schema, skipValidations = Some(List(SkipValidation.requiredUpdatedAtField)))
+      resultWithSkipValidation.attempt.map(_ shouldBe ().asRight)
+    }
+
+    "skip validation: same field type mismatch in key and value schemas" in {
+      val keySchemaString =
+        """
+          |{
+          |  "type": "record",
+          |  "name": "Date",
+          |  "fields": [
+          |    {
+          |      "name": "keyThing",
+          |      "doc": "text",
+          |      "type":{
+          |        "type": "string",
+          |        "logicalType":"uuid"
+          |      }
+          |    }
+          |  ]
+          |}
+      """.stripMargin
+      val valueSchemaString =
+        """
+          |{
+          |  "type": "record",
+          |  "name": "test",
+          |  "fields": [
+          |     {
+          |        "name":"keyThing",
+          |        "doc": "text",
+          |        "type": "string"
+          |     },
+          |    {
+          |      "name": "createdAt",
+          |      "doc": "text",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      }
+          |    },
+          |    {
+          |      "name": "updatedAt",
+          |      "doc": "text",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      }
+          |    }
+          |  ]
+          |}
+        """.stripMargin
+      val keySchema = new Schema.Parser().parse(keySchemaString)
+      val valueSchema = new Schema.Parser().parse(valueSchemaString)
+      val keyFieldSchema = keySchema.getField("keyThing").schema()
+      val valueFieldSchema = valueSchema.getField("keyThing").schema()
+
+      val result = createTopicMetadataV2Request(keySchema, valueSchema)
+      result.attempt.map(_ shouldBe IncompatibleKeyAndValueFieldNamesError("keyThing", keyFieldSchema, valueFieldSchema).asLeft)
+
+      val resultWithSkipValidation = createTopicMetadataV2Request(keySchema, valueSchema,
+        skipValidations = Some(List(SkipValidation.sameFieldsTypeMismatchInKeyValueSchemas)))
+      resultWithSkipValidation.attempt.map(_ shouldBe ().asRight)
+    }
+
+    "skip validation: nullable key schema fields" in {
+      val keySchemaString =
+        """
+          |{
+          |  "type": "record",
+          |  "name": "test",
+          |  "fields": [
+          |     {
+          |       "name": "nullableField",
+          |       "type": "null",
+          |       "doc": "text"
+          |     }
+          |  ]
+          |}
+        """.stripMargin
+
+      val keySchema = new Schema.Parser().parse(keySchemaString)
+      val fieldSchema = keySchema.getField("nullableField").schema()
+
+      val result = createTopicMetadataV2Request(keySchema = keySchema)
+      result.attempt.map(_ shouldBe KeyHasNullableFieldError("nullableField", fieldSchema).asLeft)
+
+      val resultWithSkipValidation = createTopicMetadataV2Request(keySchema = keySchema,
+        skipValidations = Some(List(SkipValidation.nullableFieldsInKeySchema)))
+      resultWithSkipValidation.attempt.map(_ shouldBe ().asRight)
+    }
+
+    "skip validation: missing default field in nullable field of value schema" in {
+      val valueSchemaString =
+        """
+          |{
+          |  "type": "record",
+          |  "name": "test",
+          |  "fields": [
+          |     {
+          |        "name":"nullableField",
+          |        "doc": "text",
+          |        "type": ["null", "string"]
+          |     },
+          |    {
+          |      "name": "createdAt",
+          |      "doc": "text",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      }
+          |    },
+          |    {
+          |      "name": "updatedAt",
+          |      "doc": "text",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      }
+          |    }
+          |  ]
+          |}
+        """.stripMargin
+      val valueSchema = new Schema.Parser().parse(valueSchemaString)
+      val valueFieldSchema = valueSchema.getField("nullableField").schema()
+
+      val result = createTopicMetadataV2Request(valueSchema=valueSchema)
+      result.attempt.map(_ shouldBe NullableFieldWithoutDefaultValueError("nullableField", valueFieldSchema).asLeft)
+
+      val resultWithSkipValidation = createTopicMetadataV2Request(valueSchema=valueSchema,
+        skipValidations = Some(List(SkipValidation.missingDefaultInNullableFieldsOfValueSchema)))
+      resultWithSkipValidation.attempt.map(_ shouldBe ().asRight)
+    }
+
+    "skip validation: unsupported logical type field in key schema" in {
+      val keySchemaString =
+        """
+          |{
+          |  "type": "record",
+          |  "name": "Date",
+          |  "fields": [
+          |    {
+          |      "name": "timestamp",
+          |      "doc": "text",
+          |      "type":{
+          |        "type": "string",
+          |        "logicalType": "iso-datetime"
+          |      }
+          |    }
+          |  ]
+          |}
+      """.stripMargin
+
+      val keySchema = new Schema.Parser().parse(keySchemaString)
+
+      val result = createTopicMetadataV2Request(keySchema = keySchema)
+      result.attempt.map(_ shouldBe UnsupportedLogicalType(keySchema.getField("timestamp"), "iso-datetime").asLeft)
+
+      val resultWithSkipValidation = createTopicMetadataV2Request(keySchema = keySchema,
+        skipValidations = Some(List(SkipValidation.unsupportedLogicalTypeFieldsInKeySchema)))
+      resultWithSkipValidation.attempt.map(_ shouldBe ().asRight)
+    }
+
+    "skip validation: unsupported logical type field in value schema" in {
+      val valueSchemaString =
+        """
+          |{
+          |  "type": "record",
+          |  "name": "Date",
+          |  "fields": [
+          |    {
+          |      "name": "timestamp",
+          |      "type":{
+          |        "type": "string",
+          |        "logicalType": "iso-datetime"
+          |      },
+          |      "doc": "text"
+          |    },
+          |    {
+          |      "name": "createdAt",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      },
+          |      "doc": "text"
+          |    },
+          |    {
+          |      "name": "updatedAt",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      },
+          |      "doc": "text"
+          |    }
+          |  ]
+          |}
+      """.stripMargin
+      val valueSchema = new Schema.Parser().parse(valueSchemaString)
+
+      val result = createTopicMetadataV2Request(valueSchema = valueSchema)
+      result.attempt.map(_ shouldBe UnsupportedLogicalType(valueSchema.getField("timestamp"), "iso-datetime").asLeft)
+
+      val resultWithSkipValidation = createTopicMetadataV2Request(valueSchema = valueSchema,
+        skipValidations = Some(List(SkipValidation.unsupportedLogicalTypeFieldsInValueSchema)))
+      resultWithSkipValidation.attempt.map(_ shouldBe ().asRight)
+    }
+
+    "skip multiple validations: all" in {
+      val valueSchemaString =
+        """
+          |{
+          |  "type": "record",
+          |  "name": "test",
+          |  "fields": [
+          |    {
+          |      "name": "updatedAt",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      }
+          |    }
+          |  ]
+          |}
+        """.stripMargin
+
+      val valueSchema = new Schema.Parser().parse(valueSchemaString)
+
+      val result = createTopicMetadataV2Request(valueSchema = valueSchema)
+      result.attempt.map(_ shouldBe
+        ValidationCombinedErrors(List(
+          RequiredSchemaValueFieldMissingError("doc", valueSchema, "Entity").message,
+          RequiredSchemaValueFieldMissingError("createdAt", valueSchema, "Entity").message
+        )).asLeft)
+
+      val resultWithSkipValidation = createTopicMetadataV2Request(valueSchema = valueSchema,
+        skipValidations = Some(List(SkipValidation.requiredDocField, SkipValidation.requiredCreatedAtField)))
+      resultWithSkipValidation.attempt.map(_ shouldBe ().asRight)
+
+      val resultWithSkipValidationAll = createTopicMetadataV2Request(valueSchema = valueSchema, skipValidations = Some(List(SkipValidation.all)))
+      resultWithSkipValidationAll.attempt.map(_ shouldBe ().asRight)
+    }
+  }
+
+  private def createTopicMetadataV2Request(keySchema: Schema = keySchema, valueSchema: Schema = valueSchema,
+                                           skipValidations: Option[List[SkipValidation]] = None) =
+    for {
+      ts <- Resource.eval(initTestServices())
+      _ <- ts.program.registerSchemas(subject, keySchema, valueSchema)
+      _ <- ts.program.createTopicResource(subject, topicDetails)
+      _ <- Resource.eval(ts.program.createTopicFromMetadataOnly(
+        subject,
+        createTopicMetadataRequest(keySchema, valueSchema),
+        withRequiredFields = true,
+        skipValidations))
+    } yield ()
+
+  private def evolveTopicMetadataV2Request(keySchema: Schema = keySchema, valueSchema: Schema = valueSchema,
+                                           evolvedKeySchema: Schema = keySchema, evolvedValueSchema: Schema = valueSchema,
+                                           skipValidations: Option[List[SkipValidation]] = None) =
+    for {
+      ts <- Resource.eval(initTestServices())
+      _ <- ts.program.registerSchemas(subject, keySchema, valueSchema)
+      _ <- ts.program.createTopicResource(subject, topicDetails)
+      _ <- Resource.eval(ts.program.createTopicFromMetadataOnly(
+        subject,
+        createTopicMetadataRequest(keySchema, valueSchema),
+        withRequiredFields = true,
+        skipValidations))
+      _ <- Resource.eval(ts.program.createTopicFromMetadataOnly(
+        subject,
+        createTopicMetadataRequest(evolvedKeySchema, evolvedValueSchema),
+        withRequiredFields = true,
+        skipValidations))
+    } yield ()
+}

From 6dbcfe2610fa5ba0b9ab72c7e725f6b344d06738 Mon Sep 17 00:00:00 2001
From: Aman Minz <aman-minz@pluralsight.com>
Date: Wed, 24 Jan 2024 10:57:46 +0530
Subject: [PATCH 045/220] Add separate enum entry for skipping default loophole
 with few more tests

---
 .../hydra/kafka/model/SkipValidation.scala    |  1 +
 .../KeyAndValueSchemaV2Validator.scala        | 12 +++---
 .../programs/RequiredFieldStructures.scala    |  4 +-
 .../endpoints/TopicMetadataEndpointSpec.scala | 19 +++++++++-
 .../kafka/programs/SkipValidationSpec.scala   | 38 +++++++++++++++++++
 5 files changed, 65 insertions(+), 9 deletions(-)

diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/model/SkipValidation.scala b/ingestors/kafka/src/main/scala/hydra/kafka/model/SkipValidation.scala
index 8dbf7c204..78667b4f0 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/model/SkipValidation.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/model/SkipValidation.scala
@@ -22,6 +22,7 @@ object SkipValidation extends Enum[SkipValidation] {
   case object missingDefaultInNullableFieldsOfValueSchema extends SkipValidation
   case object unsupportedLogicalTypeFieldsInKeySchema extends SkipValidation
   case object unsupportedLogicalTypeFieldsInValueSchema extends SkipValidation
+  case object defaultLoopholeInRequiredField extends SkipValidation
 
   override val values: immutable.IndexedSeq[SkipValidation] = findValues
 
diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/programs/KeyAndValueSchemaV2Validator.scala b/ingestors/kafka/src/main/scala/hydra/kafka/programs/KeyAndValueSchemaV2Validator.scala
index a69e863fe..dd2ab3237 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/programs/KeyAndValueSchemaV2Validator.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/programs/KeyAndValueSchemaV2Validator.scala
@@ -186,6 +186,7 @@ class KeyAndValueSchemaV2Validator[F[_]: Sync] private (schemaRegistry: SchemaRe
     val doRequiredDocFieldValidation = !skipValidations.contains(SkipValidation.requiredDocField)
     val doRequiredCreatedAtFieldValidation = !skipValidations.contains(SkipValidation.requiredCreatedAtField)
     val doRequiredUpdatedAtFieldValidation = !skipValidations.contains(SkipValidation.requiredUpdatedAtField)
+    val doDefaultLoopholeInRequiredFieldValidation = !skipValidations.contains(SkipValidation.defaultLoopholeInRequiredField)
 
     streamType match {
       case (StreamTypeV2.Entity | StreamTypeV2.Event) =>
@@ -196,10 +197,10 @@ class KeyAndValueSchemaV2Validator[F[_]: Sync] private (schemaRegistry: SchemaRe
             validateRequiredField(doRequiredDocFieldValidation, RequiredField.DOC, isKey, schema, streamType.toString),
             validateRequiredField(doRequiredCreatedAtFieldValidation, RequiredField.CREATED_AT, isKey, schema, streamType.toString),
             validateRequiredField(doRequiredUpdatedAtFieldValidation, RequiredField.UPDATED_AT, isKey, schema, streamType.toString),
-            validateDefaultFieldInRequiredField(doRequiredCreatedAtFieldValidation, RequiredField.CREATED_AT,
-              validateDefaultInRequiredField, schema, streamType.toString),
-            validateDefaultFieldInRequiredField(doRequiredUpdatedAtFieldValidation, RequiredField.UPDATED_AT,
-              validateDefaultInRequiredField, schema, streamType.toString)
+            validateDefaultFieldInRequiredField(validateDefaultInRequiredField && doDefaultLoopholeInRequiredFieldValidation,
+              RequiredField.CREATED_AT, schema, streamType.toString),
+            validateDefaultFieldInRequiredField(validateDefaultInRequiredField && doDefaultLoopholeInRequiredFieldValidation,
+              RequiredField.UPDATED_AT, schema, streamType.toString)
           ).pure
         }
       case _ =>
@@ -221,11 +222,10 @@ class KeyAndValueSchemaV2Validator[F[_]: Sync] private (schemaRegistry: SchemaRe
 
   private def validateDefaultFieldInRequiredField(doValidation: Boolean,
                                                   requiredField: RequiredField,
-                                                  validateDefaultInRequiredField: Boolean,
                                                   schema: Schema,
                                                   streamType: String): ValidationChain =
     if (doValidation) {
-      validate(defaultFieldOfRequiredFieldValidator(schema, requiredField, validateDefaultInRequiredField),
+      validate(defaultFieldOfRequiredFieldValidator(schema, requiredField),
         RequiredSchemaValueFieldWithDefaultValueError(requiredField, schema, streamType))
     } else {
       valid
diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/programs/RequiredFieldStructures.scala b/ingestors/kafka/src/main/scala/hydra/kafka/programs/RequiredFieldStructures.scala
index 024c289b4..6726539bb 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/programs/RequiredFieldStructures.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/programs/RequiredFieldStructures.scala
@@ -20,8 +20,8 @@ object RequiredFieldStructures {
   private def validateTimestampType(field: Schema.Field) =
     field.schema.getLogicalType == LogicalTypes.timestampMillis && field.schema.getType == Type.LONG
 
-  def defaultFieldOfRequiredFieldValidator(schema: Schema, field: String, validateDefaultInRequiredField: Boolean): Boolean = {
+  def defaultFieldOfRequiredFieldValidator(schema: Schema, field: String): Boolean = {
     val requiredField = schema.getFields.asScala.toList.find(_.name == field)
-    if (validateDefaultInRequiredField && requiredField.nonEmpty) requiredField.exists(!_.hasDefaultValue) else true
+    if (requiredField.nonEmpty) requiredField.exists(!_.hasDefaultValue) else true
   }
 }
diff --git a/ingestors/kafka/src/test/scala/hydra/kafka/endpoints/TopicMetadataEndpointSpec.scala b/ingestors/kafka/src/test/scala/hydra/kafka/endpoints/TopicMetadataEndpointSpec.scala
index 2e2f9807d..335855f90 100644
--- a/ingestors/kafka/src/test/scala/hydra/kafka/endpoints/TopicMetadataEndpointSpec.scala
+++ b/ingestors/kafka/src/test/scala/hydra/kafka/endpoints/TopicMetadataEndpointSpec.scala
@@ -1,7 +1,7 @@
 package hydra.kafka.endpoints
 
 import akka.actor.{Actor, ActorRef, ActorSelection, Props}
-import akka.http.javadsl.server.MalformedRequestContentRejection
+import akka.http.javadsl.server.{MalformedQueryParamRejection, MalformedRequestContentRejection}
 import akka.http.scaladsl.model.{ContentTypes, HttpEntity, StatusCodes}
 import akka.http.scaladsl.server.Route
 import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest}
@@ -428,5 +428,22 @@ class TopicMetadataEndpointSpec
         rejection shouldBe a[MalformedRequestContentRejection]
       }
     }
+
+    "reject skipValidations param with invalid values" in {
+      Put("/v2/metadata/dvs.test.subject?skipValidations=junk,moreJunk,all,emptyKeyFields",
+        HttpEntity(ContentTypes.`application/json`, validRequest)) ~> route ~> check {
+        rejection shouldBe a[MalformedQueryParamRejection]
+      }
+    }
+
+    "accept skipValidations param with valid values" in {
+      Put("/v2/metadata/dvs.test.subject?skipValidations=all,emptyKeyFields,keySchemaEvolution,valueSchemaEvolution," +
+        "requiredDocField,requiredCreatedAtField,requiredUpdatedAtField,sameFieldsTypeMismatchInKeyValueSchemas," +
+        "nullableFieldsInKeySchema,missingDefaultInNullableFieldsOfValueSchema,unsupportedLogicalTypeFieldsInKeySchema," +
+        "unsupportedLogicalTypeFieldsInValueSchema,defaultLoopholeInRequiredField",
+        HttpEntity(ContentTypes.`application/json`, validRequest)) ~> route ~> check {
+        response.status shouldBe StatusCodes.OK
+      }
+    }
   }
 }
diff --git a/ingestors/kafka/src/test/scala/hydra/kafka/programs/SkipValidationSpec.scala b/ingestors/kafka/src/test/scala/hydra/kafka/programs/SkipValidationSpec.scala
index 4f26c5191..90020cd18 100644
--- a/ingestors/kafka/src/test/scala/hydra/kafka/programs/SkipValidationSpec.scala
+++ b/ingestors/kafka/src/test/scala/hydra/kafka/programs/SkipValidationSpec.scala
@@ -474,6 +474,44 @@ class SkipValidationSpec extends AsyncFreeSpec with IOSuite {
       resultWithSkipValidation.attempt.map(_ shouldBe ().asRight)
     }
 
+    "skip validation: default loophole in a required field" in {
+      val schemaWithDefaultLoophole =
+        """
+          |{
+          |  "type": "record",
+          |  "name": "test",
+          |  "fields": [
+          |    {
+          |      "name": "createdAt",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      },
+          |      "doc": "text",
+          |      "default": 0
+          |    },
+          |    {
+          |      "name": "updatedAt",
+          |      "type":{
+          |        "type": "long",
+          |        "logicalType":"timestamp-millis"
+          |      },
+          |      "doc": "text"
+          |    }
+          |  ]
+          |}
+        """.stripMargin
+
+      val schema = new Schema.Parser().parse(schemaWithDefaultLoophole)
+
+      val result = createTopicMetadataV2Request(valueSchema = schema)
+      result.attempt.map(_ shouldBe RequiredSchemaValueFieldWithDefaultValueError("createdAt", schema, "Entity").asLeft)
+
+      val resultWithSkipValidation = createTopicMetadataV2Request(valueSchema = schema,
+        skipValidations = Some(List(SkipValidation.defaultLoopholeInRequiredField)))
+      resultWithSkipValidation.attempt.map(_ shouldBe ().asRight)
+    }
+
     "skip multiple validations: all" in {
       val valueSchemaString =
         """

From 2abb208983e441a27d0bb9e2eb2ac11cf3a023a2 Mon Sep 17 00:00:00 2001
From: Aman Minz <aman-minz@pluralsight.com>
Date: Wed, 24 Jan 2024 12:28:16 +0530
Subject: [PATCH 046/220] Trim the string provided by user and format the error
 message

---
 .../src/main/scala/hydra/kafka/model/SkipValidation.scala     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/model/SkipValidation.scala b/ingestors/kafka/src/main/scala/hydra/kafka/model/SkipValidation.scala
index 78667b4f0..fffef2cff 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/model/SkipValidation.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/model/SkipValidation.scala
@@ -29,12 +29,12 @@ object SkipValidation extends Enum[SkipValidation] {
   implicit val skipValidationUnmarshaller: Unmarshaller[String, List[SkipValidation]] =
     Unmarshaller.strict[String, List[SkipValidation]] { commaSeparatedValidations =>
       val stringValues = values.map(_.entryName)
-      val invalidValues = commaSeparatedValidations.split(",").filterNot(s => stringValues.contains(s) )
+      val invalidValues = commaSeparatedValidations.split(",").filterNot(s => stringValues.contains(s.trim))
       val validValues = commaSeparatedValidations.split(",").flatMap(s => values.find(v => v.entryName == s.trim)).toList
 
       if (invalidValues.nonEmpty) {
         throw DeserializationException(
-          s"Expected single or comma-separated values from enum[${values.mkString(", ")}] but received invalid value(s): ${invalidValues.mkString(",")}")
+          s"Expected a single or comma-separated values from enum[${values.mkString(", ")}] but received invalid value(s): [${invalidValues.mkString(",")}]")
       }
 
       validValues

From a2e790ba23c04ce7d397c37dd7d0e239cc0e4de3 Mon Sep 17 00:00:00 2001
From: Yuvi0710 <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 15:42:37 +0530
Subject: [PATCH 047/220] Updated .gitlab-ci.yml

---
 .gitlab-ci.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c541f89ff..71fa1fbe3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,10 +19,6 @@ before_script:
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   - apt-get update -yqq
   - apt-get install -yqq sbt docker.io
-  - echo "$DOCKER_REGISTRY_PASSWORD"
-  - echo "$DOCKER_REGISTRY_USERNAME"
-  #- echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
-  #- echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USER} --password-stdin
   
 stages:
   - build

From 99373cee5f5126cbdf921a033a9fca595a61eda7 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 10:34:20 +0000
Subject: [PATCH 048/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 71fa1fbe3..cf72bec76 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,7 +24,6 @@ stages:
   - build
   - package
   - publish
-  - finalize
 
 build:
   stage: build
@@ -45,7 +44,7 @@ package:
     - cd ../../
     - echo $CI_PIPELINE_IID > VERSION
     - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
-    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz']"
+    - echo "##teamcity[publishArtifacts '${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz']"
 
 publish:
   stage: publish
@@ -56,20 +55,13 @@ publish:
   before_script:
     - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
+    - pwd
+    - ls -lh
+    - cp ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz .
     - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
-
-finalize:
-  stage: finalize
-  image: harbor.vnerd.com/proxy/library/docker:cli
-  tags:
-    - ps
-    - docker
-  before_script:
-    - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
-  script:
     - docker push ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest
     - docker push ${DOCKER_REGISTRY_IMAGE}:latest
@@ -81,3 +73,4 @@ finalize:
 
 
 
+

From 90431f0648e4a4bf4dc45a13a6240e27ef464fc7 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 10:49:49 +0000
Subject: [PATCH 049/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cf72bec76..8198f61a7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -55,6 +55,8 @@ publish:
   before_script:
     - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
+    - apt-get install -yqq wget
+    - wget "https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}/-/jobs/${CI_JOB_ID}/artifacts/${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz" -O ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
     - pwd
     - ls -lh
     - cp ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz .

From 85754deaa37d727b736e1463cc9178e22db6723d Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 11:02:14 +0000
Subject: [PATCH 050/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8198f61a7..675578b4a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -55,8 +55,7 @@ publish:
   before_script:
     - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
-    - apt-get install -yqq wget
-    - wget "https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}/-/jobs/${CI_JOB_ID}/artifacts/${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz" -O ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
+    - curl -o ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz "https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}/-/jobs/${CI_JOB_ID}/artifacts/${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz"
     - pwd
     - ls -lh
     - cp ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz .

From 5803635c4604771f6f21fa2593aec128a45f4360 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 11:41:53 +0000
Subject: [PATCH 051/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 675578b4a..335965efa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -55,7 +55,7 @@ publish:
   before_script:
     - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
-    - curl -o ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz "https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}/-/jobs/${CI_JOB_ID}/artifacts/${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz"
+    - gitlab-runner download-artifact --token "$CI_JOB_TOKEN" --project "$CI_PROJECT_ID" --job "$CI_JOB_ID" --artifact ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
     - pwd
     - ls -lh
     - cp ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz .

From 330eaa9e1a6ffd962eeda3ea7799d4a07f9d8c74 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 11:44:50 +0000
Subject: [PATCH 052/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 335965efa..92d746927 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -45,6 +45,10 @@ package:
     - echo $CI_PIPELINE_IID > VERSION
     - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
     - echo "##teamcity[publishArtifacts '${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz']"
+  artifacts:
+    paths:
+      - ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
+
 
 publish:
   stage: publish
@@ -69,6 +73,8 @@ publish:
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)
     - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID}" > VERSION
+  dependencies:
+    - package
 
 
 

From 179ee9ce028211cbd837c849d7b4f2ecd3b5a188 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 12:08:16 +0000
Subject: [PATCH 053/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 92d746927..edded55ba 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -59,7 +59,6 @@ publish:
   before_script:
     - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
-    - gitlab-runner download-artifact --token "$CI_JOB_TOKEN" --project "$CI_PROJECT_ID" --job "$CI_JOB_ID" --artifact ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
     - pwd
     - ls -lh
     - cp ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz .

From 1ebebd8a77658af5ae68a898d1d1a9c54b831db7 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 12:21:58 +0000
Subject: [PATCH 054/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index edded55ba..cd3b79de1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -61,7 +61,6 @@ publish:
   script:
     - pwd
     - ls -lh
-    - cp ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz .
     - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .

From bf994e125ab3492f2b51bfddf1f0c4240f289a29 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 12:32:37 +0000
Subject: [PATCH 055/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cd3b79de1..98789ea72 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -45,10 +45,12 @@ package:
     - echo $CI_PIPELINE_IID > VERSION
     - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
     - echo "##teamcity[publishArtifacts '${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz']"
+    - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
+    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
   artifacts:
     paths:
       - ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
-
+      - ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
 
 publish:
   stage: publish
@@ -61,8 +63,6 @@ publish:
   script:
     - pwd
     - ls -lh
-    - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
-    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
     - docker push ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}

From 80a80b92b8e0cacac4c71c7e42ba0488ca78d3a4 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 13:01:37 +0000
Subject: [PATCH 056/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 98789ea72..55e191a61 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -51,6 +51,7 @@ package:
     paths:
       - ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
       - ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
+      - ps-publish/
 
 publish:
   stage: publish

From e407b2d221a6ff69252b9a4d0c257ce483850a19 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 13:13:02 +0000
Subject: [PATCH 057/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 55e191a61..fba141eed 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -59,6 +59,8 @@ publish:
   tags:
     - ps
     - docker
+  needs:
+    - package
   before_script:
     - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
@@ -72,8 +74,6 @@ publish:
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)
     - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID}" > VERSION
-  dependencies:
-    - package
 
 
 

From 23c3aacb53e99b0b00a3e50aa172016e2b3d8ae8 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 30 Jan 2024 06:02:22 +0000
Subject: [PATCH 058/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fba141eed..ed2305deb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,3 +1,7 @@
+include:
+  - project: 'pluralsight/experience/gitlab-helpers'
+    file: '/salt-deploy/helper.yml'
+
 image: openjdk:8
 
 variables:
@@ -24,6 +28,7 @@ stages:
   - build
   - package
   - publish
+  - deploy
 
 build:
   stage: build
@@ -75,6 +80,21 @@ publish:
     - CURMONTH=$(date +%-m)
     - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID}" > VERSION
 
+deploy: 
+  stage: deploy
+  extends: .saltDeploy
+  variables:
+    CI_JOB_STAGE: stage
+    SALT_TARGET: roles:hydra-publish-msk
+    SALT_PASSWORD: ${LDAP_PASS}
+    SALT_ARGUMENTS: systemd-app.deploy
+    SALT_USERNAME: tcity-data-platform
+    SALT_KWARGS: 'failhard=true test=True'
+    SALT_PILLAR: $CI_PROJECT_DIR/infrastructure/salt/pillars/expert-portal/pillar_overrides.yaml
+    BUILD_VERSION: ${CI_PIPELINE_IID}
+    SALT_URL: https://saltmaster-stage.vnerd.com:8000
+
+
 
 
 

From 52c1a58a3e9bd51ca82ba2e861494b0069aeb737 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 30 Jan 2024 06:20:44 +0000
Subject: [PATCH 059/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 26 +++++++++-----------------
 1 file changed, 9 insertions(+), 17 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ed2305deb..59b765e85 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,13 +7,9 @@ image: openjdk:8
 variables:
   ARTIFACT_NAME: hydra-publish
   IMAGE_NAME: hydra
-  DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
-  DOCKER_REGISTRY_URL: "harbor.vnerd.com/library"
-  DOCKER_HOST: tcp://localhost:2376
-  DOCKER_TLS_CERTDIR: "/certs"
-  DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
-  DOCKER_TLS_VERIFY: 1
-
+  DOCKER_REGISTRY_URL: harbor.vnerd.com/library
+  DOCKER_REGISTRY_IMAGE: ${DOCKER_REGISTRY_URL}/${IMAGE_NAME}
+  
 before_script:
   - apt-get update -yqq
   - apt-get install -yqq apt-transport-https apt-utils
@@ -21,8 +17,6 @@ before_script:
   - mkdir -p /root/.gnupg
   - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
-  - apt-get update -yqq
-  - apt-get install -yqq sbt docker.io
   
 stages:
   - build
@@ -79,6 +73,11 @@ publish:
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)
     - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID}" > VERSION
+  variables:
+    DOCKER_HOST: tcp://localhost:2376
+    DOCKER_TLS_CERTDIR: "/certs"
+    DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
+    DOCKER_TLS_VERIFY: 1
 
 deploy: 
   stage: deploy
@@ -90,13 +89,6 @@ deploy:
     SALT_ARGUMENTS: systemd-app.deploy
     SALT_USERNAME: tcity-data-platform
     SALT_KWARGS: 'failhard=true test=True'
-    SALT_PILLAR: $CI_PROJECT_DIR/infrastructure/salt/pillars/expert-portal/pillar_overrides.yaml
+    #SALT_PILLAR: $CI_PROJECT_DIR/infrastructure/salt/pillars/expert-portal/pillar_overrides.yaml
     BUILD_VERSION: ${CI_PIPELINE_IID}
     SALT_URL: https://saltmaster-stage.vnerd.com:8000
-
-
-
-
-
-
-

From 9c61ed034c1b530c3ddd166cc7b5cfd38af1ebc3 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 30 Jan 2024 06:31:11 +0000
Subject: [PATCH 060/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 59b765e85..f06baa3b7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,6 +17,9 @@ before_script:
   - mkdir -p /root/.gnupg
   - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
+  - apt-get update -yqq
+  - apt-get install -yqq sbt
+
   
 stages:
   - build

From c441a6779041d4e32f6bd0e25c5f738626610cdd Mon Sep 17 00:00:00 2001
From: Pavan Kumar Kulkarni <pavan-kumarkulkarni@pluralsight.com>
Date: Tue, 30 Jan 2024 08:21:39 +0000
Subject: [PATCH 061/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f06baa3b7..12480c5da 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -89,7 +89,7 @@ deploy:
     CI_JOB_STAGE: stage
     SALT_TARGET: roles:hydra-publish-msk
     SALT_PASSWORD: ${LDAP_PASS}
-    SALT_ARGUMENTS: systemd-app.deploy
+    SALT_ARGUMENTS: systemd-app
     SALT_USERNAME: tcity-data-platform
     SALT_KWARGS: 'failhard=true test=True'
     #SALT_PILLAR: $CI_PROJECT_DIR/infrastructure/salt/pillars/expert-portal/pillar_overrides.yaml

From 6d0c01f1034ddea64b9c4ed6d65b431652a726c3 Mon Sep 17 00:00:00 2001
From: Vikram Kunisetty <vikram-kunisetty@pluralsight.com>
Date: Thu, 1 Feb 2024 10:53:38 +0000
Subject: [PATCH 062/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 12480c5da..fe87439e2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -91,7 +91,7 @@ deploy:
     SALT_PASSWORD: ${LDAP_PASS}
     SALT_ARGUMENTS: systemd-app
     SALT_USERNAME: tcity-data-platform
-    SALT_KWARGS: 'failhard=true test=True'
+    SALT_KWARGS: 'failhard=true'
     #SALT_PILLAR: $CI_PROJECT_DIR/infrastructure/salt/pillars/expert-portal/pillar_overrides.yaml
     BUILD_VERSION: ${CI_PIPELINE_IID}
     SALT_URL: https://saltmaster-stage.vnerd.com:8000

From 87456a0b3b12107d95bec975e6e26a8d8923a19d Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 1 Feb 2024 11:12:01 +0000
Subject: [PATCH 063/220] Add new file

---
 pillar_overrides.yaml | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 pillar_overrides.yaml

diff --git a/pillar_overrides.yaml b/pillar_overrides.yaml
new file mode 100644
index 000000000..dd9a8b374
--- /dev/null
+++ b/pillar_overrides.yaml
@@ -0,0 +1,9 @@
+systemd-app:
+  artifacts:
+    expert-portal:
+      gitlab: True
+      artifactory_packaging : tgz
+      artifactory_artifact_version: latest
+      artifactory_artifact_path: hydra/publish
+      artifactory_repository: bounded-context
+      artifactory_artifact_name: hydra_publish

From 57c1b1f9d83dc423d4f237ee5777535592b5993d Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 1 Feb 2024 11:26:09 +0000
Subject: [PATCH 064/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fe87439e2..3c0c02956 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -9,6 +9,13 @@ variables:
   IMAGE_NAME: hydra
   DOCKER_REGISTRY_URL: harbor.vnerd.com/library
   DOCKER_REGISTRY_IMAGE: ${DOCKER_REGISTRY_URL}/${IMAGE_NAME}
+  ARTIFACTORY_REPOSITORY_URL: https://repository.vnerd.com/artifactory
+  ARTIFACTORY_REPOSITORY: bounded-context
+  ARTIFACTORY_ID: hydra/publish
+  ARTIFACTORY_ARTIFACT_VERSION: ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
+  ARTIFACTORY_PATH: ${ARTIFACTORY_ID}/${ARTIFACTORY_ARTIFACT_VERSION}
+  ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
+
   
 before_script:
   - apt-get update -yqq
@@ -49,6 +56,12 @@ package:
     - echo "##teamcity[publishArtifacts '${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz']"
     - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
+    - |
+      curl \
+      -H "Authorization: Bearer ${NPM_TOKEN}" \
+      -X PUT ${ARTIFACTORY_ARTIFACT_URL} \
+      -T ${ARTIFACTORY_ARTIFACT_VERSION}
+
   artifacts:
     paths:
       - ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
@@ -92,6 +105,6 @@ deploy:
     SALT_ARGUMENTS: systemd-app
     SALT_USERNAME: tcity-data-platform
     SALT_KWARGS: 'failhard=true'
-    #SALT_PILLAR: $CI_PROJECT_DIR/infrastructure/salt/pillars/expert-portal/pillar_overrides.yaml
+    SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
     BUILD_VERSION: ${CI_PIPELINE_IID}
     SALT_URL: https://saltmaster-stage.vnerd.com:8000

From 9bae0669ce75cceef2dbd09bd8352f3ac3346639 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Fri, 2 Feb 2024 05:39:24 +0000
Subject: [PATCH 065/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3c0c02956..545159a71 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -55,7 +55,7 @@ package:
     - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
     - echo "##teamcity[publishArtifacts '${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz']"
     - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
-    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
+    - echo "${ARTIFACTORY_ARTIFACT_URL}"
     - |
       curl \
       -H "Authorization: Bearer ${NPM_TOKEN}" \

From ae7cc7afdef2b3a7923d8f6ad1a7f0ed6f4bad7d Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Fri, 2 Feb 2024 09:51:56 +0000
Subject: [PATCH 066/220] Update pillar_overrides.yaml

---
 pillar_overrides.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pillar_overrides.yaml b/pillar_overrides.yaml
index dd9a8b374..fe307bcb3 100644
--- a/pillar_overrides.yaml
+++ b/pillar_overrides.yaml
@@ -1,6 +1,6 @@
 systemd-app:
   artifacts:
-    expert-portal:
+    hydra:
       gitlab: True
       artifactory_packaging : tgz
       artifactory_artifact_version: latest

From e494f2e1dee55f0c9253f073f996caab79ce0350 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Fri, 2 Feb 2024 10:37:29 +0000
Subject: [PATCH 067/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 545159a71..c6b0d274d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -12,9 +12,10 @@ variables:
   ARTIFACTORY_REPOSITORY_URL: https://repository.vnerd.com/artifactory
   ARTIFACTORY_REPOSITORY: bounded-context
   ARTIFACTORY_ID: hydra/publish
-  ARTIFACTORY_ARTIFACT_VERSION: ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
+  ARTIFACTORY_ARTIFACT_VERSION: ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz
   ARTIFACTORY_PATH: ${ARTIFACTORY_ID}/${ARTIFACTORY_ARTIFACT_VERSION}
   ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
+  BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
 
   
 before_script:
@@ -27,7 +28,6 @@ before_script:
   - apt-get update -yqq
   - apt-get install -yqq sbt
 
-  
 stages:
   - build
   - package
@@ -51,10 +51,10 @@ package:
     - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic-agent-4.4.0.jar
     - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic.yml
     - cd ../../
-    - echo $CI_PIPELINE_IID > VERSION
-    - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
-    - echo "##teamcity[publishArtifacts '${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz']"
-    - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
+    - echo $BUILD_VERSION > VERSION
+    - tar czf ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
+    - echo "##teamcity[publishArtifacts '${ARTIFACT_NAME}-${BUILD_VERSION}.tgz']"
+    - md5sum ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz > ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz.md5
     - echo "${ARTIFACTORY_ARTIFACT_URL}"
     - |
       curl \
@@ -64,8 +64,8 @@ package:
 
   artifacts:
     paths:
-      - ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
-      - ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
+      - ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz
+      - ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz.md5
       - ps-publish/
 
 publish:
@@ -82,13 +82,13 @@ publish:
     - pwd
     - ls -lh
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
-    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
-    - docker push ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
+    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${BUILD_VERSION}
+    - docker push ${DOCKER_REGISTRY_IMAGE}:${BUILD_VERSION}
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest
     - docker push ${DOCKER_REGISTRY_IMAGE}:latest
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)
-    - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID}" > VERSION
+    - echo -n "${CURYEAR}.${CURMONTH}.${BUILD_VERSION}" > VERSION
   variables:
     DOCKER_HOST: tcp://localhost:2376
     DOCKER_TLS_CERTDIR: "/certs"
@@ -98,6 +98,10 @@ publish:
 deploy: 
   stage: deploy
   extends: .saltDeploy
+  before_script:
+    - cat $SALT_PILLAR
+    - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR
+    - cat $SALT_PILLAR  
   variables:
     CI_JOB_STAGE: stage
     SALT_TARGET: roles:hydra-publish-msk
@@ -106,5 +110,4 @@ deploy:
     SALT_USERNAME: tcity-data-platform
     SALT_KWARGS: 'failhard=true'
     SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
-    BUILD_VERSION: ${CI_PIPELINE_IID}
     SALT_URL: https://saltmaster-stage.vnerd.com:8000

From c42977629c92fb056f503447c84683716339de7d Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Fri, 2 Feb 2024 10:49:26 +0000
Subject: [PATCH 068/220] Update pillar_overrides.yaml

---
 pillar_overrides.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pillar_overrides.yaml b/pillar_overrides.yaml
index fe307bcb3..222fd8018 100644
--- a/pillar_overrides.yaml
+++ b/pillar_overrides.yaml
@@ -1,9 +1,9 @@
 systemd-app:
   artifacts:
-    hydra:
+    hydra-publish:
       gitlab: True
       artifactory_packaging : tgz
       artifactory_artifact_version: latest
       artifactory_artifact_path: hydra/publish
       artifactory_repository: bounded-context
-      artifactory_artifact_name: hydra_publish
+      artifactory_artifact_name: hydra-publish

From 08cf0f01eec0fde0b7ac8e70f119c77a25fdcb26 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 13 Feb 2024 06:34:32 +0000
Subject: [PATCH 069/220] chore(PDP-1009): Updated .gitlab-ci.yml file to
 deploy on stage and prod

---
 .gitlab-ci.yml | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c6b0d274d..576f23868 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,7 +32,8 @@ stages:
   - build
   - package
   - publish
-  - deploy
+  - deploy_staging
+  - deploy_production
 
 build:
   stage: build
@@ -95,9 +96,10 @@ publish:
     DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
     DOCKER_TLS_VERIFY: 1
 
-deploy: 
-  stage: deploy
+deploy_staging: 
+  stage: deploy_staging
   extends: .saltDeploy
+  when: manual
   before_script:
     - cat $SALT_PILLAR
     - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR
@@ -111,3 +113,21 @@ deploy:
     SALT_KWARGS: 'failhard=true'
     SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
     SALT_URL: https://saltmaster-stage.vnerd.com:8000
+
+deploy_production: 
+  stage: deploy_production
+  extends: .saltDeploy
+  when: manual
+  before_script:
+    - cat $SALT_PILLAR
+    - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR
+    - cat $SALT_PILLAR  
+  variables:
+    CI_JOB_STAGE: stage
+    SALT_TARGET: roles:hydra-publish-msk
+    SALT_PASSWORD: ${LDAP_PASS}
+    SALT_ARGUMENTS: systemd-app
+    SALT_USERNAME: tcity-data-platform
+    SALT_KWARGS: 'failhard=true'
+    SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
+    SALT_URL: https://saltmaster-production.vnerd.com:8000

From 6f737d58320b7b4685a7d442dcdaa61714933ea6 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 13 Feb 2024 06:38:49 +0000
Subject: [PATCH 070/220] chore(PDP-1009): Updated .gitlab-ci.yml to move stage
 variables

---
 .gitlab-ci.yml | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 576f23868..cc43f7c08 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,6 +16,12 @@ variables:
   ARTIFACTORY_PATH: ${ARTIFACTORY_ID}/${ARTIFACTORY_ARTIFACT_VERSION}
   ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
   BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
+  SALT_TARGET: roles:hydra-publish-msk
+  SALT_PASSWORD: ${LDAP_PASS}
+  SALT_ARGUMENTS: systemd-app
+  SALT_USERNAME: tcity-data-platform
+  SALT_KWARGS: 'failhard=true'
+  SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
 
   
 before_script:
@@ -106,12 +112,6 @@ deploy_staging:
     - cat $SALT_PILLAR  
   variables:
     CI_JOB_STAGE: stage
-    SALT_TARGET: roles:hydra-publish-msk
-    SALT_PASSWORD: ${LDAP_PASS}
-    SALT_ARGUMENTS: systemd-app
-    SALT_USERNAME: tcity-data-platform
-    SALT_KWARGS: 'failhard=true'
-    SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
     SALT_URL: https://saltmaster-stage.vnerd.com:8000
 
 deploy_production: 
@@ -123,11 +123,5 @@ deploy_production:
     - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR
     - cat $SALT_PILLAR  
   variables:
-    CI_JOB_STAGE: stage
-    SALT_TARGET: roles:hydra-publish-msk
-    SALT_PASSWORD: ${LDAP_PASS}
-    SALT_ARGUMENTS: systemd-app
-    SALT_USERNAME: tcity-data-platform
-    SALT_KWARGS: 'failhard=true'
-    SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
+    CI_JOB_STAGE: production
     SALT_URL: https://saltmaster-production.vnerd.com:8000

From 0da411c2943e57a32da5e24a6f0d8862e79eb312 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 13 Feb 2024 14:43:51 +0000
Subject: [PATCH 071/220] Updated .gitlab-ci.yml file

---
 .gitlab-ci.yml | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cc43f7c08..576f23868 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,12 +16,6 @@ variables:
   ARTIFACTORY_PATH: ${ARTIFACTORY_ID}/${ARTIFACTORY_ARTIFACT_VERSION}
   ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
   BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
-  SALT_TARGET: roles:hydra-publish-msk
-  SALT_PASSWORD: ${LDAP_PASS}
-  SALT_ARGUMENTS: systemd-app
-  SALT_USERNAME: tcity-data-platform
-  SALT_KWARGS: 'failhard=true'
-  SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
 
   
 before_script:
@@ -112,6 +106,12 @@ deploy_staging:
     - cat $SALT_PILLAR  
   variables:
     CI_JOB_STAGE: stage
+    SALT_TARGET: roles:hydra-publish-msk
+    SALT_PASSWORD: ${LDAP_PASS}
+    SALT_ARGUMENTS: systemd-app
+    SALT_USERNAME: tcity-data-platform
+    SALT_KWARGS: 'failhard=true'
+    SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
     SALT_URL: https://saltmaster-stage.vnerd.com:8000
 
 deploy_production: 
@@ -123,5 +123,11 @@ deploy_production:
     - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR
     - cat $SALT_PILLAR  
   variables:
-    CI_JOB_STAGE: production
+    CI_JOB_STAGE: stage
+    SALT_TARGET: roles:hydra-publish-msk
+    SALT_PASSWORD: ${LDAP_PASS}
+    SALT_ARGUMENTS: systemd-app
+    SALT_USERNAME: tcity-data-platform
+    SALT_KWARGS: 'failhard=true'
+    SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
     SALT_URL: https://saltmaster-production.vnerd.com:8000

From 7c5968fe6e24dc0c36bfc42a9bf173b36d41093b Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 20 Feb 2024 14:22:31 +0000
Subject: [PATCH 072/220] Add new file

---
 helm/eks-dev-values.yml | 59 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 helm/eks-dev-values.yml

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
new file mode 100644
index 000000000..f01277d47
--- /dev/null
+++ b/helm/eks-dev-values.yml
@@ -0,0 +1,59 @@
+image:
+  repository: harbor.vnerd.com/library/hydra
+  pullPolicy: Always
+  tag: latest
+
+deployment:
+  apiVersion: "apps/v1"
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 0
+    type: RollingUpdate
+
+replicas: 4
+
+#Adding velero backup for k8s services
+#velero.io/exclude-from-backup" = true 
+service:
+  type: ClusterIP
+  port: 8080
+  annotations: {}
+  labels: {}
+
+# ingress:
+#   enabled: true
+#   ingressClassName: alb
+#   labels: {}
+#   hosts:
+#     - dvs.eplur-staging.vnerd.com
+#     - dvs-staging.vnerd.com
+#   tls:
+#     - hosts:
+#         - dvs.eplur-staging.vnerd.com
+#   path: /
+
+
+resources:
+  limits:
+    memory: 512Mi
+  requests:
+    cpu: 250m
+    memory: 50Mi
+
+tolerations: []
+
+nodeSelector: {}
+
+config:
+  LOG_DIR: /var/log/hydra/
+  LOG_LEVEL: DEBUG
+  HYDRA_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: b-1.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-3.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-2.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098
+  HYDRA_SCHEMA_REGISTRY_URL: https://dvs-dev-schema-registry.eplur-staging.vnerd.com:8081
+  HYDRA_V2_METADATA_CONTACT: '#dev-data-platform'
+  HYDRA_REPLICATION_FACTOR: 1
+  HYDRA_V2_METADATA_CONSUMER_GROUP: 'v2MetadataConsumer'
+  HYDRA_V2_CREATE_TOPICS_ENABLED: 'true'
+  HYDRA_V2_METADATA_CREATE_ON_STARTUP: 'true'
+  HYDRA_MIN_INSYNC_REPLICAS: 1
+
+

From 641f4088151ff31263eea5afda25d058b531ec6d Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 20 Feb 2024 14:26:15 +0000
Subject: [PATCH 073/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 576f23868..267dd5e2d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,6 +16,7 @@ variables:
   ARTIFACTORY_PATH: ${ARTIFACTORY_ID}/${ARTIFACTORY_ARTIFACT_VERSION}
   ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
   BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
+  BOUNDED_CONTEXT: adapt-dvs
 
   
 before_script:
@@ -32,6 +33,7 @@ stages:
   - build
   - package
   - publish
+  - deploy_dev
   - deploy_staging
   - deploy_production
 
@@ -96,6 +98,30 @@ publish:
     DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
     DOCKER_TLS_VERIFY: 1
 
+deploy_dev:
+    stage: deploy_dev
+    needs: ["publish"]
+    extends: .helmDeploy
+    environment:
+      name: staging
+    when: manual
+    variables:
+      HELM_CHART_NAME: ps-service
+      HELM_DEPLOY_TOKEN: ${HELM_DEPLOY_TOKEN}
+      K8S_CLUSTER_NAME: app-eks.eplur-staging.us-west-2
+      HELM_K8S_CLUSTER_URL: https://6C29C0073BB19BEF220B9437E6962AF2.gr7.us-west-2.eks.amazonaws.com
+      HELM_TARGET_NAMESPACE: ${BOUNDED_CONTEXT}
+      APPLICATION_ROLE: hydra
+      HELM_VALUES_FILE: helm/eks-dev-values.yml
+      HELM_SET_VALUES: image.tag=${BUILD_VERSION}
+    before_script:
+      #- export BUILD_VERSION="$(date +%Y).$(date +%m).$(date +%d).${CI_COMMIT_SHORT_SHA}"
+      - kubectl config set-cluster app-${CI_ENVIRONMENT_NAME%/*} --server=${HELM_K8S_CLUSTER_URL} --embed-certs --certificate-authority="$HELM_K8S_CLUSTER_CACERT"
+      - kubectl config set-credentials deploy --token=`echo ${HELM_DEPLOY_TOKEN} | base64 -d`
+      - kubectl config set-context deploy --cluster=app-${CI_ENVIRONMENT_NAME%/*} --namespace=${HELM_TARGET_NAMESPACE} --user=deploy
+      - kubectl config use-context deploy
+
+
 deploy_staging: 
   stage: deploy_staging
   extends: .saltDeploy

From aaabb83949e63429a0ef3e5506503366b818cbb0 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 20 Feb 2024 14:29:49 +0000
Subject: [PATCH 074/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 267dd5e2d..44134a960 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,8 @@
 include:
   - project: 'pluralsight/experience/gitlab-helpers'
     file: '/salt-deploy/helper.yml'
+  - project: 'pluralsight/experience/gitlab-helpers'
+    file: '/helm-deploy/helper.yml'
 
 image: openjdk:8
 

From f81dc8d7cef9077678ae89973c60dba04e4ac9d5 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 20 Feb 2024 14:50:03 +0000
Subject: [PATCH 075/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index f01277d47..d88862dad 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -35,10 +35,10 @@ service:
 
 resources:
   limits:
-    memory: 512Mi
+    memory: 1Gi
   requests:
-    cpu: 250m
-    memory: 50Mi
+    cpu: 2
+    memory: 1Gi
 
 tolerations: []
 

From 3fa8485c96233f74c38275f7d5a3bbb4ee176f7a Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 20 Feb 2024 15:04:20 +0000
Subject: [PATCH 076/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index d88862dad..e9ec57dab 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -1,7 +1,7 @@
 image:
   repository: harbor.vnerd.com/library/hydra
   pullPolicy: Always
-  tag: latest
+  tag: 1.0.70
 
 deployment:
   apiVersion: "apps/v1"

From c8db837957de29bcca1532031ebd808a0deb6ae4 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 20 Feb 2024 15:30:31 +0000
Subject: [PATCH 077/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index e9ec57dab..1b4bb2b2c 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -1,7 +1,7 @@
 image:
   repository: harbor.vnerd.com/library/hydra
   pullPolicy: Always
-  tag: 1.0.70
+  tag: 64
 
 deployment:
   apiVersion: "apps/v1"

From fbc722e7d38da3736a17ac1ba65b1e6d63d3d17b Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 22 Feb 2024 14:15:18 +0000
Subject: [PATCH 078/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 1b4bb2b2c..1c791054b 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -1,7 +1,7 @@
 image:
   repository: harbor.vnerd.com/library/hydra
   pullPolicy: Always
-  tag: 64
+  tag: '%IMAGE_TAG%'
 
 deployment:
   apiVersion: "apps/v1"

From 064f967e5ca69ec8c0cb4ce72101f679d32260dc Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 22 Feb 2024 14:20:42 +0000
Subject: [PATCH 079/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 44134a960..28461feec 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -107,6 +107,14 @@ deploy_dev:
     environment:
       name: staging
     when: manual
+    before_script:
+      - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}/g" $HELM_VALUES_FILE
+      - cat $HELM_VALUES_FILE
+      - echo 1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}
+      - kubectl config set-cluster app-${CI_ENVIRONMENT_NAME%/*} --server=${HELM_K8S_CLUSTER_URL} --embed-certs --certificate-authority="$HELM_K8S_CLUSTER_CACERT"
+      - kubectl config set-credentials deploy --token=`echo ${HELM_DEPLOY_TOKEN} | base64 -d`
+      - kubectl config set-context deploy --cluster=app-${CI_ENVIRONMENT_NAME%/*} --namespace=${HELM_TARGET_NAMESPACE} --user=deploy
+      - kubectl config use-context deploy
     variables:
       HELM_CHART_NAME: ps-service
       HELM_DEPLOY_TOKEN: ${HELM_DEPLOY_TOKEN}
@@ -115,14 +123,6 @@ deploy_dev:
       HELM_TARGET_NAMESPACE: ${BOUNDED_CONTEXT}
       APPLICATION_ROLE: hydra
       HELM_VALUES_FILE: helm/eks-dev-values.yml
-      HELM_SET_VALUES: image.tag=${BUILD_VERSION}
-    before_script:
-      #- export BUILD_VERSION="$(date +%Y).$(date +%m).$(date +%d).${CI_COMMIT_SHORT_SHA}"
-      - kubectl config set-cluster app-${CI_ENVIRONMENT_NAME%/*} --server=${HELM_K8S_CLUSTER_URL} --embed-certs --certificate-authority="$HELM_K8S_CLUSTER_CACERT"
-      - kubectl config set-credentials deploy --token=`echo ${HELM_DEPLOY_TOKEN} | base64 -d`
-      - kubectl config set-context deploy --cluster=app-${CI_ENVIRONMENT_NAME%/*} --namespace=${HELM_TARGET_NAMESPACE} --user=deploy
-      - kubectl config use-context deploy
-
 
 deploy_staging: 
   stage: deploy_staging

From df1cc7419313444409a73f2280f82fb9c9ea20c8 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 22 Feb 2024 14:36:39 +0000
Subject: [PATCH 080/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 28461feec..9605ffb4f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -20,7 +20,6 @@ variables:
   BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
   BOUNDED_CONTEXT: adapt-dvs
 
-  
 before_script:
   - apt-get update -yqq
   - apt-get install -yqq apt-transport-https apt-utils

From 9e3be0f768849696da36d886eeebda1f00a0ec1a Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Fri, 23 Feb 2024 05:02:21 +0000
Subject: [PATCH 081/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9605ffb4f..c7508e184 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -86,8 +86,8 @@ publish:
     - pwd
     - ls -lh
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
-    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${BUILD_VERSION}
-    - docker push ${DOCKER_REGISTRY_IMAGE}:${BUILD_VERSION}
+    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${BUILD_VERSION}-${CI_COMMIT_SHORT_SHA}
+    - docker push ${DOCKER_REGISTRY_IMAGE}:${BUILD_VERSION}-${CI_COMMIT_SHORT_SHA}
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest
     - docker push ${DOCKER_REGISTRY_IMAGE}:latest
     - CURYEAR=$(date +%Y)
@@ -107,7 +107,7 @@ deploy_dev:
       name: staging
     when: manual
     before_script:
-      - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}/g" $HELM_VALUES_FILE
+      - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}/g" $HELM_VALUES_FILE
       - cat $HELM_VALUES_FILE
       - echo 1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}
       - kubectl config set-cluster app-${CI_ENVIRONMENT_NAME%/*} --server=${HELM_K8S_CLUSTER_URL} --embed-certs --certificate-authority="$HELM_K8S_CLUSTER_CACERT"

From 320c93885ed729fc512291bc81588e022776331b Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Fri, 23 Feb 2024 06:09:25 +0000
Subject: [PATCH 082/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c7508e184..7e5a93145 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -107,7 +107,8 @@ deploy_dev:
       name: staging
     when: manual
     before_script:
-      - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}/g" $HELM_VALUES_FILE
+     # - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}/g" $HELM_VALUES_FILE
+      - sed -i -e "s/%IMAGE_TAG%/0.11.3.1027/g" $HELM_VALUES_FILE
       - cat $HELM_VALUES_FILE
       - echo 1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}
       - kubectl config set-cluster app-${CI_ENVIRONMENT_NAME%/*} --server=${HELM_K8S_CLUSTER_URL} --embed-certs --certificate-authority="$HELM_K8S_CLUSTER_CACERT"

From d166e9f0f68d87b6518bd964cd329e210521b86c Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Fri, 23 Feb 2024 06:35:18 +0000
Subject: [PATCH 083/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 1c791054b..1fb6d629e 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -10,7 +10,7 @@ deployment:
       maxUnavailable: 0
     type: RollingUpdate
 
-replicas: 4
+replicas: 2
 
 #Adding velero backup for k8s services
 #velero.io/exclude-from-backup" = true 
@@ -32,13 +32,12 @@ service:
 #         - dvs.eplur-staging.vnerd.com
 #   path: /
 
-
 resources:
   limits:
     memory: 1Gi
   requests:
-    cpu: 2
-    memory: 1Gi
+    cpu: 250m
+    memory: 512Mi
 
 tolerations: []
 

From 5a8457c9ffecb8deb2f9a92705126508e3f81d59 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Fri, 23 Feb 2024 06:35:45 +0000
Subject: [PATCH 084/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7e5a93145..c7508e184 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -107,8 +107,7 @@ deploy_dev:
       name: staging
     when: manual
     before_script:
-     # - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}/g" $HELM_VALUES_FILE
-      - sed -i -e "s/%IMAGE_TAG%/0.11.3.1027/g" $HELM_VALUES_FILE
+      - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}/g" $HELM_VALUES_FILE
       - cat $HELM_VALUES_FILE
       - echo 1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}
       - kubectl config set-cluster app-${CI_ENVIRONMENT_NAME%/*} --server=${HELM_K8S_CLUSTER_URL} --embed-certs --certificate-authority="$HELM_K8S_CLUSTER_CACERT"

From f5cc8201c081ffb6866065ec97ef57a9ec0275a9 Mon Sep 17 00:00:00 2001
From: Pavan Kumar Kulkarni <pavan-kumarkulkarni@pluralsight.com>
Date: Tue, 27 Feb 2024 06:54:03 +0000
Subject: [PATCH 085/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 1fb6d629e..43eb66968 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -45,7 +45,7 @@ nodeSelector: {}
 
 config:
   LOG_DIR: /var/log/hydra/
-  LOG_LEVEL: DEBUG
+  LOG_LEVEL: INFO
   HYDRA_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: b-1.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-3.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-2.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098
   HYDRA_SCHEMA_REGISTRY_URL: https://dvs-dev-schema-registry.eplur-staging.vnerd.com:8081
   HYDRA_V2_METADATA_CONTACT: '#dev-data-platform'

From 655b85b76a1a534cd0e6d8d0cf4824420b967715 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 27 Feb 2024 07:49:31 +0000
Subject: [PATCH 086/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 43eb66968..38239cff9 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -34,10 +34,10 @@ service:
 
 resources:
   limits:
-    memory: 1Gi
+    memory: 5Gi
   requests:
     cpu: 250m
-    memory: 512Mi
+    memory: 1Gi
 
 tolerations: []
 

From bc1423587fa7becc8e3be54934d09dbde7ef5d76 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 28 Feb 2024 05:53:25 +0000
Subject: [PATCH 087/220] Update Dockerfile.new

---
 Dockerfile.new | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.new b/Dockerfile.new
index 583af0638..1a20a87de 100644
--- a/Dockerfile.new
+++ b/Dockerfile.new
@@ -2,7 +2,7 @@ FROM jelastic/jetty:9.4.49-openjdk-1.8.0_352
 
 USER root
 
-ENV JAVA_OPTS="-Xmx2G"
+ENV JAVA_OPTS="-Xmx4g"
 
 ENV CONTAINER_HTTP_PORT="8088"
 
@@ -12,4 +12,4 @@ EXPOSE 8088
 
 COPY ps-publish/ /ps-publish
 
-ENTRYPOINT ["/ps-publish/bin/hydra-ingest"]
\ No newline at end of file
+ENTRYPOINT ["/ps-publish/bin/hydra-ingest"]

From f75ed09b97462881686a74dd16d57634b6a8c257 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 28 Feb 2024 06:16:11 +0000
Subject: [PATCH 088/220] Update Dockerfile.new

---
 Dockerfile.new | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile.new b/Dockerfile.new
index 1a20a87de..16d541be6 100644
--- a/Dockerfile.new
+++ b/Dockerfile.new
@@ -2,7 +2,7 @@ FROM jelastic/jetty:9.4.49-openjdk-1.8.0_352
 
 USER root
 
-ENV JAVA_OPTS="-Xmx4g"
+ENV JAVA_OPTS="-Xmx2g"
 
 ENV CONTAINER_HTTP_PORT="8088"
 

From cad5cf5a20edd868aba36db6350151461e4b5db7 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 28 Feb 2024 06:39:56 +0000
Subject: [PATCH 089/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c7508e184..14f15736e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -120,7 +120,7 @@ deploy_dev:
       K8S_CLUSTER_NAME: app-eks.eplur-staging.us-west-2
       HELM_K8S_CLUSTER_URL: https://6C29C0073BB19BEF220B9437E6962AF2.gr7.us-west-2.eks.amazonaws.com
       HELM_TARGET_NAMESPACE: ${BOUNDED_CONTEXT}
-      APPLICATION_ROLE: hydra
+      APPLICATION_ROLE: dev-hydra
       HELM_VALUES_FILE: helm/eks-dev-values.yml
 
 deploy_staging: 

From a6fc3979e188f26478c9d4b65bbb30e856d6a364 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 28 Feb 2024 07:08:50 +0000
Subject: [PATCH 090/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 38239cff9..8724d5b92 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -34,7 +34,7 @@ service:
 
 resources:
   limits:
-    memory: 5Gi
+    memory: 6Gi
   requests:
     cpu: 250m
     memory: 1Gi

From 5189e2a95eea59cedc79d8e9b8ad87b7c129076a Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 4 Mar 2024 06:32:14 +0000
Subject: [PATCH 091/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 8724d5b92..4f3882a4e 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -20,17 +20,20 @@ service:
   annotations: {}
   labels: {}
 
-# ingress:
-#   enabled: true
-#   ingressClassName: alb
-#   labels: {}
-#   hosts:
-#     - dvs.eplur-staging.vnerd.com
-#     - dvs-staging.vnerd.com
-#   tls:
-#     - hosts:
-#         - dvs.eplur-staging.vnerd.com
-#   path: /
+
+ingress:
+  enabled: true
+  ingressClassName: internal
+  annotations: {}
+  labels: {}
+  hosts:
+    - dev-hydra.eplur-staging.vnerd.com
+  tls:
+    - hosts:
+        - dev-hydra.eplur-staging.vnerd.com
+  path: /
+  pathType: Prefix
+
 
 resources:
   limits:

From 2c0343becefeffbd625b2189831e37e11cecb8a6 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 6 Mar 2024 06:31:07 +0000
Subject: [PATCH 092/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 35 +++++++++++++++++++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 14f15736e..55f48ae8a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,6 +19,15 @@ variables:
   ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
   BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
   BOUNDED_CONTEXT: adapt-dvs
+  ENV: ${CI_JOB_STAGE} 
+  SERVICE_NAME: "dev-hydra"
+  SLACK_ICON_EMOJI: ":gitlab:"
+  SLACK_CHANNEL: '#data-platform-alerts'
+  SLACK_MESSAGE: |
+    *[[SERVICE_NAME]]* deployed to *[[ENV]]*.
+    Version: *[[BUILD_VERSION]]*
+    [[PIPELINE_LINK]]
+  SLACK_USERNAME: 'GITLAB'
 
 before_script:
   - apt-get update -yqq
@@ -36,7 +45,8 @@ stages:
   - publish
   - deploy_dev
   - deploy_staging
-  - deploy_production
+  - deploy_productio
+  - notify
 
 build:
   stage: build
@@ -99,13 +109,22 @@ publish:
     DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
     DOCKER_TLS_VERIFY: 1
 
+.notify-all:
+  image: harbor.vnerd.com/library/ps-helm:latest
+  tags:
+    - ps
+    - docker 
+  script:
+    - ps-notify slack "$SLACK_MESSAGE" "$SLACK_CHANNEL"  --slack-username $SLACK_USERNAME --slack-icon-emoji $SLACK_ICON_EMOJI
+  variables:
+    NODE_ENV: $ENV
+
 deploy_dev:
     stage: deploy_dev
     needs: ["publish"]
     extends: .helmDeploy
     environment:
       name: staging
-    when: manual
     before_script:
       - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}/g" $HELM_VALUES_FILE
       - cat $HELM_VALUES_FILE
@@ -123,6 +142,17 @@ deploy_dev:
       APPLICATION_ROLE: dev-hydra
       HELM_VALUES_FILE: helm/eks-dev-values.yml
 
+slack:dev:
+  stage: notify_test
+  extends: .notify-all
+  when: on_success
+  needs: ['deploy_dev']
+  before_script:
+    - echo "Sending notification to slack"
+  variables:
+    ENV: "DEV Cluster"
+
+
 deploy_staging: 
   stage: deploy_staging
   extends: .saltDeploy
@@ -158,3 +188,4 @@ deploy_production:
     SALT_KWARGS: 'failhard=true'
     SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
     SALT_URL: https://saltmaster-production.vnerd.com:8000
+

From 2626423c37852edd4bce9e884130c1a2e7d585ae Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 6 Mar 2024 06:32:01 +0000
Subject: [PATCH 093/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 55f48ae8a..6a59dff37 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -143,7 +143,7 @@ deploy_dev:
       HELM_VALUES_FILE: helm/eks-dev-values.yml
 
 slack:dev:
-  stage: notify_test
+  stage: notify
   extends: .notify-all
   when: on_success
   needs: ['deploy_dev']

From 498acfff78a66e22b88b75af057a573f09501b82 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 6 Mar 2024 06:33:53 +0000
Subject: [PATCH 094/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6a59dff37..23fc97ae0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -45,7 +45,7 @@ stages:
   - publish
   - deploy_dev
   - deploy_staging
-  - deploy_productio
+  - deploy_production
   - notify
 
 build:

From 1e44b1c76de245a2dc6fc5bc8903f84f1a8ae671 Mon Sep 17 00:00:00 2001
From: shreedhar-kc <139169398+shreedhar-kc@users.noreply.github.com>
Date: Wed, 13 Mar 2024 20:22:57 +0530
Subject: [PATCH 095/220] Revert "Revert "ADAPT-1701: update schema-registry
 client and kafka version (#13)" (#19)"

This reverts commit b0137665a3911eeb38dace61c8718047dcbe0192.
---
 .gitignore                                    |   5 +
 .../registry/ConfluentSchemaRegistry.scala    |   2 +-
 .../registry/RedisSchemaRegistryClient.scala  | 314 ++++++++++--------
 .../hydra/avro/registry/SchemaRegistry.scala  | 204 +++++++-----
 .../ConfluentSchemaRegistrySpec.scala         |   3 +-
 .../RedisSchemaRegistryClientSpec.scala       |  48 +--
 .../avro/registry/SchemaRegistrySpec.scala    |   6 +-
 .../resource/SchemaResourceLoaderSpec.scala   |  11 +-
 build.sbt                                     |  27 +-
 .../scala/hydra/common/util/ClassUtils.scala  |   9 +
 .../hydra/core/akka/SchemaRegistryActor.scala |   6 +-
 .../scala/hydra/core/ingest/Ingestor.scala    |   2 +-
 .../core/akka/SchemaRegistryActorSpec.scala   |   7 +-
 .../ingest/http/SchemasEndpointSpec.scala     |   2 +-
 .../kafka/algebras/KafkaAdminAlgebra.scala    |   1 -
 .../hydra/kafka/producer/AvroRecord.scala     |   2 +-
 .../hydra/kafka/producer/JsonRecord.scala     |   1 -
 .../hydra/kafka/producer/KafkaRecord.scala    |   2 +-
 .../hydra/kafka/producer/StringRecord.scala   |   2 +-
 .../kafka/transport/KafkaTransport.scala      |   7 +-
 .../scala/hydra/kafka/KafkaUtilsSpec.scala    |   2 +-
 .../algebras/KafkaAdminAlgebraSpec.scala      |   2 +-
 .../algebras/KafkaClientAlgebraSpec.scala     |   2 +-
 .../consumer/KafkaConsumerProxySpec.scala     |   4 +-
 .../endpoints/BootstrapEndpointSpec.scala     |   2 +-
 .../endpoints/TopicMetadataEndpointSpec.scala |   6 +-
 .../services/TopicBootstrapActorSpec.scala    |   4 +-
 .../kafka/transport/KafkaMetricsSpec.scala    |   2 +-
 .../transport/KafkaProducerProxySpec.scala    |   4 +-
 .../kafka/transport/KafkaTransportSpec.scala  |  16 +-
 project/Dependencies.scala                    |  42 ++-
 31 files changed, 416 insertions(+), 331 deletions(-)
 create mode 100644 common/src/main/scala/hydra/common/util/ClassUtils.scala

diff --git a/.gitignore b/.gitignore
index 6ab661f5b..592cde8e6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -70,3 +70,8 @@ build.properties
 #ensime http://ensime.github.io/
 .ensime
 .ensime_cache
+
+.bsp
+
+# ignore temp files and folders
+__temp*
\ No newline at end of file
diff --git a/avro/src/main/scala/hydra/avro/registry/ConfluentSchemaRegistry.scala b/avro/src/main/scala/hydra/avro/registry/ConfluentSchemaRegistry.scala
index 56a121570..5ee245ad0 100644
--- a/avro/src/main/scala/hydra/avro/registry/ConfluentSchemaRegistry.scala
+++ b/avro/src/main/scala/hydra/avro/registry/ConfluentSchemaRegistry.scala
@@ -52,7 +52,7 @@ object ConfluentSchemaRegistry extends LoggingAdapter {
 
   private val cachedClients = CacheBuilder
     .newBuilder()
-    .build(
+    .build[SchemaRegistryClientInfo, ConfluentSchemaRegistry](
       new CacheLoader[SchemaRegistryClientInfo, ConfluentSchemaRegistry] {
 
         def load(info: SchemaRegistryClientInfo): ConfluentSchemaRegistry = {
diff --git a/avro/src/main/scala/hydra/avro/registry/RedisSchemaRegistryClient.scala b/avro/src/main/scala/hydra/avro/registry/RedisSchemaRegistryClient.scala
index 115db9a4f..f2c2fd26a 100644
--- a/avro/src/main/scala/hydra/avro/registry/RedisSchemaRegistryClient.scala
+++ b/avro/src/main/scala/hydra/avro/registry/RedisSchemaRegistryClient.scala
@@ -2,7 +2,7 @@ package hydra.avro.registry
 
 import com.typesafe.config.{Config, ConfigFactory}
 import hydra.common.config.ConfigSupport.ConfigImplicits
-import hydra.common.config.KafkaConfigUtils.SchemaRegistrySecurityConfig
+import hydra.common.config.KafkaConfigUtils.{SchemaRegistrySecurityConfig, schemaRegistrySecurityConfig}
 import io.confluent.kafka.schemaregistry.client.{SchemaMetadata, SchemaRegistryClient}
 import io.confluent.kafka.schemaregistry.client.rest.RestService
 import io.confluent.kafka.schemaregistry.client.security.SslFactory
@@ -13,8 +13,14 @@ import scalacache.modes.try_._
 import scalacache.serialization.Codec
 import scalacache.serialization.Codec.DecodingResult
 import _root_.redis.clients.jedis._
+import io.confluent.kafka.schemaregistry.ParsedSchema
+import io.confluent.kafka.schemaregistry.avro.AvroSchema
+import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference
+import io.confluent.kafka.schemaregistry.avro.AvroSchemaProvider
 
 import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, ObjectOutputStream}
+import java.util
+import java.util.Optional
 import java.util.concurrent.TimeUnit
 import scala.collection.JavaConverters._
 import scala.concurrent.duration.Duration
@@ -71,6 +77,7 @@ object RedisSchemaRegistryClient {
       new SerializableSchemaMetadata(sm.getId, sm.getVersion, sm.getSchema)
     }
   }
+
   private class SerializableSchemaMetadata(id: Int, version: Int, schema: String) extends Serializable {
     def getSchemaMetadata: SchemaMetadata = {
       new SchemaMetadata(id, version, schema)
@@ -168,6 +175,7 @@ object RedisSchemaRegistryClient {
   }
 }
 
+// scalastyle:off number.of.methods
 class RedisSchemaRegistryClient(restService: RestService,
                                 redisHost: String,
                                 redisPort: Int,
@@ -181,7 +189,7 @@ class RedisSchemaRegistryClient(restService: RestService,
   }
 
   def this(baseUrl: String, redisHost: String, redisPort: Int, ssl: Boolean) {
-    this(new RestService(baseUrl), redisHost, redisPort, Map.empty[String, String], Map.empty[String, Any],  CacheConfigs(1, 1, 1), ssl)
+    this(new RestService(baseUrl), redisHost, redisPort, Map.empty[String, String], Map.empty[String, Any], CacheConfigs(1, 1, 1), ssl)
   }
 
   def this(baseUrl: String, redisHost: String, redisPort: Int, cacheConfigs: CacheConfigs, ssl: Boolean) {
@@ -208,7 +216,8 @@ class RedisSchemaRegistryClient(restService: RestService,
     this(new RestService(baseUrl), redisHost, redisPort, httpHeaders, config, CacheConfigs(1, 1, 1), ssl)
   }
 
-  def this(baseUrl: String, redisHost: String, redisPort: Int, httpHeaders: Map[String, String], config: Map[String, Any], cacheConfigs: CacheConfigs, ssl: Boolean) {
+  def this(baseUrl: String, redisHost: String, redisPort: Int, httpHeaders: Map[String, String], config: Map[String, Any],
+           cacheConfigs: CacheConfigs, ssl: Boolean) {
     this(new RestService(baseUrl), redisHost, redisPort, httpHeaders, config, cacheConfigs, ssl)
   }
 
@@ -240,7 +249,8 @@ class RedisSchemaRegistryClient(restService: RestService,
     this(new RestService(baseUrls.asJava), redisHost, redisPort, httpHeaders, config, CacheConfigs(1, 1, 1), ssl)
   }
 
-  def this(baseUrls: List[String], redisHost: String, redisPort: Int, httpHeaders: Map[String, String], config: Map[String, Any], cacheConfigs: CacheConfigs, ssl: Boolean) {
+  def this(baseUrls: List[String], redisHost: String, redisPort: Int, httpHeaders: Map[String, String],
+           config: Map[String, Any], cacheConfigs: CacheConfigs, ssl: Boolean) {
     this(new RestService(baseUrls.asJava), redisHost, redisPort, httpHeaders, config, cacheConfigs, ssl)
   }
 
@@ -293,7 +303,7 @@ class RedisSchemaRegistryClient(restService: RestService,
     restService.setHttpHeaders(httpHeaders.asJava)
   }
 
-  if (configs!= null && configs.nonEmpty) {
+  if (configs != null && configs.nonEmpty) {
     restService.configure(configs.asJava)
 
     val srPrefix = "schema.registry."
@@ -312,34 +322,6 @@ class RedisSchemaRegistryClient(restService: RestService,
     }
   }
 
-  override def register(s: String, schema: Schema): Int = {
-    register(s, schema, 0, -1)
-  }
-
-  override def getById(i: Int): Schema = synchronized {
-    val restSchema = restService.getId(i)
-    val parser = new Schema.Parser()
-    parser.setValidateDefaults(false)
-    parser.parse(restSchema.getSchemaString)
-  }
-
-  override def getBySubjectAndId(s: String, i: Int): Schema = {
-    if (DO_NOT_CACHE_LIST.exists(s.startsWith)) {
-      restGetSchemaById(s, i)
-    } else {
-      val idKey = buildIdKey(s)
-
-      idCache.get(idKey) match {
-        case Failure(_) => restGetId(s, i)(i)
-        case Success(map) =>
-          map match {
-            case Some(m) if m.keys.exists(_ == i) => m(i)
-            case _ => getBySubjectAndIdSynchronized(s, i)
-          }
-      }
-    }
-  }
-
   private def restGetSchemaById(s: String, i: Int): Schema = {
     val restSchema = restService.getId(i)
     val parser = new Schema.Parser()
@@ -379,22 +361,15 @@ class RedisSchemaRegistryClient(restService: RestService,
     restService.getAllSubjectsById(i)
   }
 
-   def getLatestSchemaMetadata(s: String): SchemaMetadata = synchronized {
+  def getLatestSchemaMetadata(s: String): SchemaMetadata = synchronized {
     val response = restService.getLatestVersion(s)
     val id = response.getId
-    val schema: String  = response.getSchema
+    val schema: String = response.getSchema
     val version = response.getVersion
     new SchemaMetadata(id, version, schema)
   }
 
-/*  override def getSchemaMetadata(s: String, i: Int): SchemaMetadata = {
-    val response = restService.getVersion(s, i)
-    val id = response.getId
-    val schema: String = response.getSchema
-    new SchemaMetadata(id, i, schema)
-  }*/
-
-  override def getSchemaMetadata(s: String, i: Int): SchemaMetadata = {
+  override def getSchemaMetadata(subject: String, id: Int): SchemaMetadata = {
     def get(s: String, i: Int): SchemaMetadata = {
       val response = restService.getVersion(s, i)
       val id = response.getId
@@ -402,63 +377,33 @@ class RedisSchemaRegistryClient(restService: RestService,
       new SchemaMetadata(id, i, schema)
     }
 
-    if(DO_NOT_CACHE_LIST.exists(s.startsWith)) {
-      get(s, i)
+    if (DO_NOT_CACHE_LIST.exists(subject.startsWith)) {
+      get(subject, id)
     } else {
-      val metadataKey = buildMetadataKey(s)
+      val metadataKey = buildMetadataKey(subject)
 
       metadataCache.get(metadataKey) match {
         case Failure(_) =>
-          val sm = get(s, i)
-          metadataCache.put(metadataKey)(Map(i -> sm), metadataCacheDurationTtl)
+          val sm = get(subject, id)
+          metadataCache.put(metadataKey)(Map(id -> sm), metadataCacheDurationTtl)
           sm
         case Success(map) => map match {
-          case Some(m) if m.keySet.contains(i) =>
-            m(i)
+          case Some(m) if m.keySet.contains(id) =>
+            m(id)
           case Some(m) =>
-            val sm = get(s, i)
-            val concatMap: Map[Int, SchemaMetadata] = m ++ Map(i -> sm)
+            val sm = get(subject, id)
+            val concatMap: Map[Int, SchemaMetadata] = m ++ Map(id -> sm)
             metadataCache.put(metadataKey)(concatMap, metadataCacheDurationTtl)
             sm
           case None =>
-            val sm = get(s, i)
-            metadataCache.put(metadataKey)(Map(i -> sm), metadataCacheDurationTtl)
+            val sm = get(subject, id)
+            metadataCache.put(metadataKey)(Map(id -> sm), metadataCacheDurationTtl)
             sm
         }
       }
     }
   }
 
-  override def getVersion(s: String, schema: Schema): Int = synchronized {
-    if(DO_NOT_CACHE_LIST.exists(s.startsWith)) {
-      val response: io.confluent.kafka.schemaregistry.client.rest.entities.Schema = restService.lookUpSubjectVersion(schema.toString(), s, true)
-      response.getVersion.toInt
-    } else {
-      val versionKey = buildVersionKey(s)
-
-      versionCache.get(versionKey) match {
-        case Failure(_) =>
-          val response: io.confluent.kafka.schemaregistry.client.rest.entities.Schema = restService.lookUpSubjectVersion(schema.toString(), s, true)
-          versionCache.put(versionKey)(Map(schema -> response.getVersion.toInt), versionCacheDurationTtl)
-          response.getVersion.toInt
-        case Success(map) =>
-          map match {
-            case Some(m) if m.keySet.contains(schema) =>
-              m(schema)
-            case Some(m) =>
-              val response: io.confluent.kafka.schemaregistry.client.rest.entities.Schema = restService.lookUpSubjectVersion(schema.toString(), s, true)
-              val concatMap: Map[Schema, Int] = m ++ Map(schema -> response.getVersion.toInt)
-              versionCache.put(versionKey)(concatMap, versionCacheDurationTtl)
-              response.getVersion.toInt
-            case None =>
-              val response: io.confluent.kafka.schemaregistry.client.rest.entities.Schema = restService.lookUpSubjectVersion(schema.toString(), s, true)
-              versionCache.put(versionKey)(Map(schema -> response.getVersion.toInt), versionCacheDurationTtl)
-              response.getVersion.toInt
-          }
-      }
-    }
-  }
-
   override def getAllVersions(s: String): java.util.List[Integer] = {
     restService.getAllVersions(s)
   }
@@ -491,52 +436,6 @@ class RedisSchemaRegistryClient(restService: RestService,
     restService.getAllSubjects()
   }
 
-  override def getId(s: String, schema: Schema): Int = synchronized {
-    if (DO_NOT_CACHE_LIST.exists(s.startsWith)) {
-      restService.lookUpSubjectVersion(schema.toString, s, false).getId.toInt
-    } else {
-      def call(): Map[Schema, Int] = {
-        val response = restService.lookUpSubjectVersion(schema.toString, s, false)
-        Map(schema -> response.getId.toInt)
-      }
-
-      val idKey = buildIdKey(s)
-
-      def populateVersionCache(m: Map[Schema, Int]): Try[Any] = {
-        val idM: Map[Int, Schema] = m.map(kv => kv._2 -> kv._1)
-
-        idCache.get(idKey) match {
-          case Failure(_) =>
-            idCache.put(idKey)(idM, idCacheDurationTtl)
-          case Success(im) =>
-            val concatMap: Map[Int, Schema] = idM ++ im.getOrElse(Map.empty[Int, Schema])
-            idCache.put(idKey)(concatMap, idCacheDurationTtl)
-        }
-      }
-
-      val schemaKey = buildSchemaKey(s)
-
-      schemaCache.get(schemaKey) match {
-        case Failure(_) =>
-          schemaCache.put(schemaKey)(call(), schemaCacheDurationTtl)
-          call()(schema)
-        case Success(map) => map match {
-          case Some(m) if m.keys.exists(_ == schema) =>
-            populateVersionCache(m)
-            m(schema)
-          case Some(m) =>
-            val concatMaps: Map[Schema, Int] = m ++ call()
-            populateVersionCache(concatMaps)
-            schemaCache.put(schemaKey)(concatMaps, schemaCacheDurationTtl)
-            call()(schema)
-          case None =>
-            schemaCache.put(schemaKey)(call(), schemaCacheDurationTtl)
-            call()(schema)
-        }
-      }
-    }
-  }
-
   override def deleteSubject(s: String): java.util.List[Integer] = synchronized {
     deleteSubject(DEFAULT_REQUEST_PROPERTIES.asJava, s)
   }
@@ -569,18 +468,34 @@ class RedisSchemaRegistryClient(restService: RestService,
     }
   }
 
-  override def register(s: String, schema: Schema, i: Int, i1: Int): Int = synchronized {
+  override def reset(): Unit =
+    throw new UnsupportedOperationException("The reset operation unsupported for a distributed cache.")
+
+  override def parseSchema(schemaType: String, schemaString: String,
+                           references: util.List[SchemaReference]): Optional[ParsedSchema] = {
+    val parsedSchema = new AvroSchemaProvider().parseSchema(schemaString, references, false)
+    parsedSchema
+  }
+
+  override def register(subject: String, schema: ParsedSchema): Int = {
+    register(subject, schema, 0, -1)
+  }
+
+  // scalastyle:off method.length
+  override def register(subject: String, parsedSchema: ParsedSchema, version: Int, id: Int): Int = synchronized {
+    val schema = toAvroSchema(parsedSchema)
+
     def register(): Int =
-      if (i >= 0) {
-        restService.registerSchema(schema.toString(), s, i, i1)
+      if (version >= 0) {
+        restService.registerSchema(schema.toString(), subject, version, id)
       } else {
-        restService.registerSchema(schema.toString(), s)
+        restService.registerSchema(schema.toString(), subject)
       }
 
-    if(DO_NOT_CACHE_LIST.exists(s.startsWith)) {
+    if (DO_NOT_CACHE_LIST.exists(subject.startsWith)) {
       register()
     } else {
-      val idKey = buildIdKey(s)
+      val idKey = buildIdKey(subject)
 
       def populateIdCache(sc: Schema, id: Int): Unit = {
         idCache.caching(idKey)(idCacheDurationTtl) {
@@ -594,7 +509,7 @@ class RedisSchemaRegistryClient(restService: RestService,
         }
       }
 
-      val schemaKey = buildSchemaKey(s)
+      val schemaKey = buildSchemaKey(subject)
 
       schemaCache.get(schemaKey) match {
         case Failure(_) =>
@@ -605,8 +520,8 @@ class RedisSchemaRegistryClient(restService: RestService,
           case Some(m) if m.exists(_._1 == schema) =>
             val cachedId = m(schema)
 
-            if (i1 >= 0 && i1 != cachedId) {
-              throw new IllegalStateException("Schema already registered with id " + cachedId + " instead of input id " + i1)
+            if (id >= 0 && id != cachedId) {
+              throw new IllegalStateException("Schema already registered with id " + cachedId + " instead of input id " + id)
             } else {
               cachedId
             }
@@ -625,18 +540,123 @@ class RedisSchemaRegistryClient(restService: RestService,
     }
   }
 
-  override def reset(): Unit =
-    println("The reset operation unsupported for a distributed cache.")
+  override def getSchemaById(id: Int): ParsedSchema = synchronized {
+    val restSchema = restService.getId(id)
+    val parser = new Schema.Parser()
+    parser.setValidateDefaults(false)
+    toParsedSchema(parser.parse(restSchema.getSchemaString))
+  }
+
+  override def getSchemaBySubjectAndId(subject: String, id: Int): ParsedSchema = {
+    val result =
+      if (DO_NOT_CACHE_LIST.exists(subject.startsWith)) {
+        restGetSchemaById(subject, id)
+      } else {
+        val idKey = buildIdKey(subject)
+
+        idCache.get(idKey) match {
+          case Failure(_) => restGetId(subject, id)(id)
+          case Success(map) =>
+            map match {
+              case Some(m) if m.keys.exists(_ == id) => m(id)
+              case _ => getBySubjectAndIdSynchronized(subject, id)
+            }
+        }
+      }
+    toParsedSchema(result)
+  }
+
+  override def getVersion(subject: String, parsedSchema: ParsedSchema): Int = synchronized {
+    val schema = toAvroSchema(parsedSchema)
 
-  override def getByID(i: Int): Schema = {
-    getById(i)
+    if (DO_NOT_CACHE_LIST.exists(subject.startsWith)) {
+      val response: io.confluent.kafka.schemaregistry.client.rest.entities.Schema = restService.lookUpSubjectVersion(schema.toString(), subject, true)
+      response.getVersion.toInt
+    } else {
+      val versionKey = buildVersionKey(subject)
+
+      versionCache.get(versionKey) match {
+        case Failure(_) =>
+          val response: io.confluent.kafka.schemaregistry.client.rest.entities.Schema = restService.lookUpSubjectVersion(schema.toString(), subject, true)
+          versionCache.put(versionKey)(Map(schema -> response.getVersion.toInt), versionCacheDurationTtl)
+          response.getVersion.toInt
+        case Success(map) =>
+          map match {
+            case Some(m) if m.keySet.contains(schema) =>
+              m(schema)
+            case Some(m) =>
+              val response: io.confluent.kafka.schemaregistry.client.rest.entities.Schema = restService.lookUpSubjectVersion(schema.toString(), subject, true)
+              val concatMap: Map[Schema, Int] = m ++ Map(schema -> response.getVersion.toInt)
+              versionCache.put(versionKey)(concatMap, versionCacheDurationTtl)
+              response.getVersion.toInt
+            case None =>
+              val response: io.confluent.kafka.schemaregistry.client.rest.entities.Schema = restService.lookUpSubjectVersion(schema.toString(), subject, true)
+              versionCache.put(versionKey)(Map(schema -> response.getVersion.toInt), versionCacheDurationTtl)
+              response.getVersion.toInt
+          }
+      }
+    }
   }
 
-  override def getBySubjectAndID(s: String, i: Int): Schema = {
-    getBySubjectAndId(s, i)
+  override def testCompatibility(subject: String, schema: ParsedSchema): Boolean = {
+    val compatibilityErrors = restService.testCompatibility(toAvroSchema(schema).toString(), subject, "latest")
+    compatibilityErrors.isEmpty
   }
 
-  override def testCompatibility(s: String, schema: Schema): Boolean = {
-    restService.testCompatibility(schema.toString(), s, "latest")
+  override def getId(s: String, parsedSchema: ParsedSchema): Int = synchronized {
+    val schema = toAvroSchema(parsedSchema)
+    if (DO_NOT_CACHE_LIST.exists(s.startsWith)) {
+      restService.lookUpSubjectVersion(schema.toString, s, false).getId.toInt
+    } else {
+      def call(): Map[Schema, Int] = {
+        val response = restService.lookUpSubjectVersion(schema.toString, s, false)
+        Map(schema -> response.getId.toInt)
+      }
+
+      val idKey = buildIdKey(s)
+
+      def populateVersionCache(m: Map[Schema, Int]): Try[Any] = {
+        val idM: Map[Int, Schema] = m.map(kv => kv._2 -> kv._1)
+
+        idCache.get(idKey) match {
+          case Failure(_) =>
+            idCache.put(idKey)(idM, idCacheDurationTtl)
+          case Success(im) =>
+            val concatMap: Map[Int, Schema] = idM ++ im.getOrElse(Map.empty[Int, Schema])
+            idCache.put(idKey)(concatMap, idCacheDurationTtl)
+        }
+      }
+
+      val schemaKey = buildSchemaKey(s)
+
+      schemaCache.get(schemaKey) match {
+        case Failure(_) =>
+          schemaCache.put(schemaKey)(call(), schemaCacheDurationTtl)
+          call()(schema)
+        case Success(map) => map match {
+          case Some(m) if m.keys.exists(_ == schema) =>
+            populateVersionCache(m)
+            m(schema)
+          case Some(m) =>
+            val concatMaps: Map[Schema, Int] = m ++ call()
+            populateVersionCache(concatMaps)
+            schemaCache.put(schemaKey)(concatMaps, schemaCacheDurationTtl)
+            call()(schema)
+          case None =>
+            schemaCache.put(schemaKey)(call(), schemaCacheDurationTtl)
+            call()(schema)
+        }
+      }
+    }
+  }
+
+  private def toParsedSchema(avroSchema: Schema): ParsedSchema = new AvroSchema(avroSchema)
+
+  private def toAvroSchema(parsedSchema: ParsedSchema): Schema = {
+    parsedSchema match {
+      case s: AvroSchema => s.rawSchema
+      case p: ParsedSchema => throw new RuntimeException(s"Non-avro schema is not allowed: [ $p ]")
+      case _ => throw new RuntimeException(s"Encountered unknown schema: [ $parsedSchema ]")
+    }
   }
 }
diff --git a/avro/src/main/scala/hydra/avro/registry/SchemaRegistry.scala b/avro/src/main/scala/hydra/avro/registry/SchemaRegistry.scala
index f2af389d6..fbf54d6d6 100644
--- a/avro/src/main/scala/hydra/avro/registry/SchemaRegistry.scala
+++ b/avro/src/main/scala/hydra/avro/registry/SchemaRegistry.scala
@@ -4,7 +4,8 @@ import cats.Eval
 import cats.effect.Sync
 import cats.syntax.all._
 import hydra.common.config.KafkaConfigUtils._
-import io.confluent.kafka.schemaregistry.avro.AvroCompatibilityChecker
+import io.confluent.kafka.schemaregistry.CompatibilityChecker
+import io.confluent.kafka.schemaregistry.avro.AvroSchema
 import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException
 import io.confluent.kafka.schemaregistry.client.{CachedSchemaRegistryClient, MockSchemaRegistryClient, SchemaRegistryClient, SchemaRegistryClientConfig}
 import org.apache.avro.{LogicalType, LogicalTypes, Schema}
@@ -18,90 +19,100 @@ import scala.concurrent.duration._
 import scala.util.Try
 
 /**
-  * Internal interface to interact with the SchemaRegistryClient from Confluent.
-  * Abstraction allows pure functional interface for working with underlying Java implementation.
-  * Provides a live version for production usage and a test version for integration testing.
-  * @tparam F - higher kinded type - polymorphic effect type
-  */
+ * Internal interface to interact with the SchemaRegistryClient from Confluent.
+ * Abstraction allows pure functional interface for working with underlying Java implementation.
+ * Provides a live version for production usage and a test version for integration testing.
+ *
+ * @tparam F - higher kinded type - polymorphic effect type
+ */
 trait SchemaRegistry[F[_]] {
 
   import SchemaRegistry._
 
   /**
-    * Adds schema to the configured SchemaRegistry. Registration is idempotent.
-    * Equivalency is determined by taking a hash of the given schema. Any changes to the schema change the hash.
-    * @param subject - subject name for the schema found in SchemaRegistry including the suffix (-key | -value)
-    * @param schema - avro Schema which is to be added to the Schema Registry
-    * @return SchemaId for schema, whether newly created or preexisting
-    */
+   * Adds schema to the configured SchemaRegistry. Registration is idempotent.
+   * Equivalency is determined by taking a hash of the given schema. Any changes to the schema change the hash.
+   *
+   * @param subject - subject name for the schema found in SchemaRegistry including the suffix (-key | -value)
+   * @param schema  - avro Schema which is to be added to the Schema Registry
+   * @return SchemaId for schema, whether newly created or preexisting
+   */
   def registerSchema(subject: String, schema: Schema): F[SchemaId]
 
   /**
-    * Deletes schema from the configured SchemaRegistry. Deletes only the version specified and only one of the
-    * key /value, whichever was specified in the subject suffix.
-    * @param subject - subject name for the schema found in SchemaRegistry including the suffix (-key | -value)
-    * @param version - integer representing the schema version
-    * @return Unit
-    */
+   * Deletes schema from the configured SchemaRegistry. Deletes only the version specified and only one of the
+   * key /value, whichever was specified in the subject suffix.
+   *
+   * @param subject - subject name for the schema found in SchemaRegistry including the suffix (-key | -value)
+   * @param version - integer representing the schema version
+   * @return Unit
+   */
   def deleteSchemaOfVersion(subject: String, version: SchemaVersion): F[Unit]
 
   /**
-    * Deletes the subject from the versionCache, idCache, and schemaCache
-    * of the CachedSchemaRegistryClient
-    * @param subject The subject using -key or -value to delete
-    * @return Unit
-    */
+   * Deletes the subject from the versionCache, idCache, and schemaCache
+   * of the CachedSchemaRegistryClient
+   *
+   * @param subject The subject using -key or -value to delete
+   * @return Unit
+   */
   def deleteSchemaSubject(subject: String): F[Unit]
 
   /**
-    * Retrieves the SchemaVersion if the given subject and schema match an item in SchemaRegistry.
-    * The schema hash must exactly match one of the schemas stored in Schema Registry. All fields must be equal.
-    * If the schema is not found, the error will be reported in the error channel of the higher kinded type (F[_]).
-    * @param subject - subject name for the schema found in SchemaRegistry including the suffix (-key | -value)
-    * @param schema - avro Schema which is expected to be in Schema Registry
-    * @return SchemaVersion
-    */
+   * Retrieves the SchemaVersion if the given subject and schema match an item in SchemaRegistry.
+   * The schema hash must exactly match one of the schemas stored in Schema Registry. All fields must be equal.
+   * If the schema is not found, the error will be reported in the error channel of the higher kinded type (F[_]).
+   *
+   * @param subject - subject name for the schema found in SchemaRegistry including the suffix (-key | -value)
+   * @param schema  - avro Schema which is expected to be in Schema Registry
+   * @return SchemaVersion
+   */
   def getVersion(subject: String, schema: Schema): F[SchemaVersion]
 
   /**
-    * Retrieves all SchemaVersion(s) for a given subject.
-    * @param subject - subject name for the schema found in SchemaRegistry including the suffix (-key | -value)
-    * @return List[SchemaVersion] or List.empty if Subject Not Found
-    */
+   * Retrieves all SchemaVersion(s) for a given subject.
+   *
+   * @param subject - subject name for the schema found in SchemaRegistry including the suffix (-key | -value)
+   * @return List[SchemaVersion] or List.empty if Subject Not Found
+   */
   def getAllVersions(subject: String): F[List[SchemaVersion]]
 
   /**
-    * Retrieves all subjects found in the SchemaRegistry
-    * @return List[String]
-    */
+   * Retrieves all subjects found in the SchemaRegistry
+   *
+   * @return List[String]
+   */
   def getAllSubjects: F[List[String]]
 
   /**
-    * Retrieves the SchemaRegistryClient from the algebra
-    * @return SchemaRegistryClient
-    */
+   * Retrieves the SchemaRegistryClient from the algebra
+   *
+   * @return SchemaRegistryClient
+   */
   def getSchemaRegistryClient: F[SchemaRegistryClient]
 
   /**
-    * Retrieves the latest schema for the given subject name, if exists
-    * @param subject - subject name for the schema found in SchemaRegistry including the suffix (-key | -value)
-    * @return - Optional Schema for the given subject name
-    */
+   * Retrieves the latest schema for the given subject name, if exists
+   *
+   * @param subject - subject name for the schema found in SchemaRegistry including the suffix (-key | -value)
+   * @return - Optional Schema for the given subject name
+   */
   def getLatestSchemaBySubject(subject: String): F[Option[Schema]]
 
   /**
-    * Retrieves schema for the version and subject specified, if exists
-    * @param subject - subject name for the schema found in SchemaRegistry including the suffix (-key | -value)
-    * @param schemaVersion - version number for the schema
-    * @return Optional Schema for the given subject and version combination
-    */
+   * Retrieves schema for the version and subject specified, if exists
+   *
+   * @param subject       - subject name for the schema found in SchemaRegistry including the suffix (-key | -value)
+   * @param schemaVersion - version number for the schema
+   * @return Optional Schema for the given subject and version combination
+   */
   def getSchemaFor(subject: String, schemaVersion: SchemaVersion): F[Option[Schema]]
 
 }
 
 object SchemaRegistry {
 
-  private[registry] implicit class CheckKeySchemaEvolution[F[_]: Sync](schemasF: F[List[Schema]]) {
+  private[registry] implicit class CheckKeySchemaEvolution[F[_] : Sync](schemasF: F[List[Schema]]) {
     def checkKeyEvolution(subject: String, newSchema: Schema): F[List[Schema]] = schemasF.flatTap[Unit] {
       case _ if subject.endsWith("-value") => Sync[F].unit
       case Nil => Sync[F].unit
@@ -118,6 +129,7 @@ object SchemaRegistry {
     RuntimeException(message)
 
   final case class LogicalTypeBaseTypeMismatch(baseType: Schema.Type, logicalType: LogicalType, fieldName: String)
+
   final case class LogicalTypeBaseTypeMismatchErrors(errors: List[LogicalTypeBaseTypeMismatch]) extends
     RuntimeException(
       errors.map(e => s"Field named '${e.fieldName}' contains mismatch in " +
@@ -125,11 +137,12 @@ object SchemaRegistry {
     )
 
   final case class IllegalLogicalTypeChange(originalType: LogicalType, proposedType: LogicalType, fieldName: String)
+
   final case class IllegalLogicalTypeChangeErrors(errors: List[IllegalLogicalTypeChange]) extends
     RuntimeException(
       errors.map(e =>
         s"Changing logical types is not allowed. Field named '${e.fieldName}'s logical type cannot be changed from " +
-        s"logicalType of '${if(e.originalType != null) e.originalType.getName else null}' to logicalType of '${if(e.proposedType != null) e.proposedType.getName else null}'").mkString("\n")
+          s"logicalType of '${if (e.originalType != null) e.originalType.getName else null}' to logicalType of '${if (e.proposedType != null) e.proposedType.getName else null}'").mkString("\n")
     )
 
   type SchemaId = Int
@@ -137,31 +150,34 @@ object SchemaRegistry {
 
 
   private[registry] def validate(newSchema: Schema, oldSchemas: List[Schema]): Boolean = {
-    AvroCompatibilityChecker.FULL_TRANSITIVE_CHECKER.isCompatible(newSchema, oldSchemas.asJava)
+    val errors = CompatibilityChecker.FULL_TRANSITIVE_CHECKER.isCompatible(new AvroSchema(newSchema), oldSchemas.map(new AvroSchema(_)).asJava)
+    errors.isEmpty
   }
 
-  def live[F[_]: Sync: Logger: Sleep](
-      schemaRegistryBaseUrl: String,
-      maxCacheSize: Int,
-      securityConfig: SchemaRegistrySecurityConfig,
-      schemaRegistryClientRetries: Int,
-      schemaRegistryClientRetriesDelay: FiniteDuration
-  ): F[SchemaRegistry[F]] = Sync[F].delay {
-    getFromSchemaRegistryClient(new CachedSchemaRegistryClient(schemaRegistryBaseUrl, maxCacheSize,  securityConfig.toConfigMap.asJava), schemaRegistryClientRetries, schemaRegistryClientRetriesDelay)
+  def live[F[_] : Sync : Logger : Sleep](
+                                          schemaRegistryBaseUrl: String,
+                                          maxCacheSize: Int,
+                                          securityConfig: SchemaRegistrySecurityConfig,
+                                          schemaRegistryClientRetries: Int,
+                                          schemaRegistryClientRetriesDelay: FiniteDuration
+                                        ): F[SchemaRegistry[F]] = Sync[F].delay {
+    getFromSchemaRegistryClient(new CachedSchemaRegistryClient(schemaRegistryBaseUrl, maxCacheSize,
+      securityConfig.toConfigMap.asJava), schemaRegistryClientRetries, schemaRegistryClientRetriesDelay)
   }
 
-  def live[F[_] : Sync: Logger: Sleep](
-      schemaRegistryBaseUrl: String,
-      securityConfig: SchemaRegistrySecurityConfig,
-      redisUrl: String,
-      redisPort: Int,
-      ssl: Boolean,
-      idCacheTtl: Int,
-      schemaCacheTtl: Int,
-      versionCacheTtl: Int,
-      schemaRegistryClientRetries: Int,
-      schemaRegistryClientRetriesDelay: FiniteDuration
-  ): F[SchemaRegistry[F]] = Sync[F].delay {
+  // scalastyle:off parameter.number
+  def live[F[_] : Sync : Logger : Sleep](
+                                          schemaRegistryBaseUrl: String,
+                                          securityConfig: SchemaRegistrySecurityConfig,
+                                          redisUrl: String,
+                                          redisPort: Int,
+                                          ssl: Boolean,
+                                          idCacheTtl: Int,
+                                          schemaCacheTtl: Int,
+                                          versionCacheTtl: Int,
+                                          schemaRegistryClientRetries: Int,
+                                          schemaRegistryClientRetriesDelay: FiniteDuration
+                                        ): F[SchemaRegistry[F]] = Sync[F].delay {
     getFromSchemaRegistryClient(
       new RedisSchemaRegistryClient(
         schemaRegistryBaseUrl,
@@ -176,20 +192,25 @@ object SchemaRegistry {
     )
   }
 
-  def test[F[_]: Sync: Logger: Sleep]: F[SchemaRegistry[F]] = Sync[F].delay {
+  def test[F[_] : Sync : Logger : Sleep]: F[SchemaRegistry[F]] = Sync[F].delay {
     getFromSchemaRegistryClient(new MockSchemaRegistryClient, schemaRegistryClientRetries = 0, schemaRegistryClientRetriesDelay = 1.milliseconds)
   }
 
-  def test[F[_]: Sync: Logger: Sleep](mockedClient: SchemaRegistryClient, schemaRegistryClientRetries: Int = 3, schemaRegistryClientRetriesDelay: FiniteDuration = 500.milliseconds): F[SchemaRegistry[F]] = Sync[F].delay {
+  def test[F[_] : Sync : Logger : Sleep](mockedClient: SchemaRegistryClient, schemaRegistryClientRetries: Int = 3,
+                                         schemaRegistryClientRetriesDelay: FiniteDuration = 500.milliseconds): F[SchemaRegistry[F]] = Sync[F].delay {
     getFromSchemaRegistryClient(mockedClient, schemaRegistryClientRetries, schemaRegistryClientRetriesDelay)
   }
 
-  private def getFromSchemaRegistryClient[F[_]: Sync: Logger: Sleep](schemaRegistryClient: SchemaRegistryClient, schemaRegistryClientRetries: Int, schemaRegistryClientRetriesDelay: FiniteDuration): SchemaRegistry[F] =
+  // scalastyle:off method.length
+  private def getFromSchemaRegistryClient[F[_] : Sync : Logger : Sleep](schemaRegistryClient: SchemaRegistryClient,
+                                                                        schemaRegistryClientRetries: Int,
+                                                                        schemaRegistryClientRetriesDelay: FiniteDuration): SchemaRegistry[F] =
     new SchemaRegistry[F] {
       val retryPolicy = limitRetries(schemaRegistryClientRetries) |+| constantDelay[F](schemaRegistryClientRetriesDelay)
 
       private implicit class SchemaOps(sch: Schema) {
         def fields: List[Schema.Field] = fieldsEval("topLevel", box = false).value
+
         private[SchemaOps] def fieldsEval(fieldName: String, box: Boolean = false): Eval[List[Schema.Field]] = sch.getType match {
           case Schema.Type.RECORD => Eval.defer(sch.getFields.asScala.toList.flatTraverse(nf => nf.schema.fieldsEval(nf.name, box = true)))
           case Schema.Type.UNION => Eval.defer(sch.getTypes.asScala.toList.flatTraverse(_.fieldsEval(fieldName, box = true)))
@@ -202,7 +223,7 @@ object SchemaRegistry {
 
       private def checkTypesMatch(f: Schema.Field, expected: Schema.Type, logicalType: LogicalType): List[LogicalTypeBaseTypeMismatch] = {
         if (f.schema.getType == expected) {
-            List.empty
+          List.empty
         } else {
           List(LogicalTypeBaseTypeMismatch(f.schema.getType, logicalType, f.name))
         }
@@ -225,9 +246,9 @@ object SchemaRegistry {
       }
 
       override def registerSchema(
-          subject: String,
-          schema: Schema
-      ): F[SchemaId] = {
+                                   subject: String,
+                                   schema: Schema
+                                 ): F[SchemaId] = {
         for {
           versions <- getAllVersions(subject)
           schemas <- versions.traverse(getSchemaFor(subject, _)).map(_.flatten).checkKeyEvolution(subject, schema)
@@ -235,25 +256,26 @@ object SchemaRegistry {
           _ <- checkLogicalTypesCompat(schema)
           latest <- getLatestSchemaBySubject(subject)
           schemaVersion <- if (validated) {
-            if(checkForOnlyDocFieldUpdate(schema, latest)) {
+            if (checkForOnlyDocFieldUpdate(schema, latest)) {
               schemaRegistryClient.reset()
             }
             Sync[F].delay(schemaRegistryClient.register(subject, schema))
           } else {
-            Sync[F].raiseError[SchemaVersion](IncompatibleSchemaException("Incompatible Schema Evolution. You may add fields with default fields, or remove fields with default fields."))
+            Sync[F].raiseError[SchemaVersion](
+              IncompatibleSchemaException("Incompatible Schema Evolution. You may add fields with default fields, or remove fields with default fields."))
           }
         } yield schemaVersion
       }
 
       def checkForOnlyDocFieldUpdate(schema: Schema, latest: Option[Schema]): Boolean = {
-        if(latest.isEmpty) {
+        if (latest.isEmpty) {
           false
         } else {
           val realLatest = latest.get
           if (schema != null && schema.equals(realLatest)) {
-            val fieldDoc = schema.getFields.asScala.toList.map {field => field.doc()}
-            val latestDoc = realLatest.getFields.asScala.toList.map {field => field.doc()}
-            if(fieldDoc.diff(latestDoc).nonEmpty) {
+            val fieldDoc = schema.getFields.asScala.toList.map { field => field.doc() }
+            val latestDoc = realLatest.getFields.asScala.toList.map { field => field.doc() }
+            if (fieldDoc.diff(latestDoc).nonEmpty) {
               return true
             }
           }
@@ -262,17 +284,17 @@ object SchemaRegistry {
       }
 
       override def deleteSchemaOfVersion(
-          subject: String,
-          version: SchemaVersion
-      ): F[Unit] =
+                                          subject: String,
+                                          version: SchemaVersion
+                                        ): F[Unit] =
         Sync[F].delay(
           schemaRegistryClient.deleteSchemaVersion(subject, version.toString)
         )
 
       override def getVersion(
-          subject: String,
-          schema: Schema
-      ): F[SchemaVersion] =
+                               subject: String,
+                               schema: Schema
+                             ): F[SchemaVersion] =
         Sync[F].delay {
           schemaRegistryClient.getVersion(subject, schema)
         }.retryingOnAllErrors(retryPolicy, onFailure("getVersion"))
diff --git a/avro/src/test/scala/hydra/avro/registry/ConfluentSchemaRegistrySpec.scala b/avro/src/test/scala/hydra/avro/registry/ConfluentSchemaRegistrySpec.scala
index 05b2ba9c7..76475351e 100644
--- a/avro/src/test/scala/hydra/avro/registry/ConfluentSchemaRegistrySpec.scala
+++ b/avro/src/test/scala/hydra/avro/registry/ConfluentSchemaRegistrySpec.scala
@@ -17,6 +17,7 @@ package hydra.avro.registry
 
 import com.typesafe.config.ConfigFactory
 import hydra.common.config.KafkaConfigUtils.SchemaRegistrySecurityConfig
+import io.confluent.kafka.schemaregistry.avro.AvroSchema
 import io.confluent.kafka.schemaregistry.client.{CachedSchemaRegistryClient, MockSchemaRegistryClient}
 import org.apache.avro.Schema
 import org.scalatest.concurrent.ScalaFutures
@@ -52,7 +53,7 @@ class ConfluentSchemaRegistrySpec
 
   override def beforeAll(): Unit = {
     id =
-      ConfluentSchemaRegistry.mockRegistry.register(schema.getFullName, schema)
+      ConfluentSchemaRegistry.mockRegistry.register(schema.getFullName, new AvroSchema(schema))
   }
 
   describe("When creating a schema registry client") {
diff --git a/avro/src/test/scala/hydra/avro/registry/RedisSchemaRegistryClientSpec.scala b/avro/src/test/scala/hydra/avro/registry/RedisSchemaRegistryClientSpec.scala
index 733a952a2..a50bacdf5 100644
--- a/avro/src/test/scala/hydra/avro/registry/RedisSchemaRegistryClientSpec.scala
+++ b/avro/src/test/scala/hydra/avro/registry/RedisSchemaRegistryClientSpec.scala
@@ -4,9 +4,10 @@ import org.apache.avro.{Schema, SchemaBuilder}
 import org.scalatest.BeforeAndAfterAll
 import com.github.sebruck.EmbeddedRedis
 import hydra.avro.registry.RedisSchemaRegistryClient.IntSchemaMetadataMapBinCodec
+import io.confluent.kafka.schemaregistry.avro.AvroSchema
 import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException
 import io.confluent.kafka.schemaregistry.client.{CachedSchemaRegistryClient, SchemaMetadata, SchemaRegistryClient}
-import net.manub.embeddedkafka.schemaregistry.{EmbeddedKafka, EmbeddedKafkaConfig}
+import io.github.embeddedkafka.schemaregistry.{EmbeddedKafka, EmbeddedKafkaConfig}
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper
 import redis.embedded.RedisServer
@@ -60,15 +61,16 @@ class RedisSchemaRegistryClientSpec extends AnyFlatSpec with EmbeddedRedis with
         val schema1: Schema = SchemaBuilder.record("Test1").fields()
           .requiredString("testing1").endRecord()
 
-        val schema12: Schema = SchemaBuilder.record("Test12").fields()
-          .requiredString("testing1").endRecord()
+        val schema12: Schema = SchemaBuilder.record("Test1").fields()
+          .requiredString("testing1").optionalString("testing22").endRecord()
 
         val schema2: Schema = SchemaBuilder.record("Test2").fields()
           .requiredString("testing2").endRecord()
 
         //register two schemas with different clients
-        val redisClientResult = redisClient.register(topicName1, schema1)
-        val cachedClientResult = cachedClient.register(topicName2 ,schema2)
+        val redisClientResult = redisClient.register(topicName1, new AvroSchema(schema1))
+        val cachedClientResult = cachedClient.register(topicName2 , new AvroSchema(schema2))
+
 
         //test the getAllSubjectsById method
         assert(redisClient.getAllSubjectsById(redisClientResult).contains(topicName1))
@@ -77,17 +79,17 @@ class RedisSchemaRegistryClientSpec extends AnyFlatSpec with EmbeddedRedis with
         assert(cachedClient.getAllSubjectsById(cachedClientResult).contains(topicName2))
 
         //test the getById method
-        redisClient.getById(redisClientResult) shouldBe schema1
-        cachedClient.getById(redisClientResult) shouldBe schema1
-        redisClient.getById(cachedClientResult) shouldBe schema2
-        cachedClient.getById(cachedClientResult) shouldBe schema2
+        redisClient.getSchemaById(redisClientResult) shouldBe new AvroSchema(schema1)
+        cachedClient.getSchemaById(redisClientResult) shouldBe new AvroSchema(schema1)
+        redisClient.getSchemaById(cachedClientResult) shouldBe new AvroSchema(schema2)
+        cachedClient.getSchemaById(cachedClientResult) shouldBe new AvroSchema(schema2)
 
         //test the getBySubjectAndId method
         Thread.sleep(3000)
-        assert(redisClient.getBySubjectAndId(topicName1, redisClientResult).equals(schema1))
-        assert(cachedClient.getBySubjectAndId(topicName1, redisClientResult).equals(schema1))
-        assert(redisClient.getBySubjectAndId(topicName2, cachedClientResult).equals(schema2))
-        assert(cachedClient.getBySubjectAndId(topicName2, cachedClientResult).equals(schema2))
+        assert(redisClient.getSchemaBySubjectAndId(topicName1, redisClientResult).equals(new AvroSchema(schema1)))
+        assert(cachedClient.getSchemaBySubjectAndId(topicName1, redisClientResult).equals(new AvroSchema(schema1)))
+        assert(redisClient.getSchemaBySubjectAndId(topicName2, cachedClientResult).equals(new AvroSchema(schema2)))
+        assert(cachedClient.getSchemaBySubjectAndId(topicName2, cachedClientResult).equals(new AvroSchema(schema2)))
 
 
         //test the getAllSubjects method
@@ -109,10 +111,10 @@ class RedisSchemaRegistryClientSpec extends AnyFlatSpec with EmbeddedRedis with
 
         //test the getId method
         Thread.sleep(3000)
-        redisClient.getId(topicName1, schema1) shouldBe redisClientResult
-        cachedClient.getId(topicName1, schema1) shouldBe redisClientResult
-        redisClient.getId(topicName2, schema2) shouldBe cachedClientResult
-        cachedClient.getId(topicName2, schema2) shouldBe cachedClientResult
+        redisClient.getId(topicName1, new AvroSchema(schema1)) shouldBe redisClientResult
+        cachedClient.getId(topicName1, new AvroSchema(schema1)) shouldBe redisClientResult
+        redisClient.getId(topicName2, new AvroSchema(schema2)) shouldBe cachedClientResult
+        cachedClient.getId(topicName2, new AvroSchema(schema2)) shouldBe cachedClientResult
 
         //test the getLatestSchemaMetadata method
         val schemaMetadata1 = new SchemaMetadata(1, 1,schema1.toString)
@@ -123,13 +125,13 @@ class RedisSchemaRegistryClientSpec extends AnyFlatSpec with EmbeddedRedis with
         schemaMetadata1Result.getVersion shouldBe schemaMetadata1.getVersion
         schemaMetadata1Result.getSchema shouldBe schemaMetadata1.getSchema
 
-        redisClient.register(topicName1, schema12)
+        redisClient.register(topicName1, new AvroSchema(schema12))
         val schemaMetadata2Result = redisClient.getLatestSchemaMetadata(topicName1)
         schemaMetadata2Result.getId shouldBe schemaMetadata2.getId
         schemaMetadata2Result.getVersion shouldBe schemaMetadata2.getVersion
         schemaMetadata2Result.getSchema shouldBe schemaMetadata2.getSchema
 
-        //test the getSchemaMetadata method
+         //test the getSchemaMetadata method
         val metadata1 = redisClient.getSchemaMetadata(topicName1, 1)
         metadata1.getId shouldBe schemaMetadata1.getId
         metadata1.getVersion shouldBe schemaMetadata1.getVersion
@@ -143,9 +145,9 @@ class RedisSchemaRegistryClientSpec extends AnyFlatSpec with EmbeddedRedis with
 
         //test the getVersion method
         Thread.sleep(3000)
-        redisClient.getVersion(topicName1, schema1) shouldBe 1
-        redisClient.getVersion(topicName1, schema12) shouldBe 2
-        redisClient.getVersion(topicName2, schema2) shouldBe 1
+        redisClient.getVersion(topicName1, new AvroSchema(schema1)) shouldBe 1
+        redisClient.getVersion(topicName1, new AvroSchema(schema12)) shouldBe 2
+        redisClient.getVersion(topicName2, new AvroSchema(schema2)) shouldBe 1
 
         //test the deleteSchemaVersion method
         //the latest metadata is in this test -> test the getLatestSchemaMetadata method
@@ -160,7 +162,7 @@ class RedisSchemaRegistryClientSpec extends AnyFlatSpec with EmbeddedRedis with
 
         intercept[RestClientException] {
           redisClient.getLatestSchemaMetadata(topicName2)
-        }.getMessage shouldBe "Subject not found.; error code: 40401"
+        }.getMessage shouldBe "Subject 'topicB' not found.; error code: 40401"
 
         succeed
       }
diff --git a/avro/src/test/scala/hydra/avro/registry/SchemaRegistrySpec.scala b/avro/src/test/scala/hydra/avro/registry/SchemaRegistrySpec.scala
index 820574613..009027fa1 100644
--- a/avro/src/test/scala/hydra/avro/registry/SchemaRegistrySpec.scala
+++ b/avro/src/test/scala/hydra/avro/registry/SchemaRegistrySpec.scala
@@ -392,12 +392,12 @@ class SchemaRegistrySpec extends AnyFlatSpecLike with MockFactory with Matchers
   it must "do retries for getAllVersions when SchemaRegistry throws error" in {
     val expectedVersions = List(1, 2, 3)
     val mockSchemaRegistryClient = mock[SchemaRegistryClient]
-    (mockSchemaRegistryClient.getAllVersions _)
+    (mockSchemaRegistryClient.getAllVersions(_ : String))
       .expects(*)
       .throws(new RestClientException("error", 0, 50005))
       .repeat(2)
 
-    (mockSchemaRegistryClient.getAllVersions _)
+    (mockSchemaRegistryClient.getAllVersions(_ : String))
       .expects(*)
       .returns(expectedVersions.map(Integer.valueOf).asJava)
 
@@ -426,7 +426,7 @@ class SchemaRegistrySpec extends AnyFlatSpecLike with MockFactory with Matchers
 
   it must "fail if all attempts were used" in {
     val mockSchemaRegistryClient = mock[SchemaRegistryClient]
-    (mockSchemaRegistryClient.getAllVersions _)
+    (mockSchemaRegistryClient.getAllVersions(_ : String))
       .expects(*)
       .throws(new RestClientException("error", 0, 50005))
       .repeat(3)
diff --git a/avro/src/test/scala/hydra/avro/resource/SchemaResourceLoaderSpec.scala b/avro/src/test/scala/hydra/avro/resource/SchemaResourceLoaderSpec.scala
index f32404b14..f9ed3dc92 100644
--- a/avro/src/test/scala/hydra/avro/resource/SchemaResourceLoaderSpec.scala
+++ b/avro/src/test/scala/hydra/avro/resource/SchemaResourceLoaderSpec.scala
@@ -16,6 +16,7 @@
 package hydra.avro.resource
 
 import hydra.avro.registry.SchemaRegistryException
+import io.confluent.kafka.schemaregistry.avro.AvroSchema
 import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient
 import org.apache.avro.{Schema, SchemaBuilder}
 import org.scalatest.BeforeAndAfterEach
@@ -208,7 +209,7 @@ class SchemaResourceLoaderSpec
       "returns the same underlying key schema instance if the registry metadata hasn't changed"
     ) {
       val client = new MockSchemaRegistryClient
-      client.register(testKeySchema.getFullName + "-key", testKeySchema)
+      client.register(testKeySchema.getFullName + "-key", new AvroSchema(testKeySchema))
       val loader = new SchemaResourceLoader(
         "http://localhost:48223",
         client,
@@ -231,7 +232,7 @@ class SchemaResourceLoaderSpec
     ) {
       val nschema = newValueSchema(testValueSchema.getNamespace, "ntest")
       val client = new MockSchemaRegistryClient
-      client.register(nschema.getFullName + "-value", nschema)
+      client.register(nschema.getFullName + "-value", new AvroSchema(nschema))
       val loader = new SchemaResourceLoader(
         "http://localhost:48223",
         client,
@@ -258,7 +259,7 @@ class SchemaResourceLoaderSpec
         false,
         fields.asJava
       )
-      client.register(nschema.getFullName + "-value", evolvedSchema)
+      client.register(nschema.getFullName + "-value", new AvroSchema(evolvedSchema))
       eventually {
         whenReady(loader.retrieveValueSchema(nschema.getFullName)) {
           schemaResource => (schemaResource.schema eq nschema) shouldBe false
@@ -271,7 +272,7 @@ class SchemaResourceLoaderSpec
     ) {
       val nschema = newKeySchema(testKeySchema.getNamespace, "ntest")
       val client = new MockSchemaRegistryClient
-      client.register(nschema.getFullName + "-value", nschema)
+      client.register(nschema.getFullName + "-value", new AvroSchema(nschema))
       val loader = new SchemaResourceLoader(
         "http://localhost:48223",
         client,
@@ -300,7 +301,7 @@ class SchemaResourceLoaderSpec
         false,
         fields.asJava
       )
-      client.register(nschema.getFullName + "-value", evolvedSchema)
+      client.register(nschema.getFullName + "-value", new AvroSchema(evolvedSchema))
       eventually {
         whenReady(loader.retrieveValueSchema(nschema.getFullName)) {
           schemaResource => (schemaResource.schema eq nschema) shouldBe false
diff --git a/build.sbt b/build.sbt
index 2c52b32b1..44943fa47 100644
--- a/build.sbt
+++ b/build.sbt
@@ -16,11 +16,15 @@ lazy val defaultSettings = Seq(
   excludeDependencies += "org.slf4j" % "slf4j-log4j12",
   excludeDependencies += "log4j" % "log4j",
   dependencyOverrides ++= Seq(
-    "org.apache.commons" % "commons-lang3" % "3.12.0",
-    "org.apache.commons" % "commons-compress" % "1.22",
-    "org.apache.commons" % "lang3" % "3.1.0",
-    "io.confluent" % "kafka-schema-registry" % "5.4.2" % "test",
-    "io.confluent" % "kafka-avro-serializer" % "5.4.2" % "test"
+    "org.apache.commons" % "commons-compress" % "1.24.0",
+    "io.netty" % "netty-codec" % "4.1.77.Final",
+//    "org.apache.zookeeper" % "zookeeper" % "3.7.2", -- snyk vulnerability fix
+    "org.xerial.snappy" % "snappy-java" % "1.1.10.4",
+    "org.apache.avro" % "avro" % Dependencies.avroVersion,
+    "com.fasterxml.jackson.core" % "jackson-databind" % "2.13.3",
+    "org.apache.kafka" %% "kafka" % "2.8.2" % "test",
+    "io.confluent" %% "kafka-schema-registry" % "6.2.1" % "test",
+    "io.confluent" %% "kafka-avro-serializer" % "6.2.1" % "test",
   ),
   addCompilerPlugin(
     "org.typelevel" %% "kind-projector" % "0.11.3" cross CrossVersion.full
@@ -109,10 +113,10 @@ lazy val core = Project(
   .settings(
     moduleSettings,
     name := "hydra-core",
-    libraryDependencies ++= Dependencies.coreDeps ++ Dependencies.awsAuthDeps,
+    libraryDependencies ++= Dependencies.coreDeps ++ Dependencies.awsAuthDeps ++ Dependencies.kafkaSchemaRegistryDep,
     dependencyOverrides ++= Seq(
-      "io.confluent" % "kafka-schema-registry" % "5.4.2",
-      "io.confluent" % "kafka-avro-serializer" % "5.4.2"
+      "org.apache.kafka" %% "kafka" % "2.8.2",
+      "org.apache.kafka" % "kafka-clients" % "2.8.2"
     )
   )
 
@@ -126,8 +130,8 @@ lazy val kafka = Project(
     name := "hydra-kafka",
     libraryDependencies ++= Dependencies.kafkaDeps,
     dependencyOverrides ++= Seq(
-      "io.confluent" % "kafka-schema-registry" % "5.4.2",
-      "io.confluent" % "kafka-avro-serializer" % "5.4.2"
+      "org.apache.kafka" %% "kafka" % "2.8.2",
+      "org.apache.kafka" % "kafka-clients" % "2.8.2"
     )
   )
 
@@ -141,9 +145,6 @@ lazy val avro = Project(
     libraryDependencies ++= Dependencies.avroDeps
   )
 
-val sbSettings =
-  defaultSettings ++ Test.testSettings ++ noPublishSettings ++ restartSettings
-
 lazy val ingest = Project(
   id = "ingest",
   base = file("ingest")
diff --git a/common/src/main/scala/hydra/common/util/ClassUtils.scala b/common/src/main/scala/hydra/common/util/ClassUtils.scala
new file mode 100644
index 000000000..9a5b1197a
--- /dev/null
+++ b/common/src/main/scala/hydra/common/util/ClassUtils.scala
@@ -0,0 +1,9 @@
+package hydra.common.util
+
+object ClassUtils {
+
+  // Null safe access to class.getSimpleName
+  def getSimpleName(cls: Class[_]): String = {
+    if (cls == null) "" else cls.getSimpleName
+  }
+}
diff --git a/core/src/main/scala/hydra/core/akka/SchemaRegistryActor.scala b/core/src/main/scala/hydra/core/akka/SchemaRegistryActor.scala
index a48acb758..f66b4d06a 100644
--- a/core/src/main/scala/hydra/core/akka/SchemaRegistryActor.scala
+++ b/core/src/main/scala/hydra/core/akka/SchemaRegistryActor.scala
@@ -16,7 +16,7 @@ import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientExcept
 import org.apache.avro.{Schema, SchemaParseException}
 import org.apache.kafka.common.PartitionInfo
 import scalacache.cachingF
-
+import io.confluent.kafka.schemaregistry.avro.AvroSchema
 import scala.collection.JavaConverters._
 import scala.collection.immutable.Map
 import scala.concurrent.Future
@@ -173,8 +173,8 @@ class SchemaRegistryActor(
 
       val subject = getSubject(schema)
       log.debug(s"Registering schema ${schema.getFullName}: $json")
-      val schemaId = registry.registryClient.register(subject, schema)
-      val version = registry.registryClient.getVersion(subject, schema)
+      val schemaId = registry.registryClient.register(subject, new AvroSchema(schema))
+      val version = registry.registryClient.getVersion(subject, new AvroSchema(schema))
       RegisterSchemaResponse(SchemaResource(schemaId, version, schema))
     }
   }
diff --git a/core/src/main/scala/hydra/core/ingest/Ingestor.scala b/core/src/main/scala/hydra/core/ingest/Ingestor.scala
index aa4f06d8c..84d8586bb 100644
--- a/core/src/main/scala/hydra/core/ingest/Ingestor.scala
+++ b/core/src/main/scala/hydra/core/ingest/Ingestor.scala
@@ -6,7 +6,7 @@ import hydra.core.akka.InitializingActor
 import hydra.core.monitor.HydraMetrics
 import hydra.core.protocol._
 import hydra.core.transport.{AckStrategy, HydraRecord, RecordFactory}
-import org.apache.commons.lang3.ClassUtils
+import hydra.common.util.ClassUtils
 
 import scala.concurrent.Future
 import scala.util.{Success, Try}
diff --git a/core/src/test/scala/hydra/core/akka/SchemaRegistryActorSpec.scala b/core/src/test/scala/hydra/core/akka/SchemaRegistryActorSpec.scala
index a82ef6a28..afe3286b5 100644
--- a/core/src/test/scala/hydra/core/akka/SchemaRegistryActorSpec.scala
+++ b/core/src/test/scala/hydra/core/akka/SchemaRegistryActorSpec.scala
@@ -11,6 +11,7 @@ import hydra.avro.resource.SchemaResource
 import hydra.common.config.KafkaConfigUtils.SchemaRegistrySecurityConfig
 import hydra.core.akka.SchemaRegistryActor._
 import hydra.core.protocol.HydraApplicationError
+import io.confluent.kafka.schemaregistry.avro.AvroSchema
 import org.apache.avro.Schema.Parser
 import org.apache.avro.SchemaBuilder
 import org.scalatest.BeforeAndAfterAll
@@ -58,9 +59,9 @@ class SchemaRegistryActorSpec
     .endRecord
 
   override def beforeAll() = {
-    client.register("hydra.test.Tester-value", testSchema)
-    client.register("hydra.test.TesterWithKey-key", testKeySchema)
-    client.register("hydra.test.TesterWithKey-value", testSchema)
+    client.register("hydra.test.Tester-value", new AvroSchema(testSchema))
+    client.register("hydra.test.TesterWithKey-key", new AvroSchema(testKeySchema))
+    client.register("hydra.test.TesterWithKey-value", new AvroSchema(testSchema))
   }
 
   val listener = TestProbe()
diff --git a/ingest/src/test/scala/hydra/ingest/http/SchemasEndpointSpec.scala b/ingest/src/test/scala/hydra/ingest/http/SchemasEndpointSpec.scala
index 628915074..418086880 100644
--- a/ingest/src/test/scala/hydra/ingest/http/SchemasEndpointSpec.scala
+++ b/ingest/src/test/scala/hydra/ingest/http/SchemasEndpointSpec.scala
@@ -23,7 +23,7 @@ import org.apache.kafka.common.{Node, PartitionInfo}
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpecLike
 import spray.json.{JsArray, JsObject, JsValue, RootJsonFormat}
-import net.manub.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
+import io.github.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
 
 import scala.collection.immutable.Map
 import scala.concurrent.duration._
diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/algebras/KafkaAdminAlgebra.scala b/ingestors/kafka/src/main/scala/hydra/kafka/algebras/KafkaAdminAlgebra.scala
index 20e623f18..a1d9f1402 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/algebras/KafkaAdminAlgebra.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/algebras/KafkaAdminAlgebra.scala
@@ -14,7 +14,6 @@ import org.apache.kafka.common.errors.UnknownTopicOrPartitionException
 import org.typelevel.log4cats.Logger
 import hydra.common.config.KafkaConfigUtils._
 import kafka.server.KafkaConfig
-
 /**
  * Internal interface to interact with the KafkaAdminClient from FS2 Kafka.
  * Provides a live version for production usage and a test version for integration testing.
diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/producer/AvroRecord.scala b/ingestors/kafka/src/main/scala/hydra/kafka/producer/AvroRecord.scala
index 9b45f1635..c9be1bfac 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/producer/AvroRecord.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/producer/AvroRecord.scala
@@ -4,7 +4,7 @@ import com.pluralsight.hydra.avro.JsonConverter
 import hydra.core.transport.AckStrategy
 import org.apache.avro.Schema
 import org.apache.avro.generic.GenericRecord
-import org.apache.commons.lang3.StringUtils
+
 
 /**
   * Created by alexsilva on 10/30/15.
diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/producer/JsonRecord.scala b/ingestors/kafka/src/main/scala/hydra/kafka/producer/JsonRecord.scala
index e1da5603a..6a3105fd5 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/producer/JsonRecord.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/producer/JsonRecord.scala
@@ -18,7 +18,6 @@ package hydra.kafka.producer
 
 import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
 import hydra.core.transport.AckStrategy
-import org.apache.commons.lang3.StringUtils
 
 /**
   * Created by alexsilva on 11/30/15.
diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/producer/KafkaRecord.scala b/ingestors/kafka/src/main/scala/hydra/kafka/producer/KafkaRecord.scala
index b25a08324..ab1963b2a 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/producer/KafkaRecord.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/producer/KafkaRecord.scala
@@ -1,7 +1,7 @@
 package hydra.kafka.producer
 
 import hydra.core.transport.HydraRecord
-import org.apache.commons.lang3.ClassUtils
+import hydra.common.util.ClassUtils
 import org.apache.kafka.clients.producer.ProducerRecord
 
 /**
diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/producer/StringRecord.scala b/ingestors/kafka/src/main/scala/hydra/kafka/producer/StringRecord.scala
index dbb75cfe2..3cd792cd6 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/producer/StringRecord.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/producer/StringRecord.scala
@@ -17,7 +17,7 @@
 package hydra.kafka.producer
 
 import hydra.core.transport.AckStrategy
-import org.apache.commons.lang3.StringUtils
+
 
 /**
   * Created by alexsilva on 11/30/15.
diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/transport/KafkaTransport.scala b/ingestors/kafka/src/main/scala/hydra/kafka/transport/KafkaTransport.scala
index c8f9c7a46..9eea5b752 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/transport/KafkaTransport.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/transport/KafkaTransport.scala
@@ -58,8 +58,11 @@ class KafkaTransport(producerSettings: Map[String, ProducerSettings[Any, Any]])
       msgCounter.incrementAndGet()
       metrics.saveMetrics(kmd)
 
-    case e: RecordProduceError =>
-      context.system.eventStream.publish(e)
+    case e: RecordProduceError => {
+        println("Received RecordProduceError: ")
+        println(e.error.getMessage)
+        context.system.eventStream.publish(e)
+      }
 
     case p: ProducerInitializationError => context.system.eventStream.publish(p)
   }
diff --git a/ingestors/kafka/src/test/scala/hydra/kafka/KafkaUtilsSpec.scala b/ingestors/kafka/src/test/scala/hydra/kafka/KafkaUtilsSpec.scala
index 138f22eb9..80fc2a069 100644
--- a/ingestors/kafka/src/test/scala/hydra/kafka/KafkaUtilsSpec.scala
+++ b/ingestors/kafka/src/test/scala/hydra/kafka/KafkaUtilsSpec.scala
@@ -7,7 +7,7 @@ import hydra.common.config.KafkaConfigUtils.KafkaClientSecurityConfig
 import hydra.kafka.KafkaUtilsSpec.{emptyKafkaClientSecurityConfig, kafkaClientSecurityConfig}
 import hydra.kafka.util.KafkaUtils
 import hydra.kafka.util.KafkaUtils.TopicDetails
-import net.manub.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
+import io.github.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
 import org.scalatest.concurrent.{Eventually, ScalaFutures}
 import org.scalatest.BeforeAndAfterAll
 import org.scalatest.matchers.should.Matchers
diff --git a/ingestors/kafka/src/test/scala/hydra/kafka/algebras/KafkaAdminAlgebraSpec.scala b/ingestors/kafka/src/test/scala/hydra/kafka/algebras/KafkaAdminAlgebraSpec.scala
index cecdd3359..33544adfe 100644
--- a/ingestors/kafka/src/test/scala/hydra/kafka/algebras/KafkaAdminAlgebraSpec.scala
+++ b/ingestors/kafka/src/test/scala/hydra/kafka/algebras/KafkaAdminAlgebraSpec.scala
@@ -10,7 +10,7 @@ import hydra.kafka.algebras.KafkaClientAlgebra.getOptionalGenericRecordDeseriali
 import hydra.kafka.util.KafkaUtils.TopicDetails
 import org.typelevel.log4cats.SelfAwareStructuredLogger
 import org.typelevel.log4cats.slf4j.Slf4jLogger
-import net.manub.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
+import io.github.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
 import org.joda.time.DurationFieldType.seconds
 import org.scalatest.{BeforeAndAfterAll, stats}
 import org.scalatest.matchers.should.Matchers
diff --git a/ingestors/kafka/src/test/scala/hydra/kafka/algebras/KafkaClientAlgebraSpec.scala b/ingestors/kafka/src/test/scala/hydra/kafka/algebras/KafkaClientAlgebraSpec.scala
index e0c05dd47..54e18f53f 100644
--- a/ingestors/kafka/src/test/scala/hydra/kafka/algebras/KafkaClientAlgebraSpec.scala
+++ b/ingestors/kafka/src/test/scala/hydra/kafka/algebras/KafkaClientAlgebraSpec.scala
@@ -2,7 +2,7 @@ package hydra.kafka.algebras
 
 import cats.effect.{Concurrent, ContextShift, IO, Timer}
 import hydra.avro.registry.SchemaRegistry
-import net.manub.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
+import io.github.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
 import org.apache.avro.generic.GenericRecord
 import org.scalatest.BeforeAndAfterAll
 import org.scalatest.matchers.should.Matchers
diff --git a/ingestors/kafka/src/test/scala/hydra/kafka/consumer/KafkaConsumerProxySpec.scala b/ingestors/kafka/src/test/scala/hydra/kafka/consumer/KafkaConsumerProxySpec.scala
index c4f678d9d..cbf5947e7 100644
--- a/ingestors/kafka/src/test/scala/hydra/kafka/consumer/KafkaConsumerProxySpec.scala
+++ b/ingestors/kafka/src/test/scala/hydra/kafka/consumer/KafkaConsumerProxySpec.scala
@@ -18,7 +18,7 @@ package hydra.kafka.consumer
 import akka.actor.{ActorSystem, Props}
 import akka.testkit.{ImplicitSender, TestKit}
 import hydra.kafka.consumer.KafkaConsumerProxy._
-import net.manub.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
+import io.github.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
 import org.apache.kafka.common.TopicPartition
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.funspec.AnyFunSpecLike
@@ -37,7 +37,7 @@ class KafkaConsumerProxySpec
     with ImplicitSender {
 
   implicit val config =
-    EmbeddedKafkaConfig(kafkaPort = 8012, zooKeeperPort = 3111)
+    EmbeddedKafkaConfig(kafkaPort = 8012, zooKeeperPort = 3121)
 
   override def beforeAll() = {
     super.beforeAll()
diff --git a/ingestors/kafka/src/test/scala/hydra/kafka/endpoints/BootstrapEndpointSpec.scala b/ingestors/kafka/src/test/scala/hydra/kafka/endpoints/BootstrapEndpointSpec.scala
index bfb400462..599b50e1a 100644
--- a/ingestors/kafka/src/test/scala/hydra/kafka/endpoints/BootstrapEndpointSpec.scala
+++ b/ingestors/kafka/src/test/scala/hydra/kafka/endpoints/BootstrapEndpointSpec.scala
@@ -49,7 +49,7 @@ class BootstrapEndpointSpec
 
   implicit val embeddedKafkaConfig = EmbeddedKafkaConfig(
     kafkaPort = 8012,
-    zooKeeperPort = 3111,
+    zooKeeperPort = 3011,
     customBrokerProperties = Map("auto.create.topics.enable" -> "false")
   )
 
diff --git a/ingestors/kafka/src/test/scala/hydra/kafka/endpoints/TopicMetadataEndpointSpec.scala b/ingestors/kafka/src/test/scala/hydra/kafka/endpoints/TopicMetadataEndpointSpec.scala
index 335855f90..a4da94996 100644
--- a/ingestors/kafka/src/test/scala/hydra/kafka/endpoints/TopicMetadataEndpointSpec.scala
+++ b/ingestors/kafka/src/test/scala/hydra/kafka/endpoints/TopicMetadataEndpointSpec.scala
@@ -22,9 +22,11 @@ import hydra.kafka.consumer.KafkaConsumerProxy.{GetPartitionInfo, ListTopics, Li
 import hydra.kafka.marshallers.HydraKafkaJsonSupport
 import hydra.kafka.model.{DataClassification, ObsoleteDataClassification, RequiredField, SubDataClassification}
 import hydra.kafka.model.TopicMetadataV2Request.Subject
+import org.typelevel.log4cats.SelfAwareStructuredLogger
+import org.typelevel.log4cats.slf4j.Slf4jLogger
+import io.github.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
 import hydra.kafka.programs.CreateTopicProgram
 import hydra.kafka.util.KafkaUtils.TopicDetails
-import net.manub.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
 import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder}
 import org.apache.kafka.common.{Node, PartitionInfo}
 import org.scalatest.BeforeAndAfterAll
@@ -56,7 +58,7 @@ class TopicMetadataEndpointSpec
     Slf4jLogger.getLogger[F]
 
   implicit val kafkaConfig: EmbeddedKafkaConfig =
-    EmbeddedKafkaConfig(kafkaPort = 8012, zooKeeperPort = 3111)
+    EmbeddedKafkaConfig(kafkaPort = 8012, zooKeeperPort = 3789)
 
   implicit val contextShift: ContextShift[IO] = IO.contextShift(ExecutionContext.global)
   implicit val concurrent: Concurrent[IO] = IO.ioConcurrentEffect
diff --git a/ingestors/kafka/src/test/scala/hydra/kafka/services/TopicBootstrapActorSpec.scala b/ingestors/kafka/src/test/scala/hydra/kafka/services/TopicBootstrapActorSpec.scala
index db35142f3..7d569c79c 100644
--- a/ingestors/kafka/src/test/scala/hydra/kafka/services/TopicBootstrapActorSpec.scala
+++ b/ingestors/kafka/src/test/scala/hydra/kafka/services/TopicBootstrapActorSpec.scala
@@ -17,7 +17,7 @@ import hydra.kafka.model.TopicMetadata
 import hydra.kafka.producer.AvroRecord
 import hydra.kafka.services.StreamsManagerActor.{GetMetadata, GetMetadataResponse}
 import hydra.kafka.services.TopicBootstrapActor._
-import net.manub.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig, KafkaUnavailableException}
+import io.github.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
 import org.apache.avro.Schema
 import org.apache.kafka.common.serialization.StringSerializer
 import org.joda.time.DateTime
@@ -48,7 +48,7 @@ class TopicBootstrapActorSpec
 
   implicit val embeddedKafkaConfig = EmbeddedKafkaConfig(
     kafkaPort = 8012,
-    zooKeeperPort = 3111,
+    zooKeeperPort = 3241,
     customBrokerProperties = Map("auto.create.topics.enable" -> "false")
   )
 
diff --git a/ingestors/kafka/src/test/scala/hydra/kafka/transport/KafkaMetricsSpec.scala b/ingestors/kafka/src/test/scala/hydra/kafka/transport/KafkaMetricsSpec.scala
index 4ff6f427f..c90b23346 100644
--- a/ingestors/kafka/src/test/scala/hydra/kafka/transport/KafkaMetricsSpec.scala
+++ b/ingestors/kafka/src/test/scala/hydra/kafka/transport/KafkaMetricsSpec.scala
@@ -26,7 +26,7 @@ class KafkaMetricsSpec
 
   implicit val config = EmbeddedKafkaConfig(
     kafkaPort = 8012,
-    zooKeeperPort = 3111,
+    zooKeeperPort = 3114,
     customBrokerProperties = Map(
       "auto.create.topics.enable" -> "false",
       "offsets.topic.replication.factor" -> "1"
diff --git a/ingestors/kafka/src/test/scala/hydra/kafka/transport/KafkaProducerProxySpec.scala b/ingestors/kafka/src/test/scala/hydra/kafka/transport/KafkaProducerProxySpec.scala
index 8b6ef56c3..186a6b562 100644
--- a/ingestors/kafka/src/test/scala/hydra/kafka/transport/KafkaProducerProxySpec.scala
+++ b/ingestors/kafka/src/test/scala/hydra/kafka/transport/KafkaProducerProxySpec.scala
@@ -10,7 +10,7 @@ import hydra.kafka.producer.{JsonRecord, KafkaRecordMetadata, StringRecord}
 import hydra.kafka.transport.KafkaProducerProxy.ProduceToKafka
 import hydra.kafka.transport.KafkaTransport.RecordProduceError
 import hydra.kafka.util.KafkaUtils
-import net.manub.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
+import io.github.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
 import org.apache.kafka.clients.producer.RecordMetadata
 import org.apache.kafka.common.{KafkaException, TopicPartition}
 import org.scalatest.BeforeAndAfterAll
@@ -32,7 +32,7 @@ class KafkaProducerProxySpec
 
   implicit val config = EmbeddedKafkaConfig(
     kafkaPort = 8012,
-    zooKeeperPort = 3111,
+    zooKeeperPort = 3133,
     customBrokerProperties = Map("auto.create.topics.enable" -> "false")
   )
 
diff --git a/ingestors/kafka/src/test/scala/hydra/kafka/transport/KafkaTransportSpec.scala b/ingestors/kafka/src/test/scala/hydra/kafka/transport/KafkaTransportSpec.scala
index 462706665..91ee5de54 100644
--- a/ingestors/kafka/src/test/scala/hydra/kafka/transport/KafkaTransportSpec.scala
+++ b/ingestors/kafka/src/test/scala/hydra/kafka/transport/KafkaTransportSpec.scala
@@ -9,7 +9,7 @@ import hydra.core.transport.{AckStrategy, RecordMetadata, TransportCallback}
 import hydra.kafka.producer.{DeleteTombstoneRecord, JsonRecord, StringRecord}
 import hydra.kafka.transport.KafkaProducerProxy.ProducerInitializationError
 import hydra.kafka.transport.KafkaTransport.RecordProduceError
-import net.manub.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
+import io.github.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
 import org.apache.kafka.common.KafkaException
 import org.apache.kafka.common.errors.SerializationException
 import org.scalatest.matchers.should.Matchers
@@ -85,8 +85,11 @@ class KafkaTransportSpec
 
     it("forwards to the right proxy") {
       val ack: TransportCallback =
-        (d: Long, m: Option[RecordMetadata], e: Option[Throwable]) =>
-          ingestor.ref ! "DONE"
+        (d: Long, m: Option[RecordMetadata], e: Option[Throwable]) => {
+          val msg = if(e.isDefined) e.get.getMessage else "DONE"
+          ingestor.ref ! msg
+        }
+
       val rec =
         StringRecord("transport_test", "key", "payload", AckStrategy.NoAck)
       transport ! Deliver(rec, 1, ack)
@@ -95,8 +98,11 @@ class KafkaTransportSpec
 
     it("handles delete records") {
       val ack: TransportCallback =
-        (d: Long, m: Option[RecordMetadata], e: Option[Throwable]) =>
-          ingestor.ref ! "DONE"
+        (d: Long, m: Option[RecordMetadata], e: Option[Throwable]) => {
+          val msg = if(e.isDefined) e.get.getMessage else "DONE"
+          ingestor.ref ! msg
+        }
+
       val rec =
         DeleteTombstoneRecord("transport_test", "key", AckStrategy.NoAck)
       transport ! Deliver(rec, 1, ack)
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 79cd6c04a..53a06ea1f 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -12,16 +12,16 @@ object Dependencies {
   val catsRetryVersion = "2.1.0"
   val catsVersion = "2.4.2"
   val cirisVersion = "1.2.1"
-  val confluentVersion = "5.4.2"
+  val confluentVersion = "6.2.1"
   val fs2KafkaVersion = "1.4.1"
   val jacksonCoreVersion = "2.10.4"
   val jacksonDatabindVersion = "2.10.4"
-  val jodaConvertVersion = "2.2.1"
-  val jodaTimeVersion = "2.10.9"
-  val kafkaVersion = "2.4.1"
+  val jodaConvertVersion = "2.2.3"
+  val jodaTimeVersion = "2.12.5"
+  val kafkaVersion = "2.8.2"
   val kamonPVersion = "2.1.10"
   val kamonVersion = "2.1.10"
-  val log4jVersion = "2.17.1"
+  val log4jVersion = "2.22.1"
   val refinedVersion = "0.9.20"
   val reflectionsVersion = "0.9.12"
   val scalaCacheVersion = "0.28.0"
@@ -72,7 +72,8 @@ object Dependencies {
 
     val retry = "com.softwaremill.retry" %% "retry" % "0.3.3"
 
-    val embeddedKafka = "net.manub" %% "scalatest-embedded-kafka" % "2.0.0" % "test"
+    val embeddedKafka =
+      "io.github.embeddedkafka" %% "embedded-kafka" % "2.8.1" % "test"
 
     lazy val kamon = Seq(
       "io.kamon" %% "kamon-core" % kamonVersion,
@@ -86,14 +87,25 @@ object Dependencies {
       embeddedKafka
     ) ++ kafkaClients
 
-    val confluent: Seq[ModuleID] =
+    val kafkaAvroSerializer: Seq[ModuleID] =
       Seq("io.confluent" % "kafka-avro-serializer" % confluentVersion).map(
         _.excludeAll(
           ExclusionRule(organization = "org.codehaus.jackson"),
-          ExclusionRule(organization = "com.fasterxml.jackson.core")
+          ExclusionRule(organization = "com.fasterxml.jackson.core"),
+          ExclusionRule(organization = "org.apache.kafka")
         )
       )
 
+    val kafkaSchemaRegistry: Seq[ModuleID] = Seq("io.confluent" % "kafka-schema-registry-client" % confluentVersion).map(
+      _.excludeAll(
+        ExclusionRule(organization = "org.scala-lang.modules"),
+        ExclusionRule(organization = "org.apache.kafka", "kafka-clients"),
+        ExclusionRule(organization = "com.fasterxml.jackson.module"),
+        ExclusionRule(organization = "org.scala-lang.modules"),
+        ExclusionRule(organization = "com.typesafe.scala-logging")
+      )
+    )
+
     val awsMskIamAuth = Seq("software.amazon.msk" % "aws-msk-iam-auth" % "1.1.4")
 
     val awsSdk = Seq(
@@ -171,12 +183,12 @@ object Dependencies {
       val scalaMock = "org.scalamock" %% "scalamock" % scalaMockVersion % module
       val junit = "junit" % "junit" % "4.13.1" % module
 
-      val embeddedKafka =
-        "io.github.embeddedkafka" %% "embedded-kafka-schema-registry" % "5.4.1" % module
+      val embeddedKafkaSchemaRegistry =
+        "io.github.embeddedkafka" %% "embedded-kafka-schema-registry" %  confluentVersion  % module
 
       val scalatestEmbeddedRedis = "com.github.sebruck" %% "scalatest-embedded-redis" % scalaTestEmbeddedRedisVersion % module
 
-      akkaTest ++ Seq(scalaTest, scalaMock, junit, scalatestEmbeddedRedis, embeddedKafka)
+      akkaTest ++ Seq(scalaTest, scalaMock, junit, scalatestEmbeddedRedis, embeddedKafkaSchemaRegistry)
     }
 
   }
@@ -205,21 +217,23 @@ object Dependencies {
     akka ++ Seq(avro, ciris, refined, enumeratum) ++ cats ++ logging ++ joda ++ testDeps ++ kafkaClients ++ awsMskIamAuth
 
   val avroDeps: Seq[ModuleID] =
-    baseDeps ++ confluent ++ jackson ++ guavacache ++ catsEffect ++ redisCache
+    baseDeps ++ kafkaAvroSerializer ++ jackson ++ guavacache ++ catsEffect ++ redisCache
 
   val coreDeps: Seq[ModuleID] = akka ++ baseDeps ++
     Seq(
       reflections,
       retry
-    ) ++ guavacache ++ confluent ++ kamon ++ redisCache
+    ) ++ guavacache ++ kafkaAvroSerializer ++ kamon ++ redisCache
 
   val ingestDeps: Seq[ModuleID] = coreDeps ++ akkaHttpHal ++ Seq(embeddedKafka, sprayJson)
 
   val kafkaDeps: Seq[ModuleID] = coreDeps ++ Seq(
     akkaKafkaStream,
     refined
-  ) ++ kafka ++ akkaHttpHal ++ vulcan ++ fs2Kafka ++ integrationDeps
+  ) ++ kafka ++ akkaHttpHal ++ vulcan ++ fs2Kafka ++ integrationDeps ++ kafkaSchemaRegistry
 
   val awsAuthDeps: Seq[ModuleID] = awsSdk
 
+  val kafkaSchemaRegistryDep = kafkaSchemaRegistry
+
 }

From cf673b8f391b61dd863825452006e2185ea641f8 Mon Sep 17 00:00:00 2001
From: shreedhar-kc <139169398+shreedhar-kc@users.noreply.github.com>
Date: Wed, 13 Mar 2024 20:26:11 +0530
Subject: [PATCH 096/220] fix schema incompatible error on startup

---
 .../scala/hydra/avro/registry/SchemaRegistry.scala  | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/avro/src/main/scala/hydra/avro/registry/SchemaRegistry.scala b/avro/src/main/scala/hydra/avro/registry/SchemaRegistry.scala
index fbf54d6d6..13fe96fab 100644
--- a/avro/src/main/scala/hydra/avro/registry/SchemaRegistry.scala
+++ b/avro/src/main/scala/hydra/avro/registry/SchemaRegistry.scala
@@ -4,14 +4,13 @@ import cats.Eval
 import cats.effect.Sync
 import cats.syntax.all._
 import hydra.common.config.KafkaConfigUtils._
-import io.confluent.kafka.schemaregistry.CompatibilityChecker
-import io.confluent.kafka.schemaregistry.avro.AvroSchema
+import io.confluent.kafka.schemaregistry.avro.AvroCompatibilityChecker
 import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException
-import io.confluent.kafka.schemaregistry.client.{CachedSchemaRegistryClient, MockSchemaRegistryClient, SchemaRegistryClient, SchemaRegistryClientConfig}
+import io.confluent.kafka.schemaregistry.client.{CachedSchemaRegistryClient, MockSchemaRegistryClient, SchemaRegistryClient}
 import org.apache.avro.{LogicalType, LogicalTypes, Schema}
 import org.typelevel.log4cats.Logger
-import retry.syntax.all._
 import retry.RetryPolicies._
+import retry.syntax.all._
 import retry.{RetryDetails, Sleep}
 
 import scala.collection.JavaConverters._
@@ -150,8 +149,7 @@ object SchemaRegistry {
 
 
   private[registry] def validate(newSchema: Schema, oldSchemas: List[Schema]): Boolean = {
-    val errors = CompatibilityChecker.FULL_TRANSITIVE_CHECKER.isCompatible(new AvroSchema(newSchema), oldSchemas.map(new AvroSchema(_)).asJava)
-    errors.isEmpty
+    AvroCompatibilityChecker.FULL_TRANSITIVE_CHECKER.isCompatible(newSchema, oldSchemas.asJava)
   }
 
   def live[F[_] : Sync : Logger : Sleep](
@@ -262,7 +260,8 @@ object SchemaRegistry {
             Sync[F].delay(schemaRegistryClient.register(subject, schema))
           } else {
             Sync[F].raiseError[SchemaVersion](
-              IncompatibleSchemaException("Incompatible Schema Evolution. You may add fields with default fields, or remove fields with default fields."))
+              IncompatibleSchemaException(s"Incompatible Schema Evolution. You may add fields with default fields, or " +
+                s"remove fields with default fields. Subject: $subject and schema is : $schema"))
           }
         } yield schemaVersion
       }

From 9958b7a0386d6a934d6f4c3814189850fd0a4171 Mon Sep 17 00:00:00 2001
From: Yathish B <yathish-b@pluralsight.com>
Date: Wed, 20 Mar 2024 05:29:53 +0000
Subject: [PATCH 097/220] Changed the pipeline config to ignore runner cache

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 23fc97ae0..ce44123f8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -50,11 +50,13 @@ stages:
 
 build:
   stage: build
+  cache: []
   script:
     - sbt clean compile test
 
 package:
   stage: package
+  cache: []
   script:
     - sbt universal:packageBin
     - ls ingest/target/universal/*.zip | xargs -I {} unzip {} -d ps-publish

From 4afc8abf85fc76e72227f8f39511e906470f8761 Mon Sep 17 00:00:00 2001
From: Yathish B <yathish-b@pluralsight.com>
Date: Wed, 20 Mar 2024 05:34:40 +0000
Subject: [PATCH 098/220] Corrected, Added the Environment name

---
 .gitlab-ci.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ce44123f8..ce17b2797 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -126,7 +126,7 @@ deploy_dev:
     needs: ["publish"]
     extends: .helmDeploy
     environment:
-      name: staging
+      name: dev
     before_script:
       - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}/g" $HELM_VALUES_FILE
       - cat $HELM_VALUES_FILE
@@ -159,6 +159,8 @@ deploy_staging:
   stage: deploy_staging
   extends: .saltDeploy
   when: manual
+  environment:
+      name: staging
   before_script:
     - cat $SALT_PILLAR
     - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR
@@ -177,6 +179,8 @@ deploy_production:
   stage: deploy_production
   extends: .saltDeploy
   when: manual
+  environment:
+      name: production
   before_script:
     - cat $SALT_PILLAR
     - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR

From f28d7192fb3620800fa1a055c2dee37c2371f682 Mon Sep 17 00:00:00 2001
From: Yathish B <yathish-b@pluralsight.com>
Date: Thu, 21 Mar 2024 05:39:10 +0000
Subject: [PATCH 099/220] Added the notification for staging and production

---
 .gitlab-ci.yml | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ce17b2797..402814af6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -175,6 +175,16 @@ deploy_staging:
     SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
     SALT_URL: https://saltmaster-stage.vnerd.com:8000
 
+slack:staging:
+  stage: notify
+  extends: .notify-all
+  when: on_success
+  needs: ['deploy_staging']
+  before_script:
+    - echo "Sending notification to slack"
+  variables:
+    ENV: "Staging Cluster"
+
 deploy_production: 
   stage: deploy_production
   extends: .saltDeploy
@@ -195,3 +205,12 @@ deploy_production:
     SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
     SALT_URL: https://saltmaster-production.vnerd.com:8000
 
+slack:production:
+  stage: notify
+  extends: .notify-all
+  when: on_success
+  needs: ['deploy_production']
+  before_script:
+    - echo "Sending notification to slack"
+  variables:
+    ENV: "Production Cluster"

From 2d10244495b31286a98b5262f236470675a96906 Mon Sep 17 00:00:00 2001
From: Yathish B <yathish-b@pluralsight.com>
Date: Thu, 21 Mar 2024 14:00:45 +0000
Subject: [PATCH 100/220] added the env specific bounded_context

---
 .gitlab-ci.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 402814af6..d0d92fd86 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -18,7 +18,9 @@ variables:
   ARTIFACTORY_PATH: ${ARTIFACTORY_ID}/${ARTIFACTORY_ARTIFACT_VERSION}
   ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
   BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
-  BOUNDED_CONTEXT: adapt-dvs
+  BOUNDED_CONTEXT_DEV: adapt-dvs
+  BOUNDED_CONTEXT_STAGING: adapt-dvs
+  BOUNDED_CONTEXT_PROD: adapt-dvs
   ENV: ${CI_JOB_STAGE} 
   SERVICE_NAME: "dev-hydra"
   SLACK_ICON_EMOJI: ":gitlab:"
@@ -140,7 +142,7 @@ deploy_dev:
       HELM_DEPLOY_TOKEN: ${HELM_DEPLOY_TOKEN}
       K8S_CLUSTER_NAME: app-eks.eplur-staging.us-west-2
       HELM_K8S_CLUSTER_URL: https://6C29C0073BB19BEF220B9437E6962AF2.gr7.us-west-2.eks.amazonaws.com
-      HELM_TARGET_NAMESPACE: ${BOUNDED_CONTEXT}
+      HELM_TARGET_NAMESPACE: ${BOUNDED_CONTEXT_DEV}
       APPLICATION_ROLE: dev-hydra
       HELM_VALUES_FILE: helm/eks-dev-values.yml
 

From 4f4f133e5f001985d8b0f3ea8a601bfc6a686a23 Mon Sep 17 00:00:00 2001
From: Yathish B <yathish-b@pluralsight.com>
Date: Fri, 22 Mar 2024 03:54:59 +0000
Subject: [PATCH 101/220] added a retry mechanism to pipeline

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d0d92fd86..43f1bc17a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -53,6 +53,7 @@ stages:
 build:
   stage: build
   cache: []
+  retry: 3
   script:
     - sbt clean compile test
 

From d55dd6d7779f8f6993b7ad7b052d5f99e8b35ec7 Mon Sep 17 00:00:00 2001
From: Yathish B <yathish-b@pluralsight.com>
Date: Fri, 22 Mar 2024 03:56:29 +0000
Subject: [PATCH 102/220] fixed .gitlab-ci.yml pipeline

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 43f1bc17a..c3abf0b70 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -53,7 +53,7 @@ stages:
 build:
   stage: build
   cache: []
-  retry: 3
+  retry: 2
   script:
     - sbt clean compile test
 

From 7b925aa5262cd1149baf4d79cb21b99cfc5f3008 Mon Sep 17 00:00:00 2001
From: pavan-kumarkulkarni <pavan-kumarkulkarni@pluralsight.com>
Date: Wed, 27 Mar 2024 13:55:15 +0530
Subject: [PATCH 103/220] chore(PDP1-859): Deploy hydra-publish on dev
 namespace

---
 .gitlab-ci.yml          | 8 +++++---
 helm/eks-dev-values.yml | 4 ++--
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c3abf0b70..59e26b307 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -18,7 +18,7 @@ variables:
   ARTIFACTORY_PATH: ${ARTIFACTORY_ID}/${ARTIFACTORY_ARTIFACT_VERSION}
   ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
   BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
-  BOUNDED_CONTEXT_DEV: adapt-dvs
+  BOUNDED_CONTEXT_DEV: adapt-dvs-dev
   BOUNDED_CONTEXT_STAGING: adapt-dvs
   BOUNDED_CONTEXT_PROD: adapt-dvs
   ENV: ${CI_JOB_STAGE} 
@@ -80,7 +80,6 @@ package:
       -H "Authorization: Bearer ${NPM_TOKEN}" \
       -X PUT ${ARTIFACTORY_ARTIFACT_URL} \
       -T ${ARTIFACTORY_ARTIFACT_VERSION}
-
   artifacts:
     paths:
       - ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz
@@ -140,7 +139,7 @@ deploy_dev:
       - kubectl config use-context deploy
     variables:
       HELM_CHART_NAME: ps-service
-      HELM_DEPLOY_TOKEN: ${HELM_DEPLOY_TOKEN}
+      HELM_DEPLOY_TOKEN: ${HELM_DEV_DEPLOY_TOKEN}
       K8S_CLUSTER_NAME: app-eks.eplur-staging.us-west-2
       HELM_K8S_CLUSTER_URL: https://6C29C0073BB19BEF220B9437E6962AF2.gr7.us-west-2.eks.amazonaws.com
       HELM_TARGET_NAMESPACE: ${BOUNDED_CONTEXT_DEV}
@@ -156,6 +155,7 @@ slack:dev:
     - echo "Sending notification to slack"
   variables:
     ENV: "DEV Cluster"
+    SERVICE_NAME: "dev-hydra"
 
 
 deploy_staging: 
@@ -187,6 +187,7 @@ slack:staging:
     - echo "Sending notification to slack"
   variables:
     ENV: "Staging Cluster"
+    SERVICE_NAME: "staging-hydra-publish"
 
 deploy_production: 
   stage: deploy_production
@@ -217,3 +218,4 @@ slack:production:
     - echo "Sending notification to slack"
   variables:
     ENV: "Production Cluster"
+    SERVICE_NAME: "production-hydra-publish"
diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 4f3882a4e..df0cdb763 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -27,10 +27,10 @@ ingress:
   annotations: {}
   labels: {}
   hosts:
-    - dev-hydra.eplur-staging.vnerd.com
+    - hydra-publish-dev.eplur-staging.vnerd.com
   tls:
     - hosts:
-        - dev-hydra.eplur-staging.vnerd.com
+        - hydra-publish-dev.eplur-staging.vnerd.com
   path: /
   pathType: Prefix
 

From 64eacb0296a963053b04078afc3a8d1c3dfe6b1b Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 06:05:11 +0000
Subject: [PATCH 104/220] Add .gitlab-ci.yml

---
 .gitlab-ci.yml | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 .gitlab-ci.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 000000000..3074a761a
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,41 @@
+# You can copy and paste this template into a new `.gitlab-ci.yml` file.
+# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword.
+#
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Scala.gitlab-ci.yml
+
+# Official OpenJDK Java image. Look for the different tagged releases at
+# https://hub.docker.com/_/openjdk/ . A Java image is not required
+# but an image with a JVM speeds up the build a bit.
+image: openjdk:8
+
+before_script:
+  # Enable the usage of sources over https
+  - apt-get update -yqq
+  - apt-get install apt-transport-https -yqq
+  # Add keyserver for SBT
+  - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
+  - mkdir -p /root/.gnupg
+  - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
+  - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
+  # Install SBT
+  - apt-get update -yqq
+  - apt-get install sbt -yqq
+  # Log the sbt version
+  - sbt sbtVersion
+
+test:
+  script:
+    # Execute your project's tests
+    - sbt clean test
+
+build:
+  script:
+    # Execute your project's tests
+    - sbt clean compile test
+
+deploy:
+  stage: deploy
+  script: echo "Define your deployment script!"

From 6434816b8cb887c77ab551f8b2b7e949f41c5e1c Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 06:14:59 +0000
Subject: [PATCH 105/220] Updated .gitlab-ci.yml file

---
 .gitlab-ci.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3074a761a..e467986e0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,11 +26,6 @@ before_script:
   # Log the sbt version
   - sbt sbtVersion
 
-test:
-  script:
-    # Execute your project's tests
-    - sbt clean test
-
 build:
   script:
     # Execute your project's tests

From 226273fcb98c659c7ac7eb0ea5f11aac3b93d377 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 06:41:13 +0000
Subject: [PATCH 106/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e467986e0..f0c9864ed 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -30,6 +30,25 @@ build:
   script:
     # Execute your project's tests
     - sbt clean compile test
+    - sbt universal:packageBin
+
+package:
+  script:
+    # Execute your project's tests
+   - ls
+   - unzip ingest/target/universal/*.zip -d ps-publish
+   - ls
+   - mv ps-publish/hydra-ingest*/* ps-publish
+   - rm -rf ps-publish/hydra-ingest*
+   - cd ps-publish/bin
+   - ls -la
+   - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic-agent-4.4.0.jar
+   - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic.yml
+   - cd ../../
+   - echo %build.number% > VERSION
+   - tar czf %env.ARTIFACT_NAME%-%build.number%.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
+   - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-%build.number%.tgz']"
+
 
 deploy:
   stage: deploy

From 266860e62f3818ed341c77e6f37d816a52bccf92 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 06:49:08 +0000
Subject: [PATCH 107/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f0c9864ed..fe70b6487 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -11,6 +11,12 @@
 # but an image with a JVM speeds up the build a bit.
 image: openjdk:8
 
+stages:
+  - build
+  - package
+  - deploy
+
+
 before_script:
   # Enable the usage of sources over https
   - apt-get update -yqq
@@ -26,13 +32,16 @@ before_script:
   # Log the sbt version
   - sbt sbtVersion
 
+
 build:
+  stage: build
   script:
     # Execute your project's tests
     - sbt clean compile test
     - sbt universal:packageBin
 
 package:
+  stage: package
   script:
     # Execute your project's tests
    - ls
@@ -49,7 +58,6 @@ package:
    - tar czf %env.ARTIFACT_NAME%-%build.number%.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-%build.number%.tgz']"
 
-
 deploy:
   stage: deploy
   script: echo "Define your deployment script!"

From eee9775f4219130be6f44b5645814e9b60546dc7 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 06:58:03 +0000
Subject: [PATCH 108/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 41 +++++++++++++----------------------------
 1 file changed, 13 insertions(+), 28 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fe70b6487..0e8b92e55 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -11,12 +11,6 @@
 # but an image with a JVM speeds up the build a bit.
 image: openjdk:8
 
-stages:
-  - build
-  - package
-  - deploy
-
-
 before_script:
   # Enable the usage of sources over https
   - apt-get update -yqq
@@ -39,25 +33,16 @@ build:
     # Execute your project's tests
     - sbt clean compile test
     - sbt universal:packageBin
-
-package:
-  stage: package
-  script:
-    # Execute your project's tests
-   - ls
-   - unzip ingest/target/universal/*.zip -d ps-publish
-   - ls
-   - mv ps-publish/hydra-ingest*/* ps-publish
-   - rm -rf ps-publish/hydra-ingest*
-   - cd ps-publish/bin
-   - ls -la
-   - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic-agent-4.4.0.jar
-   - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic.yml
-   - cd ../../
-   - echo %build.number% > VERSION
-   - tar czf %env.ARTIFACT_NAME%-%build.number%.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
-   - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-%build.number%.tgz']"
-
-deploy:
-  stage: deploy
-  script: echo "Define your deployment script!"
+    - ls
+    - unzip ingest/target/universal/*.zip -d ps-publish
+    - ls
+    - mv ps-publish/hydra-ingest*/* ps-publish
+    - rm -rf ps-publish/hydra-ingest*
+    - cd ps-publish/bin
+    - ls -la
+    - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic-agent-4.4.0.jar
+    - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic.yml
+    - cd ../../
+    - echo %build.number% > VERSION
+    - tar czf %env.ARTIFACT_NAME%-%build.number%.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
+    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-%build.number%.tgz']"

From 91187f07efb37a2011ca69df07d1cd9dc0e66039 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 07:13:42 +0000
Subject: [PATCH 109/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0e8b92e55..b0aa46cef 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -31,8 +31,11 @@ build:
   stage: build
   script:
     # Execute your project's tests
+    # Clean compile Test
     - sbt clean compile test
+    #Package
     - sbt universal:packageBin
+    # Zip Artifact with Dependency Libs
     - ls
     - unzip ingest/target/universal/*.zip -d ps-publish
     - ls
@@ -46,3 +49,18 @@ build:
     - echo %build.number% > VERSION
     - tar czf %env.ARTIFACT_NAME%-%build.number%.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-%build.number%.tgz']"
+    # Md5sum and Publish Md5Sum Artifact
+    - md5sum %env.ARTIFACT_NAME%-%build.number%.tgz > %env.ARTIFACT_NAME%-%build.number%.tgz.md5
+    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-%build.number%.tgz.md5']"
+    # Docker Build and Docker Push
+    - pwd
+    - ls -lh
+    - docker build -t %image.name% -f Dockerfile.new .
+    - docker tag %image.name% harbor.vnerd.com/library/%image.name%:%build.number%
+    - docker push harbor.vnerd.com/library/%image.name%:%build.number%
+    - docker tag %image.name% harbor.vnerd.com/library/%image.name%:latest
+    - docker push harbor.vnerd.com/library/%image.name%:latest
+    # Update VERSION File
+    - CURYEAR=$(date +%Y)
+    - CURMONTH=$(date +%-m)
+    - echo -n "${CURYEAR}.${CURMONTH}.%env.BUILD_NUMBER%.%teamcity.build.id%" > VERSION
\ No newline at end of file

From c204de2d8d35f8c0007e70578f4fb6d5021c6948 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 07:33:01 +0000
Subject: [PATCH 110/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b0aa46cef..0f490ac6d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,6 +25,8 @@ before_script:
   - apt-get install sbt -yqq
   # Log the sbt version
   - sbt sbtVersion
+  # Install docker 
+  - sudo apt install docker-ce
 
 
 build:

From 46da60073eebfb17c214ac5c81992e7f6acf0272 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 07:35:40 +0000
Subject: [PATCH 111/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0f490ac6d..df435a9c9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,7 +26,7 @@ before_script:
   # Log the sbt version
   - sbt sbtVersion
   # Install docker 
-  - sudo apt install docker-ce
+  - apt install docker-ce
 
 
 build:

From c109228062d9814b3c892260f274dffd40324a1a Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 07:38:26 +0000
Subject: [PATCH 112/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index df435a9c9..45774e732 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,7 +26,7 @@ before_script:
   # Log the sbt version
   - sbt sbtVersion
   # Install docker 
-  - apt install docker-ce
+  - apt install docker
 
 
 build:

From 544704ace2c0b190481bb279c15a919de158b95f Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 07:39:29 +0000
Subject: [PATCH 113/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 45774e732..cf5e70db8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,7 +26,7 @@ before_script:
   # Log the sbt version
   - sbt sbtVersion
   # Install docker 
-  - apt install docker
+  - apt-get install docker -yqq
 
 
 build:

From e38b191ccf61c0c5fd62b0410e7aae9b5dd38376 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 08:24:02 +0000
Subject: [PATCH 114/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cf5e70db8..04f0e40df 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -11,6 +11,11 @@
 # but an image with a JVM speeds up the build a bit.
 image: openjdk:8
 
+variables:
+  ARTIFACT_NAME: hydra-publish
+  IMAGE_NAME: hydra
+
+
 before_script:
   # Enable the usage of sources over https
   - apt-get update -yqq
@@ -48,21 +53,21 @@ build:
     - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic-agent-4.4.0.jar
     - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic.yml
     - cd ../../
-    - echo %build.number% > VERSION
-    - tar czf %env.ARTIFACT_NAME%-%build.number%.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
-    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-%build.number%.tgz']"
+    - echo $CI_PIPELINE_IID > VERSION
+    - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
+    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz']"
     # Md5sum and Publish Md5Sum Artifact
-    - md5sum %env.ARTIFACT_NAME%-%build.number%.tgz > %env.ARTIFACT_NAME%-%build.number%.tgz.md5
-    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-%build.number%.tgz.md5']"
+    - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
+    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
     # Docker Build and Docker Push
     - pwd
     - ls -lh
-    - docker build -t %image.name% -f Dockerfile.new .
-    - docker tag %image.name% harbor.vnerd.com/library/%image.name%:%build.number%
-    - docker push harbor.vnerd.com/library/%image.name%:%build.number%
-    - docker tag %image.name% harbor.vnerd.com/library/%image.name%:latest
-    - docker push harbor.vnerd.com/library/%image.name%:latest
+    - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
+    - docker tag ${IMAGE_NAME} harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
+    - docker push harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
+    - docker tag ${IMAGE_NAME} harbor.vnerd.com/library/${IMAGE_NAME}:latest
+    - docker push harbor.vnerd.com/library/${IMAGE_NAME}:latest
     # Update VERSION File
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)
-    - echo -n "${CURYEAR}.${CURMONTH}.%env.BUILD_NUMBER%.%teamcity.build.id%" > VERSION
\ No newline at end of file
+    - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID} > VERSION
\ No newline at end of file

From bc5f1313884b40bf40ffac649454085f064eb140 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 08:51:23 +0000
Subject: [PATCH 115/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 04f0e40df..6df50668e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,6 +32,7 @@ before_script:
   - sbt sbtVersion
   # Install docker 
   - apt-get install docker -yqq
+  - docker --version
 
 
 build:

From 0d539c8eb08db99beec639ee29dbf00683fc779f Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 11:09:06 +0000
Subject: [PATCH 116/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6df50668e..470a0839f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,9 +32,9 @@ before_script:
   - sbt sbtVersion
   # Install docker 
   - apt-get install docker -yqq
+  - apt-get install apt-utils
   - docker --version
 
-
 build:
   stage: build
   script:

From 9536e2b9dd29bf20b7b0921b4ffe754d11de1cfd Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 18 Jan 2024 11:11:38 +0000
Subject: [PATCH 117/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 470a0839f..7e0909b0c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -31,8 +31,8 @@ before_script:
   # Log the sbt version
   - sbt sbtVersion
   # Install docker 
+  - apt-get install apt-utils 
   - apt-get install docker -yqq
-  - apt-get install apt-utils
   - docker --version
 
 build:

From d071b3916b2f79b41825861184664801e905d086 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 23 Jan 2024 08:19:35 +0000
Subject: [PATCH 118/220] Updated .gitlab-ci.yml file

---
 .gitlab-ci.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7e0909b0c..a228b437e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,7 +32,10 @@ before_script:
   - sbt sbtVersion
   # Install docker 
   - apt-get install apt-utils 
-  - apt-get install docker -yqq
+  - apt-get update -yqq
+  - apt-get install docker.io -yqq
+  - curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+  - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
   - docker --version
 
 build:

From 4fcb79bd4212a92b14a5bbc05f150089b6c196ae Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 23 Jan 2024 09:04:42 +0000
Subject: [PATCH 119/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a228b437e..cbc16ab20 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -37,6 +37,7 @@ before_script:
   - curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
   - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
   - docker --version
+  - sudo service docker start
 
 build:
   stage: build

From 103f1502991a0d9c8225db2f217d4ecf67ea14a4 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 23 Jan 2024 10:03:43 +0000
Subject: [PATCH 120/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cbc16ab20..7c430421e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -37,7 +37,7 @@ before_script:
   - curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
   - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
   - docker --version
-  - sudo service docker start
+  - service docker start
 
 build:
   stage: build

From 2b8cb9b78cd5a72826c0f366c46b750d87b3ff7d Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 23 Jan 2024 14:33:35 +0000
Subject: [PATCH 121/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7c430421e..e5251fb73 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -69,6 +69,7 @@ build:
     - ls -lh
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
     - docker tag ${IMAGE_NAME} harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
+    - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD harbor.vnerd.com
     - docker push harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
     - docker tag ${IMAGE_NAME} harbor.vnerd.com/library/${IMAGE_NAME}:latest
     - docker push harbor.vnerd.com/library/${IMAGE_NAME}:latest

From 7941472356304eb7f72e327d2ca93e03c01a289e Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 07:31:16 +0000
Subject: [PATCH 122/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e5251fb73..fc5bc5acf 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -69,10 +69,10 @@ build:
     - ls -lh
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
     - docker tag ${IMAGE_NAME} harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
-    - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD harbor.vnerd.com
-    - docker push harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
+    #- docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD harbor.vnerd.com
+    - docker push $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
     - docker tag ${IMAGE_NAME} harbor.vnerd.com/library/${IMAGE_NAME}:latest
-    - docker push harbor.vnerd.com/library/${IMAGE_NAME}:latest
+    - docker push $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD harbor.vnerd.com/library/${IMAGE_NAME}:latest
     # Update VERSION File
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)

From 4cbbbbd85f5b5cb856c4f2e3d9a55cb94fdd7a2c Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 11:00:40 +0000
Subject: [PATCH 123/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 73 +++++++++++++++++++-------------------------------
 1 file changed, 27 insertions(+), 46 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fc5bc5acf..f2ee03e49 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,56 +1,34 @@
-# You can copy and paste this template into a new `.gitlab-ci.yml` file.
-# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword.
-#
-# To contribute improvements to CI/CD templates, please follow the Development guide at:
-# https://docs.gitlab.com/ee/development/cicd/templates.html
-# This specific template is located at:
-# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Scala.gitlab-ci.yml
-
-# Official OpenJDK Java image. Look for the different tagged releases at
-# https://hub.docker.com/_/openjdk/ . A Java image is not required
-# but an image with a JVM speeds up the build a bit.
 image: openjdk:8
 
 variables:
   ARTIFACT_NAME: hydra-publish
   IMAGE_NAME: hydra
-
+  DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
 
 before_script:
-  # Enable the usage of sources over https
-  - apt-get update -yqq
-  - apt-get install apt-transport-https -yqq
-  # Add keyserver for SBT
-  - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
-  - mkdir -p /root/.gnupg
-  - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
-  - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
-  # Install SBT
   - apt-get update -yqq
-  - apt-get install sbt -yqq
-  # Log the sbt version
-  - sbt sbtVersion
-  # Install docker 
-  - apt-get install apt-utils 
-  - apt-get update -yqq
-  - apt-get install docker.io -yqq
+  - apt-get install -yqq apt-transport-https apt-utils sbt docker.io
   - curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
   - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
   - docker --version
   - service docker start
 
+stages:
+  - build
+  - package
+  - publish
+  - finalize
+
 build:
   stage: build
   script:
-    # Execute your project's tests
-    # Clean compile Test
     - sbt clean compile test
-    #Package
+
+package:
+  stage: package
+  script:
     - sbt universal:packageBin
-    # Zip Artifact with Dependency Libs
-    - ls
-    - unzip ingest/target/universal/*.zip -d ps-publish
-    - ls
+    - ls ingest/target/universal/*.zip | xargs -I {} unzip {} -d ps-publish
     - mv ps-publish/hydra-ingest*/* ps-publish
     - rm -rf ps-publish/hydra-ingest*
     - cd ps-publish/bin
@@ -61,19 +39,22 @@ build:
     - echo $CI_PIPELINE_IID > VERSION
     - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz']"
-    # Md5sum and Publish Md5Sum Artifact
+
+publish:
+  stage: publish
+  script:
     - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
-    # Docker Build and Docker Push
-    - pwd
-    - ls -lh
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
-    - docker tag ${IMAGE_NAME} harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
-    #- docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD harbor.vnerd.com
-    - docker push $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD harbor.vnerd.com/library/${IMAGE_NAME}:${CI_PIPELINE_IID}
-    - docker tag ${IMAGE_NAME} harbor.vnerd.com/library/${IMAGE_NAME}:latest
-    - docker push $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD harbor.vnerd.com/library/${IMAGE_NAME}:latest
-    # Update VERSION File
+    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
+
+finalize:
+  stage: finalize
+  script:
+    - docker push ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
+    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest
+    - docker push ${DOCKER_REGISTRY_IMAGE}:latest
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)
-    - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID} > VERSION
\ No newline at end of file
+    - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID}" > VERSION
+

From 6951ee368a5be639ed1568c43597fd711a026404 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 11:07:36 +0000
Subject: [PATCH 124/220] Updated .gitlab-ci.yml file

---
 .gitlab-ci.yml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f2ee03e49..d2cc1765a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,11 +7,13 @@ variables:
 
 before_script:
   - apt-get update -yqq
-  - apt-get install -yqq apt-transport-https apt-utils sbt docker.io
-  - curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-  - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
-  - docker --version
-  - service docker start
+  - apt-get install -yqq apt-transport-https apt-utils
+  - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
+  - mkdir -p /root/.gnupg
+  - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
+  - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
+  - apt-get update -yqq
+  - apt-get install -yqq sbt docker.io
 
 stages:
   - build

From f7115e310cf377f651e07233a1e4f59eb415b511 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 11:19:02 +0000
Subject: [PATCH 125/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d2cc1765a..f517ac825 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,7 +7,7 @@ variables:
 
 before_script:
   - apt-get update -yqq
-  - apt-get install -yqq apt-transport-https apt-utils
+  - apt-get install -yqq apt-transport-https apt-utils sbt docker.io
   - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
   - mkdir -p /root/.gnupg
   - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
@@ -59,4 +59,3 @@ finalize:
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)
     - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID}" > VERSION
-

From 4ff4adbca39a9a9da8f9396bfb0670cf092448cc Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 11:32:32 +0000
Subject: [PATCH 126/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f517ac825..f0b8a6030 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,7 +7,7 @@ variables:
 
 before_script:
   - apt-get update -yqq
-  - apt-get install -yqq apt-transport-https apt-utils sbt docker.io
+  - apt-get install -yqq apt-transport-https apt-utils
   - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
   - mkdir -p /root/.gnupg
   - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823

From a1ca4bfd0f49c95f9a88927557564b36ed3b8c46 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 14:37:36 +0000
Subject: [PATCH 127/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f0b8a6030..042b5a9c2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,6 +14,7 @@ before_script:
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   - apt-get update -yqq
   - apt-get install -yqq sbt docker.io
+  - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
 
 stages:
   - build
@@ -53,6 +54,7 @@ publish:
 finalize:
   stage: finalize
   script:
+    - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
     - docker push ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest
     - docker push ${DOCKER_REGISTRY_IMAGE}:latest

From 14286f52e436ad581f6907d04e5a5cf2f77f03ae Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 14:41:15 +0000
Subject: [PATCH 128/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 042b5a9c2..d98edb6b9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,8 @@ before_script:
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   - apt-get update -yqq
   - apt-get install -yqq sbt docker.io
-  - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
+  - docker login harbor.vnerd.com -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD}
+  - echo "${DOCKER_REGISTRY_USERNAME}"
 
 stages:
   - build

From 7ea3cd92ac5c0fa78db80be6e86440f541fb609e Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 14:50:58 +0000
Subject: [PATCH 129/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d98edb6b9..65074d62a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,8 +14,7 @@ before_script:
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   - apt-get update -yqq
   - apt-get install -yqq sbt docker.io
-  - docker login harbor.vnerd.com -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD}
-  - echo "${DOCKER_REGISTRY_USERNAME}"
+  - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com
 
 stages:
   - build

From c1b1b374e9d0895d91b545817f0dfe7234e2ce4c Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 24 Jan 2024 14:56:57 +0000
Subject: [PATCH 130/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 65074d62a..14aa1b5c3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,7 @@ before_script:
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   - apt-get update -yqq
   - apt-get install -yqq sbt docker.io
-  - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com
+
 
 stages:
   - build
@@ -23,9 +23,17 @@ stages:
   - finalize
 
 build:
+  #stage: build
+ #script:
+   # - sbt clean compile test
+
   stage: build
+  image: harbor.vnerd.com/proxy/library/docker:19.03.0
+  services:
+    - harbor.vnerd.com/proxy/library/docker:19.03.5-dind
   script:
-    - sbt clean compile test
+    - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com
+    - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
 
 package:
   stage: package
@@ -53,7 +61,11 @@ publish:
 
 finalize:
   stage: finalize
+  image: harbor.vnerd.com/proxy/library/docker:19.03.0
+  services:
+    - harbor.vnerd.com/proxy/library/docker:19.03.5-dind
   script:
+    - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com
     - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
     - docker push ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest

From 73728c8a37bf93550467074346e31144e49f2051 Mon Sep 17 00:00:00 2001
From: Yuvi0710 <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 11:31:46 +0530
Subject: [PATCH 131/220] Updated

---
 .gitlab-ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 14aa1b5c3..d3661702f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,14 +6,14 @@ variables:
   DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
 
 before_script:
-  - apt-get update -yqq
-  - apt-get install -yqq apt-transport-https apt-utils
+  - apk update update -yqq
+  - apk-get install -yqq apt-transport-https apt-utils
   - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
   - mkdir -p /root/.gnupg
   - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
-  - apt-get update -yqq
-  - apt-get install -yqq sbt docker.io
+  - apk-get update -yqq
+  - apk-get install -yqq sbt docker.io
 
 
 stages:

From 389f63c5c12a6627cfc41e98e6b4e9016f647487 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 06:05:09 +0000
Subject: [PATCH 132/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 31 ++++++++++++-------------------
 1 file changed, 12 insertions(+), 19 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d3661702f..c3a87a4bc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,15 +6,16 @@ variables:
   DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
 
 before_script:
-  - apk update update -yqq
-  - apk-get install -yqq apt-transport-https apt-utils
+  - apt-get update -yqq
+  - apt-get install -yqq apt-transport-https apt-utils sbt docker.io
   - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
   - mkdir -p /root/.gnupg
   - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
-  - apk-get update -yqq
-  - apk-get install -yqq sbt docker.io
-
+  - apt-get update -yqq
+  - apt-get install -yqq sbt docker.io
+  - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com
+  - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
 
 stages:
   - build
@@ -23,17 +24,9 @@ stages:
   - finalize
 
 build:
-  #stage: build
- #script:
-   # - sbt clean compile test
-
   stage: build
-  image: harbor.vnerd.com/proxy/library/docker:19.03.0
-  services:
-    - harbor.vnerd.com/proxy/library/docker:19.03.5-dind
   script:
-    - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com
-    - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
+    - sbt clean compile test
 
 package:
   stage: package
@@ -61,15 +54,15 @@ publish:
 
 finalize:
   stage: finalize
-  image: harbor.vnerd.com/proxy/library/docker:19.03.0
-  services:
-    - harbor.vnerd.com/proxy/library/docker:19.03.5-dind
   script:
-    - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com
-    - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
     - docker push ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest
     - docker push ${DOCKER_REGISTRY_IMAGE}:latest
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)
     - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID}" > VERSION
+
+
+
+
+

From 8f9d4bc95b1b5aca67b2c2f9af11f9c481b7e4a0 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 06:06:48 +0000
Subject: [PATCH 133/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c3a87a4bc..ce984e3f1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,7 +7,7 @@ variables:
 
 before_script:
   - apt-get update -yqq
-  - apt-get install -yqq apt-transport-https apt-utils sbt docker.io
+  - apt-get install -yqq apt-transport-https apt-utils
   - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
   - mkdir -p /root/.gnupg
   - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823

From b3320c9ec33644e8780d3055d0e14cfd5097ff26 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 06:09:38 +0000
Subject: [PATCH 134/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ce984e3f1..c77f4c3ef 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,7 @@ before_script:
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   - apt-get update -yqq
   - apt-get install -yqq sbt docker.io
-  - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com
+  - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com || exit 1
   - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
 
 stages:

From c1519e2df08838f24e3ffc8f614714a4e2f7120e Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 06:23:42 +0000
Subject: [PATCH 135/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c77f4c3ef..b7a431358 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,6 +4,9 @@ variables:
   ARTIFACT_NAME: hydra-publish
   IMAGE_NAME: hydra
   DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
+  DOCKER_REGISTRY_URL: "harbor.vnerd.com"
+  DOCKER_REGISTRY_USERNAME: $DOCKER_REGISTRY_USERNAME
+  DOCKER_REGISTRY_PASSWORD: $DOCKER_REGISTRY_PASSWORD
 
 before_script:
   - apt-get update -yqq
@@ -14,9 +17,8 @@ before_script:
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   - apt-get update -yqq
   - apt-get install -yqq sbt docker.io
-  - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USERNAME}" --password-stdin harbor.vnerd.com || exit 1
-  - docker login -u ${DOCKER_REGISTRY_USERNAME} -p ${DOCKER_REGISTRY_PASSWORD} harbor.vnerd.com
-
+  - echo "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin $DOCKER_REGISTRY_URL
+  
 stages:
   - build
   - package

From 2adc0196183f75dc0e53c9b2a3502416c7108a48 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 06:27:56 +0000
Subject: [PATCH 136/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b7a431358..a4a48da26 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,6 +17,8 @@ before_script:
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   - apt-get update -yqq
   - apt-get install -yqq sbt docker.io
+  - echo "$DOCKER_REGISTRY_PASSWORD"
+  - echo "$DOCKER_REGISTRY_USERNAME"
   - echo "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin $DOCKER_REGISTRY_URL
   
 stages:

From e84cf58040a475483a5bb84e9e6c9f42e2a06442 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 06:44:32 +0000
Subject: [PATCH 137/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a4a48da26..120f1dea4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,7 +19,7 @@ before_script:
   - apt-get install -yqq sbt docker.io
   - echo "$DOCKER_REGISTRY_PASSWORD"
   - echo "$DOCKER_REGISTRY_USERNAME"
-  - echo "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin $DOCKER_REGISTRY_URL
+  - echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USER} --password-stdin
   
 stages:
   - build

From d476f10164cb5e9c4bcbd03986a272a0d35ca9cb Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 06:56:25 +0000
Subject: [PATCH 138/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 120f1dea4..be83fb501 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,7 +19,8 @@ before_script:
   - apt-get install -yqq sbt docker.io
   - echo "$DOCKER_REGISTRY_PASSWORD"
   - echo "$DOCKER_REGISTRY_USERNAME"
-  - echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USER} --password-stdin
+  #- echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
+  #- echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USER} --password-stdin
   
 stages:
   - build
@@ -48,8 +49,12 @@ package:
     - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz']"
 
+
 publish:
   stage: publish
+  image: harbor.vnerd.com/proxy/library/docker:cli
+  before_script:
+    - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
     - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"

From 0cbd70cba22a291b084dea6829f24a6bd5bf85f4 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 07:12:55 +0000
Subject: [PATCH 139/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index be83fb501..78a69a68c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -23,11 +23,22 @@ before_script:
   #- echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USER} --password-stdin
   
 stages:
+  - prestep
   - build
   - package
   - publish
   - finalize
 
+prestep:
+  stage: prestep
+  image: harbor.vnerd.com/proxy/library/docker:cli
+  before_script:
+    - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
+  script:
+    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
+    - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
+    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
+
 build:
   stage: build
   script:

From 33debd6fedce0e7c295b063c5fb23d547b6d68f1 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 07:15:14 +0000
Subject: [PATCH 140/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 78a69a68c..da8c69d12 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,8 +5,6 @@ variables:
   IMAGE_NAME: hydra
   DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
   DOCKER_REGISTRY_URL: "harbor.vnerd.com"
-  DOCKER_REGISTRY_USERNAME: $DOCKER_REGISTRY_USERNAME
-  DOCKER_REGISTRY_PASSWORD: $DOCKER_REGISTRY_PASSWORD
 
 before_script:
   - apt-get update -yqq

From e1c76388d876a1fe79013f3371a210acb9848ee0 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 08:05:47 +0000
Subject: [PATCH 141/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index da8c69d12..2bd362989 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,7 +4,7 @@ variables:
   ARTIFACT_NAME: hydra-publish
   IMAGE_NAME: hydra
   DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
-  DOCKER_REGISTRY_URL: "harbor.vnerd.com"
+  DOCKER_REGISTRY_URL: "harbor.vnerd.com/library"
 
 before_script:
   - apt-get update -yqq
@@ -31,7 +31,8 @@ prestep:
   stage: prestep
   image: harbor.vnerd.com/proxy/library/docker:cli
   before_script:
-    - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
+    - echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USERNAME} --password-stdin
+    #- echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .

From 5dd372cf3e36004e624c5e485e9ecf05baf50e14 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 09:20:43 +0000
Subject: [PATCH 142/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2bd362989..5dda7b8e8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -30,6 +30,11 @@ stages:
 prestep:
   stage: prestep
   image: harbor.vnerd.com/proxy/library/docker:cli
+  variables:
+    DOCKER_HOST: tcp://localhost:2376
+    DOCKER_TLS_CERTDIR: "/certs"
+    DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
+    DOCKER_TLS_VERIFY: 1
   before_script:
     - echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USERNAME} --password-stdin
     #- echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"

From fc1c7a159d561e0f6160a3949e4d88c22e232dd2 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 09:33:20 +0000
Subject: [PATCH 143/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5dda7b8e8..ba82bf0f6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,6 +5,11 @@ variables:
   IMAGE_NAME: hydra
   DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
   DOCKER_REGISTRY_URL: "harbor.vnerd.com/library"
+  DOCKER_HOST: tcp://localhost:2375
+  DOCKER_TLS_CERTDIR: ''
+  DOCKER_DRIVER: overlay2
+  DOCKER_TLS_VERIFY: 1
+
 
 before_script:
   - apt-get update -yqq
@@ -29,12 +34,7 @@ stages:
 
 prestep:
   stage: prestep
-  image: harbor.vnerd.com/proxy/library/docker:cli
-  variables:
-    DOCKER_HOST: tcp://localhost:2376
-    DOCKER_TLS_CERTDIR: "/certs"
-    DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
-    DOCKER_TLS_VERIFY: 1
+  image: harbor.vnerd.com/proxy/library/docker:19.03.0
   before_script:
     - echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USERNAME} --password-stdin
     #- echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"

From b1129f827dcc818693d3ad377fbcd27f86d25cba Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 09:39:13 +0000
Subject: [PATCH 144/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ba82bf0f6..bef0e3e61 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,12 +5,11 @@ variables:
   IMAGE_NAME: hydra
   DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
   DOCKER_REGISTRY_URL: "harbor.vnerd.com/library"
-  DOCKER_HOST: tcp://localhost:2375
-  DOCKER_TLS_CERTDIR: ''
-  DOCKER_DRIVER: overlay2
+  DOCKER_HOST: tcp://localhost:2376
+  DOCKER_TLS_CERTDIR: "/certs"
+  DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
   DOCKER_TLS_VERIFY: 1
 
-
 before_script:
   - apt-get update -yqq
   - apt-get install -yqq apt-transport-https apt-utils
@@ -34,7 +33,10 @@ stages:
 
 prestep:
   stage: prestep
-  image: harbor.vnerd.com/proxy/library/docker:19.03.0
+  image: harbor.vnerd.com/proxy/library/docker:cli
+  tags:
+    - ps
+    - docker
   before_script:
     - echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USERNAME} --password-stdin
     #- echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"

From 904fe265420683ff6c248a4eb5cc5bc98a11f812 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 09:42:49 +0000
Subject: [PATCH 145/220] Update .gitlab-ci.yml

---
 .gitlab-ci.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bef0e3e61..7601c9e18 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -70,6 +70,9 @@ package:
 publish:
   stage: publish
   image: harbor.vnerd.com/proxy/library/docker:cli
+  tags:
+    - ps
+    - docker
   before_script:
     - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
@@ -80,6 +83,12 @@ publish:
 
 finalize:
   stage: finalize
+  image: harbor.vnerd.com/proxy/library/docker:cli
+  tags:
+    - ps
+    - docker
+  before_script:
+    - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
     - docker push ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest

From a9d0aab7b69b3ea078264c32e389a8598f387425 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 09:55:49 +0000
Subject: [PATCH 146/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7601c9e18..c541f89ff 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,26 +25,11 @@ before_script:
   #- echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USER} --password-stdin
   
 stages:
-  - prestep
   - build
   - package
   - publish
   - finalize
 
-prestep:
-  stage: prestep
-  image: harbor.vnerd.com/proxy/library/docker:cli
-  tags:
-    - ps
-    - docker
-  before_script:
-    - echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USERNAME} --password-stdin
-    #- echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
-  script:
-    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
-    - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
-    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
-
 build:
   stage: build
   script:
@@ -66,7 +51,6 @@ package:
     - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz']"
 
-
 publish:
   stage: publish
   image: harbor.vnerd.com/proxy/library/docker:cli

From 319c032eae2a9bf0a81304f491a0eb94c15818bb Mon Sep 17 00:00:00 2001
From: Yuvi0710 <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 15:42:37 +0530
Subject: [PATCH 147/220] Updated .gitlab-ci.yml

---
 .gitlab-ci.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c541f89ff..71fa1fbe3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,10 +19,6 @@ before_script:
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   - apt-get update -yqq
   - apt-get install -yqq sbt docker.io
-  - echo "$DOCKER_REGISTRY_PASSWORD"
-  - echo "$DOCKER_REGISTRY_USERNAME"
-  #- echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
-  #- echo ${DOCKER_REGISTRY_PASSWORD} | docker login ${DOCKER_REGISTRY_URL} -u ${DOCKER_REGISTRY_USER} --password-stdin
   
 stages:
   - build

From 548511d33b75fbeb7debf04b217cf046f615df33 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 10:34:20 +0000
Subject: [PATCH 148/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 71fa1fbe3..cf72bec76 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,7 +24,6 @@ stages:
   - build
   - package
   - publish
-  - finalize
 
 build:
   stage: build
@@ -45,7 +44,7 @@ package:
     - cd ../../
     - echo $CI_PIPELINE_IID > VERSION
     - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
-    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz']"
+    - echo "##teamcity[publishArtifacts '${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz']"
 
 publish:
   stage: publish
@@ -56,20 +55,13 @@ publish:
   before_script:
     - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
+    - pwd
+    - ls -lh
+    - cp ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz .
     - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
-
-finalize:
-  stage: finalize
-  image: harbor.vnerd.com/proxy/library/docker:cli
-  tags:
-    - ps
-    - docker
-  before_script:
-    - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
-  script:
     - docker push ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest
     - docker push ${DOCKER_REGISTRY_IMAGE}:latest
@@ -81,3 +73,4 @@ finalize:
 
 
 
+

From 641ab13ef2511f138f585f5ec1f7a55b0d490de8 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 10:49:49 +0000
Subject: [PATCH 149/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cf72bec76..8198f61a7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -55,6 +55,8 @@ publish:
   before_script:
     - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
+    - apt-get install -yqq wget
+    - wget "https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}/-/jobs/${CI_JOB_ID}/artifacts/${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz" -O ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
     - pwd
     - ls -lh
     - cp ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz .

From a4be257d9db4a3333b05591338df7ad21929fc4e Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 11:02:14 +0000
Subject: [PATCH 150/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8198f61a7..675578b4a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -55,8 +55,7 @@ publish:
   before_script:
     - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
-    - apt-get install -yqq wget
-    - wget "https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}/-/jobs/${CI_JOB_ID}/artifacts/${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz" -O ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
+    - curl -o ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz "https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}/-/jobs/${CI_JOB_ID}/artifacts/${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz"
     - pwd
     - ls -lh
     - cp ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz .

From 73c3227e87d61b28dd042194546916100c95553d Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 11:41:53 +0000
Subject: [PATCH 151/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 675578b4a..335965efa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -55,7 +55,7 @@ publish:
   before_script:
     - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
-    - curl -o ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz "https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}/-/jobs/${CI_JOB_ID}/artifacts/${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz"
+    - gitlab-runner download-artifact --token "$CI_JOB_TOKEN" --project "$CI_PROJECT_ID" --job "$CI_JOB_ID" --artifact ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
     - pwd
     - ls -lh
     - cp ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz .

From f0dd2d06cf8339347492fb7c3a1b926b87338b5e Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 11:44:50 +0000
Subject: [PATCH 152/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 335965efa..92d746927 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -45,6 +45,10 @@ package:
     - echo $CI_PIPELINE_IID > VERSION
     - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
     - echo "##teamcity[publishArtifacts '${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz']"
+  artifacts:
+    paths:
+      - ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
+
 
 publish:
   stage: publish
@@ -69,6 +73,8 @@ publish:
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)
     - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID}" > VERSION
+  dependencies:
+    - package
 
 
 

From 08dba94dadefe126411d0bb18c3e295a08d229d1 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 12:08:16 +0000
Subject: [PATCH 153/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 92d746927..edded55ba 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -59,7 +59,6 @@ publish:
   before_script:
     - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
-    - gitlab-runner download-artifact --token "$CI_JOB_TOKEN" --project "$CI_PROJECT_ID" --job "$CI_JOB_ID" --artifact ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
     - pwd
     - ls -lh
     - cp ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz .

From 98f02c1728f5f7276c6e6b2fd0d8101da46243f6 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 12:21:58 +0000
Subject: [PATCH 154/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index edded55ba..cd3b79de1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -61,7 +61,6 @@ publish:
   script:
     - pwd
     - ls -lh
-    - cp ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz .
     - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .

From fac55838f4345b6d957e7503523f2ccf4a93db07 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 12:32:37 +0000
Subject: [PATCH 155/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cd3b79de1..98789ea72 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -45,10 +45,12 @@ package:
     - echo $CI_PIPELINE_IID > VERSION
     - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
     - echo "##teamcity[publishArtifacts '${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz']"
+    - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
+    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
   artifacts:
     paths:
       - ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
-
+      - ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
 
 publish:
   stage: publish
@@ -61,8 +63,6 @@ publish:
   script:
     - pwd
     - ls -lh
-    - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
-    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
     - docker push ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}

From d717ad7a147e6b349e710761123f2a4b99bcf0e5 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 13:01:37 +0000
Subject: [PATCH 156/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 98789ea72..55e191a61 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -51,6 +51,7 @@ package:
     paths:
       - ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
       - ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
+      - ps-publish/
 
 publish:
   stage: publish

From 19b7a0932f916c9337bcb5267ac7df24ea83fa41 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 29 Jan 2024 13:13:02 +0000
Subject: [PATCH 157/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 55e191a61..fba141eed 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -59,6 +59,8 @@ publish:
   tags:
     - ps
     - docker
+  needs:
+    - package
   before_script:
     - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
   script:
@@ -72,8 +74,6 @@ publish:
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)
     - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID}" > VERSION
-  dependencies:
-    - package
 
 
 

From 1af0611273cf2d20c8a45aec981e9b7f249a642e Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 30 Jan 2024 06:02:22 +0000
Subject: [PATCH 158/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fba141eed..ed2305deb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,3 +1,7 @@
+include:
+  - project: 'pluralsight/experience/gitlab-helpers'
+    file: '/salt-deploy/helper.yml'
+
 image: openjdk:8
 
 variables:
@@ -24,6 +28,7 @@ stages:
   - build
   - package
   - publish
+  - deploy
 
 build:
   stage: build
@@ -75,6 +80,21 @@ publish:
     - CURMONTH=$(date +%-m)
     - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID}" > VERSION
 
+deploy: 
+  stage: deploy
+  extends: .saltDeploy
+  variables:
+    CI_JOB_STAGE: stage
+    SALT_TARGET: roles:hydra-publish-msk
+    SALT_PASSWORD: ${LDAP_PASS}
+    SALT_ARGUMENTS: systemd-app.deploy
+    SALT_USERNAME: tcity-data-platform
+    SALT_KWARGS: 'failhard=true test=True'
+    SALT_PILLAR: $CI_PROJECT_DIR/infrastructure/salt/pillars/expert-portal/pillar_overrides.yaml
+    BUILD_VERSION: ${CI_PIPELINE_IID}
+    SALT_URL: https://saltmaster-stage.vnerd.com:8000
+
+
 
 
 

From 636d5b52f90b274ae42575bcd627cd3be2f29623 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 30 Jan 2024 06:20:44 +0000
Subject: [PATCH 159/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 26 +++++++++-----------------
 1 file changed, 9 insertions(+), 17 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ed2305deb..59b765e85 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,13 +7,9 @@ image: openjdk:8
 variables:
   ARTIFACT_NAME: hydra-publish
   IMAGE_NAME: hydra
-  DOCKER_REGISTRY_IMAGE: harbor.vnerd.com/library/${IMAGE_NAME}
-  DOCKER_REGISTRY_URL: "harbor.vnerd.com/library"
-  DOCKER_HOST: tcp://localhost:2376
-  DOCKER_TLS_CERTDIR: "/certs"
-  DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
-  DOCKER_TLS_VERIFY: 1
-
+  DOCKER_REGISTRY_URL: harbor.vnerd.com/library
+  DOCKER_REGISTRY_IMAGE: ${DOCKER_REGISTRY_URL}/${IMAGE_NAME}
+  
 before_script:
   - apt-get update -yqq
   - apt-get install -yqq apt-transport-https apt-utils
@@ -21,8 +17,6 @@ before_script:
   - mkdir -p /root/.gnupg
   - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
-  - apt-get update -yqq
-  - apt-get install -yqq sbt docker.io
   
 stages:
   - build
@@ -79,6 +73,11 @@ publish:
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)
     - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID}" > VERSION
+  variables:
+    DOCKER_HOST: tcp://localhost:2376
+    DOCKER_TLS_CERTDIR: "/certs"
+    DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
+    DOCKER_TLS_VERIFY: 1
 
 deploy: 
   stage: deploy
@@ -90,13 +89,6 @@ deploy:
     SALT_ARGUMENTS: systemd-app.deploy
     SALT_USERNAME: tcity-data-platform
     SALT_KWARGS: 'failhard=true test=True'
-    SALT_PILLAR: $CI_PROJECT_DIR/infrastructure/salt/pillars/expert-portal/pillar_overrides.yaml
+    #SALT_PILLAR: $CI_PROJECT_DIR/infrastructure/salt/pillars/expert-portal/pillar_overrides.yaml
     BUILD_VERSION: ${CI_PIPELINE_IID}
     SALT_URL: https://saltmaster-stage.vnerd.com:8000
-
-
-
-
-
-
-

From 3ed095d26f6003df64260f364824943afe07c564 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 30 Jan 2024 06:31:11 +0000
Subject: [PATCH 160/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 59b765e85..f06baa3b7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,6 +17,9 @@ before_script:
   - mkdir -p /root/.gnupg
   - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
   - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
+  - apt-get update -yqq
+  - apt-get install -yqq sbt
+
   
 stages:
   - build

From 613c10991b0c07340ed071d17185b8515cde5b1f Mon Sep 17 00:00:00 2001
From: Pavan Kumar Kulkarni <pavan-kumarkulkarni@pluralsight.com>
Date: Tue, 30 Jan 2024 08:21:39 +0000
Subject: [PATCH 161/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f06baa3b7..12480c5da 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -89,7 +89,7 @@ deploy:
     CI_JOB_STAGE: stage
     SALT_TARGET: roles:hydra-publish-msk
     SALT_PASSWORD: ${LDAP_PASS}
-    SALT_ARGUMENTS: systemd-app.deploy
+    SALT_ARGUMENTS: systemd-app
     SALT_USERNAME: tcity-data-platform
     SALT_KWARGS: 'failhard=true test=True'
     #SALT_PILLAR: $CI_PROJECT_DIR/infrastructure/salt/pillars/expert-portal/pillar_overrides.yaml

From 9109212b16036114eb97863f88594ed34ff83544 Mon Sep 17 00:00:00 2001
From: Vikram Kunisetty <vikram-kunisetty@pluralsight.com>
Date: Thu, 1 Feb 2024 10:53:38 +0000
Subject: [PATCH 162/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 12480c5da..fe87439e2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -91,7 +91,7 @@ deploy:
     SALT_PASSWORD: ${LDAP_PASS}
     SALT_ARGUMENTS: systemd-app
     SALT_USERNAME: tcity-data-platform
-    SALT_KWARGS: 'failhard=true test=True'
+    SALT_KWARGS: 'failhard=true'
     #SALT_PILLAR: $CI_PROJECT_DIR/infrastructure/salt/pillars/expert-portal/pillar_overrides.yaml
     BUILD_VERSION: ${CI_PIPELINE_IID}
     SALT_URL: https://saltmaster-stage.vnerd.com:8000

From 1a4d1b8382979046f52726f5b22d7f7c867a94eb Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 1 Feb 2024 11:12:01 +0000
Subject: [PATCH 163/220] Add new file

---
 pillar_overrides.yaml | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 pillar_overrides.yaml

diff --git a/pillar_overrides.yaml b/pillar_overrides.yaml
new file mode 100644
index 000000000..dd9a8b374
--- /dev/null
+++ b/pillar_overrides.yaml
@@ -0,0 +1,9 @@
+systemd-app:
+  artifacts:
+    expert-portal:
+      gitlab: True
+      artifactory_packaging : tgz
+      artifactory_artifact_version: latest
+      artifactory_artifact_path: hydra/publish
+      artifactory_repository: bounded-context
+      artifactory_artifact_name: hydra_publish

From 04a8a605d55f1f0a4c3a140468da66919968dc18 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 1 Feb 2024 11:26:09 +0000
Subject: [PATCH 164/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fe87439e2..3c0c02956 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -9,6 +9,13 @@ variables:
   IMAGE_NAME: hydra
   DOCKER_REGISTRY_URL: harbor.vnerd.com/library
   DOCKER_REGISTRY_IMAGE: ${DOCKER_REGISTRY_URL}/${IMAGE_NAME}
+  ARTIFACTORY_REPOSITORY_URL: https://repository.vnerd.com/artifactory
+  ARTIFACTORY_REPOSITORY: bounded-context
+  ARTIFACTORY_ID: hydra/publish
+  ARTIFACTORY_ARTIFACT_VERSION: ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
+  ARTIFACTORY_PATH: ${ARTIFACTORY_ID}/${ARTIFACTORY_ARTIFACT_VERSION}
+  ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
+
   
 before_script:
   - apt-get update -yqq
@@ -49,6 +56,12 @@ package:
     - echo "##teamcity[publishArtifacts '${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz']"
     - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
     - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
+    - |
+      curl \
+      -H "Authorization: Bearer ${NPM_TOKEN}" \
+      -X PUT ${ARTIFACTORY_ARTIFACT_URL} \
+      -T ${ARTIFACTORY_ARTIFACT_VERSION}
+
   artifacts:
     paths:
       - ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
@@ -92,6 +105,6 @@ deploy:
     SALT_ARGUMENTS: systemd-app
     SALT_USERNAME: tcity-data-platform
     SALT_KWARGS: 'failhard=true'
-    #SALT_PILLAR: $CI_PROJECT_DIR/infrastructure/salt/pillars/expert-portal/pillar_overrides.yaml
+    SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
     BUILD_VERSION: ${CI_PIPELINE_IID}
     SALT_URL: https://saltmaster-stage.vnerd.com:8000

From ecc0482a224d6fb70eac91d2abd6417bfebe25c6 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Fri, 2 Feb 2024 05:39:24 +0000
Subject: [PATCH 165/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3c0c02956..545159a71 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -55,7 +55,7 @@ package:
     - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
     - echo "##teamcity[publishArtifacts '${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz']"
     - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
-    - echo "##teamcity[publishArtifacts '%env.ARTIFACT_NAME%-${CI_PIPELINE_IID}.tgz.md5']"
+    - echo "${ARTIFACTORY_ARTIFACT_URL}"
     - |
       curl \
       -H "Authorization: Bearer ${NPM_TOKEN}" \

From fa7362b64588a2f22a3379de38113bf971513a5e Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Fri, 2 Feb 2024 09:51:56 +0000
Subject: [PATCH 166/220] Update pillar_overrides.yaml

---
 pillar_overrides.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pillar_overrides.yaml b/pillar_overrides.yaml
index dd9a8b374..fe307bcb3 100644
--- a/pillar_overrides.yaml
+++ b/pillar_overrides.yaml
@@ -1,6 +1,6 @@
 systemd-app:
   artifacts:
-    expert-portal:
+    hydra:
       gitlab: True
       artifactory_packaging : tgz
       artifactory_artifact_version: latest

From b2e256b0867e0f0aff87f4326f91aaa17f963ea7 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Fri, 2 Feb 2024 10:37:29 +0000
Subject: [PATCH 167/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 545159a71..c6b0d274d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -12,9 +12,10 @@ variables:
   ARTIFACTORY_REPOSITORY_URL: https://repository.vnerd.com/artifactory
   ARTIFACTORY_REPOSITORY: bounded-context
   ARTIFACTORY_ID: hydra/publish
-  ARTIFACTORY_ARTIFACT_VERSION: ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
+  ARTIFACTORY_ARTIFACT_VERSION: ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz
   ARTIFACTORY_PATH: ${ARTIFACTORY_ID}/${ARTIFACTORY_ARTIFACT_VERSION}
   ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
+  BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
 
   
 before_script:
@@ -27,7 +28,6 @@ before_script:
   - apt-get update -yqq
   - apt-get install -yqq sbt
 
-  
 stages:
   - build
   - package
@@ -51,10 +51,10 @@ package:
     - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic-agent-4.4.0.jar
     - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic.yml
     - cd ../../
-    - echo $CI_PIPELINE_IID > VERSION
-    - tar czf ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
-    - echo "##teamcity[publishArtifacts '${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz']"
-    - md5sum ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz > ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
+    - echo $BUILD_VERSION > VERSION
+    - tar czf ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
+    - echo "##teamcity[publishArtifacts '${ARTIFACT_NAME}-${BUILD_VERSION}.tgz']"
+    - md5sum ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz > ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz.md5
     - echo "${ARTIFACTORY_ARTIFACT_URL}"
     - |
       curl \
@@ -64,8 +64,8 @@ package:
 
   artifacts:
     paths:
-      - ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz
-      - ${ARTIFACT_NAME}-${CI_PIPELINE_IID}.tgz.md5
+      - ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz
+      - ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz.md5
       - ps-publish/
 
 publish:
@@ -82,13 +82,13 @@ publish:
     - pwd
     - ls -lh
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
-    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
-    - docker push ${DOCKER_REGISTRY_IMAGE}:${CI_PIPELINE_IID}
+    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${BUILD_VERSION}
+    - docker push ${DOCKER_REGISTRY_IMAGE}:${BUILD_VERSION}
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest
     - docker push ${DOCKER_REGISTRY_IMAGE}:latest
     - CURYEAR=$(date +%Y)
     - CURMONTH=$(date +%-m)
-    - echo -n "${CURYEAR}.${CURMONTH}.${CI_PIPELINE_IID}" > VERSION
+    - echo -n "${CURYEAR}.${CURMONTH}.${BUILD_VERSION}" > VERSION
   variables:
     DOCKER_HOST: tcp://localhost:2376
     DOCKER_TLS_CERTDIR: "/certs"
@@ -98,6 +98,10 @@ publish:
 deploy: 
   stage: deploy
   extends: .saltDeploy
+  before_script:
+    - cat $SALT_PILLAR
+    - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR
+    - cat $SALT_PILLAR  
   variables:
     CI_JOB_STAGE: stage
     SALT_TARGET: roles:hydra-publish-msk
@@ -106,5 +110,4 @@ deploy:
     SALT_USERNAME: tcity-data-platform
     SALT_KWARGS: 'failhard=true'
     SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
-    BUILD_VERSION: ${CI_PIPELINE_IID}
     SALT_URL: https://saltmaster-stage.vnerd.com:8000

From cd79e3dc8c2ce4ec2632b920955ded148ec2f68b Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Fri, 2 Feb 2024 10:49:26 +0000
Subject: [PATCH 168/220] Update pillar_overrides.yaml

---
 pillar_overrides.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pillar_overrides.yaml b/pillar_overrides.yaml
index fe307bcb3..222fd8018 100644
--- a/pillar_overrides.yaml
+++ b/pillar_overrides.yaml
@@ -1,9 +1,9 @@
 systemd-app:
   artifacts:
-    hydra:
+    hydra-publish:
       gitlab: True
       artifactory_packaging : tgz
       artifactory_artifact_version: latest
       artifactory_artifact_path: hydra/publish
       artifactory_repository: bounded-context
-      artifactory_artifact_name: hydra_publish
+      artifactory_artifact_name: hydra-publish

From 9cafa2be1f4481479aa5d6cec135ac5f0186f2e1 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 13 Feb 2024 06:34:32 +0000
Subject: [PATCH 169/220] chore(PDP-1009): Updated .gitlab-ci.yml file to
 deploy on stage and prod

---
 .gitlab-ci.yml | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c6b0d274d..576f23868 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,7 +32,8 @@ stages:
   - build
   - package
   - publish
-  - deploy
+  - deploy_staging
+  - deploy_production
 
 build:
   stage: build
@@ -95,9 +96,10 @@ publish:
     DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
     DOCKER_TLS_VERIFY: 1
 
-deploy: 
-  stage: deploy
+deploy_staging: 
+  stage: deploy_staging
   extends: .saltDeploy
+  when: manual
   before_script:
     - cat $SALT_PILLAR
     - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR
@@ -111,3 +113,21 @@ deploy:
     SALT_KWARGS: 'failhard=true'
     SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
     SALT_URL: https://saltmaster-stage.vnerd.com:8000
+
+deploy_production: 
+  stage: deploy_production
+  extends: .saltDeploy
+  when: manual
+  before_script:
+    - cat $SALT_PILLAR
+    - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR
+    - cat $SALT_PILLAR  
+  variables:
+    CI_JOB_STAGE: stage
+    SALT_TARGET: roles:hydra-publish-msk
+    SALT_PASSWORD: ${LDAP_PASS}
+    SALT_ARGUMENTS: systemd-app
+    SALT_USERNAME: tcity-data-platform
+    SALT_KWARGS: 'failhard=true'
+    SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
+    SALT_URL: https://saltmaster-production.vnerd.com:8000

From 6677475ebf800e52914702f06532ac72687fdfb6 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 13 Feb 2024 06:38:49 +0000
Subject: [PATCH 170/220] chore(PDP-1009): Updated .gitlab-ci.yml to move stage
 variables

---
 .gitlab-ci.yml | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 576f23868..cc43f7c08 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,6 +16,12 @@ variables:
   ARTIFACTORY_PATH: ${ARTIFACTORY_ID}/${ARTIFACTORY_ARTIFACT_VERSION}
   ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
   BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
+  SALT_TARGET: roles:hydra-publish-msk
+  SALT_PASSWORD: ${LDAP_PASS}
+  SALT_ARGUMENTS: systemd-app
+  SALT_USERNAME: tcity-data-platform
+  SALT_KWARGS: 'failhard=true'
+  SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
 
   
 before_script:
@@ -106,12 +112,6 @@ deploy_staging:
     - cat $SALT_PILLAR  
   variables:
     CI_JOB_STAGE: stage
-    SALT_TARGET: roles:hydra-publish-msk
-    SALT_PASSWORD: ${LDAP_PASS}
-    SALT_ARGUMENTS: systemd-app
-    SALT_USERNAME: tcity-data-platform
-    SALT_KWARGS: 'failhard=true'
-    SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
     SALT_URL: https://saltmaster-stage.vnerd.com:8000
 
 deploy_production: 
@@ -123,11 +123,5 @@ deploy_production:
     - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR
     - cat $SALT_PILLAR  
   variables:
-    CI_JOB_STAGE: stage
-    SALT_TARGET: roles:hydra-publish-msk
-    SALT_PASSWORD: ${LDAP_PASS}
-    SALT_ARGUMENTS: systemd-app
-    SALT_USERNAME: tcity-data-platform
-    SALT_KWARGS: 'failhard=true'
-    SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
+    CI_JOB_STAGE: production
     SALT_URL: https://saltmaster-production.vnerd.com:8000

From ea1e43e670b60556c515631d8207569425c04221 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 13 Feb 2024 14:43:51 +0000
Subject: [PATCH 171/220] Updated .gitlab-ci.yml file

---
 .gitlab-ci.yml | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cc43f7c08..576f23868 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,12 +16,6 @@ variables:
   ARTIFACTORY_PATH: ${ARTIFACTORY_ID}/${ARTIFACTORY_ARTIFACT_VERSION}
   ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
   BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
-  SALT_TARGET: roles:hydra-publish-msk
-  SALT_PASSWORD: ${LDAP_PASS}
-  SALT_ARGUMENTS: systemd-app
-  SALT_USERNAME: tcity-data-platform
-  SALT_KWARGS: 'failhard=true'
-  SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
 
   
 before_script:
@@ -112,6 +106,12 @@ deploy_staging:
     - cat $SALT_PILLAR  
   variables:
     CI_JOB_STAGE: stage
+    SALT_TARGET: roles:hydra-publish-msk
+    SALT_PASSWORD: ${LDAP_PASS}
+    SALT_ARGUMENTS: systemd-app
+    SALT_USERNAME: tcity-data-platform
+    SALT_KWARGS: 'failhard=true'
+    SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
     SALT_URL: https://saltmaster-stage.vnerd.com:8000
 
 deploy_production: 
@@ -123,5 +123,11 @@ deploy_production:
     - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR
     - cat $SALT_PILLAR  
   variables:
-    CI_JOB_STAGE: production
+    CI_JOB_STAGE: stage
+    SALT_TARGET: roles:hydra-publish-msk
+    SALT_PASSWORD: ${LDAP_PASS}
+    SALT_ARGUMENTS: systemd-app
+    SALT_USERNAME: tcity-data-platform
+    SALT_KWARGS: 'failhard=true'
+    SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
     SALT_URL: https://saltmaster-production.vnerd.com:8000

From 0258a998eff324f1285ee82fb21697ee5fe018ba Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 20 Feb 2024 14:22:31 +0000
Subject: [PATCH 172/220] Add new file

---
 helm/eks-dev-values.yml | 59 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 helm/eks-dev-values.yml

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
new file mode 100644
index 000000000..f01277d47
--- /dev/null
+++ b/helm/eks-dev-values.yml
@@ -0,0 +1,59 @@
+image:
+  repository: harbor.vnerd.com/library/hydra
+  pullPolicy: Always
+  tag: latest
+
+deployment:
+  apiVersion: "apps/v1"
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 0
+    type: RollingUpdate
+
+replicas: 4
+
+#Adding velero backup for k8s services
+#velero.io/exclude-from-backup" = true 
+service:
+  type: ClusterIP
+  port: 8080
+  annotations: {}
+  labels: {}
+
+# ingress:
+#   enabled: true
+#   ingressClassName: alb
+#   labels: {}
+#   hosts:
+#     - dvs.eplur-staging.vnerd.com
+#     - dvs-staging.vnerd.com
+#   tls:
+#     - hosts:
+#         - dvs.eplur-staging.vnerd.com
+#   path: /
+
+
+resources:
+  limits:
+    memory: 512Mi
+  requests:
+    cpu: 250m
+    memory: 50Mi
+
+tolerations: []
+
+nodeSelector: {}
+
+config:
+  LOG_DIR: /var/log/hydra/
+  LOG_LEVEL: DEBUG
+  HYDRA_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: b-1.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-3.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-2.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098
+  HYDRA_SCHEMA_REGISTRY_URL: https://dvs-dev-schema-registry.eplur-staging.vnerd.com:8081
+  HYDRA_V2_METADATA_CONTACT: '#dev-data-platform'
+  HYDRA_REPLICATION_FACTOR: 1
+  HYDRA_V2_METADATA_CONSUMER_GROUP: 'v2MetadataConsumer'
+  HYDRA_V2_CREATE_TOPICS_ENABLED: 'true'
+  HYDRA_V2_METADATA_CREATE_ON_STARTUP: 'true'
+  HYDRA_MIN_INSYNC_REPLICAS: 1
+
+

From ec2b67d2c9ce76a96c1c14f0d731df13c4caa9c7 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 20 Feb 2024 14:26:15 +0000
Subject: [PATCH 173/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 576f23868..267dd5e2d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,6 +16,7 @@ variables:
   ARTIFACTORY_PATH: ${ARTIFACTORY_ID}/${ARTIFACTORY_ARTIFACT_VERSION}
   ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
   BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
+  BOUNDED_CONTEXT: adapt-dvs
 
   
 before_script:
@@ -32,6 +33,7 @@ stages:
   - build
   - package
   - publish
+  - deploy_dev
   - deploy_staging
   - deploy_production
 
@@ -96,6 +98,30 @@ publish:
     DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
     DOCKER_TLS_VERIFY: 1
 
+deploy_dev:
+    stage: deploy_dev
+    needs: ["publish"]
+    extends: .helmDeploy
+    environment:
+      name: staging
+    when: manual
+    variables:
+      HELM_CHART_NAME: ps-service
+      HELM_DEPLOY_TOKEN: ${HELM_DEPLOY_TOKEN}
+      K8S_CLUSTER_NAME: app-eks.eplur-staging.us-west-2
+      HELM_K8S_CLUSTER_URL: https://6C29C0073BB19BEF220B9437E6962AF2.gr7.us-west-2.eks.amazonaws.com
+      HELM_TARGET_NAMESPACE: ${BOUNDED_CONTEXT}
+      APPLICATION_ROLE: hydra
+      HELM_VALUES_FILE: helm/eks-dev-values.yml
+      HELM_SET_VALUES: image.tag=${BUILD_VERSION}
+    before_script:
+      #- export BUILD_VERSION="$(date +%Y).$(date +%m).$(date +%d).${CI_COMMIT_SHORT_SHA}"
+      - kubectl config set-cluster app-${CI_ENVIRONMENT_NAME%/*} --server=${HELM_K8S_CLUSTER_URL} --embed-certs --certificate-authority="$HELM_K8S_CLUSTER_CACERT"
+      - kubectl config set-credentials deploy --token=`echo ${HELM_DEPLOY_TOKEN} | base64 -d`
+      - kubectl config set-context deploy --cluster=app-${CI_ENVIRONMENT_NAME%/*} --namespace=${HELM_TARGET_NAMESPACE} --user=deploy
+      - kubectl config use-context deploy
+
+
 deploy_staging: 
   stage: deploy_staging
   extends: .saltDeploy

From aad7a3f1426545abccffd8eb5d6f914d438f67ce Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 20 Feb 2024 14:29:49 +0000
Subject: [PATCH 174/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 267dd5e2d..44134a960 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,8 @@
 include:
   - project: 'pluralsight/experience/gitlab-helpers'
     file: '/salt-deploy/helper.yml'
+  - project: 'pluralsight/experience/gitlab-helpers'
+    file: '/helm-deploy/helper.yml'
 
 image: openjdk:8
 

From a1ef298446a06458518ac3987ec5e3076a1e3358 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 20 Feb 2024 14:50:03 +0000
Subject: [PATCH 175/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index f01277d47..d88862dad 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -35,10 +35,10 @@ service:
 
 resources:
   limits:
-    memory: 512Mi
+    memory: 1Gi
   requests:
-    cpu: 250m
-    memory: 50Mi
+    cpu: 2
+    memory: 1Gi
 
 tolerations: []
 

From 0f17e162df5559c20470271e2b3829c2bd0f1abb Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 20 Feb 2024 15:04:20 +0000
Subject: [PATCH 176/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index d88862dad..e9ec57dab 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -1,7 +1,7 @@
 image:
   repository: harbor.vnerd.com/library/hydra
   pullPolicy: Always
-  tag: latest
+  tag: 1.0.70
 
 deployment:
   apiVersion: "apps/v1"

From 3759e4d9af1859a05f6ad68c986b4b670e3eae7e Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 20 Feb 2024 15:30:31 +0000
Subject: [PATCH 177/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index e9ec57dab..1b4bb2b2c 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -1,7 +1,7 @@
 image:
   repository: harbor.vnerd.com/library/hydra
   pullPolicy: Always
-  tag: 1.0.70
+  tag: 64
 
 deployment:
   apiVersion: "apps/v1"

From 908921518bde2d4e1577880ad6156bcfbd3e1b52 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 22 Feb 2024 14:15:18 +0000
Subject: [PATCH 178/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 1b4bb2b2c..1c791054b 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -1,7 +1,7 @@
 image:
   repository: harbor.vnerd.com/library/hydra
   pullPolicy: Always
-  tag: 64
+  tag: '%IMAGE_TAG%'
 
 deployment:
   apiVersion: "apps/v1"

From 0c73c6d393fc42d31c123cd8518b5a2f65804eea Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 22 Feb 2024 14:20:42 +0000
Subject: [PATCH 179/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 44134a960..28461feec 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -107,6 +107,14 @@ deploy_dev:
     environment:
       name: staging
     when: manual
+    before_script:
+      - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}/g" $HELM_VALUES_FILE
+      - cat $HELM_VALUES_FILE
+      - echo 1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}
+      - kubectl config set-cluster app-${CI_ENVIRONMENT_NAME%/*} --server=${HELM_K8S_CLUSTER_URL} --embed-certs --certificate-authority="$HELM_K8S_CLUSTER_CACERT"
+      - kubectl config set-credentials deploy --token=`echo ${HELM_DEPLOY_TOKEN} | base64 -d`
+      - kubectl config set-context deploy --cluster=app-${CI_ENVIRONMENT_NAME%/*} --namespace=${HELM_TARGET_NAMESPACE} --user=deploy
+      - kubectl config use-context deploy
     variables:
       HELM_CHART_NAME: ps-service
       HELM_DEPLOY_TOKEN: ${HELM_DEPLOY_TOKEN}
@@ -115,14 +123,6 @@ deploy_dev:
       HELM_TARGET_NAMESPACE: ${BOUNDED_CONTEXT}
       APPLICATION_ROLE: hydra
       HELM_VALUES_FILE: helm/eks-dev-values.yml
-      HELM_SET_VALUES: image.tag=${BUILD_VERSION}
-    before_script:
-      #- export BUILD_VERSION="$(date +%Y).$(date +%m).$(date +%d).${CI_COMMIT_SHORT_SHA}"
-      - kubectl config set-cluster app-${CI_ENVIRONMENT_NAME%/*} --server=${HELM_K8S_CLUSTER_URL} --embed-certs --certificate-authority="$HELM_K8S_CLUSTER_CACERT"
-      - kubectl config set-credentials deploy --token=`echo ${HELM_DEPLOY_TOKEN} | base64 -d`
-      - kubectl config set-context deploy --cluster=app-${CI_ENVIRONMENT_NAME%/*} --namespace=${HELM_TARGET_NAMESPACE} --user=deploy
-      - kubectl config use-context deploy
-
 
 deploy_staging: 
   stage: deploy_staging

From c74e72bc78c33e98f78e8f22f2d965184b73326c Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Thu, 22 Feb 2024 14:36:39 +0000
Subject: [PATCH 180/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 28461feec..9605ffb4f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -20,7 +20,6 @@ variables:
   BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
   BOUNDED_CONTEXT: adapt-dvs
 
-  
 before_script:
   - apt-get update -yqq
   - apt-get install -yqq apt-transport-https apt-utils

From c38d8fad9a870668642f52e3a5a979644950597b Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Fri, 23 Feb 2024 05:02:21 +0000
Subject: [PATCH 181/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9605ffb4f..c7508e184 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -86,8 +86,8 @@ publish:
     - pwd
     - ls -lh
     - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
-    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${BUILD_VERSION}
-    - docker push ${DOCKER_REGISTRY_IMAGE}:${BUILD_VERSION}
+    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${BUILD_VERSION}-${CI_COMMIT_SHORT_SHA}
+    - docker push ${DOCKER_REGISTRY_IMAGE}:${BUILD_VERSION}-${CI_COMMIT_SHORT_SHA}
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest
     - docker push ${DOCKER_REGISTRY_IMAGE}:latest
     - CURYEAR=$(date +%Y)
@@ -107,7 +107,7 @@ deploy_dev:
       name: staging
     when: manual
     before_script:
-      - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}/g" $HELM_VALUES_FILE
+      - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}/g" $HELM_VALUES_FILE
       - cat $HELM_VALUES_FILE
       - echo 1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}
       - kubectl config set-cluster app-${CI_ENVIRONMENT_NAME%/*} --server=${HELM_K8S_CLUSTER_URL} --embed-certs --certificate-authority="$HELM_K8S_CLUSTER_CACERT"

From 5573a6b00810daa5577d84c5789353adc86904b4 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Fri, 23 Feb 2024 06:09:25 +0000
Subject: [PATCH 182/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c7508e184..7e5a93145 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -107,7 +107,8 @@ deploy_dev:
       name: staging
     when: manual
     before_script:
-      - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}/g" $HELM_VALUES_FILE
+     # - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}/g" $HELM_VALUES_FILE
+      - sed -i -e "s/%IMAGE_TAG%/0.11.3.1027/g" $HELM_VALUES_FILE
       - cat $HELM_VALUES_FILE
       - echo 1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}
       - kubectl config set-cluster app-${CI_ENVIRONMENT_NAME%/*} --server=${HELM_K8S_CLUSTER_URL} --embed-certs --certificate-authority="$HELM_K8S_CLUSTER_CACERT"

From 104d6411e7f058d16d86bd71defc58ed664b03dd Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Fri, 23 Feb 2024 06:35:18 +0000
Subject: [PATCH 183/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 1c791054b..1fb6d629e 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -10,7 +10,7 @@ deployment:
       maxUnavailable: 0
     type: RollingUpdate
 
-replicas: 4
+replicas: 2
 
 #Adding velero backup for k8s services
 #velero.io/exclude-from-backup" = true 
@@ -32,13 +32,12 @@ service:
 #         - dvs.eplur-staging.vnerd.com
 #   path: /
 
-
 resources:
   limits:
     memory: 1Gi
   requests:
-    cpu: 2
-    memory: 1Gi
+    cpu: 250m
+    memory: 512Mi
 
 tolerations: []
 

From 9ca100da95c94c5058ef4fcee14ce6f998f64344 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Fri, 23 Feb 2024 06:35:45 +0000
Subject: [PATCH 184/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7e5a93145..c7508e184 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -107,8 +107,7 @@ deploy_dev:
       name: staging
     when: manual
     before_script:
-     # - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}/g" $HELM_VALUES_FILE
-      - sed -i -e "s/%IMAGE_TAG%/0.11.3.1027/g" $HELM_VALUES_FILE
+      - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}/g" $HELM_VALUES_FILE
       - cat $HELM_VALUES_FILE
       - echo 1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}
       - kubectl config set-cluster app-${CI_ENVIRONMENT_NAME%/*} --server=${HELM_K8S_CLUSTER_URL} --embed-certs --certificate-authority="$HELM_K8S_CLUSTER_CACERT"

From c1af89e580148fa57c1268f1c02eefaa27d6b68d Mon Sep 17 00:00:00 2001
From: Pavan Kumar Kulkarni <pavan-kumarkulkarni@pluralsight.com>
Date: Tue, 27 Feb 2024 06:54:03 +0000
Subject: [PATCH 185/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 1fb6d629e..43eb66968 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -45,7 +45,7 @@ nodeSelector: {}
 
 config:
   LOG_DIR: /var/log/hydra/
-  LOG_LEVEL: DEBUG
+  LOG_LEVEL: INFO
   HYDRA_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: b-1.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-3.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-2.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098
   HYDRA_SCHEMA_REGISTRY_URL: https://dvs-dev-schema-registry.eplur-staging.vnerd.com:8081
   HYDRA_V2_METADATA_CONTACT: '#dev-data-platform'

From bb17d0b6813ee95722a7ca225cc35dcf9ae5c59c Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Tue, 27 Feb 2024 07:49:31 +0000
Subject: [PATCH 186/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 43eb66968..38239cff9 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -34,10 +34,10 @@ service:
 
 resources:
   limits:
-    memory: 1Gi
+    memory: 5Gi
   requests:
     cpu: 250m
-    memory: 512Mi
+    memory: 1Gi
 
 tolerations: []
 

From 0c59cadd34bec89b31409ffd5d88ccf3867e66e9 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 28 Feb 2024 05:53:25 +0000
Subject: [PATCH 187/220] Update Dockerfile.new

---
 Dockerfile.new | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.new b/Dockerfile.new
index 583af0638..1a20a87de 100644
--- a/Dockerfile.new
+++ b/Dockerfile.new
@@ -2,7 +2,7 @@ FROM jelastic/jetty:9.4.49-openjdk-1.8.0_352
 
 USER root
 
-ENV JAVA_OPTS="-Xmx2G"
+ENV JAVA_OPTS="-Xmx4g"
 
 ENV CONTAINER_HTTP_PORT="8088"
 
@@ -12,4 +12,4 @@ EXPOSE 8088
 
 COPY ps-publish/ /ps-publish
 
-ENTRYPOINT ["/ps-publish/bin/hydra-ingest"]
\ No newline at end of file
+ENTRYPOINT ["/ps-publish/bin/hydra-ingest"]

From a5d4c3b218f12c754da4f4b4c84399c0235ab7a4 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 28 Feb 2024 06:16:11 +0000
Subject: [PATCH 188/220] Update Dockerfile.new

---
 Dockerfile.new | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile.new b/Dockerfile.new
index 1a20a87de..16d541be6 100644
--- a/Dockerfile.new
+++ b/Dockerfile.new
@@ -2,7 +2,7 @@ FROM jelastic/jetty:9.4.49-openjdk-1.8.0_352
 
 USER root
 
-ENV JAVA_OPTS="-Xmx4g"
+ENV JAVA_OPTS="-Xmx2g"
 
 ENV CONTAINER_HTTP_PORT="8088"
 

From 339354d5d5caf48732bbf111ea07312383c423e9 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 28 Feb 2024 06:39:56 +0000
Subject: [PATCH 189/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c7508e184..14f15736e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -120,7 +120,7 @@ deploy_dev:
       K8S_CLUSTER_NAME: app-eks.eplur-staging.us-west-2
       HELM_K8S_CLUSTER_URL: https://6C29C0073BB19BEF220B9437E6962AF2.gr7.us-west-2.eks.amazonaws.com
       HELM_TARGET_NAMESPACE: ${BOUNDED_CONTEXT}
-      APPLICATION_ROLE: hydra
+      APPLICATION_ROLE: dev-hydra
       HELM_VALUES_FILE: helm/eks-dev-values.yml
 
 deploy_staging: 

From ad64dd0d22b309e58fea9b175c2c8ee883ebde0f Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 28 Feb 2024 07:08:50 +0000
Subject: [PATCH 190/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 38239cff9..8724d5b92 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -34,7 +34,7 @@ service:
 
 resources:
   limits:
-    memory: 5Gi
+    memory: 6Gi
   requests:
     cpu: 250m
     memory: 1Gi

From 28b2d6d408517f0817d89f9a17fb5b2d4cf0e37e Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Mon, 4 Mar 2024 06:32:14 +0000
Subject: [PATCH 191/220] Update eks-dev-values.yml

---
 helm/eks-dev-values.yml | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 8724d5b92..4f3882a4e 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -20,17 +20,20 @@ service:
   annotations: {}
   labels: {}
 
-# ingress:
-#   enabled: true
-#   ingressClassName: alb
-#   labels: {}
-#   hosts:
-#     - dvs.eplur-staging.vnerd.com
-#     - dvs-staging.vnerd.com
-#   tls:
-#     - hosts:
-#         - dvs.eplur-staging.vnerd.com
-#   path: /
+
+ingress:
+  enabled: true
+  ingressClassName: internal
+  annotations: {}
+  labels: {}
+  hosts:
+    - dev-hydra.eplur-staging.vnerd.com
+  tls:
+    - hosts:
+        - dev-hydra.eplur-staging.vnerd.com
+  path: /
+  pathType: Prefix
+
 
 resources:
   limits:

From f3c2802e5882d277dbc3b7553fc26e30ea7661b1 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 6 Mar 2024 06:31:07 +0000
Subject: [PATCH 192/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 35 +++++++++++++++++++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 14f15736e..55f48ae8a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,6 +19,15 @@ variables:
   ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
   BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
   BOUNDED_CONTEXT: adapt-dvs
+  ENV: ${CI_JOB_STAGE} 
+  SERVICE_NAME: "dev-hydra"
+  SLACK_ICON_EMOJI: ":gitlab:"
+  SLACK_CHANNEL: '#data-platform-alerts'
+  SLACK_MESSAGE: |
+    *[[SERVICE_NAME]]* deployed to *[[ENV]]*.
+    Version: *[[BUILD_VERSION]]*
+    [[PIPELINE_LINK]]
+  SLACK_USERNAME: 'GITLAB'
 
 before_script:
   - apt-get update -yqq
@@ -36,7 +45,8 @@ stages:
   - publish
   - deploy_dev
   - deploy_staging
-  - deploy_production
+  - deploy_productio
+  - notify
 
 build:
   stage: build
@@ -99,13 +109,22 @@ publish:
     DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
     DOCKER_TLS_VERIFY: 1
 
+.notify-all:
+  image: harbor.vnerd.com/library/ps-helm:latest
+  tags:
+    - ps
+    - docker 
+  script:
+    - ps-notify slack "$SLACK_MESSAGE" "$SLACK_CHANNEL"  --slack-username $SLACK_USERNAME --slack-icon-emoji $SLACK_ICON_EMOJI
+  variables:
+    NODE_ENV: $ENV
+
 deploy_dev:
     stage: deploy_dev
     needs: ["publish"]
     extends: .helmDeploy
     environment:
       name: staging
-    when: manual
     before_script:
       - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}/g" $HELM_VALUES_FILE
       - cat $HELM_VALUES_FILE
@@ -123,6 +142,17 @@ deploy_dev:
       APPLICATION_ROLE: dev-hydra
       HELM_VALUES_FILE: helm/eks-dev-values.yml
 
+slack:dev:
+  stage: notify_test
+  extends: .notify-all
+  when: on_success
+  needs: ['deploy_dev']
+  before_script:
+    - echo "Sending notification to slack"
+  variables:
+    ENV: "DEV Cluster"
+
+
 deploy_staging: 
   stage: deploy_staging
   extends: .saltDeploy
@@ -158,3 +188,4 @@ deploy_production:
     SALT_KWARGS: 'failhard=true'
     SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
     SALT_URL: https://saltmaster-production.vnerd.com:8000
+

From 4952b5e2417a6ef2f345888ee7f4abdbff081ca1 Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 6 Mar 2024 06:32:01 +0000
Subject: [PATCH 193/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 55f48ae8a..6a59dff37 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -143,7 +143,7 @@ deploy_dev:
       HELM_VALUES_FILE: helm/eks-dev-values.yml
 
 slack:dev:
-  stage: notify_test
+  stage: notify
   extends: .notify-all
   when: on_success
   needs: ['deploy_dev']

From bf8432bd29a3b21b9a3464ec0354b6e4505ebabb Mon Sep 17 00:00:00 2001
From: Yuvaraj Govindaraju <yuvaraj-govindaraju@pluralsight.com>
Date: Wed, 6 Mar 2024 06:33:53 +0000
Subject: [PATCH 194/220] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6a59dff37..23fc97ae0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -45,7 +45,7 @@ stages:
   - publish
   - deploy_dev
   - deploy_staging
-  - deploy_productio
+  - deploy_production
   - notify
 
 build:

From e525ed3a43d025c7c469683b39c414fa7072f4d9 Mon Sep 17 00:00:00 2001
From: Yathish B <yathish-b@pluralsight.com>
Date: Wed, 20 Mar 2024 05:29:53 +0000
Subject: [PATCH 195/220] Changed the pipeline config to ignore runner cache

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 23fc97ae0..ce44123f8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -50,11 +50,13 @@ stages:
 
 build:
   stage: build
+  cache: []
   script:
     - sbt clean compile test
 
 package:
   stage: package
+  cache: []
   script:
     - sbt universal:packageBin
     - ls ingest/target/universal/*.zip | xargs -I {} unzip {} -d ps-publish

From 9c161fdf86f786b3cd126e099a7ba6470c091196 Mon Sep 17 00:00:00 2001
From: Yathish B <yathish-b@pluralsight.com>
Date: Wed, 20 Mar 2024 05:34:40 +0000
Subject: [PATCH 196/220] Corrected, Added the Environment name

---
 .gitlab-ci.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ce44123f8..ce17b2797 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -126,7 +126,7 @@ deploy_dev:
     needs: ["publish"]
     extends: .helmDeploy
     environment:
-      name: staging
+      name: dev
     before_script:
       - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}/g" $HELM_VALUES_FILE
       - cat $HELM_VALUES_FILE
@@ -159,6 +159,8 @@ deploy_staging:
   stage: deploy_staging
   extends: .saltDeploy
   when: manual
+  environment:
+      name: staging
   before_script:
     - cat $SALT_PILLAR
     - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR
@@ -177,6 +179,8 @@ deploy_production:
   stage: deploy_production
   extends: .saltDeploy
   when: manual
+  environment:
+      name: production
   before_script:
     - cat $SALT_PILLAR
     - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR

From f69e06256686fc7edb8ef8d0feabb038e738594b Mon Sep 17 00:00:00 2001
From: Yathish B <yathish-b@pluralsight.com>
Date: Thu, 21 Mar 2024 05:39:10 +0000
Subject: [PATCH 197/220] Added the notification for staging and production

---
 .gitlab-ci.yml | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ce17b2797..402814af6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -175,6 +175,16 @@ deploy_staging:
     SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
     SALT_URL: https://saltmaster-stage.vnerd.com:8000
 
+slack:staging:
+  stage: notify
+  extends: .notify-all
+  when: on_success
+  needs: ['deploy_staging']
+  before_script:
+    - echo "Sending notification to slack"
+  variables:
+    ENV: "Staging Cluster"
+
 deploy_production: 
   stage: deploy_production
   extends: .saltDeploy
@@ -195,3 +205,12 @@ deploy_production:
     SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
     SALT_URL: https://saltmaster-production.vnerd.com:8000
 
+slack:production:
+  stage: notify
+  extends: .notify-all
+  when: on_success
+  needs: ['deploy_production']
+  before_script:
+    - echo "Sending notification to slack"
+  variables:
+    ENV: "Production Cluster"

From 74c2fc7329145ef7d15081f4308696aa0d345070 Mon Sep 17 00:00:00 2001
From: Yathish B <yathish-b@pluralsight.com>
Date: Thu, 21 Mar 2024 14:00:45 +0000
Subject: [PATCH 198/220] added the env specific bounded_context

---
 .gitlab-ci.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 402814af6..d0d92fd86 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -18,7 +18,9 @@ variables:
   ARTIFACTORY_PATH: ${ARTIFACTORY_ID}/${ARTIFACTORY_ARTIFACT_VERSION}
   ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
   BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
-  BOUNDED_CONTEXT: adapt-dvs
+  BOUNDED_CONTEXT_DEV: adapt-dvs
+  BOUNDED_CONTEXT_STAGING: adapt-dvs
+  BOUNDED_CONTEXT_PROD: adapt-dvs
   ENV: ${CI_JOB_STAGE} 
   SERVICE_NAME: "dev-hydra"
   SLACK_ICON_EMOJI: ":gitlab:"
@@ -140,7 +142,7 @@ deploy_dev:
       HELM_DEPLOY_TOKEN: ${HELM_DEPLOY_TOKEN}
       K8S_CLUSTER_NAME: app-eks.eplur-staging.us-west-2
       HELM_K8S_CLUSTER_URL: https://6C29C0073BB19BEF220B9437E6962AF2.gr7.us-west-2.eks.amazonaws.com
-      HELM_TARGET_NAMESPACE: ${BOUNDED_CONTEXT}
+      HELM_TARGET_NAMESPACE: ${BOUNDED_CONTEXT_DEV}
       APPLICATION_ROLE: dev-hydra
       HELM_VALUES_FILE: helm/eks-dev-values.yml
 

From 78111267a1ea5f67400185f636fcf11babec40f6 Mon Sep 17 00:00:00 2001
From: Yathish B <yathish-b@pluralsight.com>
Date: Fri, 22 Mar 2024 03:54:59 +0000
Subject: [PATCH 199/220] added a retry mechanism to pipeline

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d0d92fd86..43f1bc17a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -53,6 +53,7 @@ stages:
 build:
   stage: build
   cache: []
+  retry: 3
   script:
     - sbt clean compile test
 

From d8e0c84146bc903f44a60042af634a56398f0a3e Mon Sep 17 00:00:00 2001
From: Yathish B <yathish-b@pluralsight.com>
Date: Fri, 22 Mar 2024 03:56:29 +0000
Subject: [PATCH 200/220] fixed .gitlab-ci.yml pipeline

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 43f1bc17a..c3abf0b70 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -53,7 +53,7 @@ stages:
 build:
   stage: build
   cache: []
-  retry: 3
+  retry: 2
   script:
     - sbt clean compile test
 

From 8208df10a3439101dfe34994f90fbc34d6a62546 Mon Sep 17 00:00:00 2001
From: pavan-kumarkulkarni <pavan-kumarkulkarni@pluralsight.com>
Date: Wed, 27 Mar 2024 13:55:15 +0530
Subject: [PATCH 201/220] chore(PDP1-859): Deploy hydra-publish on dev
 namespace

---
 .gitlab-ci.yml          | 8 +++++---
 helm/eks-dev-values.yml | 4 ++--
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c3abf0b70..59e26b307 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -18,7 +18,7 @@ variables:
   ARTIFACTORY_PATH: ${ARTIFACTORY_ID}/${ARTIFACTORY_ARTIFACT_VERSION}
   ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
   BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
-  BOUNDED_CONTEXT_DEV: adapt-dvs
+  BOUNDED_CONTEXT_DEV: adapt-dvs-dev
   BOUNDED_CONTEXT_STAGING: adapt-dvs
   BOUNDED_CONTEXT_PROD: adapt-dvs
   ENV: ${CI_JOB_STAGE} 
@@ -80,7 +80,6 @@ package:
       -H "Authorization: Bearer ${NPM_TOKEN}" \
       -X PUT ${ARTIFACTORY_ARTIFACT_URL} \
       -T ${ARTIFACTORY_ARTIFACT_VERSION}
-
   artifacts:
     paths:
       - ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz
@@ -140,7 +139,7 @@ deploy_dev:
       - kubectl config use-context deploy
     variables:
       HELM_CHART_NAME: ps-service
-      HELM_DEPLOY_TOKEN: ${HELM_DEPLOY_TOKEN}
+      HELM_DEPLOY_TOKEN: ${HELM_DEV_DEPLOY_TOKEN}
       K8S_CLUSTER_NAME: app-eks.eplur-staging.us-west-2
       HELM_K8S_CLUSTER_URL: https://6C29C0073BB19BEF220B9437E6962AF2.gr7.us-west-2.eks.amazonaws.com
       HELM_TARGET_NAMESPACE: ${BOUNDED_CONTEXT_DEV}
@@ -156,6 +155,7 @@ slack:dev:
     - echo "Sending notification to slack"
   variables:
     ENV: "DEV Cluster"
+    SERVICE_NAME: "dev-hydra"
 
 
 deploy_staging: 
@@ -187,6 +187,7 @@ slack:staging:
     - echo "Sending notification to slack"
   variables:
     ENV: "Staging Cluster"
+    SERVICE_NAME: "staging-hydra-publish"
 
 deploy_production: 
   stage: deploy_production
@@ -217,3 +218,4 @@ slack:production:
     - echo "Sending notification to slack"
   variables:
     ENV: "Production Cluster"
+    SERVICE_NAME: "production-hydra-publish"
diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 4f3882a4e..df0cdb763 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -27,10 +27,10 @@ ingress:
   annotations: {}
   labels: {}
   hosts:
-    - dev-hydra.eplur-staging.vnerd.com
+    - hydra-publish-dev.eplur-staging.vnerd.com
   tls:
     - hosts:
-        - dev-hydra.eplur-staging.vnerd.com
+        - hydra-publish-dev.eplur-staging.vnerd.com
   path: /
   pathType: Prefix
 

From aedf3eb4710ad2231b4a19ae9756136b0aea9a00 Mon Sep 17 00:00:00 2001
From: pavan-kumarkulkarni <pavan-kumarkulkarni@pluralsight.com>
Date: Tue, 16 Apr 2024 12:12:54 +0530
Subject: [PATCH 202/220] chore(PDP1-895): Implement synk for hydra-publish

---
 .gitlab-ci.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 59e26b307..c1f12747b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,6 +3,9 @@ include:
     file: '/salt-deploy/helper.yml'
   - project: 'pluralsight/experience/gitlab-helpers'
     file: '/helm-deploy/helper.yml'
+  - project: 'pluralsight/Technology/adapt/data-platform/gitlab-fragments/snyk'
+    ref: main
+    file: 'snyk-check.yml'
 
 image: openjdk:8
 
@@ -30,6 +33,11 @@ variables:
     Version: *[[BUILD_VERSION]]*
     [[PIPELINE_LINK]]
   SLACK_USERNAME: 'GITLAB'
+  SNYK_SLACK_CHANNEL: '#dataops-snyk-vulnerability-alerts'
+  SNYK_SLACK_ICON_EMOJI: ":snyk:"
+  SNYK_ORG_ID: "d8094638-7a37-413f-b1b4-ad840fb9e239"
+  SNYK_PROJECT_ID: "d243a0e6-4ced-4efe-83fc-169d03b40cc7"
+  SNYK_PROJECT_URL: "https://app.snyk.io/org/ps-data-services"
 
 before_script:
   - apt-get update -yqq

From 68fc441f49944e64f558b7aa8c289d7ad14901fc Mon Sep 17 00:00:00 2001
From: pavan-kumarkulkarni <pavan-kumarkulkarni@pluralsight.com>
Date: Tue, 16 Apr 2024 12:13:58 +0530
Subject: [PATCH 203/220] chore(PDP1-895): Implement synk for hydra-publish

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c1f12747b..63525088b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -51,6 +51,7 @@ before_script:
 
 stages:
   - build
+  - code-analysis
   - package
   - publish
   - deploy_dev

From d0a66706ec07045d23ffcae86e58ca51d24ea6d8 Mon Sep 17 00:00:00 2001
From: shreedhar-kc <139169398+shreedhar-kc@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:53:50 +0530
Subject: [PATCH 204/220] update build xmx to 5g

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index 44943fa47..7bf1f289f 100644
--- a/build.sbt
+++ b/build.sbt
@@ -5,7 +5,7 @@ val JDK = "1.8"
 val buildNumber =
   scala.util.Properties.envOrNone("version").map(v => "." + v).getOrElse("")
 val hydraVersion = "0.11.3" + buildNumber
-val jvmMaxMemoryFlag = sys.env.getOrElse("MAX_JVM_MEMORY_FLAG", "-Xmx2g")
+val jvmMaxMemoryFlag = sys.env.getOrElse("MAX_JVM_MEMORY_FLAG", "-Xmx5g")
 
 lazy val defaultSettings = Seq(
   organization := "pluralsight",

From f600bce7c7d7b76e196d6b3c39dec033b10831c4 Mon Sep 17 00:00:00 2001
From: shreedhar-kc <139169398+shreedhar-kc@users.noreply.github.com>
Date: Mon, 10 Jun 2024 15:31:09 +0530
Subject: [PATCH 205/220] add more memory to hydra dev-cluster resources

---
 helm/eks-dev-values.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index df0cdb763..25f32a362 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -37,10 +37,10 @@ ingress:
 
 resources:
   limits:
-    memory: 6Gi
+    memory: 10Gi
   requests:
     cpu: 250m
-    memory: 1Gi
+    memory: 5Gi
 
 tolerations: []
 

From 828b973e8885a76913ca502e638011c552e12452 Mon Sep 17 00:00:00 2001
From: Abhishek Verma <abhishek-verma@pluralsight.com>
Date: Mon, 10 Jun 2024 11:11:02 +0000
Subject: [PATCH 206/220] [Adapt1-1545] | Publish lag on consumer group
 internal topic as per the configured time for each node

---
 .../scala/hydra.ingest/app/AppConfig.scala    |   8 +-
 .../scala/hydra.ingest/modules/Algebras.scala |   3 +-
 .../http/TopicDeletionEndpointSpec.scala      |   3 +-
 .../algebras/ConsumerGroupsAlgebraSpec.scala  |  29 ++++-
 .../algebras/ConsumerGroupsAlgebra.scala      | 107 ++++++++++++++----
 .../kafka/algebras/KafkaClientAlgebra.scala   |   2 +-
 6 files changed, 123 insertions(+), 29 deletions(-)

diff --git a/ingest/src/main/scala/hydra.ingest/app/AppConfig.scala b/ingest/src/main/scala/hydra.ingest/app/AppConfig.scala
index 82b78153f..79ef5c5a3 100644
--- a/ingest/src/main/scala/hydra.ingest/app/AppConfig.scala
+++ b/ingest/src/main/scala/hydra.ingest/app/AppConfig.scala
@@ -209,14 +209,16 @@ object AppConfig {
   final case class ConsumerGroupsAlgebraConfig(
                                                       kafkaInternalConsumerGroupsTopic: String,
                                                       commonConsumerGroup: ConsumerGroup,
-                                                      consumerGroupsConsumerEnabled: Boolean
+                                                      consumerGroupsConsumerEnabled: Boolean,
+                                                      lagPublishInterval: FiniteDuration
                                                     )
 
   private val consumerGroupAlgebraConfig: ConfigValue[ConsumerGroupsAlgebraConfig] =
       (
         env("KAFKA_CONSUMER_GROUPS_INTERNAL_TOPIC_NAME").as[String].default("__consumer_offsets"),
-          env("HYDRA_CONSUMER_GROUPS_COMMON_CONSUMER_GROUP").as[ConsumerGroup].default("kafkaInternalConsumerGroupsTopic-ConsumerGroupName"),
-            env("CONSUMER_GROUPS_CONSUMER_ENABLED").as[Boolean].default(true)
+        env("HYDRA_CONSUMER_GROUPS_COMMON_CONSUMER_GROUP").as[ConsumerGroup].default("kafkaInternalConsumerGroupsTopic-ConsumerGroupName"),
+        env("CONSUMER_GROUPS_CONSUMER_ENABLED").as[Boolean].default(true),
+        env("CONSUMER_GROUPS_INTERNAL_TOPIC_LAG_PUBLISH_INTERVAL").as[FiniteDuration].default(1.minutes)
         ).parMapN(ConsumerGroupsAlgebraConfig)
 
   final case class IngestConfig(
diff --git a/ingest/src/main/scala/hydra.ingest/modules/Algebras.scala b/ingest/src/main/scala/hydra.ingest/modules/Algebras.scala
index 8ee343e41..c5e81b4af 100644
--- a/ingest/src/main/scala/hydra.ingest/modules/Algebras.scala
+++ b/ingest/src/main/scala/hydra.ingest/modules/Algebras.scala
@@ -64,7 +64,8 @@ object Algebras {
         kafkaClientAlgebra = kafkaClient,
         kAA = kafkaAdmin,
         sra = schemaRegistry,
-        config.kafkaClientSecurityConfig
+        config.kafkaClientSecurityConfig,
+        config.consumerGroupsAlgebraConfig.lagPublishInterval
         )
       awsIamClient <- AwsIamClient.make
       awsStsClient <- AwsStsClient.make
diff --git a/ingest/src/test/scala/hydra/ingest/http/TopicDeletionEndpointSpec.scala b/ingest/src/test/scala/hydra/ingest/http/TopicDeletionEndpointSpec.scala
index f8c4fcecc..b6d29137a 100644
--- a/ingest/src/test/scala/hydra/ingest/http/TopicDeletionEndpointSpec.scala
+++ b/ingest/src/test/scala/hydra/ingest/http/TopicDeletionEndpointSpec.scala
@@ -33,6 +33,7 @@ import scalacache.Cache
 import scalacache.guava.GuavaCache
 
 import scala.concurrent.ExecutionContext
+import scala.concurrent.duration.DurationInt
 
 
 
@@ -147,7 +148,7 @@ class TopicDeletionEndpointSpec extends Matchers with AnyWordSpecLike with Scala
     implicit val notificationSenderMock: InternalNotificationSender[IO] = getInternalNotificationSenderMock[IO]
     ConsumerGroupsAlgebra.make("",Subject.createValidated("dvs.blah.blah").get,
       Subject.createValidated("dvs.heyo.blah").get,"","","",
-      kafkaClientAlgebra,kafkaAdminAlgebra,schemaAlgebra, kafkaSecurityEmptyConfig)
+      kafkaClientAlgebra,kafkaAdminAlgebra,schemaAlgebra, kafkaSecurityEmptyConfig, 1.minutes)
   }
 
   "The deletionEndpoint path" should {
diff --git a/ingestors/kafka/src/it/scala/hydra/kafka/algebras/ConsumerGroupsAlgebraSpec.scala b/ingestors/kafka/src/it/scala/hydra/kafka/algebras/ConsumerGroupsAlgebraSpec.scala
index b7f789293..858502821 100644
--- a/ingestors/kafka/src/it/scala/hydra/kafka/algebras/ConsumerGroupsAlgebraSpec.scala
+++ b/ingestors/kafka/src/it/scala/hydra/kafka/algebras/ConsumerGroupsAlgebraSpec.scala
@@ -10,7 +10,7 @@ import hydra.common.NotificationsTestSuite
 import hydra.common.alerting.sender.{InternalNotificationSender, NotificationSender}
 import hydra.common.config.KafkaConfigUtils.{KafkaClientSecurityConfig, SchemaRegistrySecurityConfig, kafkaSecurityEmptyConfig}
 import hydra.common.alerting.sender.InternalNotificationSender
-import hydra.kafka.algebras.ConsumerGroupsAlgebra.PartitionOffsetMap
+import hydra.kafka.algebras.ConsumerGroupsAlgebra.{PartitionOffsetMap, PartitionOffsetsWithTotalLag}
 import hydra.kafka.algebras.KafkaClientAlgebra.{OffsetInfo, Record}
 import hydra.kafka.model.TopicConsumer.{TopicConsumerKey, TopicConsumerValue}
 import hydra.kafka.model.TopicConsumerOffset.{TopicConsumerOffsetKey, TopicConsumerOffsetValue}
@@ -72,7 +72,7 @@ class ConsumerGroupsAlgebraSpec extends AnyWordSpecLike with Matchers with ForAl
     kafkaClient <- KafkaClientAlgebra.live[IO](container.bootstrapServers, "https://schema-registry", schemaRegistry , kafkaSecurityEmptyConfig)
 
     consumerGroupAlgebra <- ConsumerGroupsAlgebra.make(internalKafkaConsumerTopic, dvsConsumerTopic, dvsInternalKafkaOffsetsTopic,
-      container.bootstrapServers, consumerGroup, consumerGroup, kafkaClient, kafkaAdmin, schemaRegistry, kafkaSecurityEmptyConfig)
+      container.bootstrapServers, consumerGroup, consumerGroup, kafkaClient, kafkaAdmin, schemaRegistry, kafkaSecurityEmptyConfig, 1.minutes)
     _ <- consumerGroupAlgebra.startConsumer
   } yield {
     runTests(consumerGroupAlgebra, schemaRegistry, kafkaClient, kafkaAdmin)
@@ -185,6 +185,31 @@ class ConsumerGroupsAlgebraSpec extends AnyWordSpecLike with Matchers with ForAl
           case Right(_) => succeed
         }.unsafeRunSync()
       }
+
+      "test getLagOnInternalConsumerTopic to verify no lag with commit offsets as false" in {
+        val (key1, value1) = getGenericRecords(dvsConsumerTopic.value, "abc", "123")
+
+        kafkaClient.publishMessage((key1, Some(value1), None), dvsConsumerTopic.value).unsafeRunSync()
+
+        kafkaClient.consumeMessages(dvsConsumerTopic.value, consumer1, commitOffsets = false)
+          .take(1).compile.last.unsafeRunSync() shouldBe (key1, value1.some, None).some
+
+        cga.getLagOnDvsInternalCGTopic shouldBe ( PartitionOffsetsWithTotalLag(1,1,0,0,_))
+      }
+
+      "test getLagOnInternalConsumerTopic to verify some lag with commit offsets as false" in {
+        val (key1, value1) = getGenericRecords(dvsConsumerTopic.value, "abc", "123")
+        val (key2, value2) = getGenericRecords(dvsConsumerTopic.value, "abcd", "1234")
+
+        kafkaClient.publishMessage((key1, Some(value1), None), dvsConsumerTopic.value).unsafeRunSync()
+        kafkaClient.publishMessage((key2, Some(value2), None), dvsConsumerTopic.value).unsafeRunSync()
+
+        kafkaClient.consumeMessages(dvsConsumerTopic.value, consumer1, commitOffsets = false)
+          .take(1).compile.last.unsafeRunSync() shouldBe (key1, value1.some, None).some
+
+        cga.getLagOnDvsInternalCGTopic shouldBe (PartitionOffsetsWithTotalLag(2, 1, 1, 50, _))
+      }
+
     }
   }
 
diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/algebras/ConsumerGroupsAlgebra.scala b/ingestors/kafka/src/main/scala/hydra/kafka/algebras/ConsumerGroupsAlgebra.scala
index 8b49cfd14..b9f5aeeaf 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/algebras/ConsumerGroupsAlgebra.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/algebras/ConsumerGroupsAlgebra.scala
@@ -1,21 +1,13 @@
 package hydra.kafka.algebras
 
-import cats.ApplicativeError
-
-import java.time.Instant
 import cats.effect.concurrent.Ref
 import cats.effect.{Concurrent, ConcurrentEffect, ContextShift, IO, Timer}
 import cats.implicits._
-import fs2.kafka._
 import hydra.avro.registry.SchemaRegistry
-import hydra.common.alerting.AlertProtocol.NotificationMessage
-import hydra.common.alerting.NotificationLevel
-import hydra.common.alerting.sender.{InternalNotificationSender, NotificationSender}
+import hydra.common.alerting.sender.InternalNotificationSender
 import hydra.common.config.KafkaConfigUtils.KafkaClientSecurityConfig
-import hydra.kafka.algebras.ConsumerGroupsAlgebra.{Consumer, ConsumerTopics, DetailedConsumerGroup, DetailedTopicConsumers, PartitionOffset, TopicConsumers}
 import hydra.kafka.algebras.ConsumerGroupsAlgebra._
-import hydra.kafka.algebras.HydraTag.StringJsonFormat
-import hydra.kafka.algebras.KafkaClientAlgebra.Record
+import hydra.kafka.algebras.KafkaClientAlgebra.{Offset, Partition, Record}
 import hydra.kafka.algebras.RetryableFs2Stream.RetryPolicy.Infinite
 import hydra.kafka.algebras.RetryableFs2Stream._
 import hydra.kafka.model.TopicConsumer
@@ -23,9 +15,11 @@ import hydra.kafka.model.TopicConsumer.{TopicConsumerKey, TopicConsumerValue}
 import hydra.kafka.model.TopicMetadataV2Request.Subject
 import hydra.kafka.serializers.TopicMetadataV2Parser.IntentionallyUnimplemented
 import hydra.kafka.util.ConsumerGroupsOffsetConsumer
-import org.apache.avro.generic.GenericRecord
 import org.typelevel.log4cats.Logger
 
+import java.time.Instant
+import scala.concurrent.duration.{DurationInt, FiniteDuration}
+
 trait ConsumerGroupsAlgebra[F[_]] {
   def getConsumersForTopic(topicName: String): F[TopicConsumers]
 
@@ -44,6 +38,8 @@ trait ConsumerGroupsAlgebra[F[_]] {
   def consumerGroupIsActive(str: String): F[(Boolean, String)]
 
   def getUniquePerNodeConsumerGroup: String
+
+  def getLagOnDvsInternalCGTopic: F[PartitionOffsetsWithTotalLag]
 }
 
 final case class TestConsumerGroupsAlgebra(consumerGroupMap: Map[TopicConsumerKey, (TopicConsumerValue, String)]) extends ConsumerGroupsAlgebra[IO] {
@@ -96,6 +92,11 @@ final case class TestConsumerGroupsAlgebra(consumerGroupMap: Map[TopicConsumerKe
 
   override def getUniquePerNodeConsumerGroup: String = "uniquePerNodeConsumerGroup"
 
+  override def getLagOnDvsInternalCGTopic: IO[PartitionOffsetsWithTotalLag] = {
+    IO.pure(PartitionOffsetsWithTotalLag(60, 30, 30, 50,
+      List(PartitionOffset(1, 10, 20, 10), PartitionOffset(2, 10, 20, 10), PartitionOffset(3, 10, 20, 10))
+    ))
+  }
 }
 
 object TestConsumerGroupsAlgebra {
@@ -106,6 +107,9 @@ object ConsumerGroupsAlgebra {
 
   type PartitionOffsetMap = Map[Int, Long]
 
+  final case class PartitionOffsetsWithTotalLag(totalLargestOffset: Long, totalGroupOffset: Long, totalLag: Long,
+                                       lagPercentage: Double, partitionOffsets: List[PartitionOffset])
+
   final case class PartitionOffset(partition: Int, groupOffset: Long, largestOffset: Long, partitionLag: Long)
 
   final case class TopicConsumers(topicName: String, consumers: List[Consumer])
@@ -129,22 +133,56 @@ object ConsumerGroupsAlgebra {
                                                                      kafkaClientAlgebra: KafkaClientAlgebra[F],
                                                                      kAA: KafkaAdminAlgebra[F],
                                                                      sra: SchemaRegistry[F],
-                                                                     kafkaClientSecurityConfig: KafkaClientSecurityConfig
+                                                                     kafkaClientSecurityConfig: KafkaClientSecurityConfig,
+                                                                     lagPublishInterval: FiniteDuration
                                                                    ) (implicit  notificationsService: InternalNotificationSender[F]): F[ConsumerGroupsAlgebra[F]] = {
 
-    val dvsConsumersStream: fs2.Stream[F, Record] = {
-      kafkaClientAlgebra.consumeSafelyMessages(dvsConsumersTopic.value, uniquePerNodeConsumerGroup, commitOffsets = false)
+
+    val dvsConsumersStream: fs2.Stream[F, (Record, (Partition, Offset))] = {
+      kafkaClientAlgebra.consumeSafelyWithOffsetInfo(dvsConsumersTopic.value, uniquePerNodeConsumerGroup, commitOffsets = false)
         //Ignore records with errors
         .collect { case Right(value) => value }
     }
 
     for {
       consumerGroupsStorageFacade <- Ref[F].of(ConsumerGroupsStorageFacade.empty)
+      consumerGroupsOffsetFacade <- Ref[F].of(ConsumerGroupsOffsetFacade.empty)
     } yield new ConsumerGroupsAlgebra[F] {
 
       override def getConsumersForTopic(topicName: String): F[TopicConsumers] =
         consumerGroupsStorageFacade.get.flatMap(a => addStateToTopicConsumers(a.getConsumersForTopicName(topicName)))
 
+      override def getLagOnDvsInternalCGTopic: F[PartitionOffsetsWithTotalLag] = {
+
+        def getValueFromOffsetMap(partition: Int, offsetMap: Map[Partition, Offset]): Long =
+          offsetMap.get(partition) match {
+            case Some(value) => value + 1.toLong //Adding offsets by 1 as Kafka Admin algebra's getLatestOffsets method is having the same behaviour.
+            case _ => 0
+          }
+
+        for {
+          groupOffsetMap <- consumerGroupsOffsetFacade.get.map(_.getOffsets())
+
+          partitionOffsetMapWithLag <- kAA.getLatestOffsets(dvsConsumersTopic.value)
+            .map(_.toList
+              .filter(_._2.value > 0.toLong)
+              .map(latestOffset => PartitionOffset(
+                latestOffset._1.partition,
+                getValueFromOffsetMap(latestOffset._1.partition, groupOffsetMap),
+                latestOffset._2.value,
+                latestOffset._2.value - getValueFromOffsetMap(latestOffset._1.partition, groupOffsetMap)
+              )).toList)
+
+          (totalLargestOffset, totalGroupOffset) =
+            (partitionOffsetMapWithLag.map(_.largestOffset).sum, partitionOffsetMapWithLag.map(_.groupOffset).sum)
+
+          totalLag = totalLargestOffset - totalGroupOffset
+
+          lagPercentage: Double = (totalLag.toDouble / totalLargestOffset.toDouble) * 100
+
+        } yield PartitionOffsetsWithTotalLag(totalLargestOffset, totalGroupOffset, totalLag, lagPercentage, partitionOffsetMapWithLag)
+      }
+
       private def addStateToTopicConsumers(topicConsumers: TopicConsumers): F[TopicConsumers] = {
         val detailedF: F[List[Consumer]] = topicConsumers.consumers.traverse { consumer =>
           val fState = getConsumerActiveState(consumer.consumerGroupName)
@@ -166,11 +204,19 @@ object ConsumerGroupsAlgebra {
 
       override def startConsumer: F[Unit] = {
         for {
-          _ <- Concurrent[F].start(consumeDVSConsumersTopicIntoCache(dvsConsumersStream, consumerGroupsStorageFacade))
+          _ <- Concurrent[F].start(consumeDVSConsumersTopicIntoCache(dvsConsumersStream, consumerGroupsStorageFacade, consumerGroupsOffsetFacade))
           _ <- Concurrent[F].start {
             ConsumerGroupsOffsetConsumer.start(kafkaClientAlgebra, kAA, sra, uniquePerNodeConsumerGroup, consumerOffsetsOffsetsTopicConfig,
               kafkaInternalTopic, dvsConsumersTopic, bootstrapServers, commonConsumerGroup, kafkaClientSecurityConfig)
           }
+          _ <- fs2.Stream.awakeEvery[F](lagPublishInterval).evalMap(_ => getLagOnDvsInternalCGTopic.flatMap(
+              lagInfo => Logger[F].info(
+                s"Lag details on ${dvsConsumersTopic}. Total_Offset_Lag = ${lagInfo.totalLag.toString}, " +
+                  s"Lag_percentage = ${lagInfo.lagPercentage.toString}, " +
+                  s"Total_Group_Offset = ${lagInfo.totalGroupOffset}, " +
+                  s"Total_Largest_Offset = ${lagInfo.totalLargestOffset}, " +
+                  s"Total_active_partitions = ${Option(lagInfo.partitionOffsets).map(_.size).getOrElse(0)}"
+              ))).compile.drain
         } yield ()
       }
 
@@ -206,17 +252,19 @@ object ConsumerGroupsAlgebra {
   }
 
   private def consumeDVSConsumersTopicIntoCache[F[_] : ContextShift : ConcurrentEffect : Timer : Logger](
-                                                                                                          dvsConsumersStream: fs2.Stream[F, Record],
-                                                                                                          consumerGroupsStorageFacade: Ref[F, ConsumerGroupsStorageFacade]
+                                                                                                          dvsConsumersStream:  fs2.Stream[F, (Record, (Partition, Offset))],
+                                                                                                          consumerGroupsStorageFacade: Ref[F, ConsumerGroupsStorageFacade],
+                                                                                                          consumerGroupsOffsetFacade: Ref[F, ConsumerGroupsOffsetFacade],
                                                                                                         )(implicit notificationsService: InternalNotificationSender[F]): F[Unit] = {
-    dvsConsumersStream.evalTap { case (key, value, _) =>
-      TopicConsumer.decode[F](key, value).flatMap {
+    dvsConsumersStream.evalTap {
+      case ((key, value, _),(partition, offset)) =>
+        TopicConsumer.decode[F](key, value).flatMap {
         case (topicKey, topicValue) =>
           topicValue match {
             case Some(tV) =>
-              consumerGroupsStorageFacade.update(_.addConsumerGroup(topicKey, tV))
+              consumerGroupsStorageFacade.update(_.addConsumerGroup(topicKey, tV)) *> consumerGroupsOffsetFacade.update(_.addOffset(partition, offset))
             case None =>
-              consumerGroupsStorageFacade.update(_.removeConsumerGroup(topicKey))
+              consumerGroupsStorageFacade.update(_.removeConsumerGroup(topicKey)) *> consumerGroupsOffsetFacade.update(_.addOffset(partition, offset))
           }
       }.recoverWith {
         case e => Logger[F].error(e)("Error in ConsumergroupsAlgebra consumer")
@@ -258,3 +306,20 @@ private object ConsumerGroupsStorageFacade {
   def empty: ConsumerGroupsStorageFacade = ConsumerGroupsStorageFacade(Map.empty)
 }
 
+private case class ConsumerGroupsOffsetFacade(offsetMap: Map[Partition, Offset]) {
+
+  def addOffset(key: Partition, value: Offset): ConsumerGroupsOffsetFacade =
+    this.copy(this.offsetMap + (key -> value))
+
+  def getOffsets(): Map[Partition, Offset] =
+    this.offsetMap
+
+  def removeOffset(key: Partition): ConsumerGroupsOffsetFacade =
+    this.copy(this.offsetMap - key)
+
+  }
+
+  private object ConsumerGroupsOffsetFacade {
+    def empty: ConsumerGroupsOffsetFacade = ConsumerGroupsOffsetFacade(Map.empty)
+  }
+
diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/algebras/KafkaClientAlgebra.scala b/ingestors/kafka/src/main/scala/hydra/kafka/algebras/KafkaClientAlgebra.scala
index 2392a8837..4e66f0612 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/algebras/KafkaClientAlgebra.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/algebras/KafkaClientAlgebra.scala
@@ -531,7 +531,7 @@ object KafkaClientAlgebra {
     : fs2.Stream[F, Either[Throwable, (StringRecord, (Partition, Offset))]] = ???
 
     override def consumeSafelyWithOffsetInfo(topicName: TopicName, consumerGroup: ConsumerGroup, commitOffsets: Boolean)
-    : fs2.Stream[F, Either[Throwable, ((GenericRecord, Option[GenericRecord], Option[Headers]), (Partition, Offset))]] = ???
+    : fs2.Stream[F, Either[Throwable, ((GenericRecord, Option[GenericRecord], Option[Headers]), (Partition, Offset))]] = fs2.Stream.empty
 
     override def withProducerRecordSizeLimit(sizeLimitBytes: Long): F[KafkaClientAlgebra[F]] = Sync[F].delay {
       getTestInstance(cache, schemaRegistryUrl, schemaRegistry, sizeLimitBytes.some)

From 603239f1366c6feb53be84b5a1778818a7a59d05 Mon Sep 17 00:00:00 2001
From: pavan-kumarkulkarni <pavan-kumarkulkarni@pluralsight.com>
Date: Mon, 10 Jun 2024 20:21:11 +0530
Subject: [PATCH 207/220] Adding deployment method as manual for DEV

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 63525088b..f58d659bd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -136,6 +136,7 @@ deploy_dev:
     stage: deploy_dev
     needs: ["publish"]
     extends: .helmDeploy
+    when: manual
     environment:
       name: dev
     before_script:

From bc5b646d41b172b3adeb74c724c22f26af060f2d Mon Sep 17 00:00:00 2001
From: shreedhar-kc <139169398+shreedhar-kc@users.noreply.github.com>
Date: Tue, 11 Jun 2024 10:45:33 +0530
Subject: [PATCH 208/220] update the consumer-group name

---
 helm/eks-dev-values.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 25f32a362..c28fe2121 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -40,7 +40,7 @@ resources:
     memory: 10Gi
   requests:
     cpu: 250m
-    memory: 5Gi
+    memory: 7Gi
 
 tolerations: []
 
@@ -53,9 +53,8 @@ config:
   HYDRA_SCHEMA_REGISTRY_URL: https://dvs-dev-schema-registry.eplur-staging.vnerd.com:8081
   HYDRA_V2_METADATA_CONTACT: '#dev-data-platform'
   HYDRA_REPLICATION_FACTOR: 1
-  HYDRA_V2_METADATA_CONSUMER_GROUP: 'v2MetadataConsumer'
+  HYDRA_V2_METADATA_CONSUMER_GROUP: 'dev-cluster-v2MetadataConsumer'
   HYDRA_V2_CREATE_TOPICS_ENABLED: 'true'
   HYDRA_V2_METADATA_CREATE_ON_STARTUP: 'true'
   HYDRA_MIN_INSYNC_REPLICAS: 1
 
-

From 67a89326b6b09d358f36be3b40be7680bdc79bfa Mon Sep 17 00:00:00 2001
From: shreedhar-kc <139169398+shreedhar-kc@users.noreply.github.com>
Date: Tue, 11 Jun 2024 11:19:09 +0530
Subject: [PATCH 209/220] update info to debug and add sasl config

---
 helm/eks-dev-values.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index c28fe2121..69d4e2fec 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -48,7 +48,7 @@ nodeSelector: {}
 
 config:
   LOG_DIR: /var/log/hydra/
-  LOG_LEVEL: INFO
+  LOG_LEVEL: DEBUG
   HYDRA_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: b-1.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-3.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-2.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098
   HYDRA_SCHEMA_REGISTRY_URL: https://dvs-dev-schema-registry.eplur-staging.vnerd.com:8081
   HYDRA_V2_METADATA_CONTACT: '#dev-data-platform'
@@ -57,4 +57,8 @@ config:
   HYDRA_V2_CREATE_TOPICS_ENABLED: 'true'
   HYDRA_V2_METADATA_CREATE_ON_STARTUP: 'true'
   HYDRA_MIN_INSYNC_REPLICAS: 1
+  HYDRA_KAFKA_SASL_CLIENT_CALLBACK_HANDLER_CLASS: 'software.amazon.msk.auth.iam.IAMClientCallbackHandler'
+  HYDRA_KAFKA_SASL_JAAS_CONFIG: 'software.amazon.msk.auth.iam.IAMLoginModule required;'
+  HYDRA_KAFKA_SASL_MECHANISM: 'AWS_MSK_IAM'
+  HYDRA_KAFKA_SECURITY_PROTOCOL: 'SASL_SSL'
 

From 030f8c9ce93c2852368bf2a9c08915cd984ba3f2 Mon Sep 17 00:00:00 2001
From: shreedhar-kc <139169398+shreedhar-kc@users.noreply.github.com>
Date: Tue, 11 Jun 2024 11:24:02 +0530
Subject: [PATCH 210/220] add aws config

---
 helm/eks-dev-values.yml | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 69d4e2fec..eb6ecd33c 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -47,18 +47,20 @@ tolerations: []
 nodeSelector: {}
 
 config:
-  LOG_DIR: /var/log/hydra/
-  LOG_LEVEL: DEBUG
+  AWS_IAM_SECURITY_ENABLED: 'true'
+  AWS_REGION: 'us-west-2'
   HYDRA_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: b-1.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-3.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-2.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098
+  HYDRA_KAFKA_SASL_CLIENT_CALLBACK_HANDLER_CLASS: 'software.amazon.msk.auth.iam.IAMClientCallbackHandler'
+  HYDRA_KAFKA_SASL_JAAS_CONFIG: 'software.amazon.msk.auth.iam.IAMLoginModule required;'
+  HYDRA_KAFKA_SASL_MECHANISM: 'AWS_MSK_IAM'
+  HYDRA_KAFKA_SECURITY_PROTOCOL: 'SASL_SSL'
+  HYDRA_MIN_INSYNC_REPLICAS: 1
+  HYDRA_REPLICATION_FACTOR: 1
   HYDRA_SCHEMA_REGISTRY_URL: https://dvs-dev-schema-registry.eplur-staging.vnerd.com:8081
   HYDRA_V2_METADATA_CONTACT: '#dev-data-platform'
-  HYDRA_REPLICATION_FACTOR: 1
   HYDRA_V2_METADATA_CONSUMER_GROUP: 'dev-cluster-v2MetadataConsumer'
   HYDRA_V2_CREATE_TOPICS_ENABLED: 'true'
   HYDRA_V2_METADATA_CREATE_ON_STARTUP: 'true'
-  HYDRA_MIN_INSYNC_REPLICAS: 1
-  HYDRA_KAFKA_SASL_CLIENT_CALLBACK_HANDLER_CLASS: 'software.amazon.msk.auth.iam.IAMClientCallbackHandler'
-  HYDRA_KAFKA_SASL_JAAS_CONFIG: 'software.amazon.msk.auth.iam.IAMLoginModule required;'
-  HYDRA_KAFKA_SASL_MECHANISM: 'AWS_MSK_IAM'
-  HYDRA_KAFKA_SECURITY_PROTOCOL: 'SASL_SSL'
+  LOG_DIR: /var/log/hydra/
+  LOG_LEVEL: DEBUG
 

From 22ad90e73b3725c1dd6bd874128dc82154d4e685 Mon Sep 17 00:00:00 2001
From: Abhishek <abhishek-verma@pluralsight.com>
Date: Tue, 11 Jun 2024 12:05:01 +0530
Subject: [PATCH 211/220] limit lag percentage decimal places

---
 .../main/scala/hydra/kafka/algebras/ConsumerGroupsAlgebra.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ingestors/kafka/src/main/scala/hydra/kafka/algebras/ConsumerGroupsAlgebra.scala b/ingestors/kafka/src/main/scala/hydra/kafka/algebras/ConsumerGroupsAlgebra.scala
index b9f5aeeaf..146de1e26 100644
--- a/ingestors/kafka/src/main/scala/hydra/kafka/algebras/ConsumerGroupsAlgebra.scala
+++ b/ingestors/kafka/src/main/scala/hydra/kafka/algebras/ConsumerGroupsAlgebra.scala
@@ -212,7 +212,7 @@ object ConsumerGroupsAlgebra {
           _ <- fs2.Stream.awakeEvery[F](lagPublishInterval).evalMap(_ => getLagOnDvsInternalCGTopic.flatMap(
               lagInfo => Logger[F].info(
                 s"Lag details on ${dvsConsumersTopic}. Total_Offset_Lag = ${lagInfo.totalLag.toString}, " +
-                  s"Lag_percentage = ${lagInfo.lagPercentage.toString}, " +
+                  f"Lag_percentage = ${lagInfo.lagPercentage}%2.4f, " +
                   s"Total_Group_Offset = ${lagInfo.totalGroupOffset}, " +
                   s"Total_Largest_Offset = ${lagInfo.totalLargestOffset}, " +
                   s"Total_active_partitions = ${Option(lagInfo.partitionOffsets).map(_.size).getOrElse(0)}"

From 635b7293d34b206bf672e6fb3a33f692b1bcb674 Mon Sep 17 00:00:00 2001
From: shreedhar-kc <139169398+shreedhar-kc@users.noreply.github.com>
Date: Tue, 11 Jun 2024 15:02:07 +0530
Subject: [PATCH 212/220] remove extra aws configs

---
 helm/eks-dev-values.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index eb6ecd33c..065609fbf 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -47,8 +47,6 @@ tolerations: []
 nodeSelector: {}
 
 config:
-  AWS_IAM_SECURITY_ENABLED: 'true'
-  AWS_REGION: 'us-west-2'
   HYDRA_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: b-1.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-3.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-2.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098
   HYDRA_KAFKA_SASL_CLIENT_CALLBACK_HANDLER_CLASS: 'software.amazon.msk.auth.iam.IAMClientCallbackHandler'
   HYDRA_KAFKA_SASL_JAAS_CONFIG: 'software.amazon.msk.auth.iam.IAMLoginModule required;'

From 5a9dde320e75de0247559db2af423d010b60364d Mon Sep 17 00:00:00 2001
From: shreedhar-kc <139169398+shreedhar-kc@users.noreply.github.com>
Date: Tue, 11 Jun 2024 15:53:13 +0530
Subject: [PATCH 213/220] change kafka security protocol to plaintext for
 dev-cluster

---
 helm/eks-dev-values.yml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 065609fbf..d201ae27f 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -47,11 +47,7 @@ tolerations: []
 nodeSelector: {}
 
 config:
-  HYDRA_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: b-1.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-3.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-2.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098
-  HYDRA_KAFKA_SASL_CLIENT_CALLBACK_HANDLER_CLASS: 'software.amazon.msk.auth.iam.IAMClientCallbackHandler'
-  HYDRA_KAFKA_SASL_JAAS_CONFIG: 'software.amazon.msk.auth.iam.IAMLoginModule required;'
-  HYDRA_KAFKA_SASL_MECHANISM: 'AWS_MSK_IAM'
-  HYDRA_KAFKA_SECURITY_PROTOCOL: 'SASL_SSL'
+  HYDRA_KAFKA_SECURITY_PROTOCOL: PLAINTEXT
   HYDRA_MIN_INSYNC_REPLICAS: 1
   HYDRA_REPLICATION_FACTOR: 1
   HYDRA_SCHEMA_REGISTRY_URL: https://dvs-dev-schema-registry.eplur-staging.vnerd.com:8081

From 11417499f8c86ec3562fef5a4b065d0eaeeb17c6 Mon Sep 17 00:00:00 2001
From: shreedhar-kc <139169398+shreedhar-kc@users.noreply.github.com>
Date: Tue, 11 Jun 2024 16:28:12 +0530
Subject: [PATCH 214/220] re-add kafka bootstrap servers envar

---
 helm/eks-dev-values.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index d201ae27f..e2ae9ecce 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -47,7 +47,8 @@ tolerations: []
 nodeSelector: {}
 
 config:
-  HYDRA_KAFKA_SECURITY_PROTOCOL: PLAINTEXT
+  HYDRA_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: b-1.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-3.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-2.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098
+  HYDRA_KAFKA_SECURITY_PROTOCOL: 'PLAINTEXT'
   HYDRA_MIN_INSYNC_REPLICAS: 1
   HYDRA_REPLICATION_FACTOR: 1
   HYDRA_SCHEMA_REGISTRY_URL: https://dvs-dev-schema-registry.eplur-staging.vnerd.com:8081

From 097007e093db5d27f45986bfaeb275c172c3b3d6 Mon Sep 17 00:00:00 2001
From: shreedhar-kc <139169398+shreedhar-kc@users.noreply.github.com>
Date: Tue, 11 Jun 2024 17:05:14 +0530
Subject: [PATCH 215/220] add plaintext client config port of kafka brokers

---
 helm/eks-dev-values.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index e2ae9ecce..afc310b7f 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -47,7 +47,7 @@ tolerations: []
 nodeSelector: {}
 
 config:
-  HYDRA_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: b-1.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-3.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-2.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098
+  HYDRA_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: b-3.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9092,b-2.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9092,b-1.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9092
   HYDRA_KAFKA_SECURITY_PROTOCOL: 'PLAINTEXT'
   HYDRA_MIN_INSYNC_REPLICAS: 1
   HYDRA_REPLICATION_FACTOR: 1

From d7d29348563eb78ae049d51e1534f3634310a800 Mon Sep 17 00:00:00 2001
From: shreedhar-kc <139169398+shreedhar-kc@users.noreply.github.com>
Date: Tue, 11 Jun 2024 17:29:11 +0530
Subject: [PATCH 216/220] use less heap memory

---
 build.sbt               | 2 +-
 helm/eks-dev-values.yml | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/build.sbt b/build.sbt
index 7bf1f289f..44943fa47 100644
--- a/build.sbt
+++ b/build.sbt
@@ -5,7 +5,7 @@ val JDK = "1.8"
 val buildNumber =
   scala.util.Properties.envOrNone("version").map(v => "." + v).getOrElse("")
 val hydraVersion = "0.11.3" + buildNumber
-val jvmMaxMemoryFlag = sys.env.getOrElse("MAX_JVM_MEMORY_FLAG", "-Xmx5g")
+val jvmMaxMemoryFlag = sys.env.getOrElse("MAX_JVM_MEMORY_FLAG", "-Xmx2g")
 
 lazy val defaultSettings = Seq(
   organization := "pluralsight",
diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index afc310b7f..ebf0d1e7b 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -37,10 +37,10 @@ ingress:
 
 resources:
   limits:
-    memory: 10Gi
+    memory: 7Gi
   requests:
     cpu: 250m
-    memory: 7Gi
+    memory: 2Gi
 
 tolerations: []
 

From 68cbc90e5ae6836f6d2d4158836d4867c76b31f3 Mon Sep 17 00:00:00 2001
From: shreedhar-kc <139169398+shreedhar-kc@users.noreply.github.com>
Date: Sat, 15 Jun 2024 12:28:47 +0530
Subject: [PATCH 217/220] update port number

---
 helm/eks-dev-values.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index ebf0d1e7b..639aa5ade 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -16,7 +16,7 @@ replicas: 2
 #velero.io/exclude-from-backup" = true 
 service:
   type: ClusterIP
-  port: 8080
+  port: 8088
   annotations: {}
   labels: {}
 

From a375cb9fde81c9be07a071729ea44ede9d285926 Mon Sep 17 00:00:00 2001
From: shreedhar-kc <139169398+shreedhar-kc@users.noreply.github.com>
Date: Sun, 16 Jun 2024 18:26:22 +0530
Subject: [PATCH 218/220] reduce hydra nodes replicas to zero

---
 helm/eks-dev-values.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 639aa5ade..28ac47a97 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -10,7 +10,7 @@ deployment:
       maxUnavailable: 0
     type: RollingUpdate
 
-replicas: 2
+replicas: 0
 
 #Adding velero backup for k8s services
 #velero.io/exclude-from-backup" = true 

From 4c1f5fe00eab501e57a458568d5ea60d8feb9114 Mon Sep 17 00:00:00 2001
From: shreedhar-kc <139169398+shreedhar-kc@users.noreply.github.com>
Date: Thu, 27 Jun 2024 18:35:02 +0530
Subject: [PATCH 219/220] update replicas to 2

---
 helm/eks-dev-values.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 28ac47a97..639aa5ade 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -10,7 +10,7 @@ deployment:
       maxUnavailable: 0
     type: RollingUpdate
 
-replicas: 0
+replicas: 2
 
 #Adding velero backup for k8s services
 #velero.io/exclude-from-backup" = true 

From 0a8877a7fc5e6e5669c6306a09c2fdefe509e940 Mon Sep 17 00:00:00 2001
From: shreedhar-kc <139169398+shreedhar-kc@users.noreply.github.com>
Date: Mon, 1 Jul 2024 11:45:47 +0530
Subject: [PATCH 220/220] change log level to info

---
 helm/eks-dev-values.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml
index 639aa5ade..1b8771347 100644
--- a/helm/eks-dev-values.yml
+++ b/helm/eks-dev-values.yml
@@ -57,5 +57,5 @@ config:
   HYDRA_V2_CREATE_TOPICS_ENABLED: 'true'
   HYDRA_V2_METADATA_CREATE_ON_STARTUP: 'true'
   LOG_DIR: /var/log/hydra/
-  LOG_LEVEL: DEBUG
+  LOG_LEVEL: INFO