Merge branch 'team_city_migration' into 'master'

Team city migration

Closes PDP1-895

See merge request pluralsight/Technology/adapt/data-platform/hydra!5

.gitlab-ci.yml

@@ -0,0 +1,230 @@
+include:
+  - project: 'pluralsight/experience/gitlab-helpers'
+    file: '/salt-deploy/helper.yml'
+  - project: 'pluralsight/experience/gitlab-helpers'
+    file: '/helm-deploy/helper.yml'
+  - project: 'pluralsight/Technology/adapt/data-platform/gitlab-fragments/snyk'
+    ref: main
+    file: 'snyk-check.yml'
+
+image: openjdk:8
+
+variables:
+  ARTIFACT_NAME: hydra-publish
+  IMAGE_NAME: hydra
+  DOCKER_REGISTRY_URL: harbor.vnerd.com/library
+  DOCKER_REGISTRY_IMAGE: ${DOCKER_REGISTRY_URL}/${IMAGE_NAME}
+  ARTIFACTORY_REPOSITORY_URL: https://repository.vnerd.com/artifactory
+  ARTIFACTORY_REPOSITORY: bounded-context
+  ARTIFACTORY_ID: hydra/publish
+  ARTIFACTORY_ARTIFACT_VERSION: ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz
+  ARTIFACTORY_PATH: ${ARTIFACTORY_ID}/${ARTIFACTORY_ARTIFACT_VERSION}
+  ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH}
+  BUILD_VERSION: 1.0.${CI_PIPELINE_IID}
+  BOUNDED_CONTEXT_DEV: adapt-dvs-dev
+  BOUNDED_CONTEXT_STAGING: adapt-dvs
+  BOUNDED_CONTEXT_PROD: adapt-dvs
+  ENV: ${CI_JOB_STAGE} 
+  SERVICE_NAME: "dev-hydra"
+  SLACK_ICON_EMOJI: ":gitlab:"
+  SLACK_CHANNEL: '#data-platform-alerts'
+  SLACK_MESSAGE: |
+    *[[SERVICE_NAME]]* deployed to *[[ENV]]*.
+    Version: *[[BUILD_VERSION]]*
+    [[PIPELINE_LINK]]
+  SLACK_USERNAME: 'GITLAB'
+  SNYK_SLACK_CHANNEL: '#dataops-snyk-vulnerability-alerts'
+  SNYK_SLACK_ICON_EMOJI: ":snyk:"
+  SNYK_ORG_ID: "d8094638-7a37-413f-b1b4-ad840fb9e239"
+  SNYK_PROJECT_ID: "d243a0e6-4ced-4efe-83fc-169d03b40cc7"
+  SNYK_PROJECT_URL: "https://app.snyk.io/org/ps-data-services"
+
+before_script:
+  - apt-get update -yqq
+  - apt-get install -yqq apt-transport-https apt-utils
+  - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
+  - mkdir -p /root/.gnupg
+  - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
+  - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
+  - apt-get update -yqq
+  - apt-get install -yqq sbt
+
+stages:
+  - build
+  - code-analysis
+  - package
+  - publish
+  - deploy_dev
+  - deploy_staging
+  - deploy_production
+  - notify
+
+build:
+  stage: build
+  cache: []
+  retry: 2
+  script:
+    - sbt clean compile test
+
+package:
+  stage: package
+  cache: []
+  script:
+    - sbt universal:packageBin
+    - ls ingest/target/universal/*.zip | xargs -I {} unzip {} -d ps-publish
+    - mv ps-publish/hydra-ingest*/* ps-publish
+    - rm -rf ps-publish/hydra-ingest*
+    - cd ps-publish/bin
+    - ls -la
+    - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic-agent-4.4.0.jar
+    - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic.yml
+    - cd ../../
+    - echo $BUILD_VERSION > VERSION
+    - tar czf ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/
+    - echo "##teamcity[publishArtifacts '${ARTIFACT_NAME}-${BUILD_VERSION}.tgz']"
+    - md5sum ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz > ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz.md5
+    - echo "${ARTIFACTORY_ARTIFACT_URL}"
+    - |
+      curl \
+      -H "Authorization: Bearer ${NPM_TOKEN}" \
+      -X PUT ${ARTIFACTORY_ARTIFACT_URL} \
+      -T ${ARTIFACTORY_ARTIFACT_VERSION}
+  artifacts:
+    paths:
+      - ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz
+      - ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz.md5
+      - ps-publish/
+
+publish:
+  stage: publish
+  image: harbor.vnerd.com/proxy/library/docker:cli
+  tags:
+    - ps
+    - docker
+  needs:
+    - package
+  before_script:
+    - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL"
+  script:
+    - pwd
+    - ls -lh
+    - docker build -t ${IMAGE_NAME} -f Dockerfile.new .
+    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${BUILD_VERSION}-${CI_COMMIT_SHORT_SHA}
+    - docker push ${DOCKER_REGISTRY_IMAGE}:${BUILD_VERSION}-${CI_COMMIT_SHORT_SHA}
+    - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest
+    - docker push ${DOCKER_REGISTRY_IMAGE}:latest
+    - CURYEAR=$(date +%Y)
+    - CURMONTH=$(date +%-m)
+    - echo -n "${CURYEAR}.${CURMONTH}.${BUILD_VERSION}" > VERSION
+  variables:
+    DOCKER_HOST: tcp://localhost:2376
+    DOCKER_TLS_CERTDIR: "/certs"
+    DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
+    DOCKER_TLS_VERIFY: 1
+
+.notify-all:
+  image: harbor.vnerd.com/library/ps-helm:latest
+  tags:
+    - ps
+    - docker 
+  script:
+    - ps-notify slack "$SLACK_MESSAGE" "$SLACK_CHANNEL"  --slack-username $SLACK_USERNAME --slack-icon-emoji $SLACK_ICON_EMOJI
+  variables:
+    NODE_ENV: $ENV
+
+deploy_dev:
+    stage: deploy_dev
+    needs: ["publish"]
+    extends: .helmDeploy
+    environment:
+      name: dev
+    before_script:
+      - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}/g" $HELM_VALUES_FILE
+      - cat $HELM_VALUES_FILE
+      - echo 1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}
+      - kubectl config set-cluster app-${CI_ENVIRONMENT_NAME%/*} --server=${HELM_K8S_CLUSTER_URL} --embed-certs --certificate-authority="$HELM_K8S_CLUSTER_CACERT"
+      - kubectl config set-credentials deploy --token=`echo ${HELM_DEPLOY_TOKEN} | base64 -d`
+      - kubectl config set-context deploy --cluster=app-${CI_ENVIRONMENT_NAME%/*} --namespace=${HELM_TARGET_NAMESPACE} --user=deploy
+      - kubectl config use-context deploy
+    variables:
+      HELM_CHART_NAME: ps-service
+      HELM_DEPLOY_TOKEN: ${HELM_DEV_DEPLOY_TOKEN}
+      K8S_CLUSTER_NAME: app-eks.eplur-staging.us-west-2
+      HELM_K8S_CLUSTER_URL: https://6C29C0073BB19BEF220B9437E6962AF2.gr7.us-west-2.eks.amazonaws.com
+      HELM_TARGET_NAMESPACE: ${BOUNDED_CONTEXT_DEV}
+      APPLICATION_ROLE: dev-hydra
+      HELM_VALUES_FILE: helm/eks-dev-values.yml
+
+slack:dev:
+  stage: notify
+  extends: .notify-all
+  when: on_success
+  needs: ['deploy_dev']
+  before_script:
+    - echo "Sending notification to slack"
+  variables:
+    ENV: "DEV Cluster"
+    SERVICE_NAME: "dev-hydra"
+
+
+deploy_staging: 
+  stage: deploy_staging
+  extends: .saltDeploy
+  when: manual
+  environment:
+      name: staging
+  before_script:
+    - cat $SALT_PILLAR
+    - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR
+    - cat $SALT_PILLAR  
+  variables:
+    CI_JOB_STAGE: stage
+    SALT_TARGET: roles:hydra-publish-msk
+    SALT_PASSWORD: ${LDAP_PASS}
+    SALT_ARGUMENTS: systemd-app
+    SALT_USERNAME: tcity-data-platform
+    SALT_KWARGS: 'failhard=true'
+    SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
+    SALT_URL: https://saltmaster-stage.vnerd.com:8000
+
+slack:staging:
+  stage: notify
+  extends: .notify-all
+  when: on_success
+  needs: ['deploy_staging']
+  before_script:
+    - echo "Sending notification to slack"
+  variables:
+    ENV: "Staging Cluster"
+    SERVICE_NAME: "staging-hydra-publish"
+
+deploy_production: 
+  stage: deploy_production
+  extends: .saltDeploy
+  when: manual
+  environment:
+      name: production
+  before_script:
+    - cat $SALT_PILLAR
+    - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR
+    - cat $SALT_PILLAR  
+  variables:
+    CI_JOB_STAGE: stage
+    SALT_TARGET: roles:hydra-publish-msk
+    SALT_PASSWORD: ${LDAP_PASS}
+    SALT_ARGUMENTS: systemd-app
+    SALT_USERNAME: tcity-data-platform
+    SALT_KWARGS: 'failhard=true'
+    SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml
+    SALT_URL: https://saltmaster-production.vnerd.com:8000
+
+slack:production:
+  stage: notify
+  extends: .notify-all
+  when: on_success
+  needs: ['deploy_production']
+  before_script:
+    - echo "Sending notification to slack"
+  variables:
+    ENV: "Production Cluster"
+    SERVICE_NAME: "production-hydra-publish"

Dockerfile.new

@@ -2,7 +2,7 @@ FROM jelastic/jetty:9.4.49-openjdk-1.8.0_352
 
 USER root
 
-ENV JAVA_OPTS="-Xmx2G"
+ENV JAVA_OPTS="-Xmx2g"
 
 ENV CONTAINER_HTTP_PORT="8088"
 
@@ -12,4 +12,4 @@ EXPOSE 8088
 
 COPY ps-publish/ /ps-publish
 
-ENTRYPOINT ["/ps-publish/bin/hydra-ingest"]
+ENTRYPOINT ["/ps-publish/bin/hydra-ingest"]

helm/eks-dev-values.yml

@@ -0,0 +1,61 @@
+image:
+  repository: harbor.vnerd.com/library/hydra
+  pullPolicy: Always
+  tag: '%IMAGE_TAG%'
+
+deployment:
+  apiVersion: "apps/v1"
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 0
+    type: RollingUpdate
+
+replicas: 2
+
+#Adding velero backup for k8s services
+#velero.io/exclude-from-backup" = true 
+service:
+  type: ClusterIP
+  port: 8080
+  annotations: {}
+  labels: {}
+
+
+ingress:
+  enabled: true
+  ingressClassName: internal
+  annotations: {}
+  labels: {}
+  hosts:
+    - hydra-publish-dev.eplur-staging.vnerd.com
+  tls:
+    - hosts:
+        - hydra-publish-dev.eplur-staging.vnerd.com
+  path: /
+  pathType: Prefix
+
+
+resources:
+  limits:
+    memory: 6Gi
+  requests:
+    cpu: 250m
+    memory: 1Gi
+
+tolerations: []
+
+nodeSelector: {}
+
+config:
+  LOG_DIR: /var/log/hydra/
+  LOG_LEVEL: INFO
+  HYDRA_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: b-1.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-3.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-2.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098
+  HYDRA_SCHEMA_REGISTRY_URL: https://dvs-dev-schema-registry.eplur-staging.vnerd.com:8081
+  HYDRA_V2_METADATA_CONTACT: '#dev-data-platform'
+  HYDRA_REPLICATION_FACTOR: 1
+  HYDRA_V2_METADATA_CONSUMER_GROUP: 'v2MetadataConsumer'
+  HYDRA_V2_CREATE_TOPICS_ENABLED: 'true'
+  HYDRA_V2_METADATA_CREATE_ON_STARTUP: 'true'
+  HYDRA_MIN_INSYNC_REPLICAS: 1
+
+

pillar_overrides.yaml

@@ -0,0 +1,9 @@
+systemd-app:
+  artifacts:
+    hydra-publish:
+      gitlab: True
+      artifactory_packaging : tgz
+      artifactory_artifact_version: latest
+      artifactory_artifact_path: hydra/publish
+      artifactory_repository: bounded-context
+      artifactory_artifact_name: hydra-publish