diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..63525088b --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,230 @@ +include: + - project: 'pluralsight/experience/gitlab-helpers' + file: '/salt-deploy/helper.yml' + - project: 'pluralsight/experience/gitlab-helpers' + file: '/helm-deploy/helper.yml' + - project: 'pluralsight/Technology/adapt/data-platform/gitlab-fragments/snyk' + ref: main + file: 'snyk-check.yml' + +image: openjdk:8 + +variables: + ARTIFACT_NAME: hydra-publish + IMAGE_NAME: hydra + DOCKER_REGISTRY_URL: harbor.vnerd.com/library + DOCKER_REGISTRY_IMAGE: ${DOCKER_REGISTRY_URL}/${IMAGE_NAME} + ARTIFACTORY_REPOSITORY_URL: https://repository.vnerd.com/artifactory + ARTIFACTORY_REPOSITORY: bounded-context + ARTIFACTORY_ID: hydra/publish + ARTIFACTORY_ARTIFACT_VERSION: ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz + ARTIFACTORY_PATH: ${ARTIFACTORY_ID}/${ARTIFACTORY_ARTIFACT_VERSION} + ARTIFACTORY_ARTIFACT_URL: ${ARTIFACTORY_REPOSITORY_URL}/${ARTIFACTORY_REPOSITORY}/${ARTIFACTORY_PATH} + BUILD_VERSION: 1.0.${CI_PIPELINE_IID} + BOUNDED_CONTEXT_DEV: adapt-dvs-dev + BOUNDED_CONTEXT_STAGING: adapt-dvs + BOUNDED_CONTEXT_PROD: adapt-dvs + ENV: ${CI_JOB_STAGE} + SERVICE_NAME: "dev-hydra" + SLACK_ICON_EMOJI: ":gitlab:" + SLACK_CHANNEL: '#data-platform-alerts' + SLACK_MESSAGE: | + *[[SERVICE_NAME]]* deployed to *[[ENV]]*. + Version: *[[BUILD_VERSION]]* + [[PIPELINE_LINK]] + SLACK_USERNAME: 'GITLAB' + SNYK_SLACK_CHANNEL: '#dataops-snyk-vulnerability-alerts' + SNYK_SLACK_ICON_EMOJI: ":snyk:" + SNYK_ORG_ID: "d8094638-7a37-413f-b1b4-ad840fb9e239" + SNYK_PROJECT_ID: "d243a0e6-4ced-4efe-83fc-169d03b40cc7" + SNYK_PROJECT_URL: "https://app.snyk.io/org/ps-data-services" + +before_script: + - apt-get update -yqq + - apt-get install -yqq apt-transport-https apt-utils + - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list + - mkdir -p /root/.gnupg + - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823 + - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg + - apt-get update -yqq + - apt-get install -yqq sbt + +stages: + - build + - code-analysis + - package + - publish + - deploy_dev + - deploy_staging + - deploy_production + - notify + +build: + stage: build + cache: [] + retry: 2 + script: + - sbt clean compile test + +package: + stage: package + cache: [] + script: + - sbt universal:packageBin + - ls ingest/target/universal/*.zip | xargs -I {} unzip {} -d ps-publish + - mv ps-publish/hydra-ingest*/* ps-publish + - rm -rf ps-publish/hydra-ingest* + - cd ps-publish/bin + - ls -la + - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic-agent-4.4.0.jar + - curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/4.4.0/newrelic.yml + - cd ../../ + - echo $BUILD_VERSION > VERSION + - tar czf ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz --exclude=*.tmp --exclude=*.tgz --exclude=*.tgz.md5 VERSION ps-publish/ + - echo "##teamcity[publishArtifacts '${ARTIFACT_NAME}-${BUILD_VERSION}.tgz']" + - md5sum ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz > ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz.md5 + - echo "${ARTIFACTORY_ARTIFACT_URL}" + - | + curl \ + -H "Authorization: Bearer ${NPM_TOKEN}" \ + -X PUT ${ARTIFACTORY_ARTIFACT_URL} \ + -T ${ARTIFACTORY_ARTIFACT_VERSION} + artifacts: + paths: + - ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz + - ${ARTIFACT_NAME}-${BUILD_VERSION}.tgz.md5 + - ps-publish/ + +publish: + stage: publish + image: harbor.vnerd.com/proxy/library/docker:cli + tags: + - ps + - docker + needs: + - package + before_script: + - echo -n "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin "$DOCKER_REGISTRY_URL" + script: + - pwd + - ls -lh + - docker build -t ${IMAGE_NAME} -f Dockerfile.new . + - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:${BUILD_VERSION}-${CI_COMMIT_SHORT_SHA} + - docker push ${DOCKER_REGISTRY_IMAGE}:${BUILD_VERSION}-${CI_COMMIT_SHORT_SHA} + - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY_IMAGE}:latest + - docker push ${DOCKER_REGISTRY_IMAGE}:latest + - CURYEAR=$(date +%Y) + - CURMONTH=$(date +%-m) + - echo -n "${CURYEAR}.${CURMONTH}.${BUILD_VERSION}" > VERSION + variables: + DOCKER_HOST: tcp://localhost:2376 + DOCKER_TLS_CERTDIR: "/certs" + DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" + DOCKER_TLS_VERIFY: 1 + +.notify-all: + image: harbor.vnerd.com/library/ps-helm:latest + tags: + - ps + - docker + script: + - ps-notify slack "$SLACK_MESSAGE" "$SLACK_CHANNEL" --slack-username $SLACK_USERNAME --slack-icon-emoji $SLACK_ICON_EMOJI + variables: + NODE_ENV: $ENV + +deploy_dev: + stage: deploy_dev + needs: ["publish"] + extends: .helmDeploy + environment: + name: dev + before_script: + - sed -i -e "s/%IMAGE_TAG%/1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA}/g" $HELM_VALUES_FILE + - cat $HELM_VALUES_FILE + - echo 1.0.${CI_PIPELINE_IID}-${CI_COMMIT_SHORT_SHA} + - kubectl config set-cluster app-${CI_ENVIRONMENT_NAME%/*} --server=${HELM_K8S_CLUSTER_URL} --embed-certs --certificate-authority="$HELM_K8S_CLUSTER_CACERT" + - kubectl config set-credentials deploy --token=`echo ${HELM_DEPLOY_TOKEN} | base64 -d` + - kubectl config set-context deploy --cluster=app-${CI_ENVIRONMENT_NAME%/*} --namespace=${HELM_TARGET_NAMESPACE} --user=deploy + - kubectl config use-context deploy + variables: + HELM_CHART_NAME: ps-service + HELM_DEPLOY_TOKEN: ${HELM_DEV_DEPLOY_TOKEN} + K8S_CLUSTER_NAME: app-eks.eplur-staging.us-west-2 + HELM_K8S_CLUSTER_URL: https://6C29C0073BB19BEF220B9437E6962AF2.gr7.us-west-2.eks.amazonaws.com + HELM_TARGET_NAMESPACE: ${BOUNDED_CONTEXT_DEV} + APPLICATION_ROLE: dev-hydra + HELM_VALUES_FILE: helm/eks-dev-values.yml + +slack:dev: + stage: notify + extends: .notify-all + when: on_success + needs: ['deploy_dev'] + before_script: + - echo "Sending notification to slack" + variables: + ENV: "DEV Cluster" + SERVICE_NAME: "dev-hydra" + + +deploy_staging: + stage: deploy_staging + extends: .saltDeploy + when: manual + environment: + name: staging + before_script: + - cat $SALT_PILLAR + - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR + - cat $SALT_PILLAR + variables: + CI_JOB_STAGE: stage + SALT_TARGET: roles:hydra-publish-msk + SALT_PASSWORD: ${LDAP_PASS} + SALT_ARGUMENTS: systemd-app + SALT_USERNAME: tcity-data-platform + SALT_KWARGS: 'failhard=true' + SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml + SALT_URL: https://saltmaster-stage.vnerd.com:8000 + +slack:staging: + stage: notify + extends: .notify-all + when: on_success + needs: ['deploy_staging'] + before_script: + - echo "Sending notification to slack" + variables: + ENV: "Staging Cluster" + SERVICE_NAME: "staging-hydra-publish" + +deploy_production: + stage: deploy_production + extends: .saltDeploy + when: manual + environment: + name: production + before_script: + - cat $SALT_PILLAR + - sed -i "s/latest/$BUILD_VERSION/" $SALT_PILLAR + - cat $SALT_PILLAR + variables: + CI_JOB_STAGE: stage + SALT_TARGET: roles:hydra-publish-msk + SALT_PASSWORD: ${LDAP_PASS} + SALT_ARGUMENTS: systemd-app + SALT_USERNAME: tcity-data-platform + SALT_KWARGS: 'failhard=true' + SALT_PILLAR: $CI_PROJECT_DIR/pillar_overrides.yaml + SALT_URL: https://saltmaster-production.vnerd.com:8000 + +slack:production: + stage: notify + extends: .notify-all + when: on_success + needs: ['deploy_production'] + before_script: + - echo "Sending notification to slack" + variables: + ENV: "Production Cluster" + SERVICE_NAME: "production-hydra-publish" diff --git a/Dockerfile.new b/Dockerfile.new index 583af0638..16d541be6 100644 --- a/Dockerfile.new +++ b/Dockerfile.new @@ -2,7 +2,7 @@ FROM jelastic/jetty:9.4.49-openjdk-1.8.0_352 USER root -ENV JAVA_OPTS="-Xmx2G" +ENV JAVA_OPTS="-Xmx2g" ENV CONTAINER_HTTP_PORT="8088" @@ -12,4 +12,4 @@ EXPOSE 8088 COPY ps-publish/ /ps-publish -ENTRYPOINT ["/ps-publish/bin/hydra-ingest"] \ No newline at end of file +ENTRYPOINT ["/ps-publish/bin/hydra-ingest"] diff --git a/helm/eks-dev-values.yml b/helm/eks-dev-values.yml new file mode 100644 index 000000000..df0cdb763 --- /dev/null +++ b/helm/eks-dev-values.yml @@ -0,0 +1,61 @@ +image: + repository: harbor.vnerd.com/library/hydra + pullPolicy: Always + tag: '%IMAGE_TAG%' + +deployment: + apiVersion: "apps/v1" + updateStrategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + +replicas: 2 + +#Adding velero backup for k8s services +#velero.io/exclude-from-backup" = true +service: + type: ClusterIP + port: 8080 + annotations: {} + labels: {} + + +ingress: + enabled: true + ingressClassName: internal + annotations: {} + labels: {} + hosts: + - hydra-publish-dev.eplur-staging.vnerd.com + tls: + - hosts: + - hydra-publish-dev.eplur-staging.vnerd.com + path: / + pathType: Prefix + + +resources: + limits: + memory: 6Gi + requests: + cpu: 250m + memory: 1Gi + +tolerations: [] + +nodeSelector: {} + +config: + LOG_DIR: /var/log/hydra/ + LOG_LEVEL: INFO + HYDRA_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: b-1.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-3.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098,b-2.isdvsdevblueuswest2.8xkirx.c14.kafka.us-west-2.amazonaws.com:9098 + HYDRA_SCHEMA_REGISTRY_URL: https://dvs-dev-schema-registry.eplur-staging.vnerd.com:8081 + HYDRA_V2_METADATA_CONTACT: '#dev-data-platform' + HYDRA_REPLICATION_FACTOR: 1 + HYDRA_V2_METADATA_CONSUMER_GROUP: 'v2MetadataConsumer' + HYDRA_V2_CREATE_TOPICS_ENABLED: 'true' + HYDRA_V2_METADATA_CREATE_ON_STARTUP: 'true' + HYDRA_MIN_INSYNC_REPLICAS: 1 + + diff --git a/pillar_overrides.yaml b/pillar_overrides.yaml new file mode 100644 index 000000000..222fd8018 --- /dev/null +++ b/pillar_overrides.yaml @@ -0,0 +1,9 @@ +systemd-app: + artifacts: + hydra-publish: + gitlab: True + artifactory_packaging : tgz + artifactory_artifact_version: latest + artifactory_artifact_path: hydra/publish + artifactory_repository: bounded-context + artifactory_artifact_name: hydra-publish