Feature/morphic study dataset support

.gitlab-ci.yml

@@ -12,7 +12,7 @@ Test:
   script:
     - ./gradlew verify
   artifacts:
+    when: always
     reports:
       junit:
         - build/test-results/**/TEST-*.xml
-

CI/.morphic-gitlab-ci.yml

@@ -0,0 +1,118 @@
+cache:
+  paths:
+    - .gradle/wrapper
+    - .gradle/caches
+
+before_script:
+  - echo "Performing build..."
+
+stages:
+  - build
+  - test
+  - package
+  - deploy
+
+variables:
+  AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
+  AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
+  AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
+  AWS_ACCOUNT_ID: $AWS_ACCOUNT_ID
+  EKS_CLUSTER_NAME: $EKS_CLUSTER_NAME
+  AWS_ACCESS_KEY_ID_PROD: $AWS_ACCESS_KEY_ID_PROD
+  AWS_SECRET_ACCESS_KEY_PROD: $AWS_SECRET_ACCESS_KEY_PROD
+  AWS_ACCOUNT_ID_PROD: $AWS_ACCOUNT_ID_PROD
+  EKS_CLUSTER_NAME_PROD: $EKS_CLUSTER_NAME_PROD
+
+build:
+  image: quay.io/ebi-ait/ingest-base-images:openjdk_11
+  stage: build
+  script:
+    - ./gradlew build -x test --info --stacktrace
+  artifacts:
+    paths:
+      - build/libs/*.jar
+
+test:
+  image: quay.io/ebi-ait/ingest-base-images:openjdk_11
+  stage: test
+  script:
+    - ./gradlew test
+  allow_failure: true
+  variables:
+    GRADLE_OPTS: "-Dorg.gradle.daemon=false"
+
+docker-build-dev:
+  stage: package
+  image: docker:19.03.12
+  services:
+    - docker:19.03.12-dind
+  before_script:
+    - apk add --update --no-cache python3 py3-pip
+    - pip3 install awscli
+  script:
+    - docker build -t ingest-core-morphic:$CI_COMMIT_SHA .
+    - aws ecr get-login-password --region $AWS_DEFAULT_REGION |
+      docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+    - docker tag ingest-core-morphic:$CI_COMMIT_SHA $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/ingest-core-morphic:$CI_COMMIT_SHA
+    - docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/ingest-core-morphic:$CI_COMMIT_SHA
+
+docker-build-prod:
+  stage: package
+  image: docker:19.03.12
+  services:
+    - docker:19.03.12-dind
+  variables:
+    AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID_PROD
+    AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY_PROD
+    AWS_ACCOUNT_ID: $AWS_ACCOUNT_ID_PROD
+    EKS_CLUSTER_NAME: $EKS_CLUSTER_NAME_PROD
+  before_script:
+    - export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_PROD
+    - export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_PROD
+    - export AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID_PROD
+    - apk add --update --no-cache python3 py3-pip
+    - pip3 install awscli
+  script:
+    - docker build -t ingest-core-morphic:$CI_COMMIT_SHA .
+    - aws ecr get-login-password --region $AWS_DEFAULT_REGION |
+      docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+    - docker tag ingest-core-morphic:$CI_COMMIT_SHA $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/ingest-core-morphic:$CI_COMMIT_SHA
+    - docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/ingest-core-morphic:$CI_COMMIT_SHA
+
+deploy-dev:
+  stage: deploy
+  image: alpine/k8s:1.27.13
+  before_script:
+    - apk add --update --no-cache curl jq python3 py3-pip
+    - python3 -m venv /opt/venv
+    - source /opt/venv/bin/activate
+    - pip install awscli
+    - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+    - install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+    - export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
+    - export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
+    - export AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION
+    - aws eks update-kubeconfig --name $EKS_CLUSTER_NAME --region $AWS_DEFAULT_REGION
+  script:
+    - sed -i "s|{{IMAGE}}|$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/ingest-core-morphic:$CI_COMMIT_SHA|g" CI/ingest-core-morphic-deployment-dev.yaml
+    - kubectl apply -f CI/ingest-core-morphic-deployment-dev.yaml
+    - kubectl apply -f CI/ingest-core-morphic-service.yaml -n morphic-dev
+
+deploy-prod:
+  stage: deploy
+  image: alpine/k8s:1.27.13
+  before_script:
+    - apk add --update --no-cache curl jq python3 py3-pip
+    - python3 -m venv /opt/venv
+    - source /opt/venv/bin/activate
+    - pip install awscli
+    - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+    - install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+    - export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_PROD
+    - export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_PROD
+    - export AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION
+    - aws eks update-kubeconfig --name $EKS_CLUSTER_NAME_PROD --region $AWS_DEFAULT_REGION
+  script:
+    - sed -i "s|{{IMAGE}}|$AWS_ACCOUNT_ID_PROD.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/ingest-core-morphic:$CI_COMMIT_SHA|g" CI/ingest-core-morphic-deployment-prod.yaml
+    - kubectl apply -f CI/ingest-core-morphic-deployment-prod.yaml
+    - kubectl apply -f CI/ingest-core-morphic-service.yaml -n morphic-prod

CI/ingest-core-morphic-deployment-dev.yaml

@@ -0,0 +1,44 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ingest-core-deployment
+  namespace: morphic-dev
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ingest-core
+  template:
+    metadata:
+      labels:
+        app: ingest-core
+    spec:
+      containers:
+        - name: ingest-core-container
+          image: {{IMAGE}}
+          ports:
+            - containerPort: 8080
+          env:
+            - name: SPRING_PROFILES_ACTIVE
+              value: local
+            - name: MONGO_URI
+              value: mongodb+srv://morphic-dev:mDev12345@morphicdevcluster.jduoa8p.mongodb.net/MorphicDev?retryWrites=true&w=majority
+            - name: RABBIT_HOST
+              value: rabbitmq-service
+            - name: RABBIT_PORT
+              value: "5672"
+            - name: AWS_COGNITO_DOMAIN
+              value: "https://morphic-dev-2.auth.eu-west-2.amazoncognito.com/oauth2"
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: aws-credentials
+                  key: AWS_ACCESS_KEY_ID
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: aws-credentials
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: AWS_REGION
+              value: eu-west-2  # Optionally specify AWS region as an environment variable
+          imagePullPolicy: Always

CI/ingest-core-morphic-deployment-prod.yaml

@@ -0,0 +1,44 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ingest-core-deployment
+  namespace: morphic-prod
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: ingest-core
+  template:
+    metadata:
+      labels:
+        app: ingest-core
+    spec:
+      containers:
+        - name: ingest-core-container
+          image: {{IMAGE}}
+          ports:
+            - containerPort: 8080
+          env:
+            - name: SPRING_PROFILES_ACTIVE
+              value: prod
+            - name: MONGO_URI
+              value: mongodb+srv://morphic-prod:morphicProd#12345@morphicprodcluster.gvr5bkw.mongodb.net/MorphicProd?retryWrites=true&w=majority
+            - name: RABBIT_HOST
+              value: rabbitmq-service
+            - name: RABBIT_PORT
+              value: "5672"
+            - name: AWS_COGNITO_DOMAIN
+              value: "https://morphic.auth.eu-west-2.amazoncognito.com/oauth2"
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: aws-credentials
+                  key: AWS_ACCESS_KEY_ID
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: aws-credentials
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: AWS_REGION
+              value: eu-west-2  # Optionally specify AWS region as an environment variable
+          imagePullPolicy: Always

CI/ingest-core-morphic-service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: ingest-core-service
+spec:
+  selector:
+    app: ingest-core
+  ports:
+    - name: http-ingest-core
+      protocol: TCP
+      port: 8080  # HTTP port
+      targetPort: 8080  # Should match the containerPort in the Deployment YAML
+    - name: https-ingest-core
+      protocol: TCP
+      port: 443  # HTTPS port
+      targetPort: 8080  # Should match the containerPort in the Deployment YAML
+  type: LoadBalancer

Dockerfile

@@ -19,6 +19,7 @@ ENV USR_AUTH_AUDIENCE=https://dev.data.humancellatlas.org/
 ENV GCP_JWK_PROVIDER_BASE_URL=https://www.googleapis.com/service_accounts/v1/jwk/
 ENV GCP_PROJECT_WHITELIST=hca-dcp-production.iam.gserviceaccount.com,human-cell-atlas-travis-test.iam.gserviceaccount.com,broad-dsde-mint-dev.iam.gserviceaccount.com,broad-dsde-mint-test.iam.gserviceaccount.com,broad-dsde-mint-staging.iam.gserviceaccount.com
 ENV SCHEMA_BASE_URI=https://schema.dev.data.humancellatlas.org/
+ENV AWS_COGNITO_DOMAIN=https://morphic-dev.auth.eu-west-2.amazoncognito.com/oauth2
 
 ADD gradle ./gradle
 ADD src ./src

build.gradle

@@ -3,9 +3,11 @@ import org.gradle.api.tasks.testing.logging.TestExceptionFormat
 plugins {
     id 'org.springframework.boot' version '2.1.6.RELEASE'
     id 'java'
+    id "com.diffplug.spotless" version "6.20.0"
 }
 
 apply plugin: 'io.spring.dependency-management'
+apply plugin: 'com.diffplug.spotless'
 
 group = 'com.example'
 version = '0.0.1-SNAPSHOT'
@@ -64,6 +66,14 @@ tasks.withType(Test) {
     }
 }
 
+spotless {
+    java {
+        googleJavaFormat()
+        removeUnusedImports()
+        importOrder 'java', 'javax', 'org', 'com'
+    }
+}
+
 dependencies {
     ['actuator', 'data-rest', 'hateoas', 'security', 'web', 'webflux'].forEach {
         implementation "org.springframework.boot:spring-boot-starter-${it}"
@@ -77,6 +87,7 @@ dependencies {
         exclude group: 'org.mongodb', module: 'mongodb-driver'
     }
     implementation 'org.springframework.data:spring-data-rest-hal-browser'
+    implementation 'org.springframework.boot:spring-boot-starter-mail'
 
     implementation 'org.springframework.retry:spring-retry:1.3.1'
     implementation 'org.springframework:spring-aspects:5.3.13'
@@ -102,11 +113,15 @@ dependencies {
         exclude group: 'org.mongodb', module: 'mongo-java-driver'
     }
 
-    compile 'org.projectreactor:reactor-spring:1.0.1.RELEASE'
-    compile 'org.projectlombok:lombok'
+    implementation 'com.amazonaws:aws-java-sdk-s3:1.12.705'
+    /*implementation 'software.amazon.awssdk:sts:2.17.19'*/
+
+    implementation 'org.projectreactor:reactor-spring:1.0.1.RELEASE'
+    compileOnly 'org.projectlombok:lombok'
 
     annotationProcessor 'org.projectlombok:lombok'
 
+    testCompileOnly 'org.projectlombok:lombok'
     testImplementation 'org.springframework.security:spring-security-test'
     testImplementation('org.springframework.boot:spring-boot-starter-test') {
         exclude group: 'org.assertj', module: 'assertj-core'