diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 22e408c4c..e3d7fed6c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,7 +12,7 @@ Test: script: - ./gradlew verify artifacts: + when: always reports: junit: - build/test-results/**/TEST-*.xml - diff --git a/CI/.morphic-gitlab-ci.yml b/CI/.morphic-gitlab-ci.yml new file mode 100644 index 000000000..dfb17cdbe --- /dev/null +++ b/CI/.morphic-gitlab-ci.yml @@ -0,0 +1,118 @@ +cache: + paths: + - .gradle/wrapper + - .gradle/caches + +before_script: + - echo "Performing build..." + +stages: + - build + - test + - package + - deploy + +variables: + AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION + AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY + AWS_ACCOUNT_ID: $AWS_ACCOUNT_ID + EKS_CLUSTER_NAME: $EKS_CLUSTER_NAME + AWS_ACCESS_KEY_ID_PROD: $AWS_ACCESS_KEY_ID_PROD + AWS_SECRET_ACCESS_KEY_PROD: $AWS_SECRET_ACCESS_KEY_PROD + AWS_ACCOUNT_ID_PROD: $AWS_ACCOUNT_ID_PROD + EKS_CLUSTER_NAME_PROD: $EKS_CLUSTER_NAME_PROD + +build: + image: quay.io/ebi-ait/ingest-base-images:openjdk_11 + stage: build + script: + - ./gradlew build -x test --info --stacktrace + artifacts: + paths: + - build/libs/*.jar + +test: + image: quay.io/ebi-ait/ingest-base-images:openjdk_11 + stage: test + script: + - ./gradlew test + allow_failure: true + variables: + GRADLE_OPTS: "-Dorg.gradle.daemon=false" + +docker-build-dev: + stage: package + image: docker:19.03.12 + services: + - docker:19.03.12-dind + before_script: + - apk add --update --no-cache python3 py3-pip + - pip3 install awscli + script: + - docker build -t ingest-core-morphic:$CI_COMMIT_SHA . + - aws ecr get-login-password --region $AWS_DEFAULT_REGION | + docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com + - docker tag ingest-core-morphic:$CI_COMMIT_SHA $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/ingest-core-morphic:$CI_COMMIT_SHA + - docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/ingest-core-morphic:$CI_COMMIT_SHA + +docker-build-prod: + stage: package + image: docker:19.03.12 + services: + - docker:19.03.12-dind + variables: + AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID_PROD + AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY_PROD + AWS_ACCOUNT_ID: $AWS_ACCOUNT_ID_PROD + EKS_CLUSTER_NAME: $EKS_CLUSTER_NAME_PROD + before_script: + - export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_PROD + - export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_PROD + - export AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID_PROD + - apk add --update --no-cache python3 py3-pip + - pip3 install awscli + script: + - docker build -t ingest-core-morphic:$CI_COMMIT_SHA . + - aws ecr get-login-password --region $AWS_DEFAULT_REGION | + docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com + - docker tag ingest-core-morphic:$CI_COMMIT_SHA $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/ingest-core-morphic:$CI_COMMIT_SHA + - docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/ingest-core-morphic:$CI_COMMIT_SHA + +deploy-dev: + stage: deploy + image: alpine/k8s:1.27.13 + before_script: + - apk add --update --no-cache curl jq python3 py3-pip + - python3 -m venv /opt/venv + - source /opt/venv/bin/activate + - pip install awscli + - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + - install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + - export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID + - export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY + - export AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION + - aws eks update-kubeconfig --name $EKS_CLUSTER_NAME --region $AWS_DEFAULT_REGION + script: + - sed -i "s|{{IMAGE}}|$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/ingest-core-morphic:$CI_COMMIT_SHA|g" CI/ingest-core-morphic-deployment-dev.yaml + - kubectl apply -f CI/ingest-core-morphic-deployment-dev.yaml + - kubectl apply -f CI/ingest-core-morphic-service.yaml -n morphic-dev + +deploy-prod: + stage: deploy + image: alpine/k8s:1.27.13 + before_script: + - apk add --update --no-cache curl jq python3 py3-pip + - python3 -m venv /opt/venv + - source /opt/venv/bin/activate + - pip install awscli + - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + - install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + - export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_PROD + - export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_PROD + - export AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION + - aws eks update-kubeconfig --name $EKS_CLUSTER_NAME_PROD --region $AWS_DEFAULT_REGION + script: + - sed -i "s|{{IMAGE}}|$AWS_ACCOUNT_ID_PROD.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/ingest-core-morphic:$CI_COMMIT_SHA|g" CI/ingest-core-morphic-deployment-prod.yaml + - kubectl apply -f CI/ingest-core-morphic-deployment-prod.yaml + - kubectl apply -f CI/ingest-core-morphic-service.yaml -n morphic-prod diff --git a/CI/ingest-core-morphic-deployment-dev.yaml b/CI/ingest-core-morphic-deployment-dev.yaml new file mode 100644 index 000000000..174883f07 --- /dev/null +++ b/CI/ingest-core-morphic-deployment-dev.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ingest-core-deployment + namespace: morphic-dev +spec: + replicas: 1 + selector: + matchLabels: + app: ingest-core + template: + metadata: + labels: + app: ingest-core + spec: + containers: + - name: ingest-core-container + image: {{IMAGE}} + ports: + - containerPort: 8080 + env: + - name: SPRING_PROFILES_ACTIVE + value: local + - name: MONGO_URI + value: mongodb+srv://morphic-dev:mDev12345@morphicdevcluster.jduoa8p.mongodb.net/MorphicDev?retryWrites=true&w=majority + - name: RABBIT_HOST + value: rabbitmq-service + - name: RABBIT_PORT + value: "5672" + - name: AWS_COGNITO_DOMAIN + value: "https://morphic-dev-2.auth.eu-west-2.amazoncognito.com/oauth2" + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-credentials + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-credentials + key: AWS_SECRET_ACCESS_KEY + - name: AWS_REGION + value: eu-west-2 # Optionally specify AWS region as an environment variable + imagePullPolicy: Always diff --git a/CI/ingest-core-morphic-deployment-prod.yaml b/CI/ingest-core-morphic-deployment-prod.yaml new file mode 100644 index 000000000..55516c14e --- /dev/null +++ b/CI/ingest-core-morphic-deployment-prod.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ingest-core-deployment + namespace: morphic-prod +spec: + replicas: 2 + selector: + matchLabels: + app: ingest-core + template: + metadata: + labels: + app: ingest-core + spec: + containers: + - name: ingest-core-container + image: {{IMAGE}} + ports: + - containerPort: 8080 + env: + - name: SPRING_PROFILES_ACTIVE + value: prod + - name: MONGO_URI + value: mongodb+srv://morphic-prod:morphicProd#12345@morphicprodcluster.gvr5bkw.mongodb.net/MorphicProd?retryWrites=true&w=majority + - name: RABBIT_HOST + value: rabbitmq-service + - name: RABBIT_PORT + value: "5672" + - name: AWS_COGNITO_DOMAIN + value: "https://morphic.auth.eu-west-2.amazoncognito.com/oauth2" + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-credentials + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-credentials + key: AWS_SECRET_ACCESS_KEY + - name: AWS_REGION + value: eu-west-2 # Optionally specify AWS region as an environment variable + imagePullPolicy: Always diff --git a/CI/ingest-core-morphic-service.yaml b/CI/ingest-core-morphic-service.yaml new file mode 100644 index 000000000..98e6324a3 --- /dev/null +++ b/CI/ingest-core-morphic-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: ingest-core-service +spec: + selector: + app: ingest-core + ports: + - name: http-ingest-core + protocol: TCP + port: 8080 # HTTP port + targetPort: 8080 # Should match the containerPort in the Deployment YAML + - name: https-ingest-core + protocol: TCP + port: 443 # HTTPS port + targetPort: 8080 # Should match the containerPort in the Deployment YAML + type: LoadBalancer diff --git a/Dockerfile b/Dockerfile index 123729ceb..3b2248718 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,7 @@ ENV USR_AUTH_AUDIENCE=https://dev.data.humancellatlas.org/ ENV GCP_JWK_PROVIDER_BASE_URL=https://www.googleapis.com/service_accounts/v1/jwk/ ENV GCP_PROJECT_WHITELIST=hca-dcp-production.iam.gserviceaccount.com,human-cell-atlas-travis-test.iam.gserviceaccount.com,broad-dsde-mint-dev.iam.gserviceaccount.com,broad-dsde-mint-test.iam.gserviceaccount.com,broad-dsde-mint-staging.iam.gserviceaccount.com ENV SCHEMA_BASE_URI=https://schema.dev.data.humancellatlas.org/ +ENV AWS_COGNITO_DOMAIN=https://morphic-dev.auth.eu-west-2.amazoncognito.com/oauth2 ADD gradle ./gradle ADD src ./src diff --git a/build.gradle b/build.gradle index 14e5a4b8e..bb832e879 100644 --- a/build.gradle +++ b/build.gradle @@ -3,9 +3,11 @@ import org.gradle.api.tasks.testing.logging.TestExceptionFormat plugins { id 'org.springframework.boot' version '2.1.6.RELEASE' id 'java' + id "com.diffplug.spotless" version "6.20.0" } apply plugin: 'io.spring.dependency-management' +apply plugin: 'com.diffplug.spotless' group = 'com.example' version = '0.0.1-SNAPSHOT' @@ -64,6 +66,14 @@ tasks.withType(Test) { } } +spotless { + java { + googleJavaFormat() + removeUnusedImports() + importOrder 'java', 'javax', 'org', 'com' + } +} + dependencies { ['actuator', 'data-rest', 'hateoas', 'security', 'web', 'webflux'].forEach { implementation "org.springframework.boot:spring-boot-starter-${it}" @@ -77,6 +87,7 @@ dependencies { exclude group: 'org.mongodb', module: 'mongodb-driver' } implementation 'org.springframework.data:spring-data-rest-hal-browser' + implementation 'org.springframework.boot:spring-boot-starter-mail' implementation 'org.springframework.retry:spring-retry:1.3.1' implementation 'org.springframework:spring-aspects:5.3.13' @@ -102,11 +113,15 @@ dependencies { exclude group: 'org.mongodb', module: 'mongo-java-driver' } - compile 'org.projectreactor:reactor-spring:1.0.1.RELEASE' - compile 'org.projectlombok:lombok' + implementation 'com.amazonaws:aws-java-sdk-s3:1.12.705' + /*implementation 'software.amazon.awssdk:sts:2.17.19'*/ + + implementation 'org.projectreactor:reactor-spring:1.0.1.RELEASE' + compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' + testCompileOnly 'org.projectlombok:lombok' testImplementation 'org.springframework.security:spring-security-test' testImplementation('org.springframework.boot:spring-boot-starter-test') { exclude group: 'org.assertj', module: 'assertj-core' diff --git a/docs/managed-access.md b/docs/managed-access.md index 1533d234a..2c5356590 100644 --- a/docs/managed-access.md +++ b/docs/managed-access.md @@ -3,6 +3,12 @@ ## Links - [ticket 967](https://app.zenhub.com/workspaces/dcp-ingest-product-development-5f71ca62a3cb47326bdc1b5c/issues/gh/ebi-ait/dcp-ingest-central/967) +## Terms + +* ACL - Access Control List +* DAC - Data Access Committee + +## API Access Control Flow ```mermaid sequenceDiagram @@ -10,28 +16,117 @@ sequenceDiagram participant api participant authorization_service participant operation_service + participant audit_service autonumber client ->> api: api operation note over authorization_service: new service, query
snapshot of DAC - api ->> authorization_service: operation allowed for
user and document? - - authorization_service -->> api: true - + par new + api ->> authorization_service: operation allowed for
user and document? + authorization_service -->> api: true + api ->> audit_service: record operation + end api ->> operation_service: perform operation ``` +## API Access Control - Details +There are 2 options here: +1. return all records from DB & filter the output to keep the allowed records +2. instrument the query and add a criteria to return only allowed records + +For 1st iteration, we'll go with option 1, which is easier to implement, but might +perform worse for large collections. + +```mermaid +sequenceDiagram + title 1. filter DB result + participant client + participant Controller + participant Repository + participant RowLevelSecurityAspect + participant DB + participant Authentication + participant MetadataDocument + + autonumber + + client ->> Controller: api resource + note over Controller: GET /files + Controller ->> Repository: findAll() + note over Repository: anotated with @RowLevelSecurity + Repository ->> DB: execute query + DB -->> RowLevelSecurityAspect: db result (collection) + note over RowLevelSecurityAspect: implement as an AOP Advice
to the Repository + RowLevelSecurityAspect ->> Authentication: get user roles + RowLevelSecurityAspect ->> MetadataDocument: get project uuid + RowLevelSecurityAspect ->> RowLevelSecurityAspect: filter, keep allowed + RowLevelSecurityAspect -->> Repository: filtered result + Repository -->> Controller: filtered result + Controller -->> client: api response +``` + +## ACL Update Flow ```mermaid sequenceDiagram - participant authorization_update_service - participant dac - - note left of dac: periodic update job
interface not clear yet - authorization_update_service ->> dac: snapshot access lists - authorization_update_service ->> authorization_update_service: update records + participant contributor + participant DACO + participant authorization_update_service + + par application for access + contributor ->> DACO: apply for upload
permissions to dataset + note left of DACO: name, email, project + DACO ->> DACO: assess, compare
to signed docs,
approve + end + par get ACL data into ingest + note right of DACO: periodic updates
interface not clear yet,
initially email + DACO ->> authorization_update_service: update ACL for project + authorization_update_service ->> ingest_db: update records + note over ingest_db: update roles list,
ACL audit table + end ``` +## Representing ACLs in ingest + +There are 2 options to consider. We will proceed with option 1. +1. Store a list of allowed datasets for each user, in addition to the + wrangler or contributor roles +```mermaid +classDiagram + direction LR + class User { + username + } + class Role { + name + } + User --> "1..*" Role : roles + note for Role "Wrangler, Contrbutor, Dataset A, Dataset B" +``` +2. Store a list of allowed users for each dataset. + +```mermaid +classDiagram + direction LR + + class Project { + content + } + class User { + name + roles + } + Project --> "1..*" User : allowed_users +``` + +```mermaid +graph RL + + A:::someclass --> B + classDef someclass fill:#f96; + linkStyle default fill:none,stroke-width:3px,stroke:red + +``` diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2d3811096..5b500a503 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip + diff --git a/src/integration/java/org/humancellatlas/ingest/AuditEntryTest.java b/src/integration/java/org/humancellatlas/ingest/AuditEntryTest.java deleted file mode 100644 index 8d5c443d7..000000000 --- a/src/integration/java/org/humancellatlas/ingest/AuditEntryTest.java +++ /dev/null @@ -1,97 +0,0 @@ -package org.humancellatlas.ingest; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.humancellatlas.ingest.audit.AuditEntry; -import org.humancellatlas.ingest.audit.AuditEntryRepository; -import org.humancellatlas.ingest.audit.AuditEntryService; -import org.humancellatlas.ingest.audit.AuditType; -import org.humancellatlas.ingest.bundle.BundleManifestRepository; -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.humancellatlas.ingest.core.service.MetadataCrudService; -import org.humancellatlas.ingest.core.service.MetadataUpdateService; -import org.humancellatlas.ingest.project.*; -import org.humancellatlas.ingest.schemas.SchemaService; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.security.test.context.support.WithMockUser; - - -import static org.assertj.core.api.Assertions.assertThat; - -@SpringBootTest -public class AuditEntryTest { - @Autowired - private ProjectService projectService; - - @Autowired - private AuditEntryService auditEntryService; - - @Autowired - private MongoTemplate mongoTemplate; - - @Autowired - private AuditEntryRepository auditEntryRepository; - - @MockBean - private SubmissionEnvelopeRepository submissionEnvelopeRepository; - - @MockBean - private ProjectRepository projectRepository; - - @MockBean - private MetadataCrudService metadataCrudService; - - @MockBean - private MetadataUpdateService metadataUpdateService; - - @MockBean - private SchemaService schemaService; - - @MockBean - private BundleManifestRepository bundleManifestRepository; - - @MockBean - private ProjectEventHandler projectEventHandler; - - @MockBean - MigrationConfiguration migrationConfiguration; - - - @Test - @WithMockUser(value = "test_user") - void testAuditEntryGenerationOnProjectStateUpdate() { - //given - WranglingState initialWranglingState = WranglingState.NEW; - Project project = new Project("{\"name\": \"Project 1\"}"); - project.setWranglingState(initialWranglingState); - this.mongoTemplate.save(project); - - // when - WranglingState updatedWranglingState = WranglingState.ELIGIBLE; - ObjectNode patchUpdate = new ObjectMapper() - .createObjectNode() - .put("wranglingState", updatedWranglingState.getValue()); - projectService.update(project, patchUpdate, false); - - // then - AuditEntry actual = projectService.getProjectAuditEntries(project).get(0); - - assertThat(actual) - .hasFieldOrPropertyWithValue("auditType", AuditType.STATUS_UPDATED) - .hasFieldOrPropertyWithValue("before", initialWranglingState.name()) - .hasFieldOrPropertyWithValue("after", updatedWranglingState.name()) - .returns(true, e->e.getUser().contains("Username: test_user;")); - } - - @AfterEach - private void tearDown() { - this.mongoTemplate.dropCollection(Project.class); - this.mongoTemplate.dropCollection(AuditEntry.class); - } -} diff --git a/src/integration/java/org/humancellatlas/ingest/IngestCoreApplicationTests.java b/src/integration/java/org/humancellatlas/ingest/IngestCoreApplicationTests.java deleted file mode 100644 index 62c080084..000000000 --- a/src/integration/java/org/humancellatlas/ingest/IngestCoreApplicationTests.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.humancellatlas.ingest; - -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -@SpringBootTest -public class IngestCoreApplicationTests { - @MockBean - MigrationConfiguration migrationConfiguration; - - @Test - public void contextLoads() { - } - -} diff --git a/src/integration/java/org/humancellatlas/ingest/MongoAuditingTest.java b/src/integration/java/org/humancellatlas/ingest/MongoAuditingTest.java deleted file mode 100644 index 8e710d92b..000000000 --- a/src/integration/java/org/humancellatlas/ingest/MongoAuditingTest.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.humancellatlas.ingest; - -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.security.test.context.support.WithMockUser; - -import java.util.HashMap; - -import static org.assertj.core.api.Assertions.assertThat; - -@SpringBootTest -public class MongoAuditingTest { - - @MockBean - private MigrationConfiguration migrationConfiguration; - - @Autowired - private ProjectRepository projectRepository; - - @Test - @WithMockUser("johndoe") - public void auditMongoRecord() { - //given: - Project project = new Project(new HashMap<>()); - - //when: - Project persistentProject = projectRepository.save(project); - - //then: - /* NOTE there doesn't seem to be a clean and easy way to check this without updating the UserAuditing class - itself to assume that the default principal type contains username and password. */ - assertThat(persistentProject.getUser()).contains("johndoe"); - } - -} diff --git a/src/integration/java/org/humancellatlas/ingest/archiving/web/ArchiveJobControllerTest.java b/src/integration/java/org/humancellatlas/ingest/archiving/web/ArchiveJobControllerTest.java deleted file mode 100644 index b806f860b..000000000 --- a/src/integration/java/org/humancellatlas/ingest/archiving/web/ArchiveJobControllerTest.java +++ /dev/null @@ -1,138 +0,0 @@ -package org.humancellatlas.ingest.archiving.web; - -import org.humancellatlas.ingest.archiving.entity.ArchiveJob; -import org.humancellatlas.ingest.archiving.entity.ArchiveJobRepository; -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.http.MediaType; -import org.springframework.security.test.context.support.WithMockUser; -import org.springframework.test.web.servlet.MockMvc; - -import java.time.Instant; -import java.util.*; - -import static org.hamcrest.Matchers.is; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.BDDMockito.given; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; -import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; - -@SpringBootTest -@AutoConfigureDataMongo() -@AutoConfigureMockMvc -public class ArchiveJobControllerTest { - - @Autowired - MockMvc mockMvc; - - @MockBean - private MigrationConfiguration migrationConfiguration; - - @MockBean - private ArchiveJobRepository archiveJobRepository; - - private static final ArchiveJob.ArchiveJobStatus PENDING_STATUS = ArchiveJob.ArchiveJobStatus.PENDING; - private static final ArchiveJob.ArchiveJobStatus COMPLETED_STATUS = ArchiveJob.ArchiveJobStatus.COMPLETED; - - private UUID uuid; - - private static final String ARCHIVE_JOB_ID = "1"; - private static final String SUBMISSION_UUID = "1234"; - - - @Test - public void when_request_archive_job_creation_returns_successful_response() throws Exception { - final ArchiveJob anArchiveJobById = createAnArchiveJob(ARCHIVE_JOB_ID, SUBMISSION_UUID, PENDING_STATUS); - - given(this.archiveJobRepository.save(any())) - .willReturn(anArchiveJobById); - - this.mockMvc.perform(post("/archiveJobs") - .contentType(MediaType.APPLICATION_JSON) - .content(String.format("{\"submissionUuid\": \"%s\"}", this.uuid))) - .andDo(print()) - .andExpect(status().isCreated()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON)) - .andExpect(jsonPath("$.submissionUuid", is(SUBMISSION_UUID))) - .andExpect(jsonPath("$.overallStatus", is(PENDING_STATUS.toString()))) - .andExpect(jsonPath("$.createdDate").isNotEmpty()) - .andReturn(); - } - - @Test - @WithMockUser - public void when_requesting_non_existing_archiving_job_returns_not_found_response() throws Exception { - given(this.archiveJobRepository.findById(ARCHIVE_JOB_ID)) - .willReturn(Optional.empty()); - - this.mockMvc.perform(get("/archiveJobs/{id}", ARCHIVE_JOB_ID) - .contentType(MediaType.APPLICATION_JSON)) - .andDo(print()) - .andExpect(status().isNotFound()) - .andReturn(); - } - - @Test - @WithMockUser - public void when_existing_archiving_job_in_pending_status_returns_valid_response() throws Exception { - final ArchiveJob anArchiveJob = createAnArchiveJob(ARCHIVE_JOB_ID, SUBMISSION_UUID, PENDING_STATUS); - - given(this.archiveJobRepository.findById(ARCHIVE_JOB_ID)) - .willReturn(Optional.of(anArchiveJob)); - - this.mockMvc.perform(get("/archiveJobs/{id}", ARCHIVE_JOB_ID) - .contentType(MediaType.APPLICATION_JSON)) - .andDo(print()) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.submissionUuid", is(SUBMISSION_UUID))) - .andExpect(jsonPath("$.overallStatus", is(PENDING_STATUS.toString()))) - .andExpect(jsonPath("$.createdDate").isNotEmpty()) - .andExpect(jsonPath("$.resultsFromArchives").doesNotExist()) - .andReturn(); - } - - @Test - @WithMockUser - public void when_existing_archiving_job_in_completed_status_returns_valid_response() throws Exception { - final ArchiveJob anArchiveJob = createAnArchiveJob(ARCHIVE_JOB_ID, SUBMISSION_UUID, COMPLETED_STATUS); - setArchiveResult(anArchiveJob); - - given(this.archiveJobRepository.findById(ARCHIVE_JOB_ID)) - .willReturn(Optional.of(anArchiveJob)); - - this.mockMvc.perform(get("/archiveJobs/{id}", ARCHIVE_JOB_ID) - .contentType(MediaType.APPLICATION_JSON)) - .andDo(print()) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.submissionUuid", is(SUBMISSION_UUID))) - .andExpect(jsonPath("$.overallStatus", is(COMPLETED_STATUS.toString()))) - .andExpect(jsonPath("$.createdDate").isNotEmpty()) - .andExpect(jsonPath("$.resultsFromArchives").isNotEmpty()) - .andReturn(); - } - - private ArchiveJob createAnArchiveJob(String id, String submissionUuid, ArchiveJob.ArchiveJobStatus status) { - ArchiveJob archiveJob = new ArchiveJob(); - archiveJob.setId(ARCHIVE_JOB_ID); - archiveJob.setCreatedDate(Instant.now()); - archiveJob.setOverallStatus(status); - archiveJob.setSubmissionUuid(submissionUuid); - - return archiveJob; - } - - private void setArchiveResult(ArchiveJob anArchiveJob) { - Map resultByArchive = new HashMap<>(); - Map>> experimentsResult = new HashMap<>(); - experimentsResult.put("experiments", List.of(Map.of("accession", "1234"), Map.of("uuid", "1-2-3-4"))); - resultByArchive.put("hca_assays", experimentsResult); - anArchiveJob.setResultsFromArchives(resultByArchive); - } -} diff --git a/src/integration/java/org/humancellatlas/ingest/biomaterial/BiomaterialControllerTest.java b/src/integration/java/org/humancellatlas/ingest/biomaterial/BiomaterialControllerTest.java deleted file mode 100644 index 329ed412b..000000000 --- a/src/integration/java/org/humancellatlas/ingest/biomaterial/BiomaterialControllerTest.java +++ /dev/null @@ -1,308 +0,0 @@ -package org.humancellatlas.ingest.biomaterial; - -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.core.service.ValidationStateChangeService; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.state.SubmissionState; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.http.MediaType; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.web.servlet.support.ServletUriComponentsBuilder; -import org.springframework.web.util.UriComponentsBuilder; - -import java.util.Arrays; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -@SpringBootTest -@AutoConfigureDataMongo() -@AutoConfigureMockMvc(printOnlyOnFailure = false) -public class BiomaterialControllerTest { - - @MockBean - ValidationStateChangeService validationStateChangeService; - - @Autowired - private MockMvc webApp; - - @Autowired - private ProcessRepository processRepository; - - @Autowired - private BiomaterialRepository biomaterialRepository; - - @Autowired - private SubmissionEnvelopeRepository submissionEnvelopeRepository; - - @Autowired - private ProjectRepository projectRepository; - - @MockBean - private MigrationConfiguration migrationConfiguration; - - @MockBean - private MessageRouter messageRouter; - - Process process1; - - Process process2; - - Process process3; - - Biomaterial biomaterial; - - UriComponentsBuilder uriBuilder; - - SubmissionEnvelope submissionEnvelope; - - Project project; - - @BeforeEach - void setUp() { - submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.setUuid(Uuid.newUuid()); - submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALID); - submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); - - project = new Project(null); - project.getSubmissionEnvelopes().add(submissionEnvelope); - project = projectRepository.save(project); - - process1 = processRepository.save(new Process(null)); - process2 = processRepository.save(new Process(null)); - process3 = processRepository.save(new Process(null)); - - biomaterial = new Biomaterial(); - biomaterial.setSubmissionEnvelope(submissionEnvelope); - biomaterial = biomaterialRepository.save(biomaterial); - - uriBuilder = ServletUriComponentsBuilder.fromCurrentContextPath(); - } - - @AfterEach - void tearDown() { - processRepository.deleteAll(); - biomaterialRepository.deleteAll(); - submissionEnvelopeRepository.deleteAll(); - projectRepository.deleteAll(); - } - - @Test - public void newBiomaterialInSubmissionLinksToSubmissionAndProject() throws Exception { - //given - biomaterialRepository.deleteAll(); - processRepository.deleteAll(); - - // when - webApp.perform( - post("/submissionEnvelopes/{id}/biomaterials", submissionEnvelope.getId()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"content\": {}}") - ).andExpect(status().isAccepted()); - - //then - assertThat(biomaterialRepository.findAll()).hasSize(1); - assertThat(biomaterialRepository.findAllBySubmissionEnvelope(submissionEnvelope)).hasSize(1); - assertThat(biomaterialRepository.findByProject(project)).hasSize(1); - - var newBiomaterial = biomaterialRepository.findAll().get(0); - assertThat(newBiomaterial.getSubmissionEnvelope().getId()).isEqualTo(submissionEnvelope.getId()); - assertThat(newBiomaterial.getProject().getId()).isEqualTo(project.getId()); - assertThat(newBiomaterial.getProjects()).hasSize(1); - assertThat(newBiomaterial.getProjects().stream().findFirst().get().getId()).isEqualTo(project.getId()); - } - - @Test - public void newBiomaterialInSubmissionDoesNotFailIfSubmissionHasNoProject() throws Exception { - //given - biomaterialRepository.deleteAll(); - processRepository.deleteAll(); - projectRepository.deleteAll(); - - // when - webApp.perform( - post("/submissionEnvelopes/{id}/biomaterials", submissionEnvelope.getId()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"content\": {}}") - ).andExpect(status().isAccepted()); - - //then - assertThat(biomaterialRepository.findAll()).hasSize(1); - assertThat(biomaterialRepository.findAllBySubmissionEnvelope(submissionEnvelope)).hasSize(1); - - var newBiomaterial = biomaterialRepository.findAll().get(0); - assertThat(newBiomaterial.getSubmissionEnvelope().getId()).isEqualTo(submissionEnvelope.getId()); - assertThat(newBiomaterial.getProject()).isNull(); - assertThat(newBiomaterial.getProjects()).isEmpty(); - } - - @Test - public void testLinkBiomaterialAsInputToProcessesUsingPostMethodWithManyProcessesInPayload() throws Exception { - // when - webApp.perform(post("/biomaterials/{id}/inputToProcesses/", biomaterial.getId()) - .contentType("text/uri-list") - .content(uriBuilder.build().toUriString() + "/processes/" + process1.getId() - + '\n' + uriBuilder.build().toUriString() + "/processes/" + process2.getId())) - .andExpect(status().isOk()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(biomaterial); - Biomaterial updatedBiomaterial = biomaterialRepository.findById(biomaterial.getId()).get(); - assertThat(updatedBiomaterial.getInputToProcesses()) - .usingElementComparatorOnFields("id") - .containsExactly(process1, process2); - } - - @Test - public void testLinkBiomaterialAsInputToProcessesUsingPutMethodWithManyProcessesInPayload() throws Exception { - // given - biomaterial.addAsInputToProcess(process1); - biomaterialRepository.save(biomaterial); - - // when - webApp.perform(put("/biomaterials/{id}/inputToProcesses/", biomaterial.getId()) - .contentType("text/uri-list") - .content(uriBuilder.build().toUriString() + "/processes/" + process2.getId() - + '\n' + uriBuilder.build().toUriString() + "/processes/" + process3.getId())) - .andExpect(status().isOk()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(biomaterial); - Biomaterial updatedBiomaterial = biomaterialRepository.findById(biomaterial.getId()).get(); - assertThat(updatedBiomaterial.getInputToProcesses()) - .usingElementComparatorOnFields("id") - .containsExactly(process2, process3); - } - - - @Test - public void testLinkBiomaterialAsInputToProcessesUsingPostMethodWithOneProcessInPayload() throws Exception { - //when - webApp.perform(post("/biomaterials/{id}/inputToProcesses/", biomaterial.getId()) - .contentType("text/uri-list") - .content(uriBuilder.build().toUriString() + "/processes/" + process1.getId())) - .andExpect(status().isOk()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(biomaterial); - Biomaterial updatedBiomaterial = biomaterialRepository.findById(biomaterial.getId()).get(); - assertThat(updatedBiomaterial.getInputToProcesses()) - .usingElementComparatorOnFields("id") - .containsExactly(process1); - } - - @Test - public void testLinkBiomaterialAsDerivedByProcessesUsingPostMethodWithManyProcessesInPayload() throws Exception { - // when - webApp.perform(post("/biomaterials/{id}/derivedByProcesses/", biomaterial.getId()) - .contentType("text/uri-list") - .content(uriBuilder.build().toUriString() + "/processes/" + process1.getId() - + '\n' + uriBuilder.build().toUriString() + "/processes/" + process2.getId())) - .andExpect(status().isOk()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(biomaterial); - Biomaterial updatedBiomaterial = biomaterialRepository.findById(biomaterial.getId()).get(); - assertThat(updatedBiomaterial.getDerivedByProcesses()) - .usingElementComparatorOnFields("id") - .containsExactly(process1, process2); - } - - @Test - public void testLinkBiomaterialAsDerivedByProcessesUsingPutMethodWithManyProcessesInPayload() throws Exception { - // given - biomaterial.addAsDerivedByProcess(process1); - biomaterialRepository.save(biomaterial); - - // when - webApp.perform(put("/biomaterials/{id}/derivedByProcesses/", biomaterial.getId()) - .contentType("text/uri-list") - .content(uriBuilder.build().toUriString() + "/processes/" + process2.getId() - + '\n' + uriBuilder.build().toUriString() + "/processes/" + process3.getId())) - .andExpect(status().isOk()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(biomaterial); - Biomaterial updatedBiomaterial = biomaterialRepository.findById(biomaterial.getId()).get(); - assertThat(updatedBiomaterial.getDerivedByProcesses()) - .usingElementComparatorOnFields("id") - .containsExactly(process2, process3); - } - - @Test - public void testLinkBiomaterialAsDerivedByProcessesUsingPostMethodWithOneProcessInPayload() throws Exception { - // when - webApp.perform(post("/biomaterials/{id}/derivedByProcesses/", biomaterial.getId()) - .contentType("text/uri-list") - .content(uriBuilder.build().toUriString() + "/processes/" + process1.getId())) - .andExpect(status().isOk()); - - verifyThatValidationStateChangedToDraftWhenGraphValid(biomaterial); - - // then - Biomaterial updatedBiomaterial = biomaterialRepository.findById(biomaterial.getId()).get(); - assertThat(updatedBiomaterial.getDerivedByProcesses()) - .usingElementComparatorOnFields("id") - .containsExactly(process1); - } - - @Test - public void testUnlinkBiomaterialAsInputToProcesses() throws Exception { - // given - biomaterial.addAsInputToProcess(process1); - biomaterialRepository.save(biomaterial); - - // when - webApp.perform(delete("/biomaterials/{id}/inputToProcesses/{processId}", biomaterial.getId(), process1.getId())) - .andExpect(status().isNoContent()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(biomaterial); - - Biomaterial updatedBiomaterial = biomaterialRepository.findById(biomaterial.getId()).get(); - assertThat(updatedBiomaterial.getInputToProcesses()).doesNotContain(process1); - } - - @Test - public void testUnlinkBiomaterialAsDerivedByProcesses() throws Exception { - // given - biomaterial.addAsDerivedByProcess(process1); - biomaterialRepository.save(biomaterial); - - // when - webApp.perform(delete("/biomaterials/{id}/derivedByProcesses/{processId}", biomaterial.getId(), process1.getId())) - .andExpect(status().isNoContent()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(biomaterial); - Biomaterial updatedBiomaterial = biomaterialRepository.findById(biomaterial.getId()).get(); - assertThat(updatedBiomaterial.getDerivedByProcesses()).doesNotContain(process1); - } - - private void verifyThatValidationStateChangedToDraftWhenGraphValid(MetadataDocument... values) { - Arrays.stream(values).forEach( - value -> verify(validationStateChangeService, times(1)) - .changeValidationState(value.getType(), value.getId(), ValidationState.DRAFT) - ); - } -} \ No newline at end of file diff --git a/src/integration/java/org/humancellatlas/ingest/export/job/ExportJobControllerTest.java b/src/integration/java/org/humancellatlas/ingest/export/job/ExportJobControllerTest.java deleted file mode 100644 index 6f087e7cc..000000000 --- a/src/integration/java/org/humancellatlas/ingest/export/job/ExportJobControllerTest.java +++ /dev/null @@ -1,167 +0,0 @@ -package org.humancellatlas.ingest.export.job; - -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.export.destination.ExportDestination; -import org.humancellatlas.ingest.exporter.Exporter; -import org.humancellatlas.ingest.state.SubmissionState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.json.simple.JSONObject; -import org.junit.Ignore; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.web.servlet.MockMvc; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.humancellatlas.ingest.export.destination.ExportDestinationName.DCP; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; -import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -@SpringBootTest -@AutoConfigureDataMongo() -@AutoConfigureMockMvc() -public class ExportJobControllerTest { - public static final String STARTED = "STARTED"; - public static final String COMPLETE = "COMPLETE"; - @Autowired - private MockMvc webApp; - - @Autowired - private SubmissionEnvelopeRepository submissionEnvelopeRepository; - - @Autowired - private ExportJobRepository exportJobRepository; - - @Autowired - private ExportJobService exportJobService; - - @MockBean - private Exporter exporter; - - // Adding MigrationConfiguration as a MockBean is needed as otherwise MigrationConfiguration won't be initialised. - @MockBean - private MigrationConfiguration migrationConfiguration; - - SubmissionEnvelope submissionEnvelope; - - ExportJob exportJob; - - Uuid projectUuid; - - @BeforeEach - void setUp() { - submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.setUuid(Uuid.newUuid()); - submissionEnvelope.enactStateTransition(SubmissionState.EXPORTING); - submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); - - projectUuid = Uuid.newUuid(); - var destinationContext = new JSONObject(); - destinationContext.put("projectUuid", projectUuid); - - var exportJobContext = new JSONObject(); - exportJobContext.put("dataFileTransfer", false); - exportJob = ExportJob.builder() - .id("export-job-id") - .submission(submissionEnvelope) - .destination(new ExportDestination(DCP, "v2", destinationContext)) - .context(exportJobContext) - .build(); - exportJob = exportJobRepository.save(exportJob); - } - - @AfterEach - void tearDown() { - exportJobRepository.deleteAll(); - submissionEnvelopeRepository.deleteAll(); - reset(exporter); - } - - @Test - void testDataTransferCallbackOnlyProgressesOnComplete() throws Exception { - webApp.perform( - // when - patch("/exportJobs/{id}/context", exportJob.getId()) - .contentType(APPLICATION_JSON_VALUE) - .content("{\"dataFileTransfer\": \"" + STARTED + "\"}") - ) // then - .andExpect(status().isAccepted()); - verify(exporter, never()).generateSpreadsheet(any(ExportJob.class)); - - var savedJob = exportJobRepository.findById(exportJob.getId()).orElseThrow(); - assertThat(savedJob.getContext().get("dataFileTransfer")).isEqualTo(STARTED); - } - - @Ignore - void testDataTransferCallbackEndpoint() throws Exception { - webApp.perform( - // when - patch("/exportJobs/{id}/context", exportJob.getId()) - .contentType(APPLICATION_JSON_VALUE) - .content("{\"dataFileTransfer\": \"" + COMPLETE + "\"}") - ) // then - .andExpect(status().isAccepted()); - var argumentCaptor = ArgumentCaptor.forClass(ExportJob.class); - verify(exporter).generateSpreadsheet(argumentCaptor.capture()); - - var capturedArgument = argumentCaptor.getValue(); - assertThat(capturedArgument.getId()).isEqualTo(exportJob.getId()); - assertThat(capturedArgument.getDestination().getContext().get("projectUuid")).isEqualTo(projectUuid); - assertThat(capturedArgument.getContext().get("dataFileTransfer")).isEqualTo(COMPLETE); - } - - @Test - void testSpreadsheetGenerationCallbackOnlyProgressesOnComplete() throws Exception { - // given - exportJob.getContext().put("dataFileTransfer", COMPLETE); - exportJob = exportJobRepository.save(exportJob); - - webApp.perform( - // when - patch("/exportJobs/{id}/context", exportJob.getId()) - .contentType(APPLICATION_JSON_VALUE) - .content("{\"spreadsheetGeneration\": \"" + STARTED + "\"}") - ) // then - .andExpect(status().isAccepted()); - verify(exporter, never()).exportMetadata(any(ExportJob.class)); - - var savedJob = exportJobRepository.findById(exportJob.getId()).orElseThrow(); - assertThat(savedJob.getContext().get("dataFileTransfer")).isEqualTo(COMPLETE); - assertThat(savedJob.getContext().get("spreadsheetGeneration")).isEqualTo(STARTED); - } - - @Ignore - void testSpreadsheetGenerationCallbackEndpoint() throws Exception { - // given - exportJob.getContext().put("dataFileTransfer", COMPLETE); - exportJob.getContext().put("spreadsheetGeneration", STARTED); - exportJob = exportJobRepository.save(exportJob); - - webApp.perform( - // when - patch("/exportJobs/{id}/context", exportJob.getId()) - .contentType(APPLICATION_JSON_VALUE) - .content("{\"spreadsheetGeneration\": \"" + COMPLETE + "\"}") - ) // then - .andExpect(status().isAccepted()); - var argumentCaptor = ArgumentCaptor.forClass(ExportJob.class); - verify(exporter).exportMetadata(argumentCaptor.capture()); - - var capturedArgument = argumentCaptor.getValue(); - assertThat(capturedArgument.getId()).isEqualTo(exportJob.getId()); - assertThat(capturedArgument.getDestination().getContext().get("projectUuid")).isEqualTo(projectUuid); - assertThat(capturedArgument.getContext().get("dataFileTransfer")).isEqualTo(COMPLETE); - assertThat(capturedArgument.getContext().get("spreadsheetGeneration")).isEqualTo(COMPLETE); - } -} diff --git a/src/integration/java/org/humancellatlas/ingest/file/FileControllerTest.java b/src/integration/java/org/humancellatlas/ingest/file/FileControllerTest.java deleted file mode 100644 index 40c519f66..000000000 --- a/src/integration/java/org/humancellatlas/ingest/file/FileControllerTest.java +++ /dev/null @@ -1,384 +0,0 @@ -package org.humancellatlas.ingest.file; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.core.service.ValidationStateChangeService; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.state.SubmissionState; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.http.MediaType; -import org.springframework.mock.web.MockHttpServletResponse; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.MvcResult; -import org.springframework.web.servlet.support.ServletUriComponentsBuilder; -import org.springframework.web.util.UriComponentsBuilder; - -import java.util.Arrays; -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; - -@SpringBootTest -@AutoConfigureDataMongo() -@AutoConfigureMockMvc(printOnlyOnFailure = false) -public class FileControllerTest { - @MockBean - ValidationStateChangeService validationStateChangeService; - - @Autowired - private MockMvc webApp; - - @Autowired - private ProcessRepository processRepository; - - @Autowired - private SubmissionEnvelopeRepository submissionEnvelopeRepository; - - @Autowired - private FileRepository fileRepository; - - @Autowired - private ProjectRepository projectRepository; - - @Autowired - private ObjectMapper objectMapper; - - @MockBean - private MigrationConfiguration migrationConfiguration; - - @MockBean - private MessageRouter messageRouter; - - Process process1; - - Process process2; - - Process process3; - - File file; - - UriComponentsBuilder uriBuilder; - - SubmissionEnvelope submissionEnvelope; - - Project project; - - @BeforeEach - void setUp() { - submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.setUuid(Uuid.newUuid()); - submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALID); - submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); - - project = new Project(null); - project.setSubmissionEnvelope(submissionEnvelope); - project.getSubmissionEnvelopes().add(submissionEnvelope); - project = projectRepository.save(project); - - process1 = processRepository.save(new Process(null)); - process2 = processRepository.save(new Process(null)); - process3 = processRepository.save(new Process(null)); - - file = new File(null, "fileName"); - file.setSubmissionEnvelope(submissionEnvelope); - file = fileRepository.save(file); - - uriBuilder = ServletUriComponentsBuilder.fromCurrentContextPath(); - } - - @AfterEach - void tearDown() { - processRepository.deleteAll(); - fileRepository.deleteAll(); - submissionEnvelopeRepository.deleteAll(); - } - - @Test - public void newFileInSubmissionLinksToSubmissionAndProject() throws Exception { - //given - fileRepository.deleteAll(); - - // when - webApp.perform( - post("/submissionEnvelopes/{id}/files", submissionEnvelope.getId()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"content\": {}}") - ).andExpect(status().isAccepted()); - - //then - assertThat(fileRepository.findAll()).hasSize(1); - assertThat(fileRepository.findAllBySubmissionEnvelope(submissionEnvelope)).hasSize(1); - assertThat(fileRepository.findByProject(project)).hasSize(1); - - var newFile = fileRepository.findAll().get(0); - assertThat(newFile.getSubmissionEnvelope().getId()).isEqualTo(submissionEnvelope.getId()); - assertThat(newFile.getProject().getId()).isEqualTo(project.getId()); - } - - @Test - public void newFileInSubmissionDoesNotFailIfSubmissionHasNoProject() throws Exception { - //given - fileRepository.deleteAll(); - projectRepository.deleteAll(); - - // when - webApp.perform( - post("/submissionEnvelopes/{id}/files", submissionEnvelope.getId()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"content\": {}}") - ).andExpect(status().isAccepted()); - - //then - assertThat(fileRepository.findAll()).hasSize(1); - assertThat(fileRepository.findAllBySubmissionEnvelope(submissionEnvelope)).hasSize(1); - - var newFile = fileRepository.findAll().get(0); - assertThat(newFile.getSubmissionEnvelope().getId()).isEqualTo(submissionEnvelope.getId()); - assertThat(newFile.getProject()).isNull(); - } - - @Test - public void testLinkFileAsInputToProcessesUsingPutMethodWithManyProcessesInPayload() throws Exception { - file.addAsInputToProcess(process1); - fileRepository.save(file); - - webApp.perform(put("/files/{fileId}/inputToProcesses/", file.getId()) - .contentType("text/uri-list") - .content(uriBuilder.build().toUriString() + "/processes/" + process2.getId() + '\n' - + uriBuilder.build().toUriString() + "/processes/" + process3.getId())) - .andExpect(status().isOk()); - - verifyThatValidationStateChangedToDraftWhenGraphValid(file); - - File updatedFile = fileRepository.findById(file.getId()).get(); - assertThat(updatedFile.getInputToProcesses()) - .usingElementComparatorOnFields("id") - .containsExactly(process2, process3); - } - - @Test - public void testLinkFileAsInputToMultipleProcessesUsingPostMethodWithManyProcessesInPayload() throws Exception { - // when - webApp.perform(post("/files/{fileId}/inputToProcesses/", file.getId()) - .contentType("text/uri-list") - .content(uriBuilder.build().toUriString() + "/processes/" + process1.getId() - + '\n' + uriBuilder.build().toUriString() + "/processes/" + process2.getId())) - .andExpect(status().isOk()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(file); - File updatedFile = fileRepository.findById(file.getId()).get(); - assertThat(updatedFile.getInputToProcesses()) - .usingElementComparatorOnFields("id") - .containsExactly(process1, process2); - } - - @Test - public void testLinkFileAsInputToProcessesUsingPostMethodWithOneProcessInPayload() throws Exception { - // when - webApp.perform(post("/files/{fileId}/inputToProcesses/", file.getId()) - .contentType("text/uri-list") - .content(uriBuilder.build().toUriString() + "/processes/" + process1.getId())) - .andExpect(status().isOk()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(file); - File updatedFile = fileRepository.findById(file.getId()).get(); - assertThat(updatedFile.getInputToProcesses()) - .usingElementComparatorOnFields("id") - .containsExactly(process1); - } - - @Test - public void testLinkFileAsDerivedByProcessesUsingPostMethodWithOneProcessInPayload() throws Exception { - // when - webApp.perform(post("/files/{fileId}/derivedByProcesses/", file.getId()) - .contentType("text/uri-list") - .content(uriBuilder.build().toUriString() + "/processes/" + process1.getId())) - .andExpect(status().isOk()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(file); - File updatedFile = fileRepository.findById(file.getId()).get(); - assertThat(updatedFile.getDerivedByProcesses()) - .usingElementComparatorOnFields("id") - .contains(process1); - } - - @Test - public void testLinkFileAsDerivedByProcessesUsingPostMethodWithManyProcessesInPayload() throws Exception { - // when - webApp.perform(post("/files/{fileId}/derivedByProcesses/", file.getId()) - .contentType("text/uri-list") - .content(uriBuilder.build().toUriString() + "/processes/" + process1.getId() - + '\n' + uriBuilder.build().toUriString() + "/processes/" + process2.getId())) - .andExpect(status().isOk()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(file); - File updatedFile = fileRepository.findById(file.getId()).get(); - assertThat(updatedFile.getDerivedByProcesses()) - .usingElementComparatorOnFields("id") - .containsExactly(process1, process2); - } - - @Test - public void testLinkFileAsDerivedByProcessesUsingPutMethodWithManyProcessesInPayload() throws Exception { - // given - file.addAsDerivedByProcess(process1); - fileRepository.save(file); - - // when - webApp.perform(put("/files/{fileId}/derivedByProcesses/", file.getId()) - .contentType("text/uri-list") - .content(uriBuilder.build().toUriString() + "/processes/" + process2.getId() + '\n' - + uriBuilder.build().toUriString() + "/processes/" + process3.getId())) - .andExpect(status().isOk()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(file); - File updatedFile = fileRepository.findById(file.getId()).get(); - assertThat(updatedFile.getDerivedByProcesses()) - .usingElementComparatorOnFields("id") - .containsExactly(process2, process3); - } - - @Test - public void testUnlinkFileAsDerivedByProcesses() throws Exception { - // given - file.addAsDerivedByProcess(process1); - fileRepository.save(file); - - // when - webApp.perform(delete("/files/{fileId}/derivedByProcesses/{processId}", file.getId(), process1.getId())) - .andExpect(status().isNoContent()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(file); - File updatedFile = fileRepository.findById(file.getId()).get(); - assertThat(updatedFile.getDerivedByProcesses()).doesNotContain(process1); - } - - @Test - public void testUnlinkFileAsInputToProcesses() throws Exception { - // given - file.addAsInputToProcess(process1); - fileRepository.save(file); - - // when - webApp.perform(delete("/files/{fileId}/inputToProcesses/{processId}", file.getId(), process1.getId())) - .andExpect(status().isNoContent()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(file); - - File updatedFile = fileRepository.findById(file.getId()).get(); - assertThat(updatedFile.getInputToProcesses()).doesNotContain(process1); - } - - private void verifyThatValidationStateChangedToDraftWhenGraphValid(MetadataDocument... values) { - Arrays.stream(values).forEach(value -> { - verify(validationStateChangeService, times(1)).changeValidationState(value.getType(), value.getId(), ValidationState.DRAFT); - }); - } - - @Test - public void testValidationJobPatch() throws Exception { - //given: - File file = new File(null, "test"); - file.setSubmissionEnvelope(submissionEnvelope); - file = fileRepository.save(file); - - //when: - String patch = "{ \"validationJob\": { \"validationReport\": { \"validationState\": \"Valid\" }}}"; - - MvcResult result = webApp - .perform(patch("/files/{id}", file.getId()) - .contentType(APPLICATION_JSON_VALUE) - .content(patch)) - .andReturn(); - - //expect: - MockHttpServletResponse response = result.getResponse(); - assertThat(response.getContentType()).containsPattern("application/.*json.*"); - - //and: - file = fileRepository.findById(file.getId()).get(); - assertThat(file.getValidationJob().getValidationReport().getValidationState()).isEqualTo(ValidationState.VALID); - } - - @Test - public void when_new_File_ctor__pass() throws Exception { - String filePayload = objectMapper.writeValueAsString(new File()); - webApp.perform( - post("/submissionEnvelopes/{id}/files", submissionEnvelope.getId()) - .contentType(MediaType.APPLICATION_JSON) - .content(filePayload) - ).andExpect(status().isAccepted()); - } - - @Test - public void when_DataFileUuid_is_null__accepted_with_random() throws Exception { - ObjectNode patch = createPayloadhNoDataFileUuid(); - webApp.perform( - post("/submissionEnvelopes/{id}/files", submissionEnvelope.getId()) - .contentType(MediaType.APPLICATION_JSON) - .content(objectMapper.writeValueAsString(patch)) - ).andExpect(status().isAccepted()) - .andExpect(jsonPath("$.dataFileUuid").isNotEmpty()); - } - - @Test - public void when_payload_is_good__pass() throws Exception { - ObjectNode patch = createValidFilePayload(); - webApp.perform( - post("/submissionEnvelopes/{id}/files", submissionEnvelope.getId()) - .contentType(MediaType.APPLICATION_JSON) - .content(objectMapper.writeValueAsString(patch)) - ).andExpect(status().isAccepted()); - } - - - private ObjectNode createValidFilePayload() { - ObjectMapper mapper = new ObjectMapper(); - ObjectNode newFilePayload = mapper.createObjectNode(); - - newFilePayload - .put("dataFileUuid", UUID.randomUUID().toString()) - .put("fileName", "test-file") - .put("fileContentType", "text/plain"); - return newFilePayload; - } - - private ObjectNode createPayloadhNoDataFileUuid() { - ObjectMapper mapper = new ObjectMapper(); - ObjectNode newFilePayload = mapper.createObjectNode(); - - newFilePayload - .put("fileName", "test-file") - .put("fileContentType", "text/plain"); - return newFilePayload; - } -} \ No newline at end of file diff --git a/src/integration/java/org/humancellatlas/ingest/file/FileServiceTest.java b/src/integration/java/org/humancellatlas/ingest/file/FileServiceTest.java deleted file mode 100644 index 60f61b319..000000000 --- a/src/integration/java/org/humancellatlas/ingest/file/FileServiceTest.java +++ /dev/null @@ -1,169 +0,0 @@ -package org.humancellatlas.ingest.file; - -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.humancellatlas.ingest.core.Checksums; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.core.exception.CoreEntityNotFoundException; -import org.humancellatlas.ingest.core.service.MetadataCrudService; -import org.humancellatlas.ingest.core.service.MetadataUpdateService; -import org.humancellatlas.ingest.file.web.FileMessage; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.state.MetadataDocumentEventHandler; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.ApplicationContext; -import org.springframework.dao.OptimisticLockingFailureException; - -import java.util.ArrayList; -import java.util.UUID; -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatExceptionOfType; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; - - -@SpringBootTest -public class FileServiceTest { - @MockBean - MigrationConfiguration migrationConfiguration; - - @MockBean - FileRepository fileRepository; - - @MockBean - SubmissionEnvelopeRepository submissionEnvelopeRepository; - - @MockBean - BiomaterialRepository biomaterialRepository; - - @MockBean - ProcessRepository processRepository; - - @MockBean - ProjectRepository projectRepository; - - @MockBean - MetadataDocumentEventHandler metadataDocumentEventHandler; - - @MockBean - MetadataCrudService metadataCrudService; - - @MockBean - MetadataUpdateService metadataUpdateService; - - @Autowired - FileService fileService; - - @Autowired - private ApplicationContext applicationContext; - - FileMessage fileMessage; - - SubmissionEnvelope submissionEnvelope; - - File file; - - Project project; - - @BeforeEach - void setUp() { - applicationContext.getBeansWithAnnotation(MockBean.class).forEach(Mockito::reset); - - Checksums checksums = new Checksums("sha1", "sha256", "crc32c", "s3Etag"); - String submissionUuid = UUID.randomUUID().toString(); - String filename = "filename"; - fileMessage = new FileMessage("cloudUrl", filename, submissionUuid, "content_type", checksums, 123); - - submissionEnvelope = new SubmissionEnvelope(); - - project = spy(new Project(null)); - when(project.getId()).thenReturn("projectId"); - - file = new File(null, filename); - var files = new ArrayList(); - files.add(file); - - when(submissionEnvelopeRepository.findByUuid(any(Uuid.class))).thenReturn(submissionEnvelope); - when(projectRepository.findBySubmissionEnvelopesContains(submissionEnvelope)).thenReturn(Stream.of(project)); - when(fileRepository.findBySubmissionEnvelopeAndFileName(submissionEnvelope, fileMessage.getFileName())).thenReturn(files); - when(fileRepository.save(file)).thenReturn(file); - } - - @Test - public void testCreateFileFromFileMessage() throws CoreEntityNotFoundException { - //given: - when(fileRepository.findBySubmissionEnvelopeAndFileName(submissionEnvelope, fileMessage.getFileName())).thenReturn(new ArrayList<>()); - - //when: - fileService.createFileFromFileMessage(fileMessage); - - //then: - verify(metadataCrudService).addToSubmissionEnvelopeAndSave(any(File.class), any(SubmissionEnvelope.class)); - } - - @Test - public void testCreateFileFromFileMessageNotCreated() throws CoreEntityNotFoundException { - //when: - fileService.createFileFromFileMessage(fileMessage); - - //then: - verify(metadataCrudService, never()).addToSubmissionEnvelopeAndSave(any(File.class), any(SubmissionEnvelope.class)); - } - - - @Test - public void testUpdateFileFromFileMessage() throws CoreEntityNotFoundException { - //when: - fileService.updateFileFromFileMessage(fileMessage); - - //then: - verify(fileRepository).save(file); - assertThat(file.getCloudUrl()).isEqualTo(fileMessage.getCloudUrl()); - assertThat(file.getChecksums()).isEqualTo(fileMessage.getChecksums()); - assertThat(file.getFileContentType()).isEqualTo(fileMessage.getContentType()); - assertThat(file.getValidationState()).isEqualTo(ValidationState.DRAFT); - } - - @Test - public void testUpdateFileFromFileMessageRetry() throws CoreEntityNotFoundException { - //given: - when(fileRepository.save(file)) - .thenThrow(new OptimisticLockingFailureException("Error")) - .thenReturn(file); - - //when: - fileService.updateFileFromFileMessage(fileMessage); - - //then: - verify(fileRepository, times(2)).save(file); - } - - @Test - public void testUpdateFileFromFileMessageMaxRetries() throws CoreEntityNotFoundException { - //given: - when(fileRepository.save(file)) - .thenThrow(new OptimisticLockingFailureException("Error")); - - //when: - assertThatExceptionOfType(OptimisticLockingFailureException.class).isThrownBy(() -> { - fileService.updateFileFromFileMessage(fileMessage); - }); - - //then: - verify(fileRepository, times(5)).save(file); - } - -} - diff --git a/src/integration/java/org/humancellatlas/ingest/process/ProcessControllerTest.java b/src/integration/java/org/humancellatlas/ingest/process/ProcessControllerTest.java deleted file mode 100644 index 8aaacbc98..000000000 --- a/src/integration/java/org/humancellatlas/ingest/process/ProcessControllerTest.java +++ /dev/null @@ -1,254 +0,0 @@ -package org.humancellatlas.ingest.process; - -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.core.service.ValidationStateChangeService; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.humancellatlas.ingest.state.SubmissionState; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.http.MediaType; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.web.servlet.support.ServletUriComponentsBuilder; -import org.springframework.web.util.UriComponentsBuilder; - -import java.util.Arrays; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -@SpringBootTest -@AutoConfigureDataMongo() -@AutoConfigureMockMvc(printOnlyOnFailure = false) -class ProcessControllerTest { - @MockBean - ValidationStateChangeService validationStateChangeService; - - @MockBean - private MessageRouter messageRouter; - - @MockBean - private MigrationConfiguration migrationConfiguration; - - @Autowired - private MockMvc webApp; - - @Autowired - private ProcessRepository processRepository; - - @Autowired - private ProtocolRepository protocolRepository; - - @Autowired - private ProjectRepository projectRepository; - - @Autowired - private SubmissionEnvelopeRepository submissionEnvelopeRepository; - - Protocol protocol1; - - Protocol protocol2; - - Protocol protocol3; - - Project project; - - Process process; - - UriComponentsBuilder uriBuilder; - - SubmissionEnvelope submissionEnvelope; - - @BeforeEach - void setUp() { - submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.setUuid(Uuid.newUuid()); - submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALID); - submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); - - protocol1 = protocolRepository.save(new Protocol(null)); - protocol2 = protocolRepository.save(new Protocol(null)); - protocol3 = protocolRepository.save(new Protocol(null)); - - project = new Project(null); - project.setSubmissionEnvelope(submissionEnvelope); - project.getSubmissionEnvelopes().add(submissionEnvelope); - project = projectRepository.save(project); - - process = new Process(null); - process.setSubmissionEnvelope(submissionEnvelope); - process = processRepository.save(process); - - uriBuilder = ServletUriComponentsBuilder.fromCurrentContextPath(); - } - - @AfterEach - void tearDown() { - submissionEnvelopeRepository.deleteAll(); - processRepository.deleteAll(); - protocolRepository.deleteAll(); - projectRepository.deleteAll(); - } - - @Test - public void newProcessInSubmissionLinksToSubmissionAndProject() throws Exception { - //given - processRepository.deleteAll(); - - // when - webApp.perform( - post("/submissionEnvelopes/{id}/processes", submissionEnvelope.getId()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"content\": {}}") - ).andExpect(status().isAccepted()); - - //then - assertThat(processRepository.findAll()).hasSize(1); - assertThat(processRepository.findAllBySubmissionEnvelope(submissionEnvelope)).hasSize(1); - assertThat(processRepository.findByProject(project)).hasSize(1); - - var newProcess = processRepository.findAll().get(0); - assertThat(newProcess.getSubmissionEnvelope().getId()).isEqualTo(submissionEnvelope.getId()); - assertThat(newProcess.getProject().getId()).isEqualTo(project.getId()); - assertThat(newProcess.getProjects()).hasSize(1); - assertThat(newProcess.getProjects().stream().findFirst().get().getId()).isEqualTo(project.getId()); - } - - @Test - public void newProcessInSubmissionDoesNotFailIfSubmissionHasNoProject() throws Exception { - //given - processRepository.deleteAll(); - projectRepository.deleteAll(); - - // when - webApp.perform( - post("/submissionEnvelopes/{id}/processes", submissionEnvelope.getId()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"content\": {}}") - ).andExpect(status().isAccepted()); - - //then - assertThat(processRepository.findAll()).hasSize(1); - assertThat(processRepository.findAllBySubmissionEnvelope(submissionEnvelope)).hasSize(1); - - var newProcess = processRepository.findAll().get(0); - assertThat(newProcess.getSubmissionEnvelope().getId()).isEqualTo(submissionEnvelope.getId()); - assertThat(newProcess.getProject()).isNull(); - assertThat(newProcess.getProjects()).isEmpty(); - } - - @Test - public void testLinkProtocolsToProcessUsingPutMethodWithManyProtocolsInPayload() throws Exception { - // given - process.addProtocol(protocol1); - processRepository.save(process); - - // when - webApp.perform(put("/processes/{id}/protocols/", process.getId()) - .contentType("text/uri-list") - .content(uriBuilder.build().toUriString() + "/protocols/" + protocol2.getId() - + '\n' + uriBuilder.build().toUriString() + "/protocols/" + protocol3.getId())) - .andExpect(status().isOk()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(process); - Process updatedProcess = processRepository.findById(process.getId()).get(); - assertThat(updatedProcess.getProtocols()) - .usingElementComparatorOnFields("id") - .containsExactly(protocol2, protocol3); - } - - @Test - public void testLinkProtocolsToProcessUsingPostMethodWithManyProtocolsInPayload() throws Exception { - // when - webApp.perform(post("/processes/{id}/protocols/", process.getId()) - .contentType("text/uri-list") - .content(uriBuilder.build().toUriString() + "/protocols/" + protocol1.getId() - + '\n' + uriBuilder.build().toUriString() + "/protocols/" + protocol2.getId())) - .andExpect(status().isOk()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(process); - Process updatedProcess = processRepository.findById(process.getId()).get(); - assertThat(updatedProcess.getProtocols()) - .usingElementComparatorOnFields("id") - .containsExactly(protocol1, protocol2); - } - - @Test - public void testLinkProtocolsToProcessUsingPostMethodWithOneProtocolInPayload() throws Exception { - // when - webApp.perform(post("/processes/{processId}/protocols/", process.getId()) - .contentType("text/uri-list") - .content(uriBuilder.build().toUriString() + "/protocols/" + protocol1.getId())) - .andExpect(status().isOk()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(process); - Process updatedProcess = processRepository.findById(process.getId()).get(); - assertThat(updatedProcess.getProtocols()) - .usingElementComparatorOnFields("id") - .containsExactly(protocol1); - } - - @Test - public void testUnlinkProtocolFromProcess() throws Exception { - // given - process.addProtocol(protocol1); - processRepository.save(process); - - // when - webApp.perform(delete("/processes/{processId}/protocols/{protocolId}", process.getId(), protocol1.getId())) - .andExpect(status().isNoContent()); - - // then - verifyThatValidationStateChangedToDraftWhenGraphValid(process); - - Process updatedProcess = processRepository.findById(process.getId()).get(); - assertThat(updatedProcess.getProtocols()).doesNotContain(protocol1); - } - - @Test - public void testLinkProjectToProcessDoesNotChangeTheirValidationStatesToDraft() throws Exception { - webApp.perform(put("/processes/{processId}/project", process.getId()) - .contentType("text/uri-list") - .content(uriBuilder.build().toUriString() + "/projects/" + project.getId())) - .andExpect(status().isNoContent()); - - verify(validationStateChangeService, times(0)).changeValidationState(any(), any(), eq(ValidationState.DRAFT)); - } - - @Test - public void testUnlinkProjectFromProcessDoesNotChangeTheirValidationStatesToDraft() throws Exception { - webApp.perform(delete("/processes/{processId}/project/{projectId}", process.getId(), project.getId())) - .andExpect(status().isNoContent()); - - verify(validationStateChangeService, times(0)).changeValidationState(any(), any(), eq(ValidationState.DRAFT)); - } - - private void verifyThatValidationStateChangedToDraftWhenGraphValid(MetadataDocument... values) { - Arrays.stream(values).forEach( - value -> verify(validationStateChangeService, times(1)) - .changeValidationState(value.getType(), value.getId(), ValidationState.DRAFT) - ); - } -} \ No newline at end of file diff --git a/src/integration/java/org/humancellatlas/ingest/process/ProcessRepositoryTest.java b/src/integration/java/org/humancellatlas/ingest/process/ProcessRepositoryTest.java deleted file mode 100644 index 065a656e9..000000000 --- a/src/integration/java/org/humancellatlas/ingest/process/ProcessRepositoryTest.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.humancellatlas.ingest.process; - -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.data.mongo.DataMongoTest; -import org.springframework.boot.test.mock.mockito.MockBean; - -import java.util.Optional; - -import static java.util.Arrays.asList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assumptions.assumeThat; - -@DataMongoTest -public class ProcessRepositoryTest { - - @Autowired - private ProcessRepository processRepository; - - @Autowired - private ProtocolRepository protocolRepository; - - @MockBean - private MigrationConfiguration migrationConfiguration; - - @MockBean - private MessageRouter messageRouter; - - @AfterEach - private void tearDown() { - processRepository.deleteAll(); - protocolRepository.deleteAll(); - } - - @Test - public void findFirstByProtocolNonUnique() { - //given: - Protocol protocol = protocolRepository.save(new Protocol(null)); - - Process process1 = new Process(null); - process1.addProtocol(protocol); - process1 = processRepository.save(process1); - - Process process2 = new Process(null); - process2.addProtocol(protocol); - process2 = processRepository.save(process2); - - //and: - assumeThat(processRepository.findAll()).hasSize(2); - - //when: - Optional first = processRepository.findFirstByProtocolsContains(protocol); - - // then - assertThat(first.isPresent()).isTrue(); - assertThat(first.get().getId()).isIn(asList(process1.getId(), process2.getId())); - } -} diff --git a/src/integration/java/org/humancellatlas/ingest/project/ProjectFilterTest.java b/src/integration/java/org/humancellatlas/ingest/project/ProjectFilterTest.java deleted file mode 100644 index ccefc29be..000000000 --- a/src/integration/java/org/humancellatlas/ingest/project/ProjectFilterTest.java +++ /dev/null @@ -1,534 +0,0 @@ -package org.humancellatlas.ingest.project; - -import org.humancellatlas.ingest.audit.AuditEntryService; -import org.humancellatlas.ingest.bundle.BundleManifestRepository; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.core.service.MetadataCrudService; -import org.humancellatlas.ingest.core.service.MetadataUpdateService; -import org.humancellatlas.ingest.project.web.SearchFilter; -import org.humancellatlas.ingest.project.web.SearchType; -import org.humancellatlas.ingest.schemas.SchemaService; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.data.mongo.DataMongoTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Pageable; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.index.TextIndexDefinition; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; - -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; - -@DataMongoTest -class ProjectFilterTest { - - // class under test - private ProjectService projectService; - - // participants - @Autowired - private MongoTemplate mongoTemplate; - - @MockBean - private SubmissionEnvelopeRepository submissionEnvelopeRepository; - - @MockBean - private ProjectRepository projectRepository; - - @MockBean - private MetadataCrudService metadataCrudService; - - @MockBean - private MetadataUpdateService metadataUpdateService; - - @MockBean - private SchemaService schemaService; - - @MockBean - private BundleManifestRepository bundleManifestRepository; - - @MockBean - private ProjectEventHandler projectEventHandler; - - @MockBean - private AuditEntryService auditEntryService; - - private Project project1; - private Project project2; - private Project project3; - - Comparator upToMillies = Comparator.comparing(d -> d.truncatedTo(ChronoUnit.MILLIS)); - - @Test - void test_raw_criteria() { - Query query = new Query().addCriteria(Criteria.where("content.project_core.project_title").regex("project1", "i")); - Project actual = this.mongoTemplate.findOne(query, Project.class); - assertThat(actual) - .usingComparatorForFields(upToMillies, "contentLastModified") - .isEqualToComparingFieldByFieldRecursively(project1); - - } - - @Test - void test_criteria_building() { - SearchFilter searchFilter = SearchFilter.builder().wranglingState("NEW").build(); - Query query = ProjectQueryBuilder.buildProjectsQuery(searchFilter); - Project actual = this.mongoTemplate.find(query, Project.class).get(0); - assertThat(actual) - .usingComparatorForFields(upToMillies, "contentLastModified") - .isEqualToComparingFieldByFieldRecursively(project1); - } - - @Test - void test_criteria_building_with_pageable() { - SearchFilter searchFilter = SearchFilter.builder().search("project1").build(); - Query query = ProjectQueryBuilder.buildProjectsQuery(searchFilter); - Pageable pageable = PageRequest.of(0, 10); - Project actual = this.mongoTemplate.find(query.with(pageable), Project.class).get(0); - assertThat(actual) - .usingComparatorForFields(upToMillies, "contentLastModified") - .isEqualToComparingFieldByFieldRecursively(project1); - } - - @Test - void filter_by_state() { - Project project4 = makeProject("project4"); - project4.setWranglingState(WranglingState.IN_PROGRESS); - this.mongoTemplate.save(project4); - - //when - SearchFilter filterNew = SearchFilter.builder().wranglingState("NEW").build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(filterNew, pageable); - - // then - assertThat(result.getContent()) - .hasSize(3) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingRecursiveFieldByFieldElementComparator() - .containsExactlyInAnyOrder(project1, project2, project3); - } - - @Test - void filter_by_wrangler() { - // given - //when - SearchFilter searchFilter = SearchFilter.builder().primaryWrangler(this.project2.getPrimaryWrangler()).build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getTotalElements()).isEqualTo(1); - assertThat(result.getContent()) - .hasSize(1) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingRecursiveFieldByFieldElementComparator() - .containsExactly(this.project2); - } - - @Test - void filter_by_wrangling_priority() { - //given - Project project4 = makeProject("project4"); - project4.setWranglingPriority(3); - this.mongoTemplate.save(project4); - //when - SearchFilter searchFilter = SearchFilter.builder().wranglingPriority(this.project1.getWranglingPriority()).build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getTotalElements()).isEqualTo(3); - assertThat(result.getContent()) - .hasSize(3) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingRecursiveFieldByFieldElementComparator() - .containsExactlyInAnyOrder(project1, project2, project3); - } - - @Test - void filter_by_official_hca_publication() { - //given - Project project4 = makeProject("project4"); - var content = Map.of( - "project_core", Map.of("project_title", "Project 4"), - "publications", List.of(Map.of("official_hca_publication", true)) - ); - project4.setContent(content); - this.mongoTemplate.save(project4); - - //when - SearchFilter searchFilter = SearchFilter.builder().hasOfficialHcaPublication(true).build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getTotalElements()).isEqualTo(1); - assertThat(result.getContent()) - .hasSize(1) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingRecursiveFieldByFieldElementComparator() - .containsExactly(project4); - } - - @Test - void filter_by_identifying_organisms() { - //given - Project project4 = makeProject("project4"); - String human = "Human"; - project4.setIdentifyingOrganisms(List.of(human)); - this.mongoTemplate.save(project4); - - //when - SearchFilter searchFilter = SearchFilter.builder().identifyingOrganism(human).build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getTotalElements()).isEqualTo(1); - assertThat(result.getContent()) - .hasSize(1) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingRecursiveFieldByFieldElementComparator() - .containsExactly(project4); - } - - @Test - void filter_by_organ_ontology() { - //given - Project project4 = makeProject("project4"); - String ontologyTerm = "AN_ONTOLOGY"; - project4.setOrgan(Map.of("ontologies", List.of(Map.of("ontology", ontologyTerm)))); - this.mongoTemplate.save(project4); - //when - SearchFilter searchFilter = SearchFilter.builder().organOntology(ontologyTerm).build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getTotalElements()).isEqualTo(1); - assertThat(result.getContent()) - .hasSize(1) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingRecursiveFieldByFieldElementComparator() - .containsExactly(project4); - } - - @Test - void filter_by_cell_count() { - //given - Project project4 = makeProject("project4"); - project4.setCellCount(1000); - this.mongoTemplate.save(project4); - //when - SearchFilter searchFilter = SearchFilter.builder().maxCellCount(project1.getCellCount()).minCellCount(0).build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getTotalElements()).isEqualTo(3); - assertThat(result.getContent()) - .hasSize(3) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingRecursiveFieldByFieldElementComparator() - .containsExactlyInAnyOrder(project1, project2, project3); - } - - @Test - void filter_by_data_access() { - //given - Project project4 = makeProject("project4"); - project4.setDataAccess(Map.of("type", DataAccessTypes.OPEN.getLabel())); - this.mongoTemplate.save(project4); - //when - SearchFilter searchFilter = SearchFilter.builder().dataAccess(DataAccessTypes.OPEN).build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getTotalElements()).isEqualTo(1); - assertThat(result.getContent()) - .hasSize(1) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingRecursiveFieldByFieldElementComparator() - .containsExactly(project4); - } - - @Test - void filter_by_text() { - // given - //when - SearchFilter searchFilter = SearchFilter.builder().search("project1").build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getContent()) - .hasSize(1) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingElementComparatorIgnoringFields("supplementaryFiles", "submissionEnvelopes") - .containsExactly(project1); - } - - @Test - void query_all_keywords(){ - SearchFilter searchFilter = SearchFilter.builder().search("human liver").build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getContent()) - .hasSize(1) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingElementComparatorIgnoringFields("supplementaryFiles", "submissionEnvelopes") - .containsExactly(project1); - } - @Test - void query_all_keywords__order_independent(){ - SearchFilter searchFilter = SearchFilter.builder() - .search("liver human") - .searchType(SearchType.AllKeywords) - .build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getContent()) - .hasSize(1) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingElementComparatorIgnoringFields("supplementaryFiles", "submissionEnvelopes") - .containsExactly(project1); - } - @Test - void query_any_keywords(){ - SearchFilter searchFilter = SearchFilter.builder() - .search("liver mouse") - .searchType(SearchType.AnyKeyword) - .build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - Page resultFromReverse = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getContent()) - .hasSize(2) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingElementComparatorIgnoringFields("supplementaryFiles", "submissionEnvelopes") - .containsExactlyInAnyOrder(project1, project2); - } - @Test - void query_exact_phrase__correct_order(){ - SearchFilter searchFilter = SearchFilter.builder() - .search("human liver") - .searchType(SearchType.ExactMatch) - .build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - Page resultFromReverse = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getContent()) - .hasSize(1) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingElementComparatorIgnoringFields("supplementaryFiles", "submissionEnvelopes") - .containsExactly(project1); - } - - @Test - void query_exact_phrase__reverse_order(){ - SearchFilter searchFilter = SearchFilter.builder() - .search("liver human") - .searchType(SearchType.ExactMatch) - .build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getContent()) - .hasSize(0); - } - - @Test - void lookup_by_uuid() { - String uuidString = project1.getUuid().getUuid().toString(); - SearchFilter searchFilter = SearchFilter.builder() - .search(uuidString) - .build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getContent()) - .hasSize(1) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingElementComparatorIgnoringFields("supplementaryFiles", "submissionEnvelopes") - .containsExactly(project1); - } - - @Test - void filter_by_release() { - //given - Project project4 = makeProject("project4"); - project4.setDcpReleaseNumber(11); - this.mongoTemplate.save(project4); - //when - SearchFilter searchFilter = SearchFilter.builder().dcpReleaseNumber(11).build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getTotalElements()).isEqualTo(1); - assertThat(result.getContent()) - .hasSize(1) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingRecursiveFieldByFieldElementComparator() - .containsExactlyInAnyOrder(project4); - } - - @Test - void filter_by_project_labels() { - //given - Project project4 = makeProject("project4"); - project4.setProjectLabels(List.of("CellxGene")); - this.mongoTemplate.save(project4); - //when - SearchFilter searchFilter = SearchFilter.builder().projectLabels("CellxGene").build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getTotalElements()).isEqualTo(1); - assertThat(result.getContent()) - .hasSize(1) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingRecursiveFieldByFieldElementComparator() - .containsExactlyInAnyOrder(project4); - } - - @Test - void filter_by_project_networks() { - //given - Project project5 = makeProject("project5"); - project5.setProjectNetworks(List.of("Lung")); - this.mongoTemplate.save(project5); - //when - SearchFilter searchFilter = SearchFilter.builder().projectNetworks("Lung").build(); - - Pageable pageable = PageRequest.of(0, 10); - Page result = projectService.filterProjects(searchFilter, pageable); - - // then - assertThat(result.getTotalElements()).isEqualTo(1); - assertThat(result.getContent()) - .hasSize(1) - .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) - .usingRecursiveFieldByFieldElementComparator() - .containsExactlyInAnyOrder(project5); - } - - @Test - void all_args_constructor() { - new SearchFilter( - "a", - "b", - "c", - 1, - false, - "Human", - "AN_ONTOLOGY_TERM", - 0, - 10000, - 1, - DataAccessTypes.MANAGED, - "a label", - "a network", - false, - SearchType.AllKeywords - ); - } - - private static Project makeProject(String title) { - Map projectJson = ProjectJson.fromTitle(title).toMap(); - Project project = new Project(projectJson.get("content")); - project.setIsUpdate(false); - project.setPrimaryWrangler("wrangler_" + title); - project.setWranglingState(WranglingState.NEW); - project.setUuid(Uuid.newUuid()); - project.setCellCount(100); - project.setWranglingPriority(1); - project.setDcpReleaseNumber(1); - return project; - } - - @BeforeEach - private void setup() { - initProjectService(); - initTestData(); - } - - private void initTestData() { - this.project1 = makeProject("project1 human liver"); - this.project2 = makeProject("project2 mouse liver"); - this.project3 = makeProject("project3 lung human"); - Arrays.asList(project1, project2, project3).forEach(project -> { - this.mongoTemplate.save(project); - this.projectService.register(project); - }); - this.mongoTemplate.indexOps(Project.class).ensureIndex( - new TextIndexDefinition.TextIndexDefinitionBuilder() - .onField("content.project_core.project_title") - .build() - ); - assertThat(this.mongoTemplate.findAll(Project.class)).hasSize(3); - } - - private void initProjectService() { - this.projectService = new ProjectService( - mongoTemplate, - submissionEnvelopeRepository, - projectRepository, - metadataCrudService, - metadataUpdateService, - schemaService, - bundleManifestRepository, - auditEntryService, - projectEventHandler - ); - assertThat(this.mongoTemplate.findAll(Project.class)).hasSize(0); - } - - @AfterEach - private void tearDown() { - this.mongoTemplate.dropCollection(Project.class); - } -} diff --git a/src/integration/java/org/humancellatlas/ingest/project/ProjectJson.java b/src/integration/java/org/humancellatlas/ingest/project/ProjectJson.java deleted file mode 100644 index 0b017b694..000000000 --- a/src/integration/java/org/humancellatlas/ingest/project/ProjectJson.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.humancellatlas.ingest.project; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; - -import java.util.Map; - -public class ProjectJson { - String title; - - public static ProjectJson fromTitle(String title){ - ProjectJson project = new ProjectJson(); - project.title = title; - return project; - } - - public ObjectNode toObjectNode() { - ObjectMapper mapper = new ObjectMapper(); - ObjectNode content = mapper.createObjectNode(); - ObjectNode projectCore0 = content.putObject("project_core"); - projectCore0.put("project_title", this.title); - - ObjectNode metadata = mapper.createObjectNode(); - metadata.set("content", content); - - return metadata; - } - - public Map toMap() { - ObjectMapper mapper = new ObjectMapper(); - ObjectNode project = this.toObjectNode(); - return mapper.convertValue(project, new TypeReference>(){}); - } -} diff --git a/src/integration/java/org/humancellatlas/ingest/project/web/ProjectControllerTest.java b/src/integration/java/org/humancellatlas/ingest/project/web/ProjectControllerTest.java deleted file mode 100644 index 10703a64d..000000000 --- a/src/integration/java/org/humancellatlas/ingest/project/web/ProjectControllerTest.java +++ /dev/null @@ -1,251 +0,0 @@ -package org.humancellatlas.ingest.project.web; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.assertj.core.api.Assertions; -import org.assertj.core.data.MapEntry; -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectEventHandler; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.schemas.SchemaService; -import org.humancellatlas.ingest.state.ValidationState; -import org.jetbrains.annotations.NotNull; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import org.mockito.ArgumentCaptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.boot.test.mock.mockito.SpyBean; -import org.springframework.http.HttpStatus; -import org.springframework.mock.web.MockHttpServletResponse; -import org.springframework.security.test.context.support.WithMockUser; -import org.springframework.security.test.context.support.WithUserDetails; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.MvcResult; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Consumer; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.entry; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; -import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.handler; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -@SpringBootTest -@AutoConfigureMockMvc(printOnlyOnFailure = false) -class ProjectControllerTest { - - @Autowired - private MockMvc webApp; - - @Autowired - private ProjectRepository repository; - - @Autowired - private ObjectMapper objectMapper; - - @SpyBean - private ProjectEventHandler projectEventHandler; - - @MockBean - private MigrationConfiguration migrationConfiguration; - - @MockBean - private SchemaService schemaService; - - @AfterEach - private void tearDown() { - repository.deleteAll(); - } - - @Nested - class Update { - - @Test - void updateSuccess() throws Exception { - doTestUpdate("/projects/{id}", project -> { - var projectCaptor = ArgumentCaptor.forClass(Project.class); - verify(projectEventHandler).editedProjectMetadata(projectCaptor.capture()); - Project handledProject = projectCaptor.getValue(); - assertThat(handledProject.getId()).isEqualTo(project.getId()); - }); - } - - @Test - void partialUpdateSuccess() throws Exception { - doTestUpdate("/projects/{id}?partial=true", project -> { - verify(projectEventHandler, never()).editedProjectMetadata(any()); - }); - } - - private void doTestUpdate(String patchUrl, Consumer postCondition) throws Exception { - //given: - var content = Map.of( - "description", "test", - "attr2", "should be deleted after patch"); - Project originalProject = repository.save(new Project(content)); - - //when: - - Map patch = Map.of("description", "test updated"); - MvcResult result = webApp - .perform(patch(patchUrl, originalProject.getId()) - .contentType(APPLICATION_JSON_VALUE) - .content("{\"content\": " + objectMapper.writeValueAsString(patch) + "}")) - .andReturn(); - - //expect: - MockHttpServletResponse response = result.getResponse(); - assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); - assertThat(response.getContentType()).containsPattern("application/.*json.*"); - - //and: - Project updated = objectMapper.readValue(response.getContentAsString(), Project.class); - assertThat(updated.getContent()).isInstanceOf(Map.class); - MapEntry updatedDescription = entry("description", "test updated"); - assertThat((Map) updated.getContent()).containsOnly(updatedDescription); - - //and: - repository.findById(originalProject.getId()) - .ifPresentOrElse(project -> { - assertThat((Map) project.getContent()).containsOnly(updatedDescription); - postCondition.accept(project); - }, () -> Assertions.fail("project {} not found", originalProject.getId())); - - //and: - } - - @Test - void onlyUpdateAllowedFields() throws Exception { - //given: - var content = new HashMap(); - content.put("description", "test"); - Project project = new Project(content); - project = repository.save(project); - - //when: - content.put("description", "test updated"); - MvcResult result = webApp - .perform(patch("/projects/{id}", project.getId()) - .contentType(APPLICATION_JSON_VALUE) - .content("{\"content\": " + objectMapper.writeValueAsString(content) + ", \"validationState\": \"METADATA_VALID\"}")) - .andReturn(); - - //expect: - MockHttpServletResponse response = result.getResponse(); - assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); - assertThat(response.getContentType()).containsPattern("application/.*json.*"); - - //and: - //Using Map here because reading directly to Project converts the entire JSON to Project.content. - Map updated = objectMapper.readValue(response.getContentAsString(), Map.class); - assertThat(updated.get("content")).isInstanceOf(Map.class); - MapEntry updatedDescription = entry("description", "test updated"); - assertThat((Map) updated.get("content")).containsOnly(updatedDescription); - assertThat(updated.get("validationState")).isEqualTo("DRAFT"); - - //and: - project = repository.findById(project.getId()).get(); - assertThat((Map) project.getContent()).containsOnly(updatedDescription); - assertThat(project.getValidationState()).isEqualTo(ValidationState.DRAFT); - } - - } - - @Nested - @WithMockUser(roles = "WRNAGLER") - class Filter { - @BeforeEach - public void setup(){ - Project project = makeProject(); - repository.save(project); - } - - @NotNull - private Project makeProject() { - var content = new HashMap(); - content.put("description", "test kw1"); - Project project = new Project(content); - return project; - } - - @ParameterizedTest( - name = "[{index}] all values, some null: {arguments}" - ) - @CsvSource({ - "kw1,null,null,AllKeywords", - "kw1,null,null,AnyKeyword", - "kw1,null,null,UnsuppportedSearchType", - "kw1,null,null,null", - }) - @WithMockUser - public void allValuesSetSomeNull(String search, String wrangler, String wranglingState, String searchType) throws Exception { - //given: - var content = Map.of( - "search", search, - "wrangler", wrangler, - "wranglingState", wranglingState, - "searchType", searchType - ); - - webApp - .perform(get("/projects/filter") - .contentType(APPLICATION_JSON_VALUE) - .content(objectMapper.writeValueAsString(content))) - .andDo(print()) - .andExpect(handler().handlerType(ProjectController.class)) - .andExpect(status().isOk()); - } - - @ParameterizedTest( - name = "[{index}] nulls missing from filter payload: {arguments}" - ) - @CsvSource({ - "kw1,null,null,AllKeywords", - "kw1,null,null,AnyKeyword", - "kw1,null,null,Unsuppported", - "kw1,null,null,null", - "kw1,Amnon,null,null", - "null,null,NEW,null", - "null,null,Unsupported,null", - "null,null,null,null", - }) - @WithMockUser - public void nullsAreMissingFromPayload(String search, String wrangler, String wranglingState, String searchType) throws Exception { - var content = new HashMap(); - putIfNotNull(content, search, "search"); - putIfNotNull(content, wrangler, "wrangler"); - putIfNotNull(content, wranglingState, "wranglingState"); - putIfNotNull(content, searchType, "searchType"); - webApp - .perform(get("/projects/filter") - .contentType(APPLICATION_JSON_VALUE) - .content(objectMapper.writeValueAsString(content))) - .andDo(print()) - .andExpect(handler().handlerType(ProjectController.class)) - .andExpect(status().isOk()); - - } - - private void putIfNotNull(HashMap content, String value, String key) { - if(!"null".equals(value)){ - content.put(key, value); - } - } - } - -} diff --git a/src/integration/java/org/humancellatlas/ingest/protocol/ProtocolControllerTest.java b/src/integration/java/org/humancellatlas/ingest/protocol/ProtocolControllerTest.java deleted file mode 100644 index 26ba2116a..000000000 --- a/src/integration/java/org/humancellatlas/ingest/protocol/ProtocolControllerTest.java +++ /dev/null @@ -1,121 +0,0 @@ -package org.humancellatlas.ingest.protocol; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.state.SubmissionState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.http.MediaType; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.web.servlet.support.ServletUriComponentsBuilder; -import org.springframework.web.util.UriComponentsBuilder; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -@SpringBootTest -@AutoConfigureDataMongo() -@AutoConfigureMockMvc() -public class ProtocolControllerTest { - @Autowired - private MockMvc webApp; - - @Autowired - private SubmissionEnvelopeRepository submissionEnvelopeRepository; - - @Autowired - private ProjectRepository projectRepository; - - @Autowired - private ProtocolRepository protocolRepository; - - @Autowired - private ObjectMapper objectMapper; - - @MockBean - private MigrationConfiguration migrationConfiguration; - - @MockBean - private MessageRouter messageRouter; - - SubmissionEnvelope submissionEnvelope; - - Project project; - - UriComponentsBuilder uriBuilder; - - @BeforeEach - void setUp() { - submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.setUuid(Uuid.newUuid()); - submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALID); - submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); - - project = new Project(null); - project.setSubmissionEnvelope(submissionEnvelope); - project.getSubmissionEnvelopes().add(submissionEnvelope); - project = projectRepository.save(project); - - uriBuilder = ServletUriComponentsBuilder.fromCurrentContextPath(); - } - - @AfterEach - void tearDown() { - submissionEnvelopeRepository.deleteAll(); - projectRepository.deleteAll(); - protocolRepository.deleteAll(); - } - - @Test - public void newProtocolInSubmissionLinksToSubmissionAndProject() throws Exception { - // when - webApp.perform( - post("/submissionEnvelopes/{id}/protocols", submissionEnvelope.getId()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"content\": {}}") - ).andExpect(status().isAccepted()); - - //then - assertThat(protocolRepository.findAll()).hasSize(1); - assertThat(protocolRepository.findAllBySubmissionEnvelope(submissionEnvelope)).hasSize(1); - assertThat(protocolRepository.findByProject(project)).hasSize(1); - - var newProtocol = protocolRepository.findAll().get(0); - assertThat(newProtocol.getSubmissionEnvelope().getId()).isEqualTo(submissionEnvelope.getId()); - assertThat(newProtocol.getProject().getId()).isEqualTo(project.getId()); - } - - @Test - public void newProtocolInSubmissionDoesNotFailIfSubmissionHasNoProject() throws Exception { - //given - projectRepository.deleteAll(); - - // when - webApp.perform( - post("/submissionEnvelopes/{id}/protocols", submissionEnvelope.getId()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"content\": {}}") - ).andExpect(status().isAccepted()); - - //then - assertThat(protocolRepository.findAll()).hasSize(1); - assertThat(protocolRepository.findAllBySubmissionEnvelope(submissionEnvelope)).hasSize(1); - - var newProtocol = protocolRepository.findAll().get(0); - assertThat(newProtocol.getSubmissionEnvelope().getId()).isEqualTo(submissionEnvelope.getId()); - assertThat(newProtocol.getProject()).isNull(); - } -} diff --git a/src/integration/java/org/humancellatlas/ingest/schemas/SchemaScraperTest.java b/src/integration/java/org/humancellatlas/ingest/schemas/SchemaScraperTest.java deleted file mode 100644 index 147820256..000000000 --- a/src/integration/java/org/humancellatlas/ingest/schemas/SchemaScraperTest.java +++ /dev/null @@ -1,225 +0,0 @@ -package org.humancellatlas.ingest.schemas; - -import com.github.tomakehurst.wiremock.WireMockServer; -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.humancellatlas.ingest.schemas.schemascraper.SchemaScraper; -import org.humancellatlas.ingest.schemas.schemascraper.impl.S3BucketSchemaScraper; - -import org.humancellatlas.ingest.schemas.schemascraper.impl.SchemaScrapeException; -import org.junit.jupiter.api.*; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.boot.test.mock.mockito.SpyBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.mock.env.MockEnvironment; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.io.File; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collection; - -import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.doReturn; - - -/** - * Created by rolando on 19/04/2018. - */ -@ExtendWith(SpringExtension.class) -@SpringBootTest -public class SchemaScraperTest { - @SpyBean - SchemaService schemaService; - - @MockBean SchemaRepository schemaRepository; - - @MockBean MigrationConfiguration migrationConfiguration; - - WireMockServer wireMockServer; - - @BeforeEach - public void setupWireMockServer() { - wireMockServer = new WireMockServer(8089); - wireMockServer.start(); - } - - @AfterEach - public void teardownWireMockServer() { - wireMockServer.stop(); - wireMockServer.resetAll(); - } - - - String mockSchemaUri = "http://localhost:8089"; - - @Test - public void testSchemaScrape() throws Exception { - // given - // an s3 bucket files listing as XML - SchemaScraper schemaScraper = new S3BucketSchemaScraper(); - - // when - wireMockServer.stubFor( - get(urlEqualTo("/")) - .willReturn(aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/xml") - .withBody(new String(Files.readAllBytes(Paths.get(new File(".").getAbsolutePath() + "/src/test/resources/testfiles/TestBucketListing.xml")))))); - - Collection mockSchemaUris = schemaScraper.getAllSchemaURIs(URI.create(mockSchemaUri)); - - // we know there are 107 schemas in the test file - assert mockSchemaUris.size() == 107; - } - - @Test - public void testSchemaParse_BundleUris() { - try { - // when - schemaService.schemaDescriptionFromSchemaUris(Arrays.asList(URI.create("bundle/1.2.3/biomaterial"), - URI.create("bundle/2.3.4/links"), - URI.create("bundle/1.0/protocols"))); - } catch (Exception e) { - assert false; - } - - assert true; - } - - @Test - public void testSchemaParse_ModuleUris() { - try { - // when - schemaService.schemaDescriptionFromSchemaUris(Arrays.asList(URI.create("module/biomaterial/5.1.0/growth_condition"), - URI.create("module/ontology/5.0.0/biological_macromolecule_ontology"), - URI.create("module/process/5.1.0/purchased_reagents"))); - } catch (Exception e) { - assert false; - } - - assert true; - } - - @Test - public void testSchemaParse_TypeUris() { - try { - // when - schemaService.schemaDescriptionFromSchemaUris(Arrays.asList(URI.create("type/biomaterial/5.0.1/cell_line"), - URI.create("type/biomaterial/5.1.0/organoid"), - URI.create("type/file/5.0.0/sequence_file"))); - } catch (Exception e) { - assert false; - } - - assert true; - } - - @Test - public void testSchemaParse_SubdomainTypeUris() { - try { - // when - schemaService.schemaDescriptionFromSchemaUris(Arrays.asList(URI.create("type/process/biomaterial_collection/5.1.0/collection_process"), - URI.create("type/process/sequencing/5.0.0/sequencing_process"), - URI.create("type/process/sequencing/5.1.0/sequencing_process"))); - } catch (Exception e) { - assert false; - } - - assert true; - } - - @Test - public void testSchemaParse() throws Exception { - // pre-given - SchemaScraper schemaScraper = new S3BucketSchemaScraper(); - - wireMockServer.stubFor(get(urlEqualTo("/")) - .willReturn(aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/xml") - .withBody(new String(Files.readAllBytes(Paths.get(new File(".").getAbsolutePath() + "/src/test/resources/testfiles/TestBucketListing.xml")))))); - // given - Collection mockSchemaUris = schemaScraper.getAllSchemaURIs(URI.create(mockSchemaUri)); - - try { - // when - schemaService.schemaDescriptionFromSchemaUris(mockSchemaUris); - } catch (Exception e) { - assert false; - } - - assert true; - } - - @Test - public void testGetLatestSchemas() { - Schema mockSchemaA = new Schema("mockHighLevel-A", "2.0","mockDomain-A","mockSubdomain-A","mockConcrete-A", "mock.io/mock-schema-a"); - Schema mockSchemaB = new Schema("mockHighLevel-B", "1.9","mockDomain-B","mockSubdomain-B","mockConcrete-B", "mock.io/mock-schema-a"); - Schema mockSchemaOldA = new Schema("mockHighLevel-A", "1.9","mockDomain-A","mockSubdomain-A","mockConcrete-A", "mock.io/mock-schema-duplicate-a"); - - doReturn(Arrays.asList(mockSchemaA, mockSchemaB, mockSchemaOldA)) - .when(schemaRepository).findAll(); - - Collection latestSchemas = schemaService.getLatestSchemas(); - assert latestSchemas.size() == 2; - latestSchemas.forEach(schema -> { - assert !schema.getSchemaUri().equals("mock.io/mock-schema-duplicate-a"); - }); - assert true; - } - - @Test - public void testFilterLatestSchemas() { - Schema mockSchemaA = new Schema("mockHighLevel-A", "2.0","mockDomain-A","mockSubdomain-A","mockConcrete-A", "mock.io/mock-schema-a"); - Schema mockSchemaB = new Schema("mockHighLevel-B", "1.9","mockDomain-B","mockSubdomain-B","mockConcrete-B", "mock.io/mock-schema-a"); - Schema mockSchemaOldA = new Schema("mockHighLevel-A", "1.9","mockDomain-A","mockSubdomain-A","mockConcrete-A", "mock.io/mock-schema-duplicate-a"); - - doReturn(Arrays.asList(mockSchemaA, mockSchemaB, mockSchemaOldA)) - .when(schemaRepository).findAll(); - - Collection latestSchemas = schemaService.filterLatestSchemas("mockHighLevel-B"); - assert latestSchemas.size() == 1; - latestSchemas.forEach(schema -> { - assert schema.getHighLevelEntity().equals("mockHighLevel-B"); - }); - } - - @Test - public void testEmptyEnvironmentVariable() { - doReturn(null).when(schemaService).getSchemaBaseUri(); - - Exception exception = assertThrows(SchemaScrapeException.class, () -> - schemaService.updateSchemasCollection() - ); - - String expectedMessage = "SCHEMA_BASE_URI environmental variable should not be null."; - String actualMessage = exception.getMessage(); - - assertEquals(actualMessage, expectedMessage); - } - - @Configuration - class MockConfiguration { - @Autowired SchemaScraper schemaScraper; - @Autowired MockEnvironment mockEnvironment; - - @Bean - SchemaService schemaService() { - return new SchemaService(schemaRepository, schemaScraper, mockEnvironment); - } - - @Bean - SchemaScraper schemaScraper() { - return new S3BucketSchemaScraper(); - } - - } - -} diff --git a/src/integration/java/org/humancellatlas/ingest/security/SecurityTest.java b/src/integration/java/org/humancellatlas/ingest/security/SecurityTest.java deleted file mode 100644 index 2717cb6d4..000000000 --- a/src/integration/java/org/humancellatlas/ingest/security/SecurityTest.java +++ /dev/null @@ -1,87 +0,0 @@ -package org.humancellatlas.ingest.security; - -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.security.test.context.support.WithMockUser; -import org.springframework.test.web.servlet.MockMvc; - -import java.util.stream.Stream; - -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -@SpringBootTest -@AutoConfigureDataMongo() -@AutoConfigureMockMvc(printOnlyOnFailure = false) -public class SecurityTest { - @Autowired - private MockMvc webApp; - - public static Stream metadataTypes() { - return Stream.of( - Arguments.of("projects"), - Arguments.of("files"), - Arguments.of("biomaterials"), - Arguments.of("protocols"), - Arguments.of("processes") - ); - } - - @MockBean - // NOTE: Adding MigrationConfiguration as a MockBean is needed - // as otherwise MigrationConfiguration won't be initialised. - private MigrationConfiguration migrationConfiguration; - - @Nested - class Authorised { - @ParameterizedTest - @MethodSource("org.humancellatlas.ingest.security.SecurityTest#metadataTypes") - @WithMockUser - public void autorizedApiAccessWithTrailingSlashIsPermitted(String metadataTypePlural) throws Exception { - checkGetUrlIsAuthorized("/" + metadataTypePlural + "/"); - } - - @ParameterizedTest - @MethodSource("org.humancellatlas.ingest.security.SecurityTest#metadataTypes") - @WithMockUser - public void unautorizedApiAccessNoTrailingSlashIsPermitted(String metadataTypePlural) throws Exception { - checkGetUrlIsAuthorized("/" + metadataTypePlural); - } - - } - @Nested - class Unauthorised { - - @ParameterizedTest - @MethodSource("org.humancellatlas.ingest.security.SecurityTest#metadataTypes") - public void unautorizedApiAccessWithTrailingSlashIsBlocked(String metadataTypePlural) throws Exception { - checkGetUrlIsUnauthorized("/" + metadataTypePlural + "/"); - } - - @ParameterizedTest - @MethodSource("org.humancellatlas.ingest.security.SecurityTest#metadataTypes") - - public void unautorizedApiAccessNoTrailingSlashIsBlocked(String metadataTypePlural) throws Exception { - checkGetUrlIsUnauthorized("/" + metadataTypePlural); - } - } - - private void checkGetUrlIsUnauthorized(String url) throws Exception { - webApp.perform( - get(url) - ).andExpect(status().isUnauthorized()); - } - private void checkGetUrlIsAuthorized(String url) throws Exception { - webApp.perform( - get(url) - ).andExpect(status().isOk()); - } -} diff --git a/src/integration/java/org/humancellatlas/ingest/stagingjob/StagingJobRepositoryTest.java b/src/integration/java/org/humancellatlas/ingest/stagingjob/StagingJobRepositoryTest.java deleted file mode 100644 index d083c1281..000000000 --- a/src/integration/java/org/humancellatlas/ingest/stagingjob/StagingJobRepositoryTest.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.humancellatlas.ingest.stagingjob; - -import org.assertj.core.api.Assertions; -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.dao.DuplicateKeyException; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.util.UUID; - - -@ExtendWith(SpringExtension.class) -@SpringBootTest -public class StagingJobRepositoryTest { - @MockBean - MigrationConfiguration migrationConfiguration; - - @Autowired - StagingJobRepository stagingJobRepository; - - - @AfterEach - private void tearDown() { - stagingJobRepository.deleteAll(); - } - - @Test - public void testJpaExceptionWhenInsertingMultipleCompoundKey() { - UUID testStagingAreaUuid = UUID.randomUUID(); - String testFileName = "test.fastq.gz"; - - stagingJobRepository.save(new StagingJob(testStagingAreaUuid, testFileName)); - - Assertions.assertThatExceptionOfType(DuplicateKeyException.class).isThrownBy(() -> { - stagingJobRepository.save(new StagingJob(testStagingAreaUuid, testFileName)); - }); - } - - @Test - public void testRegisteringJobsWithDifferentCompoundKey() { - UUID testStagingAreaUuid_1 = UUID.randomUUID(); - String testFileName_1 = "test_1.fastq.gz"; - - UUID testStagingAreaUuid_2 = UUID.randomUUID(); - String testFileName_2 = "test_2.fastq.gz"; - - stagingJobRepository.save(new StagingJob(testStagingAreaUuid_1, testFileName_1)); - stagingJobRepository.save(new StagingJob(testStagingAreaUuid_1, testFileName_2)); - - stagingJobRepository.save(new StagingJob(testStagingAreaUuid_2, testFileName_1)); - stagingJobRepository.save(new StagingJob(testStagingAreaUuid_2, testFileName_2)); - } -} diff --git a/src/integration/java/org/humancellatlas/ingest/submission/web/ProjectStatusUpdateTest.java b/src/integration/java/org/humancellatlas/ingest/submission/web/ProjectStatusUpdateTest.java deleted file mode 100644 index 4a3d8e22c..000000000 --- a/src/integration/java/org/humancellatlas/ingest/submission/web/ProjectStatusUpdateTest.java +++ /dev/null @@ -1,141 +0,0 @@ -package org.humancellatlas.ingest.submission.web; - -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.project.WranglingState; -import org.humancellatlas.ingest.security.Role; -import org.humancellatlas.ingest.state.ValidationState; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.http.MediaType; -import org.springframework.security.test.context.support.WithMockUser; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.MvcResult; -import org.springframework.web.servlet.support.ServletUriComponentsBuilder; -import org.springframework.web.util.UriComponentsBuilder; - -import static org.humancellatlas.ingest.project.WranglingState.IN_PROGRESS; -import static org.humancellatlas.ingest.project.WranglingState.SUBMITTED; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -@SpringBootTest -@AutoConfigureDataMongo -@AutoConfigureMockMvc -@WithMockUser(username = "test_user", authorities={"WRANGLER"}) -public class ProjectStatusUpdateTest { - @Autowired - private MockMvc webApp; - @Autowired - private ProjectRepository projectRepository; - - // NOTE: Adding MigrationConfiguration as a MockBean is needed as otherwise MigrationConfiguration won't be - // initialised. This is very un-elegant and should be fixed. - @MockBean - private MigrationConfiguration migrationConfiguration; - UriComponentsBuilder uriBuilder; - - @BeforeEach - void setUp() { - uriBuilder = ServletUriComponentsBuilder.fromCurrentContextPath(); - } - - @Test - public void test_statusIsInProgress_afterSubmissionCreation() throws Exception { - // given - Project project = createProject(); - - // when - String submissionUrl = createSubmission(); - connectSubmissionToProject(project, submissionUrl); - - // then - assertProjectStatus(project, IN_PROGRESS); - } - - @Test - public void test_statusIsSubmitted_afterSubmissionIsExported() throws Exception { - // given - Project project = createProject(); - String submissionUrl = createSubmission(); - connectSubmissionToProject(project, submissionUrl); - - // when - setSubmissionToExported(submissionUrl); - - // then - assertProjectStatus(project, SUBMITTED); - } - - @Test - public void test_deleteSubmissionWorks() throws Exception { - // given - Project project = createProject(); - String submissionUrl = createSubmission(); - connectSubmissionToProject(project, submissionUrl); - - // when - deleteSubmissionFromProject(submissionUrl); - String submissionUrl2 = createSubmission(); - connectSubmissionToProject(project, submissionUrl2); - - // then - // no errors - } - - private void deleteSubmissionFromProject(String submissionUrl) throws Exception { - webApp.perform(delete(submissionUrl)).andExpect(status().isAccepted()); - } - - - private void setSubmissionToExported(String submissionUrl) throws Exception { - webApp.perform( - put(submissionUrl + Links.COMMIT_EXPORTED_URL) - ).andExpect(status().isAccepted()); - } - - private void assertProjectStatus(Project project, WranglingState wranglingState) throws Exception { - webApp.perform( - get("/projects/{id}",project.getId()) - ).andExpect(status().isOk()) - .andExpect(jsonPath("$.wranglingState") - .value(wranglingState.getValue())); - } - - private void connectSubmissionToProject(Project project, String submissionUrl) throws Exception { - webApp.perform( - post("/projects/{id}/submissionEnvelopes", project.getId()) - .contentType("text/uri-list") - .content(submissionUrl) - ).andExpect(status().isNoContent()); - } - - @Nullable - private String createSubmission() throws Exception { - MvcResult mvcResult = webApp.perform( - post("/submissionEnvelopes/") - .contentType(MediaType.APPLICATION_JSON) - .content("{}")) - .andExpect(status().isCreated()) - .andReturn(); - String submissionUrl = mvcResult.getResponse().getHeader("Location"); - return submissionUrl; - } - - @NotNull - private Project createProject() { - Project project = new Project(null); - project.setWranglingState(WranglingState.ELIGIBLE); - return projectRepository.save(project); - } -} diff --git a/src/integration/java/org/humancellatlas/ingest/submission/web/SubmissionControllerTest.java b/src/integration/java/org/humancellatlas/ingest/submission/web/SubmissionControllerTest.java deleted file mode 100644 index 389df8ab7..000000000 --- a/src/integration/java/org/humancellatlas/ingest/submission/web/SubmissionControllerTest.java +++ /dev/null @@ -1,189 +0,0 @@ -package org.humancellatlas.ingest.submission.web; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.humancellatlas.ingest.state.SubmissionState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; -import org.junit.jupiter.params.provider.ValueSource; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.http.MediaType; -import org.springframework.security.test.context.support.WithMockUser; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.web.servlet.support.ServletUriComponentsBuilder; -import org.springframework.web.util.UriComponentsBuilder; - -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -@SpringBootTest -@AutoConfigureDataMongo() -@AutoConfigureMockMvc() -public class SubmissionControllerTest { - @Autowired - private MockMvc webApp; - - @Autowired - private SubmissionEnvelopeRepository submissionEnvelopeRepository; - - @Autowired - private ProjectRepository projectRepository; - - @Autowired - private BiomaterialRepository biomaterialRepository; - - @Autowired - private ProcessRepository processRepository; - - @Autowired - private ProtocolRepository protocolRepository; - - @Autowired - private FileRepository fileRepository; - - @Autowired - private ObjectMapper objectMapper; - - @MockBean - private MigrationConfiguration migrationConfiguration; - - @MockBean - private MessageRouter messageRouter; - - SubmissionEnvelope submissionEnvelope; - - Project project; - - Biomaterial biomaterial; - - Process process; - - Protocol protocol; - - File file; - - UriComponentsBuilder uriBuilder; - - @BeforeEach - void setUp() { - submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.setUuid(Uuid.newUuid()); - submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALID); - submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); - - project = new Project(null); - project.getSubmissionEnvelopes().add(submissionEnvelope); - project = projectRepository.save(project); - - biomaterial = new Biomaterial(null); - biomaterial.setSubmissionEnvelope(submissionEnvelope); - biomaterial = biomaterialRepository.save(biomaterial); - - process = new Process(null); - process.setSubmissionEnvelope(submissionEnvelope); - process = processRepository.save(process); - - protocol = new Protocol(null); - protocol.setSubmissionEnvelope(submissionEnvelope); - protocol = protocolRepository.save(protocol); - - file = new File(null, "fileName"); - file.setSubmissionEnvelope(submissionEnvelope); - file = fileRepository.save(file); - - uriBuilder = ServletUriComponentsBuilder.fromCurrentContextPath(); - } - - @AfterEach - void tearDown() { - submissionEnvelopeRepository.deleteAll(); - projectRepository.deleteAll(); - biomaterialRepository.deleteAll(); - processRepository.deleteAll(); - protocolRepository.deleteAll(); - fileRepository.deleteAll(); - } - - @ParameterizedTest - @ValueSource(strings = { - "biomaterials", - "processes", - "protocols", - "files" - }) - public void testAdditionToNonEditableSubmissionThrowsErrorForAllEntityTypes(String endpoint) throws Exception { - // given - submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALIDATION_REQUESTED); - submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); - - // when - webApp.perform( - post("/submissionEnvelopes/{id}/" + endpoint, submissionEnvelope.getId()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"content\": {}}") - ).andExpect(status().isForbidden()); - } - - @ParameterizedTest - @EnumSource(value = SubmissionState.class, names = { - "GRAPH_VALIDATION_REQUESTED", - "GRAPH_VALIDATING", - "EXPORTING", - "PROCESSING", - "ARCHIVED", - "SUBMITTED" - }) - public void testAdditionToNonEditableSubmissionThrowsErrorInAllStates(SubmissionState state) throws Exception { - // given - submissionEnvelope.enactStateTransition(state); - submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); - - // when - webApp.perform( - post("/submissionEnvelopes/{id}/biomaterials", submissionEnvelope.getId()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"content\": {}}") - ).andExpect(status().isForbidden()); - } - - @ParameterizedTest - @ValueSource(strings = { - "/submissionEnvelopes/{id}/projects", - "/submissionEnvelopes/{id}/relatedProjects" - }) - @WithMockUser - public void testProjectsAreReturnedWhenTheyIncludeTheSubmissionInTheirEnvelopes(String endpoint) throws Exception { - webApp.perform( - // when - get(endpoint, submissionEnvelope.getId()) - .contentType(MediaType.APPLICATION_JSON) - ) // then - .andExpect(status().isOk()) - .andExpect(jsonPath("$._embedded.projects", hasSize(1))) - .andExpect(jsonPath("$._embedded.projects[0].uuid.uuid", is(project.getUuid().getUuid().toString()))); - } -} diff --git a/src/integration/java/org/humancellatlas/ingest/submission/web/SubmissionLinkMapControllerTest.java b/src/integration/java/org/humancellatlas/ingest/submission/web/SubmissionLinkMapControllerTest.java deleted file mode 100644 index 8c2bc18e0..000000000 --- a/src/integration/java/org/humancellatlas/ingest/submission/web/SubmissionLinkMapControllerTest.java +++ /dev/null @@ -1,140 +0,0 @@ -package org.humancellatlas.ingest.submission.web; - -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.config.MigrationConfiguration; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.junit.Ignore; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.data.mongodb.repository.MongoRepository; - -import java.util.HashSet; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; - -@Ignore("ignoring because $toString mongo aggregation operator is not supported by the in memory mongo version we use. See dcp-936") -@SpringBootTest -@AutoConfigureDataMongo() -public class SubmissionLinkMapControllerTest { - @Autowired - private SubmissionLinkMapController controller; - - @Autowired - BiomaterialRepository biomaterialRepository; - - @Autowired - FileRepository fileRepository; - - @Autowired - ProcessRepository processRepository; - - @Autowired - ProtocolRepository protocolRepository; - - @Autowired - SubmissionEnvelopeRepository submissionEnvelopeRepository; - - @MockBean - private MigrationConfiguration migrationConfiguration; - - @MockBean - private MessageRouter messageRouter; - - @AfterEach - private void tearDown() { - List.of( - biomaterialRepository, - fileRepository, - processRepository, - protocolRepository, - submissionEnvelopeRepository - ).forEach(MongoRepository::deleteAll); - } - - /** - * @Ignore("ignoring because $toString mongo aggregation operator is not supported by the in memory mongo version we use. See dcp-936") - */ - public void testSubmissionLinkMap() { - //given: - SubmissionEnvelope submissionEnvelope = submissionEnvelopeRepository.save(new SubmissionEnvelope()); - - Biomaterial donor = biomaterialRepository.save(new Biomaterial(null)); - Biomaterial specimen = biomaterialRepository.save(new Biomaterial(null)); - Biomaterial cellSuspension = biomaterialRepository.save(new Biomaterial(null)); - - Process donorSpecimen = processRepository.save(new Process(null)); - Process cellSuspensionSequenceFile = processRepository.save(new Process(null)); - Process sequenceFileAnalysisFile = processRepository.save(new Process(null)); - - Protocol collectionProtocol = protocolRepository.save(new Protocol(null)); - Protocol sequencingProtocol = protocolRepository.save(new Protocol(null)); - Protocol analysisProtocol = protocolRepository.save(new Protocol(null)); - - File sequencingFile = fileRepository.save(new File(null, "sequenceFile")); - File analysisFile = fileRepository.save(new File(null, "analysisFile")); - - List.of(donor, - specimen, - cellSuspension, - donorSpecimen, - cellSuspensionSequenceFile, - sequenceFileAnalysisFile, - collectionProtocol, - sequencingProtocol, - analysisProtocol, - sequencingFile, - analysisFile - ).forEach(entity-> entity.setSubmissionEnvelope(submissionEnvelope)); - - specimen.addAsDerivedByProcess(donorSpecimen); - sequencingFile.addAsDerivedByProcess(cellSuspensionSequenceFile); - analysisFile.addAsDerivedByProcess(sequenceFileAnalysisFile); - - donor.addAsInputToProcess(donorSpecimen); - cellSuspension.addAsInputToProcess(cellSuspensionSequenceFile); - sequencingFile.addAsInputToProcess(sequenceFileAnalysisFile); - - donorSpecimen.addProtocol(collectionProtocol); - cellSuspensionSequenceFile.addProtocol(sequencingProtocol); - sequenceFileAnalysisFile.addProtocol(analysisProtocol); - - submissionEnvelopeRepository.save(submissionEnvelope); - biomaterialRepository.saveAll(List.of(donor, specimen, cellSuspension)); - protocolRepository.saveAll(List.of(collectionProtocol, sequencingProtocol, analysisProtocol)); - processRepository.saveAll(List.of(donorSpecimen, cellSuspensionSequenceFile, sequenceFileAnalysisFile)); - fileRepository.saveAll(List.of(sequencingFile, analysisFile)); - - //when: - SubmissionLinkMapController.SubmissionLinkingMap submissionLinkMap = controller.getSubmissionLinkMap(submissionEnvelope); - - //then: - assertThat(submissionLinkMap).isNotNull(); - assertThat(submissionLinkMap.processes.get(donorSpecimen.getId()).protocols).isEqualTo(new HashSet<>(List.of(collectionProtocol.getId()))); - assertThat(submissionLinkMap.processes.get(donorSpecimen.getId()).inputBiomaterials).isEqualTo(new HashSet<>(List.of(donor.getId()))); - assertThat(submissionLinkMap.processes.get(cellSuspensionSequenceFile.getId()).protocols).isEqualTo(new HashSet<>(List.of(sequencingProtocol.getId()))); - assertThat(submissionLinkMap.processes.get(cellSuspensionSequenceFile.getId()).inputBiomaterials).isEqualTo(new HashSet<>(List.of(cellSuspension.getId()))); - assertThat(submissionLinkMap.processes.get(cellSuspensionSequenceFile.getId()).inputFiles).isEmpty(); - assertThat(submissionLinkMap.processes.get(sequenceFileAnalysisFile.getId()).protocols).isEqualTo(new HashSet<>(List.of(analysisProtocol.getId()))); - assertThat(submissionLinkMap.processes.get(sequenceFileAnalysisFile.getId()).inputBiomaterials).isEmpty(); - assertThat(submissionLinkMap.processes.get(sequenceFileAnalysisFile.getId()).inputFiles).isEqualTo(new HashSet<>(List.of(sequencingFile.getId()))); - assertThat(submissionLinkMap.biomaterials.get(donor.getId()).inputToProcesses).isEqualTo(new HashSet<>(List.of(donorSpecimen.getId()))); - assertThat(submissionLinkMap.biomaterials.get(cellSuspension.getId()).inputToProcesses).isEqualTo(new HashSet<>(List.of(cellSuspensionSequenceFile.getId()))); - assertThat(submissionLinkMap.files.get(sequencingFile.getId()).inputToProcesses).isEqualTo(new HashSet<>(List.of(sequenceFileAnalysisFile.getId()))); - } - - -} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/AuditEntryTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/AuditEntryTest.java new file mode 100644 index 000000000..fc80984b7 --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/AuditEntryTest.java @@ -0,0 +1,86 @@ +package uk.ac.ebi.subs.ingest; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.security.test.context.support.WithMockUser; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import uk.ac.ebi.subs.ingest.audit.AuditEntry; +import uk.ac.ebi.subs.ingest.audit.AuditEntryRepository; +import uk.ac.ebi.subs.ingest.audit.AuditEntryService; +import uk.ac.ebi.subs.ingest.audit.AuditType; +import uk.ac.ebi.subs.ingest.bundle.BundleManifestRepository; +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.project.*; +import uk.ac.ebi.subs.ingest.schemas.SchemaService; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@SpringBootTest +public class AuditEntryTest { + @Autowired private ProjectService projectService; + + @Autowired private AuditEntryService auditEntryService; + + @Autowired private MongoTemplate mongoTemplate; + + @Autowired private AuditEntryRepository auditEntryRepository; + + @MockBean private SubmissionEnvelopeRepository submissionEnvelopeRepository; + + @MockBean private ProjectRepository projectRepository; + + @MockBean private MetadataCrudService metadataCrudService; + + @MockBean private MetadataUpdateService metadataUpdateService; + + @MockBean private SchemaService schemaService; + + @MockBean private BundleManifestRepository bundleManifestRepository; + + @MockBean private ProjectEventHandler projectEventHandler; + + @MockBean MigrationConfiguration migrationConfiguration; + + @Test + @WithMockUser(value = "test_user") + void testAuditEntryGenerationOnProjectStateUpdate() { + // given + WranglingState initialWranglingState = WranglingState.NEW; + Project project = new Project("{\"name\": \"Project 1\"}"); + project.setWranglingState(initialWranglingState); + this.mongoTemplate.save(project); + + // when + WranglingState updatedWranglingState = WranglingState.ELIGIBLE; + ObjectNode patchUpdate = + new ObjectMapper() + .createObjectNode() + .put("wranglingState", updatedWranglingState.getValue()); + projectService.update(project, patchUpdate, false); + + // then + AuditEntry actual = projectService.getProjectAuditEntries(project).get(0); + + assertThat(actual) + .hasFieldOrPropertyWithValue("auditType", AuditType.STATUS_UPDATED) + .hasFieldOrPropertyWithValue("before", initialWranglingState.name()) + .hasFieldOrPropertyWithValue("after", updatedWranglingState.name()) + .returns(true, e -> e.getUser().contains("Username: test_user;")); + } + + @AfterEach + private void tearDown() { + this.mongoTemplate.dropCollection(Project.class); + this.mongoTemplate.dropCollection(AuditEntry.class); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/IngestCoreApplicationTests.java b/src/integration/java/uk/ac/ebi/subs/ingest/IngestCoreApplicationTests.java new file mode 100644 index 000000000..64fecaae8 --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/IngestCoreApplicationTests.java @@ -0,0 +1,15 @@ +package uk.ac.ebi.subs.ingest; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; + +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; + +@SpringBootTest +public class IngestCoreApplicationTests { + @MockBean MigrationConfiguration migrationConfiguration; + + @Test + public void contextLoads() {} +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/MongoAuditingTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/MongoAuditingTest.java new file mode 100644 index 000000000..70f6a662e --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/MongoAuditingTest.java @@ -0,0 +1,38 @@ +package uk.ac.ebi.subs.ingest; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.HashMap; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.security.test.context.support.WithMockUser; + +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; + +@SpringBootTest +public class MongoAuditingTest { + + @MockBean private MigrationConfiguration migrationConfiguration; + + @Autowired private ProjectRepository projectRepository; + + @Test + @WithMockUser("johndoe") + public void auditMongoRecord() { + // given: + Project project = new Project(new HashMap<>()); + + // when: + Project persistentProject = projectRepository.save(project); + + // then: + /* NOTE there doesn't seem to be a clean and easy way to check this without updating the UserAuditing class + itself to assume that the default principal type contains username and password. */ + assertThat(persistentProject.getUser()).contains("johndoe"); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/TestingHelper.java b/src/integration/java/uk/ac/ebi/subs/ingest/TestingHelper.java new file mode 100644 index 000000000..1597f12bb --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/TestingHelper.java @@ -0,0 +1,15 @@ +package uk.ac.ebi.subs.ingest; + +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.test.context.TestSecurityContextHolder; + +public class TestingHelper { + /** + * WebMvc resets the security context when it finishes, so the user declared in WithMockUser is + * deleted. see this StackOVerflow post: + * https://stackoverflow.com/questions/51622300/mockmvc-seems-to-be-clear-securitycontext-after-performing-request-java-lang-il + */ + public static void resetTestingSecurityContext() { + SecurityContextHolder.setContext(TestSecurityContextHolder.getContext()); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/archiving/web/ArchiveJobControllerTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/archiving/web/ArchiveJobControllerTest.java new file mode 100644 index 000000000..e76535e0f --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/archiving/web/ArchiveJobControllerTest.java @@ -0,0 +1,143 @@ +package uk.ac.ebi.subs.ingest.archiving.web; + +import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.given; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; + +import java.time.Instant; +import java.util.*; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.security.test.context.support.WithMockUser; +import org.springframework.test.web.servlet.MockMvc; + +import uk.ac.ebi.subs.ingest.archiving.entity.ArchiveJob; +import uk.ac.ebi.subs.ingest.archiving.entity.ArchiveJobRepository; +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; + +@SpringBootTest +@AutoConfigureDataMongo() +@AutoConfigureMockMvc +public class ArchiveJobControllerTest { + + @Autowired MockMvc mockMvc; + + @MockBean private MigrationConfiguration migrationConfiguration; + + @MockBean private ArchiveJobRepository archiveJobRepository; + + private static final ArchiveJob.ArchiveJobStatus PENDING_STATUS = + ArchiveJob.ArchiveJobStatus.PENDING; + private static final ArchiveJob.ArchiveJobStatus COMPLETED_STATUS = + ArchiveJob.ArchiveJobStatus.COMPLETED; + + private UUID uuid; + + private static final String ARCHIVE_JOB_ID = "1"; + private static final String SUBMISSION_UUID = "1234"; + + @Test + public void when_request_archive_job_creation_returns_successful_response() throws Exception { + final ArchiveJob anArchiveJobById = + createAnArchiveJob(ARCHIVE_JOB_ID, SUBMISSION_UUID, PENDING_STATUS); + + given(this.archiveJobRepository.save(any())).willReturn(anArchiveJobById); + + this.mockMvc + .perform( + post("/archiveJobs") + .contentType(MediaType.APPLICATION_JSON) + .content(String.format("{\"submissionUuid\": \"%s\"}", this.uuid))) + .andDo(print()) + .andExpect(status().isCreated()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andExpect(jsonPath("$.submissionUuid", is(SUBMISSION_UUID))) + .andExpect(jsonPath("$.overallStatus", is(PENDING_STATUS.toString()))) + .andExpect(jsonPath("$.createdDate").isNotEmpty()) + .andReturn(); + } + + @Test + @WithMockUser + public void when_requesting_non_existing_archiving_job_returns_not_found_response() + throws Exception { + given(this.archiveJobRepository.findById(ARCHIVE_JOB_ID)).willReturn(Optional.empty()); + + this.mockMvc + .perform(get("/archiveJobs/{id}", ARCHIVE_JOB_ID).contentType(MediaType.APPLICATION_JSON)) + .andDo(print()) + .andExpect(status().isNotFound()) + .andReturn(); + } + + @Test + @WithMockUser + public void when_existing_archiving_job_in_pending_status_returns_valid_response() + throws Exception { + final ArchiveJob anArchiveJob = + createAnArchiveJob(ARCHIVE_JOB_ID, SUBMISSION_UUID, PENDING_STATUS); + + given(this.archiveJobRepository.findById(ARCHIVE_JOB_ID)).willReturn(Optional.of(anArchiveJob)); + + this.mockMvc + .perform(get("/archiveJobs/{id}", ARCHIVE_JOB_ID).contentType(MediaType.APPLICATION_JSON)) + .andDo(print()) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.submissionUuid", is(SUBMISSION_UUID))) + .andExpect(jsonPath("$.overallStatus", is(PENDING_STATUS.toString()))) + .andExpect(jsonPath("$.createdDate").isNotEmpty()) + .andExpect(jsonPath("$.resultsFromArchives").doesNotExist()) + .andReturn(); + } + + @Test + @WithMockUser + public void when_existing_archiving_job_in_completed_status_returns_valid_response() + throws Exception { + final ArchiveJob anArchiveJob = + createAnArchiveJob(ARCHIVE_JOB_ID, SUBMISSION_UUID, COMPLETED_STATUS); + setArchiveResult(anArchiveJob); + + given(this.archiveJobRepository.findById(ARCHIVE_JOB_ID)).willReturn(Optional.of(anArchiveJob)); + + this.mockMvc + .perform(get("/archiveJobs/{id}", ARCHIVE_JOB_ID).contentType(MediaType.APPLICATION_JSON)) + .andDo(print()) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.submissionUuid", is(SUBMISSION_UUID))) + .andExpect(jsonPath("$.overallStatus", is(COMPLETED_STATUS.toString()))) + .andExpect(jsonPath("$.createdDate").isNotEmpty()) + .andExpect(jsonPath("$.resultsFromArchives").isNotEmpty()) + .andReturn(); + } + + private ArchiveJob createAnArchiveJob( + String id, String submissionUuid, ArchiveJob.ArchiveJobStatus status) { + ArchiveJob archiveJob = new ArchiveJob(); + archiveJob.setId(ARCHIVE_JOB_ID); + archiveJob.setCreatedDate(Instant.now()); + archiveJob.setOverallStatus(status); + archiveJob.setSubmissionUuid(submissionUuid); + + return archiveJob; + } + + private void setArchiveResult(ArchiveJob anArchiveJob) { + Map resultByArchive = new HashMap<>(); + Map>> experimentsResult = new HashMap<>(); + experimentsResult.put( + "experiments", List.of(Map.of("accession", "1234"), Map.of("uuid", "1-2-3-4"))); + resultByArchive.put("hca_assays", experimentsResult); + anArchiveJob.setResultsFromArchives(resultByArchive); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/biomaterial/BiomaterialControllerTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/biomaterial/BiomaterialControllerTest.java new file mode 100644 index 000000000..0571549ae --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/biomaterial/BiomaterialControllerTest.java @@ -0,0 +1,427 @@ +package uk.ac.ebi.subs.ingest.biomaterial; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.Resources; +import org.springframework.http.MediaType; +import org.springframework.security.test.context.support.WithMockUser; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.web.servlet.support.ServletUriComponentsBuilder; +import org.springframework.web.util.UriComponentsBuilder; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import uk.ac.ebi.subs.ingest.TestingHelper; +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.service.ValidationStateChangeService; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.project.*; +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@SpringBootTest +@AutoConfigureDataMongo() +@AutoConfigureMockMvc(printOnlyOnFailure = false) +@WithMockUser( + username = "alice", + roles = {"WRANGLER"}) +public class BiomaterialControllerTest { + + @MockBean ValidationStateChangeService validationStateChangeService; + + @Autowired private MockMvc webApp; + + @Autowired private ProcessRepository processRepository; + + @Autowired private BiomaterialRepository biomaterialRepository; + + @Autowired private SubmissionEnvelopeRepository submissionEnvelopeRepository; + + @Autowired private ProjectRepository projectRepository; + + @Autowired private ProjectService projectService; + + @Autowired BiomaterialService biomaterialService; + + @Autowired private ObjectMapper objectMapper; + + @MockBean private MigrationConfiguration migrationConfiguration; + + @MockBean private MessageRouter messageRouter; + + Process process1; + + Process process2; + + Process process3; + + Biomaterial biomaterial; + + UriComponentsBuilder uriBuilder; + + SubmissionEnvelope submissionEnvelope; + + Project project; + + @BeforeEach + void setUp() { + submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.setUuid(Uuid.newUuid()); + submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALID); + submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); + + project = new Project(new HashMap<>()); + ((Map) project.getContent()) + .put("dataAccess", new ObjectToMapConverter().asMap(new DataAccess(DataAccessTypes.OPEN))); + project.setUuid(Uuid.newUuid()); + project.getSubmissionEnvelopes().add(submissionEnvelope); + project = projectRepository.save(project); + + projectService.addProjectToSubmissionEnvelope(submissionEnvelope, project); + + process1 = processRepository.save(new Process(null)); + process2 = processRepository.save(new Process(null)); + process3 = processRepository.save(new Process(null)); + + biomaterial = new Biomaterial(new HashMap<>()); + biomaterialService.addBiomaterialToSubmissionEnvelope(submissionEnvelope, biomaterial); + + uriBuilder = ServletUriComponentsBuilder.fromCurrentContextPath(); + } + + @AfterEach + void tearDown() { + processRepository.deleteAll(); + biomaterialRepository.deleteAll(); + submissionEnvelopeRepository.deleteAll(); + projectRepository.deleteAll(); + } + + @Test + public void newBiomaterialInSubmissionLinksToSubmissionAndProject() throws Exception { + // given + biomaterialRepository.deleteAll(); + processRepository.deleteAll(); + + // when + webApp + .perform( + post("/submissionEnvelopes/{id}/biomaterials", submissionEnvelope.getId()) + .contentType(MediaType.APPLICATION_JSON) + .content("{\"content\": {}}")) + .andExpect(status().isAccepted()); + TestingHelper.resetTestingSecurityContext(); + + // then + assertThat(biomaterialRepository.findAll()).hasSize(1); + + MvcResult allBiomaterialsResult = + webApp + .perform(get("/biomaterials").accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.page.totalElements").value("1")) + .andReturn(); + TestingHelper.resetTestingSecurityContext(); + + verifyBiomaterial(allBiomaterialsResult); + } + + private void verifyBiomaterial(MvcResult allBiomaterialsResult) throws Exception { + webApp + .perform( + get( + "/biomaterials/search/findBySubmissionEnvelope?submissionEnvelope=http://localhost/submissionEnvelopes/{id}", + submissionEnvelope.getId()) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.page.totalElements").value("1")); + TestingHelper.resetTestingSecurityContext(); + + assertThat(biomaterialRepository.findByProject(project)).hasSize(1); + + String content = allBiomaterialsResult.getResponse().getContentAsString(); + + Resources> halResponse = + objectMapper.readValue(content, new TypeReference<>() {}); + + halResponse.getContent().stream() + .map(r -> r.getContent()) + .forEach( + newBiomaterial -> { + assertThat(newBiomaterial.getSubmissionEnvelope().getId()) + .isEqualTo(submissionEnvelope.getId()); + assertThat(newBiomaterial.getProject().getId()).isEqualTo(project.getId()); + assertThat(newBiomaterial.getProjects()).hasSize(1); + assertThat(newBiomaterial.getProjects().stream().findFirst().get().getId()) + .isEqualTo(project.getId()); + }); + } + + @Test + public void newBiomaterialInSubmissionDoesNotFailIfSubmissionHasNoProject() throws Exception { + // given + biomaterialRepository.deleteAll(); + processRepository.deleteAll(); + projectRepository.deleteAll(); + + // when + webApp + .perform( + post("/submissionEnvelopes/{id}/biomaterials", submissionEnvelope.getId()) + .contentType(MediaType.APPLICATION_JSON) + .content("{\"content\": {}}")) + .andExpect(status().isAccepted()); + TestingHelper.resetTestingSecurityContext(); + + // then + assertThat(biomaterialRepository.findAll()).hasSize(1); + assertThat(biomaterialRepository.findAllBySubmissionEnvelope(submissionEnvelope)).hasSize(1); + + var newBiomaterial = biomaterialRepository.findAll().get(0); + assertThat(newBiomaterial.getSubmissionEnvelope().getId()) + .isEqualTo(submissionEnvelope.getId()); + assertThat(newBiomaterial.getProject()).isNull(); + assertThat(newBiomaterial.getProjects()).isEmpty(); + } + + @Test + public void testLinkBiomaterialAsInputToProcessesUsingPostMethodWithManyProcessesInPayload() + throws Exception { + // when + webApp + .perform( + post("/biomaterials/{id}/inputToProcesses/", biomaterial.getId()) + .contentType("text/uri-list") + .content( + uriBuilder.build().toUriString() + + "/processes/" + + process1.getId() + + '\n' + + uriBuilder.build().toUriString() + + "/processes/" + + process2.getId())) + .andExpect(status().isOk()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(biomaterial); + Biomaterial updatedBiomaterial = biomaterialRepository.findById(biomaterial.getId()).get(); + assertThat(updatedBiomaterial.getInputToProcesses()) + .usingElementComparatorOnFields("id") + .containsExactly(process1, process2); + } + + @Test + public void testLinkBiomaterialAsInputToProcessesUsingPutMethodWithManyProcessesInPayload() + throws Exception { + // given + biomaterial.addAsInputToProcess(process1); + biomaterialRepository.save(biomaterial); + TestingHelper.resetTestingSecurityContext(); + + // when + webApp + .perform( + put("/biomaterials/{id}/inputToProcesses/", biomaterial.getId()) + .contentType("text/uri-list") + .content( + uriBuilder.build().toUriString() + + "/processes/" + + process2.getId() + + '\n' + + uriBuilder.build().toUriString() + + "/processes/" + + process3.getId())) + .andExpect(status().isOk()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(biomaterial); + Biomaterial updatedBiomaterial = biomaterialRepository.findById(biomaterial.getId()).get(); + assertThat(updatedBiomaterial.getInputToProcesses()) + .usingElementComparatorOnFields("id") + .containsExactly(process2, process3); + } + + @Test + public void testLinkBiomaterialAsInputToProcessesUsingPostMethodWithOneProcessInPayload() + throws Exception { + // when + webApp + .perform( + post("/biomaterials/{id}/inputToProcesses/", biomaterial.getId()) + .contentType("text/uri-list") + .content(uriBuilder.build().toUriString() + "/processes/" + process1.getId())) + .andExpect(status().isOk()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(biomaterial); + Biomaterial updatedBiomaterial = biomaterialRepository.findById(biomaterial.getId()).get(); + assertThat(updatedBiomaterial.getInputToProcesses()) + .usingElementComparatorOnFields("id") + .containsExactly(process1); + } + + @Test + public void testLinkBiomaterialAsDerivedByProcessesUsingPostMethodWithManyProcessesInPayload() + throws Exception { + // when + webApp + .perform( + post("/biomaterials/{id}/derivedByProcesses/", biomaterial.getId()) + .contentType("text/uri-list") + .content( + uriBuilder.build().toUriString() + + "/processes/" + + process1.getId() + + '\n' + + uriBuilder.build().toUriString() + + "/processes/" + + process2.getId())) + .andExpect(status().isOk()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(biomaterial); + Biomaterial updatedBiomaterial = biomaterialRepository.findById(biomaterial.getId()).get(); + assertThat(updatedBiomaterial.getDerivedByProcesses()) + .usingElementComparatorOnFields("id") + .containsExactly(process1, process2); + } + + @Test + public void testLinkBiomaterialAsDerivedByProcessesUsingPutMethodWithManyProcessesInPayload() + throws Exception { + // given + biomaterial.addAsDerivedByProcess(process1); + biomaterialRepository.save(biomaterial); + + // when + webApp + .perform( + put("/biomaterials/{id}/derivedByProcesses/", biomaterial.getId()) + .contentType("text/uri-list") + .content( + uriBuilder.build().toUriString() + + "/processes/" + + process2.getId() + + '\n' + + uriBuilder.build().toUriString() + + "/processes/" + + process3.getId())) + .andExpect(status().isOk()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(biomaterial); + Biomaterial updatedBiomaterial = biomaterialRepository.findById(biomaterial.getId()).get(); + assertThat(updatedBiomaterial.getDerivedByProcesses()) + .usingElementComparatorOnFields("id") + .containsExactly(process2, process3); + } + + @Test + @WithMockUser( + username = "alice", + roles = {"WRANGLER"}) + public void testLinkBiomaterialAsDerivedByProcessesUsingPostMethodWithOneProcessInPayload() + throws Exception { + // when + webApp + .perform( + post("/biomaterials/{id}/derivedByProcesses/", biomaterial.getId()) + .contentType("text/uri-list") + .content(uriBuilder.build().toUriString() + "/processes/" + process1.getId())) + .andExpect(status().isOk()); + + TestingHelper.resetTestingSecurityContext(); + + verifyThatValidationStateChangedToDraftWhenGraphValid(biomaterial); + + // then + Biomaterial updatedBiomaterial = biomaterialRepository.findById(biomaterial.getId()).get(); + assertThat(updatedBiomaterial.getDerivedByProcesses()) + .usingElementComparatorOnFields("id") + .containsExactly(process1); + } + + @Test + public void testUnlinkBiomaterialAsInputToProcesses() throws Exception { + // given + biomaterial.addAsInputToProcess(process1); + biomaterialRepository.save(biomaterial); + + // when + webApp + .perform( + delete( + "/biomaterials/{id}/inputToProcesses/{processId}", + biomaterial.getId(), + process1.getId())) + .andExpect(status().isNoContent()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(biomaterial); + + Biomaterial updatedBiomaterial = biomaterialRepository.findById(biomaterial.getId()).get(); + assertThat(updatedBiomaterial.getInputToProcesses()).doesNotContain(process1); + } + + @Test + public void testUnlinkBiomaterialAsDerivedByProcesses() throws Exception { + // given + biomaterial.addAsDerivedByProcess(process1); + biomaterialRepository.save(biomaterial); + + // when + webApp + .perform( + delete( + "/biomaterials/{id}/derivedByProcesses/{processId}", + biomaterial.getId(), + process1.getId())) + .andExpect(status().isNoContent()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(biomaterial); + Biomaterial updatedBiomaterial = biomaterialRepository.findById(biomaterial.getId()).get(); + assertThat(updatedBiomaterial.getDerivedByProcesses()).doesNotContain(process1); + } + + private void verifyThatValidationStateChangedToDraftWhenGraphValid(MetadataDocument... values) { + Arrays.stream(values) + .forEach( + value -> + verify(validationStateChangeService, times(1)) + .changeValidationState(value.getType(), value.getId(), ValidationState.DRAFT)); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/dataset/web/DatasetControllerTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/dataset/web/DatasetControllerTest.java new file mode 100644 index 000000000..7a2e213f1 --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/dataset/web/DatasetControllerTest.java @@ -0,0 +1,301 @@ +package uk.ac.ebi.subs.ingest.dataset.web; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; +import static org.junit.Assert.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.verify; +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +import org.assertj.core.data.MapEntry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.http.HttpStatus; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.security.test.context.support.WithMockUser; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.dataset.Dataset; +import uk.ac.ebi.subs.ingest.dataset.DatasetEventHandler; +import uk.ac.ebi.subs.ingest.dataset.DatasetRepository; +import uk.ac.ebi.subs.ingest.dataset.util.UploadAreaUtil; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; + +@SpringBootTest +@AutoConfigureMockMvc(printOnlyOnFailure = false) +@WithMockUser +class DatasetControllerTest { + + @Autowired private MockMvc webApp; + + @Autowired private DatasetRepository repository; + + @Autowired private FileRepository fileRepository; + + @Autowired private BiomaterialRepository biomaterialRepository; + + @Autowired private ProcessRepository processRepository; + + @Autowired private ProtocolRepository protocolRepository; + + @Autowired private MetadataCrudService metadataCrudService; + + @Autowired private ObjectMapper objectMapper; + + @SpyBean private DatasetEventHandler eventHandler; + + @MockBean private UploadAreaUtil uploadAreaUtil; + + @AfterEach + private void tearDown() { + repository.deleteAll(); + } + + @Nested + class Registration { + @Test + @DisplayName("Register Dataset - Success") + @WithMockUser + void registerSuccess() throws Exception { + doTestRegister( + "/datasets", + dataset -> { + var datasetArgumentCaptor = ArgumentCaptor.forClass(Dataset.class); + verify(eventHandler).registeredDataset(datasetArgumentCaptor.capture()); + Dataset handledDataset = datasetArgumentCaptor.getValue(); + assertThat(handledDataset.getId()).isNotNull(); + }); + } + + private void doTestRegister(String registerUrl, Consumer postCondition) + throws Exception { + // given: + var content = new HashMap(); + content.put("name", "Test Dataset"); + + doAnswer(invocation -> null).when(uploadAreaUtil).createDataFilesUploadArea(any()); + + // when: + MvcResult result = + webApp + .perform( + post(registerUrl) + .contentType(APPLICATION_JSON_VALUE) + .content("{\"content\": " + objectMapper.writeValueAsString(content) + "}")) + .andReturn(); + + // then: + MockHttpServletResponse response = result.getResponse(); + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + assertThat(response.getContentType()).containsPattern("application/.*json.*"); + + // and: verify the registered dataset content + Map registeredDataset = + objectMapper.readValue(response.getContentAsString(), Map.class); + assertThat(registeredDataset.get("content")).isInstanceOf(Map.class); + MapEntry nameEntry = entry("name", "Test Dataset"); + assertThat((Map) registeredDataset.get("content")).containsOnly(nameEntry); + + // and: verify the dataset is stored in the repository + List datasets = repository.findAll(); + assertThat(datasets).hasSize(1); + Dataset storedDataset = datasets.get(0); + assertThat((Map) storedDataset.getContent()).containsOnly(nameEntry); + + // and: + postCondition.accept(storedDataset); + } + } + + @Nested + class Update { + @Test + @DisplayName("Update Dataset - Success") + void updateSuccess() throws Exception { + doTestUpdate( + "/datasets/{datasetId}", + dataset -> { + var datasetCaptor = ArgumentCaptor.forClass(Dataset.class); + verify(eventHandler).updatedDataset(datasetCaptor.capture()); + Dataset handledDataset = datasetCaptor.getValue(); + assertThat(handledDataset.getId()).isEqualTo(dataset.getId()); + }); + } + + private void doTestUpdate(String patchUrl, Consumer postCondition) throws Exception { + // given: + var content = new HashMap(); + content.put("description", "test"); + Dataset dataset = new Dataset(content); + dataset = repository.save(dataset); + + // when: + content.put("description", "test updated"); + MvcResult result = + webApp + .perform( + patch(patchUrl, dataset.getId()) + .contentType(APPLICATION_JSON_VALUE) + .content("{\"content\": " + objectMapper.writeValueAsString(content) + "}")) + .andReturn(); + + // expect: + MockHttpServletResponse response = result.getResponse(); + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + assertThat(response.getContentType()).containsPattern("application/.*json.*"); + + // and: + // Using Map here because reading directly to dataset converts the entire JSON to dataset + // content. + Map updated = + objectMapper.readValue(response.getContentAsString(), Map.class); + assertThat(updated.get("content")).isInstanceOf(Map.class); + MapEntry updatedDescription = entry("description", "test updated"); + assertThat((Map) updated.get("content")).containsOnly(updatedDescription); + + // and: + dataset = repository.findById(dataset.getId()).get(); + assertThat((Map) dataset.getContent()).containsOnly(updatedDescription); + + // and: + postCondition.accept(dataset); + } + + @Test + @DisplayName("Update Dataset - Not Found") + void updateDatasetNotFound() throws Exception { + // given: + var content = new HashMap(); + content.put("description", "nonExistentDataset"); + + Dataset nonExistentDataset = new Dataset(content); + + // when: + MvcResult result = + webApp + .perform( + patch("/datasets/{datasetId}", nonExistentDataset.getId()) + .contentType(APPLICATION_JSON_VALUE) + .content("{\"content\": {\"description\": \"Updated Description\"}}")) + .andReturn(); + + // then: + MockHttpServletResponse response = result.getResponse(); + assertThat(response.getStatus()).isEqualTo(HttpStatus.NOT_FOUND.value()); + } + } + + @Nested + class Delete { + @Test + @DisplayName("Delete dataset - Success") + void deleteSuccess() throws Exception { + // given: + String content = "{\"name\": \"delete dataset\"}"; + Dataset persistentDataset = new Dataset(content); + repository.save(persistentDataset); + String existingDatasetId = persistentDataset.getId(); + + // when: + webApp + .perform(delete("/datasets/{datasetId}", existingDatasetId)) + .andExpect(status().isNoContent()); + + // then: + assertThat(repository.findById(existingDatasetId)).isEmpty(); + MetadataDocument document = + metadataCrudService.findOriginalByUuid( + String.valueOf(persistentDataset.getUuid()), EntityType.DATASET); + assertNull(document); + verify(eventHandler).deletedDataset(existingDatasetId); + } + + @Test + @DisplayName("Delete dataset - Not Found") + void deleteDatasetNotFound() throws Exception { + // given: a non-existent dataset id + String nonExistentDatasetId = "nonExistentId"; + + // when: + MvcResult result = + webApp.perform(delete("/datasets/{datasetId}", nonExistentDatasetId)).andReturn(); + + // then: + MockHttpServletResponse response = result.getResponse(); + assertThat(response.getStatus()).isEqualTo(HttpStatus.NOT_FOUND.value()); + } + } + + @Nested + class Link { + @Test + @WithMockUser + @DisplayName("Link file to dataset - Success") + void lintFileToDatasetSuccess() throws Exception { + // given: + String datasetContent = "{\"name\": \"dataset\"}"; + Dataset persistentDataset = new Dataset(datasetContent); + repository.save(persistentDataset); + String datasetId = persistentDataset.getId(); + + String fileContent = "{\"name\": \"path\", \"file_type\"}"; + File persistentFile = new File(fileContent, "test.fasta"); + fileRepository.save(persistentFile); + String fileId = persistentFile.getId(); + + // when: + webApp + .perform(put("/datasets/{dataset_id}/files/{file_id}", datasetId, fileId)) + .andExpect(status().isAccepted()); + } + + @Test + @WithMockUser + @DisplayName("Link biomaterial to dataset - Success") + void lintBiomaterialToDatasetSuccess() throws Exception { + // given: + String datasetContent = "{\"name\": \"dataset\"}"; + Dataset persistentDataset = new Dataset(datasetContent); + repository.save(persistentDataset); + String datasetId = persistentDataset.getId(); + + String biomaterialContent = "{\"name\": \"test\"}"; + Biomaterial persistentBiomaterial = new Biomaterial(biomaterialContent); + biomaterialRepository.save(persistentBiomaterial); + String biomaterialId = persistentBiomaterial.getId(); + + // when: + webApp + .perform( + put("/datasets/{dataset_id}/biomaterials/{biomaterial_id}", datasetId, biomaterialId)) + .andExpect(status().isAccepted()); + } + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/export/job/ExportJobControllerTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/export/job/ExportJobControllerTest.java new file mode 100644 index 000000000..a1975804b --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/export/job/ExportJobControllerTest.java @@ -0,0 +1,166 @@ +package uk.ac.ebi.subs.ingest.export.job; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static uk.ac.ebi.subs.ingest.export.destination.ExportDestinationName.DCP; + +import org.json.simple.JSONObject; +import org.junit.Ignore; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.web.servlet.MockMvc; + +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.export.destination.ExportDestination; +import uk.ac.ebi.subs.ingest.exporter.Exporter; +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@SpringBootTest +@AutoConfigureDataMongo() +@AutoConfigureMockMvc() +public class ExportJobControllerTest { + public static final String STARTED = "STARTED"; + public static final String COMPLETE = "COMPLETE"; + @Autowired private MockMvc webApp; + + @Autowired private SubmissionEnvelopeRepository submissionEnvelopeRepository; + + @Autowired private ExportJobRepository exportJobRepository; + + @Autowired private ExportJobService exportJobService; + + @MockBean private Exporter exporter; + + // Adding MigrationConfiguration as a MockBean is needed as otherwise MigrationConfiguration won't + // be initialised. + @MockBean private MigrationConfiguration migrationConfiguration; + + SubmissionEnvelope submissionEnvelope; + + ExportJob exportJob; + + Uuid projectUuid; + + @BeforeEach + void setUp() { + submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.setUuid(Uuid.newUuid()); + submissionEnvelope.enactStateTransition(SubmissionState.EXPORTING); + submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); + + projectUuid = Uuid.newUuid(); + var destinationContext = new JSONObject(); + destinationContext.put("projectUuid", projectUuid); + + var exportJobContext = new JSONObject(); + exportJobContext.put("dataFileTransfer", false); + exportJob = + ExportJob.builder() + .id("export-job-id") + .submission(submissionEnvelope) + .destination(new ExportDestination(DCP, "v2", destinationContext)) + .context(exportJobContext) + .build(); + exportJob = exportJobRepository.save(exportJob); + } + + @AfterEach + void tearDown() { + exportJobRepository.deleteAll(); + submissionEnvelopeRepository.deleteAll(); + reset(exporter); + } + + @Test + void testDataTransferCallbackOnlyProgressesOnComplete() throws Exception { + webApp + .perform( + // when + patch("/exportJobs/{id}/context", exportJob.getId()) + .contentType(APPLICATION_JSON_VALUE) + .content("{\"dataFileTransfer\": \"" + STARTED + "\"}")) // then + .andExpect(status().isAccepted()); + verify(exporter, never()).generateSpreadsheet(any(ExportJob.class)); + + var savedJob = exportJobRepository.findById(exportJob.getId()).orElseThrow(); + assertThat(savedJob.getContext().get("dataFileTransfer")).isEqualTo(STARTED); + } + + @Ignore + void testDataTransferCallbackEndpoint() throws Exception { + webApp + .perform( + // when + patch("/exportJobs/{id}/context", exportJob.getId()) + .contentType(APPLICATION_JSON_VALUE) + .content("{\"dataFileTransfer\": \"" + COMPLETE + "\"}")) // then + .andExpect(status().isAccepted()); + var argumentCaptor = ArgumentCaptor.forClass(ExportJob.class); + verify(exporter).generateSpreadsheet(argumentCaptor.capture()); + + var capturedArgument = argumentCaptor.getValue(); + assertThat(capturedArgument.getId()).isEqualTo(exportJob.getId()); + assertThat(capturedArgument.getDestination().getContext().get("projectUuid")) + .isEqualTo(projectUuid); + assertThat(capturedArgument.getContext().get("dataFileTransfer")).isEqualTo(COMPLETE); + } + + @Test + void testSpreadsheetGenerationCallbackOnlyProgressesOnComplete() throws Exception { + // given + exportJob.getContext().put("dataFileTransfer", COMPLETE); + exportJob = exportJobRepository.save(exportJob); + + webApp + .perform( + // when + patch("/exportJobs/{id}/context", exportJob.getId()) + .contentType(APPLICATION_JSON_VALUE) + .content("{\"spreadsheetGeneration\": \"" + STARTED + "\"}")) // then + .andExpect(status().isAccepted()); + verify(exporter, never()).exportMetadata(any(ExportJob.class)); + + var savedJob = exportJobRepository.findById(exportJob.getId()).orElseThrow(); + assertThat(savedJob.getContext().get("dataFileTransfer")).isEqualTo(COMPLETE); + assertThat(savedJob.getContext().get("spreadsheetGeneration")).isEqualTo(STARTED); + } + + @Ignore + void testSpreadsheetGenerationCallbackEndpoint() throws Exception { + // given + exportJob.getContext().put("dataFileTransfer", COMPLETE); + exportJob.getContext().put("spreadsheetGeneration", STARTED); + exportJob = exportJobRepository.save(exportJob); + + webApp + .perform( + // when + patch("/exportJobs/{id}/context", exportJob.getId()) + .contentType(APPLICATION_JSON_VALUE) + .content("{\"spreadsheetGeneration\": \"" + COMPLETE + "\"}")) // then + .andExpect(status().isAccepted()); + var argumentCaptor = ArgumentCaptor.forClass(ExportJob.class); + verify(exporter).exportMetadata(argumentCaptor.capture()); + + var capturedArgument = argumentCaptor.getValue(); + assertThat(capturedArgument.getId()).isEqualTo(exportJob.getId()); + assertThat(capturedArgument.getDestination().getContext().get("projectUuid")) + .isEqualTo(projectUuid); + assertThat(capturedArgument.getContext().get("dataFileTransfer")).isEqualTo(COMPLETE); + assertThat(capturedArgument.getContext().get("spreadsheetGeneration")).isEqualTo(COMPLETE); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/file/FileControllerTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/file/FileControllerTest.java new file mode 100644 index 000000000..b31e35cea --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/file/FileControllerTest.java @@ -0,0 +1,454 @@ +package uk.ac.ebi.subs.ingest.file; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.security.test.context.support.WithMockUser; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.web.servlet.support.ServletUriComponentsBuilder; +import org.springframework.web.util.UriComponentsBuilder; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import uk.ac.ebi.subs.ingest.TestingHelper; +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.service.ValidationStateChangeService; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.project.*; +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@SpringBootTest +@AutoConfigureDataMongo() +@AutoConfigureMockMvc(printOnlyOnFailure = false) +@WithMockUser( + username = "alice", + roles = {"WRANGLER"}) +public class FileControllerTest { + @MockBean ValidationStateChangeService validationStateChangeService; + + @Autowired private MockMvc webApp; + + @Autowired private ProcessRepository processRepository; + + @Autowired private SubmissionEnvelopeRepository submissionEnvelopeRepository; + + @Autowired private FileRepository fileRepository; + + @Autowired private ProjectRepository projectRepository; + + @Autowired private ObjectMapper objectMapper; + + @MockBean private MigrationConfiguration migrationConfiguration; + + @MockBean private MessageRouter messageRouter; + + Process process1; + + Process process2; + + Process process3; + + File file; + + UriComponentsBuilder uriBuilder; + + SubmissionEnvelope submissionEnvelope; + + Project project; + + @BeforeEach + void setUp() { + submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.setUuid(Uuid.newUuid()); + submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALID); + submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); + + project = new Project(new HashMap<>()); + project.setUuid(Uuid.newUuid()); + ((Map) project.getContent()) + .put("dataAccess", new ObjectToMapConverter().asMap(new DataAccess(DataAccessTypes.OPEN))); + + project.setSubmissionEnvelope(submissionEnvelope); + project.getSubmissionEnvelopes().add(submissionEnvelope); + project = projectRepository.save(project); + + process1 = processRepository.save(new Process(null)); + process2 = processRepository.save(new Process(null)); + process3 = processRepository.save(new Process(null)); + + file = new File(null, "fileName"); + file.setSubmissionEnvelope(submissionEnvelope); + file = fileRepository.save(file); + + uriBuilder = ServletUriComponentsBuilder.fromCurrentContextPath(); + } + + @AfterEach + void tearDown() { + processRepository.deleteAll(); + fileRepository.deleteAll(); + submissionEnvelopeRepository.deleteAll(); + } + + @Test + public void newFileInSubmissionLinksToSubmissionAndProject() throws Exception { + // given + fileRepository.deleteAll(); + + // when + webApp + .perform( + post("/submissionEnvelopes/{id}/files", submissionEnvelope.getId()) + .contentType(MediaType.APPLICATION_JSON) + .content("{\"content\": {}}")) + .andExpect(status().isAccepted()); + + TestingHelper.resetTestingSecurityContext(); + // then + assertThat(fileRepository.findAll()).hasSize(1); + assertThat(fileRepository.findAllBySubmissionEnvelope(submissionEnvelope)).hasSize(1); + assertThat(fileRepository.findByProject(project)).hasSize(1); + + var newFile = fileRepository.findAll().get(0); + assertThat(newFile.getSubmissionEnvelope().getId()).isEqualTo(submissionEnvelope.getId()); + assertThat(newFile.getProject().getId()).isEqualTo(project.getId()); + } + + @Test + public void newFileInSubmissionDoesNotFailIfSubmissionHasNoProject() throws Exception { + // given + fileRepository.deleteAll(); + projectRepository.deleteAll(); + + // when + webApp + .perform( + post("/submissionEnvelopes/{id}/files", submissionEnvelope.getId()) + .contentType(MediaType.APPLICATION_JSON) + .content("{\"content\": {}}")) + .andExpect(status().isAccepted()); + TestingHelper.resetTestingSecurityContext(); + + // then + assertThat(fileRepository.findAll()).hasSize(1); + assertThat(fileRepository.findAllBySubmissionEnvelope(submissionEnvelope)).hasSize(1); + + var newFile = fileRepository.findAll().get(0); + assertThat(newFile.getSubmissionEnvelope().getId()).isEqualTo(submissionEnvelope.getId()); + assertThat(newFile.getProject()).isNull(); + } + + @Test + public void testLinkFileAsInputToProcessesUsingPutMethodWithManyProcessesInPayload() + throws Exception { + file.addAsInputToProcess(process1); + fileRepository.save(file); + + webApp + .perform( + put("/files/{fileId}/inputToProcesses/", file.getId()) + .contentType("text/uri-list") + .content( + uriBuilder.build().toUriString() + + "/processes/" + + process2.getId() + + '\n' + + uriBuilder.build().toUriString() + + "/processes/" + + process3.getId())) + .andExpect(status().isOk()); + TestingHelper.resetTestingSecurityContext(); + + verifyThatValidationStateChangedToDraftWhenGraphValid(file); + + File updatedFile = fileRepository.findById(file.getId()).get(); + assertThat(updatedFile.getInputToProcesses()) + .usingElementComparatorOnFields("id") + .containsExactly(process2, process3); + } + + @Test + public void testLinkFileAsInputToMultipleProcessesUsingPostMethodWithManyProcessesInPayload() + throws Exception { + // when + webApp + .perform( + post("/files/{fileId}/inputToProcesses/", file.getId()) + .contentType("text/uri-list") + .content( + uriBuilder.build().toUriString() + + "/processes/" + + process1.getId() + + '\n' + + uriBuilder.build().toUriString() + + "/processes/" + + process2.getId())) + .andExpect(status().isOk()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(file); + File updatedFile = fileRepository.findById(file.getId()).get(); + assertThat(updatedFile.getInputToProcesses()) + .usingElementComparatorOnFields("id") + .containsExactly(process1, process2); + } + + @Test + public void testLinkFileAsInputToProcessesUsingPostMethodWithOneProcessInPayload() + throws Exception { + // when + webApp + .perform( + post("/files/{fileId}/inputToProcesses/", file.getId()) + .contentType("text/uri-list") + .content(uriBuilder.build().toUriString() + "/processes/" + process1.getId())) + .andExpect(status().isOk()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(file); + File updatedFile = fileRepository.findById(file.getId()).get(); + assertThat(updatedFile.getInputToProcesses()) + .usingElementComparatorOnFields("id") + .containsExactly(process1); + } + + @Test + public void testLinkFileAsDerivedByProcessesUsingPostMethodWithOneProcessInPayload() + throws Exception { + // when + webApp + .perform( + post("/files/{fileId}/derivedByProcesses/", file.getId()) + .contentType("text/uri-list") + .content(uriBuilder.build().toUriString() + "/processes/" + process1.getId())) + .andExpect(status().isOk()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(file); + File updatedFile = fileRepository.findById(file.getId()).get(); + assertThat(updatedFile.getDerivedByProcesses()) + .usingElementComparatorOnFields("id") + .contains(process1); + } + + @Test + public void testLinkFileAsDerivedByProcessesUsingPostMethodWithManyProcessesInPayload() + throws Exception { + // when + webApp + .perform( + post("/files/{fileId}/derivedByProcesses/", file.getId()) + .contentType("text/uri-list") + .content( + uriBuilder.build().toUriString() + + "/processes/" + + process1.getId() + + '\n' + + uriBuilder.build().toUriString() + + "/processes/" + + process2.getId())) + .andExpect(status().isOk()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(file); + File updatedFile = fileRepository.findById(file.getId()).get(); + assertThat(updatedFile.getDerivedByProcesses()) + .usingElementComparatorOnFields("id") + .containsExactly(process1, process2); + } + + @Test + public void testLinkFileAsDerivedByProcessesUsingPutMethodWithManyProcessesInPayload() + throws Exception { + // given + file.addAsDerivedByProcess(process1); + fileRepository.save(file); + + // when + webApp + .perform( + put("/files/{fileId}/derivedByProcesses/", file.getId()) + .contentType("text/uri-list") + .content( + uriBuilder.build().toUriString() + + "/processes/" + + process2.getId() + + '\n' + + uriBuilder.build().toUriString() + + "/processes/" + + process3.getId())) + .andExpect(status().isOk()); + + TestingHelper.resetTestingSecurityContext(); + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(file); + File updatedFile = fileRepository.findById(file.getId()).get(); + assertThat(updatedFile.getDerivedByProcesses()) + .usingElementComparatorOnFields("id") + .containsExactly(process2, process3); + } + + @Test + public void testUnlinkFileAsDerivedByProcesses() throws Exception { + // given + file.addAsDerivedByProcess(process1); + fileRepository.save(file); + + // when + webApp + .perform( + delete( + "/files/{fileId}/derivedByProcesses/{processId}", file.getId(), process1.getId())) + .andExpect(status().isNoContent()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(file); + File updatedFile = fileRepository.findById(file.getId()).get(); + assertThat(updatedFile.getDerivedByProcesses()).doesNotContain(process1); + } + + @Test + public void testUnlinkFileAsInputToProcesses() throws Exception { + // given + file.addAsInputToProcess(process1); + fileRepository.save(file); + + // when + webApp + .perform( + delete("/files/{fileId}/inputToProcesses/{processId}", file.getId(), process1.getId())) + .andExpect(status().isNoContent()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(file); + + File updatedFile = fileRepository.findById(file.getId()).get(); + assertThat(updatedFile.getInputToProcesses()).doesNotContain(process1); + } + + private void verifyThatValidationStateChangedToDraftWhenGraphValid(MetadataDocument... values) { + Arrays.stream(values) + .forEach( + value -> { + verify(validationStateChangeService, times(1)) + .changeValidationState(value.getType(), value.getId(), ValidationState.DRAFT); + }); + } + + @Test + public void testValidationJobPatch() throws Exception { + // given: + File file = new File(null, "test"); + file.setSubmissionEnvelope(submissionEnvelope); + file = fileRepository.save(file); + + // when: + String patch = + "{ \"validationJob\": { \"validationReport\": { \"validationState\": \"Valid\" }}}"; + + MvcResult result = + webApp + .perform( + patch("/files/{id}", file.getId()) + .contentType(APPLICATION_JSON_VALUE) + .content(patch)) + .andReturn(); + TestingHelper.resetTestingSecurityContext(); + + // expect: + MockHttpServletResponse response = result.getResponse(); + assertThat(response.getContentType()).containsPattern("application/.*json.*"); + + // and: + file = fileRepository.findById(file.getId()).get(); + assertThat(file.getValidationJob().getValidationReport().getValidationState()) + .isEqualTo(ValidationState.VALID); + } + + @Test + public void when_new_File_ctor__pass() throws Exception { + String filePayload = objectMapper.writeValueAsString(new File()); + webApp + .perform( + post("/submissionEnvelopes/{id}/files", submissionEnvelope.getId()) + .contentType(MediaType.APPLICATION_JSON) + .content(filePayload)) + .andExpect(status().isAccepted()); + } + + @Test + public void when_DataFileUuid_is_null__accepted_with_random() throws Exception { + ObjectNode patch = createPayloadhNoDataFileUuid(); + webApp + .perform( + post("/submissionEnvelopes/{id}/files", submissionEnvelope.getId()) + .contentType(MediaType.APPLICATION_JSON) + .content(objectMapper.writeValueAsString(patch))) + .andExpect(status().isAccepted()) + .andExpect(jsonPath("$.dataFileUuid").isNotEmpty()); + } + + @Test + public void when_payload_is_good__pass() throws Exception { + ObjectNode patch = createValidFilePayload(); + webApp + .perform( + post("/submissionEnvelopes/{id}/files", submissionEnvelope.getId()) + .contentType(MediaType.APPLICATION_JSON) + .content(objectMapper.writeValueAsString(patch))) + .andExpect(status().isAccepted()); + } + + private ObjectNode createValidFilePayload() { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode newFilePayload = mapper.createObjectNode(); + + newFilePayload + .put("dataFileUuid", UUID.randomUUID().toString()) + .put("fileName", "test-file") + .put("fileContentType", "text/plain"); + return newFilePayload; + } + + private ObjectNode createPayloadhNoDataFileUuid() { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode newFilePayload = mapper.createObjectNode(); + + newFilePayload.put("fileName", "test-file").put("fileContentType", "text/plain"); + return newFilePayload; + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/file/FileServiceTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/file/FileServiceTest.java new file mode 100644 index 000000000..ec3f43233 --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/file/FileServiceTest.java @@ -0,0 +1,164 @@ +package uk.ac.ebi.subs.ingest.file; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +import java.util.ArrayList; +import java.util.UUID; +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.ApplicationContext; +import org.springframework.dao.OptimisticLockingFailureException; + +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.core.Checksums; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.exception.CoreEntityNotFoundException; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.file.web.FileMessage; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.state.MetadataDocumentEventHandler; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@SpringBootTest +public class FileServiceTest { + @MockBean MigrationConfiguration migrationConfiguration; + + @MockBean FileRepository fileRepository; + + @MockBean SubmissionEnvelopeRepository submissionEnvelopeRepository; + + @MockBean BiomaterialRepository biomaterialRepository; + + @MockBean ProcessRepository processRepository; + + @MockBean ProjectRepository projectRepository; + + @MockBean MetadataDocumentEventHandler metadataDocumentEventHandler; + + @MockBean MetadataCrudService metadataCrudService; + + @MockBean MetadataUpdateService metadataUpdateService; + + @Autowired FileService fileService; + + @Autowired private ApplicationContext applicationContext; + + FileMessage fileMessage; + + SubmissionEnvelope submissionEnvelope; + + File file; + + Project project; + + @BeforeEach + void setUp() { + applicationContext.getBeansWithAnnotation(MockBean.class).forEach(Mockito::reset); + + Checksums checksums = new Checksums("sha1", "sha256", "crc32c", "s3Etag"); + String submissionUuid = UUID.randomUUID().toString(); + String filename = "filename"; + fileMessage = + new FileMessage("cloudUrl", filename, submissionUuid, "content_type", checksums, 123); + + submissionEnvelope = new SubmissionEnvelope(); + + project = spy(new Project(null)); + when(project.getId()).thenReturn("projectId"); + + file = new File(null, filename); + var files = new ArrayList(); + files.add(file); + + when(submissionEnvelopeRepository.findByUuid(any(Uuid.class))).thenReturn(submissionEnvelope); + when(projectRepository.findBySubmissionEnvelopesContains(submissionEnvelope)) + .thenReturn(Stream.of(project)); + when(fileRepository.findBySubmissionEnvelopeAndFileName( + submissionEnvelope, fileMessage.getFileName())) + .thenReturn(files); + when(fileRepository.save(file)).thenReturn(file); + } + + @Test + public void testCreateFileFromFileMessage() throws CoreEntityNotFoundException { + // given: + when(fileRepository.findBySubmissionEnvelopeAndFileName( + submissionEnvelope, fileMessage.getFileName())) + .thenReturn(new ArrayList<>()); + + // when: + fileService.createFileFromFileMessage(fileMessage); + + // then: + verify(metadataCrudService) + .addToSubmissionEnvelopeAndSave(any(File.class), any(SubmissionEnvelope.class)); + } + + @Test + public void testCreateFileFromFileMessageNotCreated() throws CoreEntityNotFoundException { + // when: + fileService.createFileFromFileMessage(fileMessage); + + // then: + verify(metadataCrudService, never()) + .addToSubmissionEnvelopeAndSave(any(File.class), any(SubmissionEnvelope.class)); + } + + @Test + public void testUpdateFileFromFileMessage() throws CoreEntityNotFoundException { + // when: + fileService.updateFileFromFileMessage(fileMessage); + + // then: + verify(fileRepository).save(file); + assertThat(file.getCloudUrl()).isEqualTo(fileMessage.getCloudUrl()); + assertThat(file.getChecksums()).isEqualTo(fileMessage.getChecksums()); + assertThat(file.getFileContentType()).isEqualTo(fileMessage.getContentType()); + assertThat(file.getValidationState()).isEqualTo(ValidationState.DRAFT); + } + + @Test + public void testUpdateFileFromFileMessageRetry() throws CoreEntityNotFoundException { + // given: + when(fileRepository.save(file)) + .thenThrow(new OptimisticLockingFailureException("Error")) + .thenReturn(file); + + // when: + fileService.updateFileFromFileMessage(fileMessage); + + // then: + verify(fileRepository, times(2)).save(file); + } + + @Test + public void testUpdateFileFromFileMessageMaxRetries() throws CoreEntityNotFoundException { + // given: + when(fileRepository.save(file)).thenThrow(new OptimisticLockingFailureException("Error")); + + // when: + assertThatExceptionOfType(OptimisticLockingFailureException.class) + .isThrownBy( + () -> { + fileService.updateFileFromFileMessage(fileMessage); + }); + + // then: + verify(fileRepository, times(5)).save(file); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/process/ProcessControllerTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/process/ProcessControllerTest.java new file mode 100644 index 000000000..8638d1a78 --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/process/ProcessControllerTest.java @@ -0,0 +1,302 @@ +package uk.ac.ebi.subs.ingest.process; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.security.test.context.support.WithMockUser; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.web.servlet.support.ServletUriComponentsBuilder; +import org.springframework.web.util.UriComponentsBuilder; + +import uk.ac.ebi.subs.ingest.TestingHelper; +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.service.ValidationStateChangeService; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.project.*; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@SpringBootTest +@AutoConfigureDataMongo() +@AutoConfigureMockMvc(printOnlyOnFailure = false) +@WithMockUser( + username = "alice", + roles = {"WRANGLER"}) +class ProcessControllerTest { + @MockBean ValidationStateChangeService validationStateChangeService; + + @MockBean private MessageRouter messageRouter; + + @MockBean private MigrationConfiguration migrationConfiguration; + + @Autowired private MockMvc webApp; + + @Autowired private ProcessRepository processRepository; + + @Autowired private ProtocolRepository protocolRepository; + + @Autowired private ProjectRepository projectRepository; + + @Autowired private SubmissionEnvelopeRepository submissionEnvelopeRepository; + + @Autowired private ProjectService projectService; + Protocol protocol1; + + Protocol protocol2; + + Protocol protocol3; + + Project project; + + Process process; + + UriComponentsBuilder uriBuilder; + + SubmissionEnvelope submissionEnvelope; + + @BeforeEach + void setUp() { + submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.setUuid(Uuid.newUuid()); + submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALID); + submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); + + protocol1 = protocolRepository.save(new Protocol(null)); + protocol2 = protocolRepository.save(new Protocol(null)); + protocol3 = protocolRepository.save(new Protocol(null)); + + project = new Project(new HashMap<>()); + ((Map) project.getContent()) + .put("dataAccess", new ObjectToMapConverter().asMap(new DataAccess(DataAccessTypes.OPEN))); + projectService.addProjectToSubmissionEnvelope(submissionEnvelope, project); + + project.setSubmissionEnvelope(submissionEnvelope); + project.getSubmissionEnvelopes().add(submissionEnvelope); + project = projectRepository.save(project); + + process = new Process(null); + process.setSubmissionEnvelope(submissionEnvelope); + process.setProject(project); + + process = processRepository.save(process); + + uriBuilder = ServletUriComponentsBuilder.fromCurrentContextPath(); + } + + @AfterEach + void tearDown() { + submissionEnvelopeRepository.deleteAll(); + processRepository.deleteAll(); + protocolRepository.deleteAll(); + projectRepository.deleteAll(); + } + + @Test + public void newProcessInSubmissionLinksToSubmissionAndProject() throws Exception { + // given + processRepository.deleteAll(); + + // when + webApp + .perform( + post("/submissionEnvelopes/{id}/processes", submissionEnvelope.getId()) + .contentType(MediaType.APPLICATION_JSON) + .content("{\"content\": {}}")) + .andExpect(status().isAccepted()); + TestingHelper.resetTestingSecurityContext(); + + // then + assertThat(processRepository.findAll()).hasSize(1); + assertThat(processRepository.findAllBySubmissionEnvelope(submissionEnvelope)).hasSize(1); + assertThat(processRepository.findByProject(project)).hasSize(1); + + var newProcess = processRepository.findAll().get(0); + assertThat(newProcess.getSubmissionEnvelope().getId()).isEqualTo(submissionEnvelope.getId()); + assertThat(newProcess.getProject().getId()).isEqualTo(project.getId()); + assertThat(newProcess.getProjects()).hasSize(1); + assertThat(newProcess.getProjects().stream().findFirst().get().getId()) + .isEqualTo(project.getId()); + } + + @Test + public void newProcessInSubmissionDoesNotFailIfSubmissionHasNoProject() throws Exception { + // given + processRepository.deleteAll(); + projectRepository.deleteAll(); + + // when + webApp + .perform( + post("/submissionEnvelopes/{id}/processes", submissionEnvelope.getId()) + .contentType(MediaType.APPLICATION_JSON) + .content("{\"content\": {}}")) + .andExpect(status().isAccepted()); + TestingHelper.resetTestingSecurityContext(); + + // then + assertThat(processRepository.findAll()).hasSize(1); + assertThat(processRepository.findAllBySubmissionEnvelope(submissionEnvelope)).hasSize(1); + + var newProcess = processRepository.findAll().get(0); + assertThat(newProcess.getSubmissionEnvelope().getId()).isEqualTo(submissionEnvelope.getId()); + assertThat(newProcess.getProject()).isNull(); + assertThat(newProcess.getProjects()).isEmpty(); + } + + @Test + public void testLinkProtocolsToProcessUsingPutMethodWithManyProtocolsInPayload() + throws Exception { + // given + process.addProtocol(protocol1); + processRepository.save(process); + + // when + webApp + .perform( + put("/processes/{id}/protocols/", process.getId()) + .contentType("text/uri-list") + .content( + uriBuilder.build().toUriString() + + "/protocols/" + + protocol2.getId() + + '\n' + + uriBuilder.build().toUriString() + + "/protocols/" + + protocol3.getId())) + .andExpect(status().isOk()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(process); + Process updatedProcess = processRepository.findById(process.getId()).get(); + assertThat(updatedProcess.getProtocols()) + .usingElementComparatorOnFields("id") + .containsExactly(protocol2, protocol3); + } + + @Test + public void testLinkProtocolsToProcessUsingPostMethodWithManyProtocolsInPayload() + throws Exception { + // when + webApp + .perform( + post("/processes/{id}/protocols/", process.getId()) + .contentType("text/uri-list") + .content( + uriBuilder.build().toUriString() + + "/protocols/" + + protocol1.getId() + + '\n' + + uriBuilder.build().toUriString() + + "/protocols/" + + protocol2.getId())) + .andExpect(status().isOk()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(process); + Process updatedProcess = processRepository.findById(process.getId()).get(); + assertThat(updatedProcess.getProtocols()) + .usingElementComparatorOnFields("id") + .containsExactly(protocol1, protocol2); + } + + @Test + public void testLinkProtocolsToProcessUsingPostMethodWithOneProtocolInPayload() throws Exception { + // when + webApp + .perform( + post("/processes/{processId}/protocols/", process.getId()) + .contentType("text/uri-list") + .content(uriBuilder.build().toUriString() + "/protocols/" + protocol1.getId())) + .andExpect(status().isOk()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(process); + Process updatedProcess = processRepository.findById(process.getId()).get(); + assertThat(updatedProcess.getProtocols()) + .usingElementComparatorOnFields("id") + .containsExactly(protocol1); + } + + @Test + public void testUnlinkProtocolFromProcess() throws Exception { + // given + process.addProtocol(protocol1); + processRepository.save(process); + + // when + webApp + .perform( + delete( + "/processes/{processId}/protocols/{protocolId}", + process.getId(), + protocol1.getId())) + .andExpect(status().isNoContent()); + TestingHelper.resetTestingSecurityContext(); + + // then + verifyThatValidationStateChangedToDraftWhenGraphValid(process); + + Process updatedProcess = processRepository.findById(process.getId()).get(); + assertThat(updatedProcess.getProtocols()).doesNotContain(protocol1); + } + + @Test + public void testLinkProjectToProcessDoesNotChangeTheirValidationStatesToDraft() throws Exception { + webApp + .perform( + put("/processes/{processId}/project", process.getId()) + .contentType("text/uri-list") + .content(uriBuilder.build().toUriString() + "/projects/" + project.getId())) + .andExpect(status().isNoContent()); + + verify(validationStateChangeService, times(0)) + .changeValidationState(any(), any(), eq(ValidationState.DRAFT)); + } + + @Test + public void testUnlinkProjectFromProcessDoesNotChangeTheirValidationStatesToDraft() + throws Exception { + webApp + .perform( + delete("/processes/{processId}/project/{projectId}", process.getId(), project.getId())) + .andExpect(status().isNoContent()); + + verify(validationStateChangeService, times(0)) + .changeValidationState(any(), any(), eq(ValidationState.DRAFT)); + } + + private void verifyThatValidationStateChangedToDraftWhenGraphValid(MetadataDocument... values) { + Arrays.stream(values) + .forEach( + value -> + verify(validationStateChangeService, times(1)) + .changeValidationState(value.getType(), value.getId(), ValidationState.DRAFT)); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/process/ProcessRepositoryTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/process/ProcessRepositoryTest.java new file mode 100644 index 000000000..0ac645510 --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/process/ProcessRepositoryTest.java @@ -0,0 +1,60 @@ +package uk.ac.ebi.subs.ingest.process; + +import static java.util.Arrays.asList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; + +import java.util.Optional; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.data.mongo.DataMongoTest; +import org.springframework.boot.test.mock.mockito.MockBean; + +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; + +@DataMongoTest +public class ProcessRepositoryTest { + + @Autowired private ProcessRepository processRepository; + + @Autowired private ProtocolRepository protocolRepository; + + @MockBean private MigrationConfiguration migrationConfiguration; + + @MockBean private MessageRouter messageRouter; + + @AfterEach + private void tearDown() { + processRepository.deleteAll(); + protocolRepository.deleteAll(); + } + + @Test + public void findFirstByProtocolNonUnique() { + // given: + Protocol protocol = protocolRepository.save(new Protocol(null)); + + Process process1 = new Process(null); + process1.addProtocol(protocol); + process1 = processRepository.save(process1); + + Process process2 = new Process(null); + process2.addProtocol(protocol); + process2 = processRepository.save(process2); + + // and: + assumeThat(processRepository.findAll()).hasSize(2); + + // when: + Optional first = processRepository.findFirstByProtocolsContains(protocol); + + // then + assertThat(first.isPresent()).isTrue(); + assertThat(first.get().getId()).isIn(asList(process1.getId(), process2.getId())); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/project/ProjectFilterTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/project/ProjectFilterTest.java new file mode 100644 index 000000000..30f4cf3ff --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/project/ProjectFilterTest.java @@ -0,0 +1,539 @@ +package uk.ac.ebi.subs.ingest.project; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.data.mongo.DataMongoTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Import; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.index.TextIndexDefinition; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.security.test.context.support.WithMockUser; + +import uk.ac.ebi.subs.ingest.audit.AuditEntryService; +import uk.ac.ebi.subs.ingest.bundle.BundleManifestRepository; +import uk.ac.ebi.subs.ingest.config.ConvertersConfiguration; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.dataset.DatasetRepository; +import uk.ac.ebi.subs.ingest.project.web.SearchFilter; +import uk.ac.ebi.subs.ingest.project.web.SearchType; +import uk.ac.ebi.subs.ingest.schemas.SchemaService; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@DataMongoTest +@WithMockUser( + username = "alice", + roles = {"WRANGLER"}) +@Import(ConvertersConfiguration.class) +class ProjectFilterTest { + + // class under test + private ProjectService projectService; + + // participants + @Autowired private MongoTemplate mongoTemplate; + + @MockBean private SubmissionEnvelopeRepository submissionEnvelopeRepository; + + @MockBean private ProjectRepository projectRepository; + + @MockBean private DatasetRepository datasetRepository; + + @MockBean private MetadataCrudService metadataCrudService; + + @MockBean private MetadataUpdateService metadataUpdateService; + + @MockBean private SchemaService schemaService; + + @MockBean private BundleManifestRepository bundleManifestRepository; + + @MockBean private ProjectEventHandler projectEventHandler; + + @MockBean private AuditEntryService auditEntryService; + + private Project project1; + private Project project2; + private Project project3; + + Comparator upToMillies = Comparator.comparing(d -> d.truncatedTo(ChronoUnit.MILLIS)); + + @Test + void test_raw_criteria() { + Query query = + new Query() + .addCriteria( + Criteria.where("content.project_core.project_title").regex("project1", "i")); + Project actual = this.mongoTemplate.findOne(query, Project.class); + assertThat(actual) + .usingComparatorForFields(upToMillies, "contentLastModified") + .isEqualToComparingFieldByFieldRecursively(project1); + } + + @Test + void test_criteria_building() { + SearchFilter searchFilter = SearchFilter.builder().wranglingState("NEW").build(); + Query query = ProjectQueryBuilder.buildProjectsQuery(searchFilter); + Project actual = this.mongoTemplate.find(query, Project.class).get(0); + assertThat(actual) + .usingComparatorForFields(upToMillies, "contentLastModified") + .isEqualToComparingFieldByFieldRecursively(project1); + } + + @Test + void test_criteria_building_with_pageable() { + SearchFilter searchFilter = SearchFilter.builder().search("project1").build(); + Query query = ProjectQueryBuilder.buildProjectsQuery(searchFilter); + Pageable pageable = PageRequest.of(0, 10); + Project actual = this.mongoTemplate.find(query.with(pageable), Project.class).get(0); + assertThat(actual) + .usingComparatorForFields(upToMillies, "contentLastModified") + .isEqualToComparingFieldByFieldRecursively(project1); + } + + @Test + void filter_by_state() { + Project project4 = makeProject("project4"); + project4.setWranglingState(WranglingState.IN_PROGRESS); + this.mongoTemplate.save(project4); + + // when + SearchFilter filterNew = SearchFilter.builder().wranglingState("NEW").build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(filterNew, pageable); + + // then + assertThat(result.getContent()) + .hasSize(3) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder(project1, project2, project3); + } + + @Test + void filter_by_wrangler() { + // given + // when + SearchFilter searchFilter = + SearchFilter.builder().primaryWrangler(this.project2.getPrimaryWrangler()).build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getTotalElements()).isEqualTo(1); + assertThat(result.getContent()) + .hasSize(1) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingRecursiveFieldByFieldElementComparator() + .containsExactly(this.project2); + } + + @Test + void filter_by_wrangling_priority() { + // given + Project project4 = makeProject("project4"); + project4.setWranglingPriority(3); + this.mongoTemplate.save(project4); + // when + SearchFilter searchFilter = + SearchFilter.builder().wranglingPriority(this.project1.getWranglingPriority()).build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getTotalElements()).isEqualTo(3); + assertThat(result.getContent()) + .hasSize(3) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder(project1, project2, project3); + } + + @Test + void filter_by_official_hca_publication() { + // given + Project project4 = makeProject("project4"); + var content = + Map.of( + "project_core", Map.of("project_title", "Project 4"), + "publications", List.of(Map.of("official_hca_publication", true))); + project4.setContent(content); + this.mongoTemplate.save(project4); + + // when + SearchFilter searchFilter = SearchFilter.builder().hasOfficialHcaPublication(true).build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getTotalElements()).isEqualTo(1); + assertThat(result.getContent()) + .hasSize(1) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingRecursiveFieldByFieldElementComparator() + .containsExactly(project4); + } + + @Test + void filter_by_identifying_organisms() { + // given + Project project4 = makeProject("project4"); + String human = "Human"; + project4.setIdentifyingOrganisms(List.of(human)); + this.mongoTemplate.save(project4); + + // when + SearchFilter searchFilter = SearchFilter.builder().identifyingOrganism(human).build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getTotalElements()).isEqualTo(1); + assertThat(result.getContent()) + .hasSize(1) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingRecursiveFieldByFieldElementComparator() + .containsExactly(project4); + } + + @Test + void filter_by_organ_ontology() { + // given + Project project4 = makeProject("project4"); + String ontologyTerm = "AN_ONTOLOGY"; + project4.setOrgan(Map.of("ontologies", List.of(Map.of("ontology", ontologyTerm)))); + this.mongoTemplate.save(project4); + // when + SearchFilter searchFilter = SearchFilter.builder().organOntology(ontologyTerm).build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getTotalElements()).isEqualTo(1); + assertThat(result.getContent()) + .hasSize(1) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingRecursiveFieldByFieldElementComparator() + .containsExactly(project4); + } + + @Test + void filter_by_cell_count() { + // given + Project project4 = makeProject("project4"); + project4.setCellCount(1000); + this.mongoTemplate.save(project4); + // when + SearchFilter searchFilter = + SearchFilter.builder().maxCellCount(project1.getCellCount()).minCellCount(0).build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getTotalElements()).isEqualTo(3); + assertThat(result.getContent()) + .hasSize(3) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder(project1, project2, project3); + } + + @Test + void filter_by_data_access() { + // given + Project project4 = makeProject("project4"); + ((Map) project4.getContent()) + .put("dataAccess", new DataAccess(DataAccessTypes.MANAGED)); + this.mongoTemplate.save(project4); + // when + SearchFilter searchFilter = SearchFilter.builder().dataAccess(DataAccessTypes.MANAGED).build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getTotalElements()).isEqualTo(1); + assertThat(result.getContent()) + .hasSize(1) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingRecursiveFieldByFieldElementComparator() + .containsExactly(project4); + } + + @Test + void filter_by_text() { + // given + // when + SearchFilter searchFilter = SearchFilter.builder().search("project1").build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getContent()) + .hasSize(1) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingElementComparatorIgnoringFields("supplementaryFiles", "submissionEnvelopes") + .containsExactly(project1); + } + + @Test + void query_all_keywords() { + SearchFilter searchFilter = SearchFilter.builder().search("human liver").build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getContent()) + .hasSize(1) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingElementComparatorIgnoringFields("supplementaryFiles", "submissionEnvelopes") + .containsExactly(project1); + } + + @Test + void query_all_keywords__order_independent() { + SearchFilter searchFilter = + SearchFilter.builder().search("liver human").searchType(SearchType.AllKeywords).build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getContent()) + .hasSize(1) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingElementComparatorIgnoringFields("supplementaryFiles", "submissionEnvelopes") + .containsExactly(project1); + } + + @Test + void query_any_keywords() { + SearchFilter searchFilter = + SearchFilter.builder().search("liver mouse").searchType(SearchType.AnyKeyword).build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + Page resultFromReverse = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getContent()) + .hasSize(2) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingElementComparatorIgnoringFields("supplementaryFiles", "submissionEnvelopes") + .containsExactlyInAnyOrder(project1, project2); + } + + @Test + void query_exact_phrase__correct_order() { + SearchFilter searchFilter = + SearchFilter.builder().search("human liver").searchType(SearchType.ExactMatch).build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + Page resultFromReverse = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getContent()) + .hasSize(1) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingElementComparatorIgnoringFields("supplementaryFiles", "submissionEnvelopes") + .containsExactly(project1); + } + + @Test + void query_exact_phrase__reverse_order() { + SearchFilter searchFilter = + SearchFilter.builder().search("liver human").searchType(SearchType.ExactMatch).build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getContent()).hasSize(0); + } + + @Test + void lookup_by_uuid() { + String uuidString = project1.getUuid().getUuid().toString(); + SearchFilter searchFilter = SearchFilter.builder().search(uuidString).build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getContent()) + .hasSize(1) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingElementComparatorIgnoringFields("supplementaryFiles", "submissionEnvelopes") + .containsExactly(project1); + } + + @Test + void filter_by_release() { + // given + Project project4 = makeProject("project4"); + project4.setDcpReleaseNumber(11); + this.mongoTemplate.save(project4); + // when + SearchFilter searchFilter = SearchFilter.builder().dcpReleaseNumber(11).build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getTotalElements()).isEqualTo(1); + assertThat(result.getContent()) + .hasSize(1) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder(project4); + } + + @Test + void filter_by_project_labels() { + // given + Project project4 = makeProject("project4"); + project4.setProjectLabels(List.of("CellxGene")); + this.mongoTemplate.save(project4); + // when + SearchFilter searchFilter = SearchFilter.builder().projectLabels("CellxGene").build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getTotalElements()).isEqualTo(1); + assertThat(result.getContent()) + .hasSize(1) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder(project4); + } + + @Test + void filter_by_project_networks() { + // given + Project project5 = makeProject("project5"); + project5.setProjectNetworks(List.of("Lung")); + this.mongoTemplate.save(project5); + // when + SearchFilter searchFilter = SearchFilter.builder().projectNetworks("Lung").build(); + + Pageable pageable = PageRequest.of(0, 10); + Page result = projectService.filterProjects(searchFilter, pageable); + + // then + assertThat(result.getTotalElements()).isEqualTo(1); + assertThat(result.getContent()) + .hasSize(1) + .usingComparatorForElementFieldsWithType(upToMillies, Instant.class) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder(project5); + } + + @Test + void all_args_constructor() { + new SearchFilter( + "a", + "b", + "c", + 1, + false, + "Human", + "AN_ONTOLOGY_TERM", + 0, + 10000, + 1, + DataAccessTypes.OPEN, + "a label", + "a network", + false, + SearchType.AllKeywords); + } + + private static Project makeProject(String title) { + Map projectJson = ProjectJson.fromTitle(title).toMap(); + Project project = new Project(projectJson.get("content")); + project.setIsUpdate(false); + project.setPrimaryWrangler("wrangler_" + title); + project.setWranglingState(WranglingState.NEW); + project.setUuid(Uuid.newUuid()); + ((Map) project.getContent()) + .put("dataAccess", new DataAccess(DataAccessTypes.OPEN)); + project.setCellCount(100); + project.setWranglingPriority(1); + project.setDcpReleaseNumber(1); + return project; + } + + @BeforeEach + public void setup() { + initProjectService(); + initTestData(); + } + + private void initTestData() { + this.project1 = makeProject("project1 human liver"); + this.project2 = makeProject("project2 mouse liver"); + this.project3 = makeProject("project3 lung human"); + Arrays.asList(project1, project2, project3) + .forEach( + project -> { + this.mongoTemplate.save(project); + this.projectService.register(project); + }); + this.mongoTemplate + .indexOps(Project.class) + .ensureIndex( + new TextIndexDefinition.TextIndexDefinitionBuilder() + .onField("content.project_core.project_title") + .build()); + assertThat(this.mongoTemplate.findAll(Project.class)).hasSize(3); + } + + private void initProjectService() { + this.projectService = + new ProjectService( + mongoTemplate, + submissionEnvelopeRepository, + projectRepository, + datasetRepository, + metadataCrudService, + metadataUpdateService, + schemaService, + bundleManifestRepository, + auditEntryService, + projectEventHandler); + assertThat(this.mongoTemplate.findAll(Project.class)).hasSize(0); + } + + @AfterEach + private void tearDown() { + this.mongoTemplate.dropCollection(Project.class); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/project/ProjectJson.java b/src/integration/java/uk/ac/ebi/subs/ingest/project/ProjectJson.java new file mode 100644 index 000000000..ea7370dd6 --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/project/ProjectJson.java @@ -0,0 +1,34 @@ +package uk.ac.ebi.subs.ingest.project; + +import java.util.Map; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class ProjectJson { + String title; + + public static ProjectJson fromTitle(String title) { + ProjectJson project = new ProjectJson(); + project.title = title; + return project; + } + + public ObjectNode toObjectNode() { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode content = mapper.createObjectNode(); + ObjectNode projectCore0 = content.putObject("project_core"); + projectCore0.put("project_title", this.title); + + ObjectNode metadata = mapper.createObjectNode(); + metadata.set("content", content); + return metadata; + } + + public Map toMap() { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode project = this.toObjectNode(); + return mapper.convertValue(project, new TypeReference>() {}); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/project/web/ProjectControllerTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/project/web/ProjectControllerTest.java new file mode 100644 index 000000000..b50ffa7af --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/project/web/ProjectControllerTest.java @@ -0,0 +1,286 @@ +package uk.ac.ebi.subs.ingest.project.web; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.handler; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +import org.assertj.core.api.Assertions; +import org.assertj.core.data.MapEntry; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.http.HttpStatus; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.security.test.context.support.WithMockUser; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.dataset.Dataset; +import uk.ac.ebi.subs.ingest.dataset.DatasetRepository; +import uk.ac.ebi.subs.ingest.project.*; +import uk.ac.ebi.subs.ingest.schemas.SchemaService; +import uk.ac.ebi.subs.ingest.state.ValidationState; + +@SpringBootTest +@AutoConfigureMockMvc(printOnlyOnFailure = false) +class ProjectControllerTest { + + @Autowired private MockMvc webApp; + + @Autowired private ProjectRepository projectRepository; + + @Autowired private DatasetRepository datasetRepository; + + @Autowired private ObjectMapper objectMapper; + + @SpyBean private ProjectEventHandler projectEventHandler; + + @MockBean private MigrationConfiguration migrationConfiguration; + + @MockBean private SchemaService schemaService; + + @AfterEach + private void tearDown() { + projectRepository.deleteAll(); + } + + @Nested + class Link { + @Test + @DisplayName("Link dataset to project - Success") + void listDatasetToProjectSuccess() throws Exception { + // given: + String projectContent = "{\"name\": \"project\"}"; + Project project = new Project(projectContent); + + projectRepository.save(project); + String projectId = project.getId(); + + String datasetContent = "{\"name\": \"dataset\"}"; + Dataset persistentDataset = new Dataset(datasetContent); + + datasetRepository.save(persistentDataset); + String datasetId = persistentDataset.getId(); + + // when: + webApp + .perform(put("/projects/{project_id}/datasets/{dataset_id}", projectId, datasetId)) + .andExpect(status().isAccepted()); + } + } + + @Nested + class Update { + + @Test + void updateSuccess() throws Exception { + doTestUpdate( + "/projects/{id}", + project -> { + var projectCaptor = ArgumentCaptor.forClass(Project.class); + verify(projectEventHandler).editedProjectMetadata(projectCaptor.capture()); + Project handledProject = projectCaptor.getValue(); + assertThat(handledProject.getId()).isEqualTo(project.getId()); + }); + } + + @Test + void partialUpdateSuccess() throws Exception { + doTestUpdate( + "/projects/{id}?partial=true", + project -> { + verify(projectEventHandler, never()).editedProjectMetadata(any()); + }); + } + + private void doTestUpdate(String patchUrl, Consumer postCondition) throws Exception { + // given: + var content = + Map.of( + "description", "test", + "attr2", "should be deleted after patch"); + Project originalProject = projectRepository.save(new Project(content)); + + // when: + + Map patch = Map.of("description", "test updated"); + MvcResult result = + webApp + .perform( + patch(patchUrl, originalProject.getId()) + .contentType(APPLICATION_JSON_VALUE) + .content("{\"content\": " + objectMapper.writeValueAsString(patch) + "}")) + .andReturn(); + + // expect: + MockHttpServletResponse response = result.getResponse(); + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + assertThat(response.getContentType()).containsPattern("application/.*json.*"); + + // and: + Project updated = objectMapper.readValue(response.getContentAsString(), Project.class); + assertThat(updated.getContent()).isInstanceOf(Map.class); + MapEntry updatedDescription = entry("description", "test updated"); + assertThat((Map) updated.getContent()).containsOnly(updatedDescription); + + // and: + projectRepository + .findById(originalProject.getId()) + .ifPresentOrElse( + project -> { + assertThat((Map) project.getContent()).containsOnly(updatedDescription); + postCondition.accept(project); + }, + () -> Assertions.fail("project {} not found", originalProject.getId())); + + // and: + } + + @Test + void onlyUpdateAllowedFields() throws Exception { + // given: + var content = new HashMap(); + content.put("description", "test"); + Project project = new Project(content); + project = projectRepository.save(project); + + // when: + content.put("description", "test updated"); + MvcResult result = + webApp + .perform( + patch("/projects/{id}", project.getId()) + .contentType(APPLICATION_JSON_VALUE) + .content( + "{\"content\": " + + objectMapper.writeValueAsString(content) + + ", \"validationState\": \"METADATA_VALID\"}")) + .andReturn(); + + // expect: + MockHttpServletResponse response = result.getResponse(); + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + assertThat(response.getContentType()).containsPattern("application/.*json.*"); + + // and: + // Using Map here because reading directly to Project converts the entire JSON to + // Project.content. + Map updated = + objectMapper.readValue(response.getContentAsString(), Map.class); + assertThat(updated.get("content")).isInstanceOf(Map.class); + MapEntry updatedDescription = entry("description", "test updated"); + assertThat((Map) updated.get("content")).containsOnly(updatedDescription); + assertThat(updated.get("validationState")).isEqualTo("DRAFT"); + + // and: + project = projectRepository.findById(project.getId()).get(); + assertThat((Map) project.getContent()).containsOnly(updatedDescription); + assertThat(project.getValidationState()).isEqualTo(ValidationState.DRAFT); + } + } + + @Nested + @WithMockUser(roles = "WRNAGLER") + class Filter { + @BeforeEach + public void setup() { + Project project = makeProject(); + projectRepository.save(project); + } + + @NotNull + private Project makeProject() { + var content = new HashMap(); + content.put("description", "test kw1"); + Project project = new Project(content); + ((Map) project.getContent()) + .put("dataAccess", new DataAccess(DataAccessTypes.OPEN)); + + return project; + } + + @ParameterizedTest(name = "[{index}] all values, some null: {arguments}") + @CsvSource({ + "kw1,null,null,AllKeywords", + "kw1,null,null,AnyKeyword", + "kw1,null,null,UnsuppportedSearchType", + "kw1,null,null,null", + }) + @WithMockUser + public void allValuesSetSomeNull( + String search, String wrangler, String wranglingState, String searchType) throws Exception { + // given: + var content = + Map.of( + "search", search, + "wrangler", wrangler, + "wranglingState", wranglingState, + "searchType", searchType); + + webApp + .perform( + get("/projects/filter") + .contentType(APPLICATION_JSON_VALUE) + .content(objectMapper.writeValueAsString(content))) + .andDo(print()) + .andExpect(handler().handlerType(ProjectController.class)) + .andExpect(status().isOk()); + } + + @ParameterizedTest(name = "[{index}] nulls missing from filter payload: {arguments}") + @CsvSource({ + "kw1,null,null,AllKeywords", + "kw1,null,null,AnyKeyword", + "kw1,null,null,Unsuppported", + "kw1,null,null,null", + "kw1,Amnon,null,null", + "null,null,NEW,null", + "null,null,Unsupported,null", + "null,null,null,null", + }) + @WithMockUser + public void nullsAreMissingFromPayload( + String search, String wrangler, String wranglingState, String searchType) throws Exception { + var content = new HashMap(); + putIfNotNull(content, search, "search"); + putIfNotNull(content, wrangler, "wrangler"); + putIfNotNull(content, wranglingState, "wranglingState"); + putIfNotNull(content, searchType, "searchType"); + webApp + .perform( + get("/projects/filter") + .contentType(APPLICATION_JSON_VALUE) + .content(objectMapper.writeValueAsString(content))) + .andDo(print()) + .andExpect(handler().handlerType(ProjectController.class)) + .andExpect(status().isOk()); + } + + private void putIfNotNull(HashMap content, String value, String key) { + if (!"null".equals(value)) { + content.put(key, value); + } + } + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/protocol/ProtocolControllerTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/protocol/ProtocolControllerTest.java new file mode 100644 index 000000000..08a21ba71 --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/protocol/ProtocolControllerTest.java @@ -0,0 +1,131 @@ +package uk.ac.ebi.subs.ingest.protocol; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.security.test.context.support.WithMockUser; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.web.servlet.support.ServletUriComponentsBuilder; +import org.springframework.web.util.UriComponentsBuilder; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import uk.ac.ebi.subs.ingest.TestingHelper; +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.project.*; +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@SpringBootTest +@AutoConfigureDataMongo() +@AutoConfigureMockMvc() +@WithMockUser( + username = "alice", + roles = {"WRANGLER"}) +public class ProtocolControllerTest { + @Autowired private MockMvc webApp; + + @Autowired private SubmissionEnvelopeRepository submissionEnvelopeRepository; + + @Autowired private ProjectRepository projectRepository; + + @Autowired private ProtocolRepository protocolRepository; + + @Autowired private ObjectMapper objectMapper; + + @MockBean private MigrationConfiguration migrationConfiguration; + + @MockBean private MessageRouter messageRouter; + + SubmissionEnvelope submissionEnvelope; + + Project project; + + UriComponentsBuilder uriBuilder; + + @BeforeEach + void setUp() { + submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.setUuid(Uuid.newUuid()); + submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALID); + submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); + + project = new Project(new HashMap()); + project.setSubmissionEnvelope(submissionEnvelope); + project.getSubmissionEnvelopes().add(submissionEnvelope); + ((Map) project.getContent()) + .put("dataAccess", new ObjectToMapConverter().asMap(new DataAccess(DataAccessTypes.OPEN))); + + project = projectRepository.save(project); + + uriBuilder = ServletUriComponentsBuilder.fromCurrentContextPath(); + } + + @AfterEach + void tearDown() { + submissionEnvelopeRepository.deleteAll(); + projectRepository.deleteAll(); + protocolRepository.deleteAll(); + } + + @Test + @WithMockUser() + public void newProtocolInSubmissionLinksToSubmissionAndProject() throws Exception { + // when + webApp + .perform( + post("/submissionEnvelopes/{id}/protocols", submissionEnvelope.getId()) + .contentType(MediaType.APPLICATION_JSON) + .content("{\"content\": {}}")) + .andExpect(status().isAccepted()); + TestingHelper.resetTestingSecurityContext(); + + // then + assertThat(protocolRepository.findAll()).hasSize(1); + assertThat(protocolRepository.findAllBySubmissionEnvelope(submissionEnvelope)).hasSize(1); + assertThat(protocolRepository.findByProject(project)).hasSize(1); + + var newProtocol = protocolRepository.findAll().get(0); + assertThat(newProtocol.getSubmissionEnvelope().getId()).isEqualTo(submissionEnvelope.getId()); + assertThat(newProtocol.getProject().getId()).isEqualTo(project.getId()); + } + + @Test + public void newProtocolInSubmissionDoesNotFailIfSubmissionHasNoProject() throws Exception { + // given + projectRepository.deleteAll(); + + // when + webApp + .perform( + post("/submissionEnvelopes/{id}/protocols", submissionEnvelope.getId()) + .contentType(MediaType.APPLICATION_JSON) + .content("{\"content\": {}}")) + .andExpect(status().isAccepted()); + TestingHelper.resetTestingSecurityContext(); + + // then + assertThat(protocolRepository.findAll()).hasSize(1); + assertThat(protocolRepository.findAllBySubmissionEnvelope(submissionEnvelope)).hasSize(1); + + var newProtocol = protocolRepository.findAll().get(0); + assertThat(newProtocol.getSubmissionEnvelope().getId()).isEqualTo(submissionEnvelope.getId()); + assertThat(newProtocol.getProject()).isNull(); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/schemas/SchemaScraperTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/schemas/SchemaScraperTest.java new file mode 100644 index 000000000..a4e296c79 --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/schemas/SchemaScraperTest.java @@ -0,0 +1,301 @@ +package uk.ac.ebi.subs.ingest.schemas; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.doReturn; + +import java.io.File; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collection; + +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.mock.env.MockEnvironment; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.github.tomakehurst.wiremock.WireMockServer; + +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.schemas.schemascraper.SchemaScraper; +import uk.ac.ebi.subs.ingest.schemas.schemascraper.impl.S3BucketSchemaScraper; +import uk.ac.ebi.subs.ingest.schemas.schemascraper.impl.SchemaScrapeException; + +/** Created by rolando on 19/04/2018. */ +@ExtendWith(SpringExtension.class) +@SpringBootTest +public class SchemaScraperTest { + @SpyBean SchemaService schemaService; + + @MockBean SchemaRepository schemaRepository; + + @MockBean MigrationConfiguration migrationConfiguration; + + WireMockServer wireMockServer; + + @BeforeEach + public void setupWireMockServer() { + wireMockServer = new WireMockServer(8089); + wireMockServer.start(); + } + + @AfterEach + public void teardownWireMockServer() { + wireMockServer.stop(); + wireMockServer.resetAll(); + } + + String mockSchemaUri = "http://localhost:8089"; + + @Test + public void testSchemaScrape() throws Exception { + // given + // an s3 bucket files listing as XML + SchemaScraper schemaScraper = new S3BucketSchemaScraper(); + + // when + wireMockServer.stubFor( + get(urlEqualTo("/")) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/xml") + .withBody( + new String( + Files.readAllBytes( + Paths.get( + new File(".").getAbsolutePath() + + "/src/test/resources/testfiles/TestBucketListing.xml")))))); + + Collection mockSchemaUris = schemaScraper.getAllSchemaURIs(URI.create(mockSchemaUri)); + + // we know there are 107 schemas in the test file + assert mockSchemaUris.size() == 107; + } + + @Test + public void testSchemaParse_BundleUris() { + try { + // when + schemaService.schemaDescriptionFromSchemaUris( + Arrays.asList( + URI.create("bundle/1.2.3/biomaterial"), + URI.create("bundle/2.3.4/links"), + URI.create("bundle/1.0/protocols"))); + } catch (Exception e) { + assert false; + } + + assert true; + } + + @Test + public void testSchemaParse_ModuleUris() { + try { + // when + schemaService.schemaDescriptionFromSchemaUris( + Arrays.asList( + URI.create("module/biomaterial/5.1.0/growth_condition"), + URI.create("module/ontology/5.0.0/biological_macromolecule_ontology"), + URI.create("module/process/5.1.0/purchased_reagents"))); + } catch (Exception e) { + assert false; + } + + assert true; + } + + @Test + public void testSchemaParse_TypeUris() { + try { + // when + schemaService.schemaDescriptionFromSchemaUris( + Arrays.asList( + URI.create("type/biomaterial/5.0.1/cell_line"), + URI.create("type/biomaterial/5.1.0/organoid"), + URI.create("type/file/5.0.0/sequence_file"))); + } catch (Exception e) { + assert false; + } + + assert true; + } + + @Test + public void testSchemaParse_SubdomainTypeUris() { + try { + // when + schemaService.schemaDescriptionFromSchemaUris( + Arrays.asList( + URI.create("type/process/biomaterial_collection/5.1.0/collection_process"), + URI.create("type/process/sequencing/5.0.0/sequencing_process"), + URI.create("type/process/sequencing/5.1.0/sequencing_process"))); + } catch (Exception e) { + assert false; + } + + assert true; + } + + @Test + public void testSchemaParse() throws Exception { + // pre-given + SchemaScraper schemaScraper = new S3BucketSchemaScraper(); + + wireMockServer.stubFor( + get(urlEqualTo("/")) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/xml") + .withBody( + new String( + Files.readAllBytes( + Paths.get( + new File(".").getAbsolutePath() + + "/src/test/resources/testfiles/TestBucketListing.xml")))))); + // given + Collection mockSchemaUris = schemaScraper.getAllSchemaURIs(URI.create(mockSchemaUri)); + + try { + // when + schemaService.schemaDescriptionFromSchemaUris(mockSchemaUris); + } catch (Exception e) { + assert false; + } + + assert true; + } + + @Test + public void testGetLatestSchemas() { + Schema mockSchemaA = + new Schema( + "mockHighLevel-A", + "2.0", + "mockDomain-A", + "mockSubdomain-A", + "mockConcrete-A", + "mock.io/mock-schema-a"); + Schema mockSchemaB = + new Schema( + "mockHighLevel-B", + "1.9", + "mockDomain-B", + "mockSubdomain-B", + "mockConcrete-B", + "mock.io/mock-schema-a"); + Schema mockSchemaOldA = + new Schema( + "mockHighLevel-A", + "1.9", + "mockDomain-A", + "mockSubdomain-A", + "mockConcrete-A", + "mock.io/mock-schema-duplicate-a"); + + doReturn(Arrays.asList(mockSchemaA, mockSchemaB, mockSchemaOldA)) + .when(schemaRepository) + .findAll(); + + Collection latestSchemas = schemaService.getLatestSchemas(); + assert latestSchemas.size() == 2; + latestSchemas.forEach( + schema -> { + assert !schema.getSchemaUri().equals("mock.io/mock-schema-duplicate-a"); + }); + assert true; + } + + @Test + public void testFilterLatestSchemas() { + Schema mockSchemaA = + new Schema( + "mockHighLevel-A", + "2.0", + "mockDomain-A", + "mockSubdomain-A", + "mockConcrete-A", + "mock.io/mock-schema-a"); + Schema mockSchemaB = + new Schema( + "mockHighLevel-B", + "1.9", + "mockDomain-B", + "mockSubdomain-B", + "mockConcrete-B", + "mock.io/mock-schema-a"); + Schema mockSchemaOldA = + new Schema( + "mockHighLevel-A", + "1.9", + "mockDomain-A", + "mockSubdomain-A", + "mockConcrete-A", + "mock.io/mock-schema-duplicate-a"); + + doReturn(Arrays.asList(mockSchemaA, mockSchemaB, mockSchemaOldA)) + .when(schemaRepository) + .findAll(); + + Collection latestSchemas = schemaService.filterLatestSchemas("mockHighLevel-B"); + assert latestSchemas.size() == 1; + latestSchemas.forEach( + schema -> { + assert schema.getHighLevelEntity().equals("mockHighLevel-B"); + }); + } + + @Test + public void testEmptyEnvironmentVariable() { + doReturn(null).when(schemaService).getSchemaBaseUri(); + + Exception exception = + assertThrows(SchemaScrapeException.class, () -> schemaService.updateSchemasCollection()); + + String expectedMessage = "SCHEMA_BASE_URI environmental variable should not be null."; + String actualMessage = exception.getMessage(); + + assertEquals(actualMessage, expectedMessage); + } + + @Configuration + class MockConfiguration { + @Autowired SchemaScraper schemaScraper; + @Autowired MockEnvironment mockEnvironment; + + @Bean + SchemaService schemaService() { + return new SchemaService(schemaRepository, schemaScraper, mockEnvironment); + } + + @Bean + SchemaScraper schemaScraper() { + return new S3BucketSchemaScraper(); + } + } + + @Test + public void testSchemaParse_MorphicSchemas() { + try { + // when + schemaService.schemaDescriptionFromSchemaUris( + Arrays.asList( + URI.create("type/1.0.0/biomaterial/cell_line"), + URI.create("type/0.9.0/file/sequence_file"), + URI.create("type/2.0.0/project/study"))); + } catch (Exception e) { + assert false; + } + + assert true; + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/security/ManagedAccessConfigurationTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/security/ManagedAccessConfigurationTest.java new file mode 100644 index 000000000..c6bbe7ff9 --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/security/ManagedAccessConfigurationTest.java @@ -0,0 +1,52 @@ +package uk.ac.ebi.subs.ingest.security; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Arrays; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; + +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; + +@SpringBootTest +@AutoConfigureDataMongo() +@AutoConfigureMockMvc(printOnlyOnFailure = false) +public class ManagedAccessConfigurationTest { + @Autowired FileRepository fileRepository; + + @Autowired BiomaterialRepository biomaterialRepository; + + @Autowired ProtocolRepository protocolRepository; + + @Autowired ProcessRepository processRepository; + // NOTE: Adding MigrationConfiguration as a MockBean is needed + // as otherwise MigrationConfiguration won't be initialised. + @MockBean private MigrationConfiguration migrationConfiguration; + + @Test + public void testDBRepositoriesHaveAnnotation() { + Stream.builder() + .add(fileRepository) + .add(biomaterialRepository) + .add(protocolRepository) + .add(processRepository) + .build() + .forEach( + r -> + assertThat( + Arrays.stream(r.getClass().getAnnotations()) + .anyMatch( + annotation -> + annotation.annotationType().equals(RowLevelFilterSecurity.class)))); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/security/ManagedAccessTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/security/ManagedAccessTest.java new file mode 100644 index 000000000..0325bc6b6 --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/security/ManagedAccessTest.java @@ -0,0 +1,425 @@ +package uk.ac.ebi.subs.ingest.security; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static uk.ac.ebi.subs.ingest.TestingHelper.resetTestingSecurityContext; + +import java.lang.reflect.InvocationTargetException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import org.assertj.core.api.Assertions; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.data.repository.CrudRepository; +import org.springframework.http.MediaType; +import org.springframework.security.test.context.support.WithMockUser; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.ResultActions; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.core.AbstractEntity; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.project.*; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@SpringBootTest +@AutoConfigureDataMongo() +@AutoConfigureMockMvc(printOnlyOnFailure = false) +public class ManagedAccessTest { + @Autowired private MockMvc webApp; + @Autowired private ProjectRepository projectRepository; + @Autowired FileRepository fileRepository; + + @Autowired BiomaterialRepository biomaterialRepository; + + @Autowired ProtocolRepository protocolRepository; + + @Autowired ProcessRepository processRepository; + + @Autowired SubmissionEnvelopeRepository submissionEnvelopeRepository; + + @Autowired ObjectMapper objectMapper; + + @MockBean + // NOTE: Adding MigrationConfiguration as a MockBean is needed + // as otherwise MigrationConfiguration won't be initialised. + private MigrationConfiguration migrationConfiguration; + + @BeforeEach + @WithMockUser(roles = "WRANGLER") + public void setupTestData() throws Exception { + + // datasets A, B - managed access + List> projects = TestDataHelper.createManagedAccessProjects(); + // dataset C - open access + projects.add(TestDataHelper.createOpenAccessProjects()); + + projects.stream() + .map(TestDataHelper::mapAsJsonString) + .forEach( + p -> { + try { + webApp + .perform(post("/projects").contentType(MediaType.APPLICATION_JSON).content(p)) + .andExpect(status().isOk()); + resetTestingSecurityContext(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + Stream.of("a", "b", "c") + .map(TestDataHelper::makeUuid) + .forEach( + uuidString -> + addMetadataToProjectByProjectUuid( + uuidString, + List.of(File.class, Biomaterial.class, Protocol.class, Process.class))); + } + + @AfterEach + @WithMockUser(roles = "WRANGLER") + public void tearDown() { + Stream.builder() + .add(projectRepository) + .add(fileRepository) + .add(biomaterialRepository) + .add(protocolRepository) + .add(processRepository) + .add(submissionEnvelopeRepository) + .build() + .forEach(r -> ((CrudRepository) r).deleteAll()); + } + + @Test + @WithMockUser( + username = "alice", + roles = {"CONTRIBUTOR", "access_aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}) + public void testDataAccessTypeFieldDeserialization() { + projectRepository + .findByUuid(new Uuid(TestDataHelper.makeUuid("a"))) + .forEach( + p -> + Assertions.assertThat(p.getContent()) + .extracting("dataAccess") + .containsExactly( + new ObjectToMapConverter().asMap(new DataAccess(DataAccessTypes.MANAGED)))); + } + + /** checks access to sub resources of a project, `/projects/{id}/{metadata-type}` */ + @Nested + class MetadataFromProjectAccessControl { + + @ParameterizedTest + @MethodSource("uk.ac.ebi.subs.ingest.security.SecurityTest#metadataTypes") + @WithMockUser( + username = "alice", + roles = {"CONTRIBUTOR", "access_aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}) + public void userOnProjectAList_CanSeeProjectMetadata(String metadataTypePlural) + throws Exception { + String projectMetadataUrl = + getProjectMetadataUrl(metadataTypePlural, TestDataHelper.makeUuid("a")); + + webApp.perform(get(projectMetadataUrl)).andExpect(status().isOk()); + } + + @ParameterizedTest + @MethodSource("uk.ac.ebi.subs.ingest.security.SecurityTest#metadataTypes") + @WithMockUser( + username = "alice", + roles = {"CONTRIBUTOR", "access_aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}) + public void userOnProjectAList_CanSeeOpenProjectMetadata(String metadataTypePlural) + throws Exception { + String openAccessProjectMetadataUrl = + getProjectMetadataUrl(metadataTypePlural, TestDataHelper.makeUuid("c")); + + webApp + .perform(get(openAccessProjectMetadataUrl).contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + } + + @ParameterizedTest + @MethodSource("uk.ac.ebi.subs.ingest.security.SecurityTest#metadataTypes") + @WithMockUser( + username = "bob", + roles = {"CONTRIBUTOR", "access_bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"}) + public void userNotOnProjectAList_CannotSeeMetadata(String metadataTypePlural) + throws Exception { + String projectMetadataUrl = + getProjectMetadataUrl(metadataTypePlural, TestDataHelper.makeUuid("a")); + + webApp.perform(get(projectMetadataUrl)).andExpect(status().isForbidden()); + } + } + + @NotNull + private String getProjectMetadataUrl(String metadataTypePlural, String uuid) { + return projectRepository + .findByUuid(new Uuid(uuid)) + .findFirst() + .map(Project::getId) + .map(projectId -> String.format("/projects/%s/%s", projectId, metadataTypePlural)) + .get(); + } + + @Nested + class ProjectAccessControl { + @Test + @WithMockUser( + username = "alice", + roles = {"CONTRIBUTOR", "access_aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}) + public void userOnProjectAListCanSeeAllProjects() throws Exception { + + webApp.perform(get("/projects")).andExpect(jsonPath("$.page.totalElements").value("3")); + } + + @Test + @WithMockUser( + username = "bob", + roles = {"CONTRIBUTOR"}) + public void userNotOnProjectAListCanSeeAllProjects() throws Exception { + webApp.perform(get("/projects")).andExpect(jsonPath("$.page.totalElements").value("3")); + } + } + + private void addMetadataToProjectByProjectUuid( + String uuidString, List> metadataTypes) { + Project project = projectRepository.findByUuid(new Uuid(uuidString)).findFirst().get(); + try { + final String submissionUrl = createSubmissionAndGetUrl(); + linkSubmissionToProject(project, submissionUrl); + metadataTypes.forEach( + metadataType -> { + try { + MetadataDocument metadataDocument = newMetadataInstance(metadataType); + String lowerCaseMetadataType = metadataDocument.getType().toString().toLowerCase(); + setDocumentProperties(uuidString, metadataDocument); + + String submissionMetadataDocumentsUrl = + buildSubmissionMetadataUrl(submissionUrl, lowerCaseMetadataType); + Map documentAsMap = + new ObjectToMapConverter(objectMapper) + .asMap(metadataDocument, List.of("contentLastModified")); + webApp + .perform( + post(submissionMetadataDocumentsUrl) + .contentType(MediaType.APPLICATION_JSON) + .content(TestDataHelper.mapAsJsonString(documentAsMap))) + .andExpect(status().isAccepted()); + resetTestingSecurityContext(); + } catch (Exception e) { + throw new RuntimeException( + "problem crating metadata document " + metadataType.getSimpleName(), e); + } + }); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static String buildSubmissionMetadataUrl( + String submissionUrl, String lowerCaseMetadataType) { + return submissionUrl + + "/" + + lowerCaseMetadataType + + (lowerCaseMetadataType.endsWith("s") ? "es" : "s"); + } + + @NotNull + private static void setDocumentProperties(String uuidString, MetadataDocument metadataDocument) { + String lowerCaseMetadataType = metadataDocument.getType().toString().toLowerCase(); + Map> content = + Map.of( + lowerCaseMetadataType + "_core", + Map.of( + lowerCaseMetadataType + "_name", + lowerCaseMetadataType + " in project " + uuidString)); + metadataDocument.setContent(content); + metadataDocument.setUuid(Uuid.newUuid()); + } + + @NotNull + private static MetadataDocument newMetadataInstance( + Class metadataType) + throws InstantiationException, + IllegalAccessException, + InvocationTargetException, + NoSuchMethodException { + MetadataDocument metadataDocument; + try { + // try the single arg ctor with null content arg + // content will be set later + metadataDocument = metadataType.getConstructor(Object.class).newInstance(new Object[] {null}); + } catch (IllegalArgumentException + | InvocationTargetException + | NoSuchMethodException + | SecurityException e) { + // fallback to no-args ctor + metadataDocument = metadataType.getConstructor().newInstance(); + } + return metadataDocument; + } + + @NotNull + private ResultActions linkSubmissionToProject(Project project, String submissionUrl) + throws Exception { + return webApp + .perform( + post("/projects/{id}/submissionEnvelopes", project.getId()) + .contentType("text/uri-list") + .content(submissionUrl)) + .andExpect(status().isNoContent()); + } + + @Nullable + private String createSubmissionAndGetUrl() throws Exception { + return webApp + .perform(post("/submissionEnvelopes").contentType(MediaType.APPLICATION_JSON).content("{}")) + .andExpect(status().isCreated()) + .andReturn() + .getResponse() + .getHeader("Location"); + } + + @Nested + class MetadataRepositoryAccessControl { + + @ParameterizedTest + @MethodSource("uk.ac.ebi.subs.ingest.security.SecurityTest#metadataTypes") + @WithMockUser( + username = "alice", + roles = {"CONTRIBUTOR", "access_aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}) + public void userOnProjectAList_CanSeeOnlyOpenAndProjectAMetadata(String metadataTypePlural) + throws Exception { + String metadataCollectionUrl = "/" + metadataTypePlural; + webApp + .perform(get(metadataCollectionUrl)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.page.totalElements").value("2")); + } + } + + @Nested + class MetadataFromSubmissionAccessControl { + @ParameterizedTest + @MethodSource("uk.ac.ebi.subs.ingest.security.SecurityTest#metadataTypesWithProject") + @WithMockUser( + username = "alice", + roles = {"CONTRIBUTOR", "access_aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}) + public void userOnProjectAList_CanSeeSubmissionMetadata(String metadataTypePlural) + throws Exception { + String submissionMetadataUrl = + getSubmissionMetadataUrl(metadataTypePlural, TestDataHelper.makeUuid("a")); + webApp.perform(get(submissionMetadataUrl)).andExpect(status().isOk()); + } + } + + @NotNull + private String getSubmissionMetadataUrl(String metadataTypePlural, String uuid) { + return projectRepository + .findByUuid(new Uuid(uuid)) + .map(Project::getSubmissionEnvelopes) + .flatMap(Collection::stream) + .map(AbstractEntity::getId) + .map( + submissionId -> + String.format("/submissionEnvelopes/%s/%s", submissionId, metadataTypePlural)) + .findFirst() + .get(); + } + + @Nested + @WithMockUser( + username = "service", + roles = {"SERVICE"}) + class ServiceUserAccessControl { + @ParameterizedTest + @MethodSource("uk.ac.ebi.subs.ingest.security.SecurityTest#metadataTypesWithProject") + public void canSeeSubmissionMetadata(String metadataTypePlural) throws Exception { + String submissionMetadataUrl = + getSubmissionMetadataUrl(metadataTypePlural, TestDataHelper.makeUuid("a")); + webApp.perform(get(submissionMetadataUrl)).andExpect(status().isOk()); + } + + @ParameterizedTest + @MethodSource("uk.ac.ebi.subs.ingest.security.SecurityTest#metadataTypes") + public void canSeeProjectMetadata(String metadataTypePlural) throws Exception { + String projectMetadataUrl = + getProjectMetadataUrl(metadataTypePlural, TestDataHelper.makeUuid("a")); + webApp.perform(get(projectMetadataUrl)).andExpect(status().isOk()); + } + + @ParameterizedTest + @MethodSource("uk.ac.ebi.subs.ingest.security.SecurityTest#metadataTypes") + public void canSeeOnlyOpenAndProjectAMetadata(String metadataTypePlural) throws Exception { + String metadataCollectionUrl = "/" + metadataTypePlural; + webApp + .perform(get(metadataCollectionUrl)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.page.totalElements").value("3")); + } + + @Test + public void canCreateProject() throws Exception { + final String submissionUrl = createSubmissionAndGetUrl(); + webApp + .perform( + post(submissionUrl + "/projects") + .content("{}") + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isAccepted()); + } + + @Test + public void canCreateSubmissionManifest() throws Exception { + final String submissionUrl = createSubmissionAndGetUrl(); + webApp + .perform( + post(submissionUrl + "/submissionManifest") + .content("{}") + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isAccepted()); + } + + @Test + public void canGetSubmissionManifest() throws Exception { + final String submissionUrl = createSubmissionAndGetUrl(); + webApp + .perform( + post(submissionUrl + "/submissionManifest") + .content("{}") + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isAccepted()); + webApp + .perform(get(submissionUrl + "/submissionManifest").accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + } + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/security/RowLevelFilterSecurityAspectTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/security/RowLevelFilterSecurityAspectTest.java new file mode 100644 index 000000000..895afb08f --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/security/RowLevelFilterSecurityAspectTest.java @@ -0,0 +1,79 @@ +package uk.ac.ebi.subs.ingest.security; + +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.reset; + +import java.util.stream.Stream; + +import org.junit.Ignore; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.security.test.context.support.WithMockUser; + +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; + +@SpringBootTest +@TestInstance(PER_CLASS) +@WithMockUser +class RowLevelFilterSecurityAspectTest { + @Autowired FileRepository fileRepository; + @Autowired BiomaterialRepository biomaterialRepository; + @Autowired ProcessRepository processRepository; + @Autowired ProtocolRepository protocolRepository; + + @SpyBean private RowLevelFilterSecurityAspect rowLevelFilterSecurityAspect; + + @MockBean + // NOTE: Adding MigrationConfiguration as a MockBean is needed + // as otherwise MigrationConfiguration won't be initialised. + private MigrationConfiguration migrationConfiguration; + + @AfterEach + public void resetSpy() { + reset(rowLevelFilterSecurityAspect); + } + + // TODO fix failing test testAdviceOnRepositoryDeclaredMethod + @Ignore + public void testAdviceOnRepositoryDeclaredMethod() throws Throwable { + try { + fileRepository.findByProject(Project.builder().emptyProject().build()); + } catch (Exception e) { + // ignore exceptions, we are just testing whether the Advice is called + } + + Mockito.verify(rowLevelFilterSecurityAspect, atLeast(1)).applyRowLevelSecurity(Mockito.any()); + } + + @ParameterizedTest(name = "{index} {1}") + @MethodSource("repositoryBeans") + public void testAdviceOnRepositoryInheritedMethod(MongoRepository repository, String metadataType) + throws Throwable { + repository.findAll(); + + Mockito.verify(rowLevelFilterSecurityAspect, atLeast(1)).applyRowLevelSecurity(Mockito.any()); + } + + private Stream repositoryBeans() { + return Stream.of( + Arguments.of(fileRepository, "file"), + Arguments.of(biomaterialRepository, "biomaterial"), + Arguments.of(protocolRepository, "protocol"), + Arguments.of(processRepository, "process")); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/security/SecurityTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/security/SecurityTest.java new file mode 100644 index 000000000..5d16a71e2 --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/security/SecurityTest.java @@ -0,0 +1,133 @@ +package uk.ac.ebi.subs.ingest.security; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; + +import java.util.stream.Stream; + +import org.hamcrest.CoreMatchers; +import org.junit.Ignore; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.security.test.context.support.WithMockUser; +import org.springframework.test.web.servlet.MockMvc; + +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; + +@SpringBootTest +@AutoConfigureMockMvc(printOnlyOnFailure = false) +public class SecurityTest { + @Autowired private MockMvc webApp; + + public static Stream metadataTypes() { + return Stream.of( + Arguments.of("files"), + Arguments.of("biomaterials"), + Arguments.of("protocols"), + Arguments.of("processes")); + } + + public static Stream metadataTypesWithProject() { + return Stream.concat(metadataTypes(), Stream.of(Arguments.of("projects"))); + } + + @MockBean + // NOTE: Adding MigrationConfiguration as a MockBean is needed + // as otherwise MigrationConfiguration won't be initialised. + private MigrationConfiguration migrationConfiguration; + + @Nested + class Authorised { + @ParameterizedTest + @MethodSource("uk.ac.ebi.subs.ingest.security.SecurityTest#metadataTypes") + @WithMockUser + public void apiAccessWithTrailingSlashIsPermitted(String metadataTypePlural) throws Exception { + checkGetUrlIsOk("/" + metadataTypePlural + "/"); + } + + @ParameterizedTest + @MethodSource("uk.ac.ebi.subs.ingest.security.SecurityTest#metadataTypes") + @WithMockUser + public void apiAccessNoTrailingSlashIsPermitted(String metadataTypePlural) throws Exception { + checkGetUrlIsOk("/" + metadataTypePlural); + } + } + + @Nested + class Unauthorised { + + @ParameterizedTest + @MethodSource("uk.ac.ebi.subs.ingest.security.SecurityTest#metadataTypes") + public void apiAccessWithTrailingSlashIsBlocked(String metadataTypePlural) throws Exception { + checkGetUrlIsUnauthorized("/" + metadataTypePlural + "/"); + } + + @ParameterizedTest + @MethodSource("uk.ac.ebi.subs.ingest.security.SecurityTest#metadataTypes") + public void apiAccessNoTrailingSlashIsBlocked(String metadataTypePlural) throws Exception { + checkGetUrlIsUnauthorized("/" + metadataTypePlural); + } + } + + @Nested + class RootResource { + @Test + public void checkUnauthenticatedJson_IsAllowed() throws Exception { + webApp + .perform(get("/").accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("_links").hasJsonPath()); + } + + @Test + public void checkUnauthenticatedHtml_IsAllowed() throws Exception { + webApp + .perform(get("/browser/index.html").accept(MediaType.TEXT_HTML)) + .andExpect(status().isOk()) + .andExpect( + content() + .string(CoreMatchers.containsString("The HAL Browser (for Spring Data REST)"))); + } + } + + @Nested + class ManagementResources { + // @ParameterizedTest + @Ignore + @ValueSource(strings = {"health", "info", "prometheus"}) + public void checkUnauthenticatedJson_IsAllowed(String endpoint) throws Exception { + webApp.perform(get("/" + endpoint)).andExpect(status().isOk()); + } + } + + @Nested + class SchemaResource { + + @Test + public void checkUnauthenticate_IsAllowed() throws Exception { + webApp.perform(get("/schemas")).andExpect(status().isOk()); + } + + @Test + public void checkUnauthenticatedSubResource_IsAllowed() throws Exception { + webApp.perform(get("/schemas/search")).andExpect(status().isOk()); + } + } + + private void checkGetUrlIsUnauthorized(String url) throws Exception { + webApp.perform(get(url)).andExpect(status().isUnauthorized()); + } + + private void checkGetUrlIsOk(String url) throws Exception { + webApp.perform(get(url)).andExpect(status().isOk()); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/security/TestDataHelper.java b/src/integration/java/uk/ac/ebi/subs/ingest/security/TestDataHelper.java new file mode 100644 index 000000000..81f872106 --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/security/TestDataHelper.java @@ -0,0 +1,63 @@ +package uk.ac.ebi.subs.ingest.security; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.jetbrains.annotations.NotNull; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import uk.ac.ebi.subs.ingest.project.Project; + +public class TestDataHelper { + @NotNull + static String makeUuid(String s) { + if (s.length() != 1) { + throw new IllegalArgumentException("use a single character"); + } + return s.repeat(8) + + "-" + + s.repeat(4) + + "-" + + s.repeat(4) + + "-" + + s.repeat(4) + + "-" + + s.repeat(12); + } + + static Map createOpenAccessProjects() { + // TODO amnon: exclusion of contentLastModified needed because of serialization problem. Not + // sure why.\n" + return Project.builder() + .withOpenAccess() + .withShortName("dataset C open") + .withUuid(makeUuid("C")) + .asMap(); + } + + @NotNull + static List> createManagedAccessProjects() { + return Stream.of("A", "B") + .map( + s -> + Project.builder() + .withManagedAccess() + .withShortName("dataset " + s + " managed") + .withUuid(makeUuid(s)) + .asMap()) + .collect(Collectors.toList()); + } + + static String mapAsJsonString(Map value) { + ObjectMapper objectMapper = new ObjectMapper(); + try { + return objectMapper.writeValueAsString(value); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/stagingjob/StagingJobRepositoryTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/stagingjob/StagingJobRepositoryTest.java new file mode 100644 index 000000000..d6ab4e413 --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/stagingjob/StagingJobRepositoryTest.java @@ -0,0 +1,57 @@ +package uk.ac.ebi.subs.ingest.stagingjob; + +import java.util.UUID; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; + +@ExtendWith(SpringExtension.class) +@SpringBootTest +public class StagingJobRepositoryTest { + @MockBean MigrationConfiguration migrationConfiguration; + + @Autowired StagingJobRepository stagingJobRepository; + + @AfterEach + private void tearDown() { + stagingJobRepository.deleteAll(); + } + + @Test + public void testJpaExceptionWhenInsertingMultipleCompoundKey() { + UUID testStagingAreaUuid = UUID.randomUUID(); + String testFileName = "test.fastq.gz"; + + stagingJobRepository.save(new StagingJob(testStagingAreaUuid, testFileName)); + + Assertions.assertThatExceptionOfType(DuplicateKeyException.class) + .isThrownBy( + () -> { + stagingJobRepository.save(new StagingJob(testStagingAreaUuid, testFileName)); + }); + } + + @Test + public void testRegisteringJobsWithDifferentCompoundKey() { + UUID testStagingAreaUuid_1 = UUID.randomUUID(); + String testFileName_1 = "test_1.fastq.gz"; + + UUID testStagingAreaUuid_2 = UUID.randomUUID(); + String testFileName_2 = "test_2.fastq.gz"; + + stagingJobRepository.save(new StagingJob(testStagingAreaUuid_1, testFileName_1)); + stagingJobRepository.save(new StagingJob(testStagingAreaUuid_1, testFileName_2)); + + stagingJobRepository.save(new StagingJob(testStagingAreaUuid_2, testFileName_1)); + stagingJobRepository.save(new StagingJob(testStagingAreaUuid_2, testFileName_2)); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/study/web/StudyControllerTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/study/web/StudyControllerTest.java new file mode 100644 index 000000000..d1b46250e --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/study/web/StudyControllerTest.java @@ -0,0 +1,312 @@ +package uk.ac.ebi.subs.ingest.study.web; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.verify; +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +import org.assertj.core.data.MapEntry; +import org.junit.jupiter.api.*; +import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.http.HttpStatus; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.security.test.context.support.WithMockUser; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.dataset.Dataset; +import uk.ac.ebi.subs.ingest.dataset.DatasetRepository; +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.study.Study; +import uk.ac.ebi.subs.ingest.study.StudyEventHandler; +import uk.ac.ebi.subs.ingest.study.StudyRepository; +import uk.ac.ebi.subs.ingest.study.StudyService; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@SpringBootTest +@AutoConfigureMockMvc(printOnlyOnFailure = false) +class StudyControllerTest { + @Autowired private MockMvc webApp; + + @Autowired private StudyRepository studyRepository; + + @Autowired private StudyService studyService; + + @Autowired private DatasetRepository datasetRepository; + + @Autowired private MetadataCrudService metadataCrudService; + + @Autowired private ObjectMapper objectMapper; + + @Autowired private SubmissionEnvelopeRepository submissionEnvelopeRepository; + + @SpyBean private StudyEventHandler studyEventHandler; + + @MockBean private MigrationConfiguration migrationConfiguration; + + SubmissionEnvelope submissionEnvelope; + + @AfterEach + private void tearDown() { + studyRepository.deleteAll(); + } + + @Nested + @Disabled("Test class is currently deactivated - Need to be linked to Submission Envelope") + class Registration { + @Test + @DisplayName("Register Study - Success") + @WithMockUser + void registerSuccess() throws Exception { + doTestRegister( + "/studies", + study -> { + var studyCaptor = ArgumentCaptor.forClass(Study.class); + verify(studyEventHandler).registeredStudy(studyCaptor.capture()); + Study handledStudy = studyCaptor.getValue(); + assertThat(handledStudy.getId()).isNotNull(); + }); + } + + private void doTestRegister(String registerUrl, Consumer postCondition) + throws Exception { + // given: + submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.setUuid(Uuid.newUuid()); + submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALID); + submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); + + var content = new HashMap(); + content.put("name", "Test Study"); + content.put("submissionEnvelope", submissionEnvelope.getId()); + + Study study = + new Study( + "https://dev.schema.morphic.bio/type/0.0.1/project/study", "0.0.1", "study", content); + + // Link study to submission envelope + study.getSubmissionEnvelopes().add(submissionEnvelope); + study = studyRepository.save(study); + + studyService.addStudyToSubmissionEnvelope(submissionEnvelope, study); + + // when: + MvcResult result = + webApp + .perform( + post(registerUrl) + .contentType(APPLICATION_JSON_VALUE) + .content("{\"content\": " + objectMapper.writeValueAsString(content) + "}")) + .andReturn(); + + // then: + MockHttpServletResponse response = result.getResponse(); + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + assertThat(response.getContentType()).containsPattern("application/.*json.*"); + + // and: verify the registered study content + Map registeredStudy = + objectMapper.readValue(response.getContentAsString(), Map.class); + assertThat(registeredStudy.get("content")).isInstanceOf(Map.class); + MapEntry nameEntry = entry("name", "Test Study"); + assertThat((Map) registeredStudy.get("content")).containsOnly(nameEntry); + + // and: verify the study is stored in the repository + List studies = studyRepository.findAll(); + assertThat(studies).hasSize(1); + Study storedStudy = studies.get(0); + assertThat((Map) storedStudy.getContent()).containsOnly(nameEntry); + + // and: + postCondition.accept(storedStudy); + } + } + + @Nested + class Update { + @Test + @DisplayName("Update Study - Success") + void updateSuccess() throws Exception { + doTestUpdate( + "/studies/{studyId}", + study -> { + var studyCaptor = ArgumentCaptor.forClass(Study.class); + verify(studyEventHandler).updatedStudy(studyCaptor.capture()); + Study handledStudy = studyCaptor.getValue(); + assertThat(handledStudy.getId()).isEqualTo(study.getId()); + }); + } + + private void doTestUpdate(String patchUrl, Consumer postCondition) throws Exception { + // given: + submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.setUuid(Uuid.newUuid()); + submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALID); + submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); + + var content = new HashMap(); + content.put("description", "test"); + Study study = + new Study( + "https://dev.schema.morphic.bio/type/0.0.1/project/study", "0.0.1", "study", content); + study.getSubmissionEnvelopes().add(submissionEnvelope); + study = studyRepository.save(study); + + studyService.addStudyToSubmissionEnvelope(submissionEnvelope, study); + + // when: + content.put("description", "test updated"); + MvcResult result = + webApp + .perform( + patch(patchUrl, study.getId()) + .contentType(APPLICATION_JSON_VALUE) + .content("{\"content\": " + objectMapper.writeValueAsString(content) + "}")) + .andReturn(); + + // expect: + MockHttpServletResponse response = result.getResponse(); + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + assertThat(response.getContentType()).containsPattern("application/.*json.*"); + + // and: + Map updated = + objectMapper.readValue(response.getContentAsString(), Map.class); + assertThat(updated.get("described_by")) + .isEqualTo("https://dev.schema.morphic.bio/type/0.0.1/project/study"); + assertThat(updated.get("schema_version")).isEqualTo("0.0.1"); + assertThat(updated.get("schema_type")).isEqualTo("study"); + assertThat(updated.containsKey("content")).isTrue(); + assertThat(((Map) updated.get("content")).get("description")) + .isEqualTo("test updated"); + + // and: + study = studyRepository.findById(study.getId()).get(); + + assertThat(study.getDescribedBy()) + .isEqualTo("https://dev.schema.morphic.bio/type/0.0.1/project/study"); + assertThat(study.getSchemaVersion()).isEqualTo("0.0.1"); + assertThat(study.getSchemaType()).isEqualTo("study"); + assertThat(((Map) study.getContent()).get("description")) + .isEqualTo("test updated"); + + // and: + postCondition.accept(study); + } + + @Test + @DisplayName("Update Study - Not Found") + void updateStudyNotFound() throws Exception { + // given: + String nonExistentStudyId = "nonExistentId"; + + // when: + MvcResult result = + webApp + .perform( + patch("/studies/{studyId}", nonExistentStudyId) + .contentType(APPLICATION_JSON_VALUE) + .content("{\"content\": {\"description\": \"Updated Description\"}}")) + .andReturn(); + + // then: + MockHttpServletResponse response = result.getResponse(); + assertThat(response.getStatus()).isEqualTo(HttpStatus.NOT_FOUND.value()); + } + } + + @Nested + class Delete { + @Test + @DisplayName("Delete Study - Success") + void deleteSuccess() throws Exception { + // given: + String content = "{\"name\": \"delete study\"}"; + Study persistentStudy = + new Study( + "https://dev.schema.morphic.bio/type/0.0.1/project/study", "0.0.1", "study", content); + studyRepository.save(persistentStudy); + String existingStudyId = persistentStudy.getId(); + + // when: + webApp + .perform(delete("/studies/{studyId}", existingStudyId)) + .andExpect(status().isNoContent()); + + // then: + assertThat(studyRepository.findById(existingStudyId)).isEmpty(); + // Expect the ResourceNotFoundException when attempting to find the study after deletion + assertThrows( + ResourceNotFoundException.class, + () -> { + metadataCrudService.findOriginalByUuid( + String.valueOf(persistentStudy.getUuid()), EntityType.STUDY); + }); + verify(studyEventHandler).deletedStudy(existingStudyId); + } + + @Test + @DisplayName("Delete Study - Not Found") + void deleteStudyNotFound() throws Exception { + // given: a non-existent study id + String nonExistentStudyId = "nonExistentId"; + + // when: + MvcResult result = + webApp.perform(delete("/studies/{studyId}", nonExistentStudyId)).andReturn(); + + // then: + MockHttpServletResponse response = result.getResponse(); + assertThat(response.getStatus()).isEqualTo(HttpStatus.NOT_FOUND.value()); + } + } + + @Nested + class Link { + @Test + @DisplayName("Link dataset to study - Success") + void listDatasetToStudySuccess() throws Exception { + // given: + String studyContent = "{\"name\": \"study\"}"; + Study persistentStudy = + new Study( + "https://dev.schema.morphic.bio/type/0.0.1/project/study", + "0.0.1", + "study", + studyContent); + studyRepository.save(persistentStudy); + String studyId = persistentStudy.getId(); + + String datasetContent = "{\"name\": \"dataset\"}"; + Dataset persistentDataset = new Dataset(datasetContent); + datasetRepository.save(persistentDataset); + String datasetId = persistentDataset.getId(); + + // when: + webApp + .perform(put("/studies/{stud_id}/datasets/{dataset_id}", studyId, datasetId)) + .andExpect(status().isAccepted()); + } + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/submission/web/ProjectStatusUpdateTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/submission/web/ProjectStatusUpdateTest.java new file mode 100644 index 000000000..6b66aace4 --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/submission/web/ProjectStatusUpdateTest.java @@ -0,0 +1,138 @@ +package uk.ac.ebi.subs.ingest.submission.web; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static uk.ac.ebi.subs.ingest.project.WranglingState.IN_PROGRESS; +import static uk.ac.ebi.subs.ingest.project.WranglingState.SUBMITTED; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.security.test.context.support.WithMockUser; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.web.servlet.support.ServletUriComponentsBuilder; +import org.springframework.web.util.UriComponentsBuilder; + +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.project.WranglingState; + +@SpringBootTest +@AutoConfigureDataMongo +@AutoConfigureMockMvc +@WithMockUser( + username = "test_user", + authorities = {"WRANGLER"}) +public class ProjectStatusUpdateTest { + @Autowired private MockMvc webApp; + @Autowired private ProjectRepository projectRepository; + + // NOTE: Adding MigrationConfiguration as a MockBean is needed as otherwise MigrationConfiguration + // won't be + // initialised. This is very un-elegant and should be fixed. + @MockBean private MigrationConfiguration migrationConfiguration; + UriComponentsBuilder uriBuilder; + + @BeforeEach + void setUp() { + uriBuilder = ServletUriComponentsBuilder.fromCurrentContextPath(); + } + + @Test + public void test_statusIsInProgress_afterSubmissionCreation() throws Exception { + // given + Project project = createProject(); + + // when + String submissionUrl = createSubmission(); + connectSubmissionToProject(project, submissionUrl); + + // then + assertProjectStatus(project, IN_PROGRESS); + } + + @Test + public void test_statusIsSubmitted_afterSubmissionIsExported() throws Exception { + // given + Project project = createProject(); + String submissionUrl = createSubmission(); + connectSubmissionToProject(project, submissionUrl); + + // when + setSubmissionToExported(submissionUrl); + + // then + assertProjectStatus(project, SUBMITTED); + } + + @Test + public void test_deleteSubmissionWorks() throws Exception { + // given + Project project = createProject(); + String submissionUrl = createSubmission(); + connectSubmissionToProject(project, submissionUrl); + + // when + deleteSubmissionFromProject(submissionUrl); + String submissionUrl2 = createSubmission(); + connectSubmissionToProject(project, submissionUrl2); + + // then + // no errors + } + + private void deleteSubmissionFromProject(String submissionUrl) throws Exception { + webApp.perform(delete(submissionUrl)).andExpect(status().isAccepted()); + } + + private void setSubmissionToExported(String submissionUrl) throws Exception { + webApp.perform(put(submissionUrl + Links.COMMIT_EXPORTED_URL)).andExpect(status().isAccepted()); + } + + private void assertProjectStatus(Project project, WranglingState wranglingState) + throws Exception { + webApp + .perform(get("/projects/{id}", project.getId())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.wranglingState").value(wranglingState.getValue())); + } + + private void connectSubmissionToProject(Project project, String submissionUrl) throws Exception { + webApp + .perform( + post("/projects/{id}/submissionEnvelopes", project.getId()) + .contentType("text/uri-list") + .content(submissionUrl)) + .andExpect(status().isNoContent()); + } + + @Nullable + private String createSubmission() throws Exception { + MvcResult mvcResult = + webApp + .perform( + post("/submissionEnvelopes/").contentType(MediaType.APPLICATION_JSON).content("{}")) + .andExpect(status().isCreated()) + .andReturn(); + String submissionUrl = mvcResult.getResponse().getHeader("Location"); + return submissionUrl; + } + + @NotNull + private Project createProject() { + Project project = new Project(null); + project.setWranglingState(WranglingState.ELIGIBLE); + return projectRepository.save(project); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionControllerTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionControllerTest.java new file mode 100644 index 000000000..fb1be8de3 --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionControllerTest.java @@ -0,0 +1,196 @@ +package uk.ac.ebi.subs.ingest.submission.web; + +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.security.test.context.support.WithMockUser; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.web.servlet.support.ServletUriComponentsBuilder; +import org.springframework.web.util.UriComponentsBuilder; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.dataset.Dataset; +import uk.ac.ebi.subs.ingest.dataset.DatasetRepository; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.study.Study; +import uk.ac.ebi.subs.ingest.study.StudyRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@SpringBootTest +@AutoConfigureDataMongo() +@AutoConfigureMockMvc() +@WithMockUser +public class SubmissionControllerTest { + @Autowired private MockMvc webApp; + + @Autowired private SubmissionEnvelopeRepository submissionEnvelopeRepository; + + @Autowired private ProjectRepository projectRepository; + + @Autowired private BiomaterialRepository biomaterialRepository; + + @Autowired private ProcessRepository processRepository; + + @Autowired private ProtocolRepository protocolRepository; + + @Autowired private FileRepository fileRepository; + + @Autowired private StudyRepository studyRepository; + + @Autowired private DatasetRepository datasetRepository; + + @Autowired private ObjectMapper objectMapper; + + @MockBean private MigrationConfiguration migrationConfiguration; + + @MockBean private MessageRouter messageRouter; + + SubmissionEnvelope submissionEnvelope; + + Project project; + + Biomaterial biomaterial; + + Process process; + + Protocol protocol; + + File file; + + Study study; + + Dataset dataset; + + UriComponentsBuilder uriBuilder; + + @BeforeEach + void setUp() { + submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.setUuid(Uuid.newUuid()); + submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALID); + submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); + + project = new Project(null); + project.getSubmissionEnvelopes().add(submissionEnvelope); + project = projectRepository.save(project); + + biomaterial = new Biomaterial(null); + biomaterial.setSubmissionEnvelope(submissionEnvelope); + biomaterial = biomaterialRepository.save(biomaterial); + + process = new Process(null); + process.setSubmissionEnvelope(submissionEnvelope); + process = processRepository.save(process); + + protocol = new Protocol(null); + protocol.setSubmissionEnvelope(submissionEnvelope); + protocol = protocolRepository.save(protocol); + + file = new File(null, "fileName"); + file.setSubmissionEnvelope(submissionEnvelope); + file = fileRepository.save(file); + + study = new Study(null, null, null, null); + study.getSubmissionEnvelopes().add(submissionEnvelope); + study = studyRepository.save(study); + + dataset = new Dataset(null); + dataset = datasetRepository.save(dataset); + + uriBuilder = ServletUriComponentsBuilder.fromCurrentContextPath(); + } + + @AfterEach + void tearDown() { + submissionEnvelopeRepository.deleteAll(); + projectRepository.deleteAll(); + biomaterialRepository.deleteAll(); + processRepository.deleteAll(); + protocolRepository.deleteAll(); + fileRepository.deleteAll(); + } + + @ParameterizedTest + @ValueSource(strings = {"biomaterials", "processes", "protocols", "files"}) + public void testAdditionToNonEditableSubmissionThrowsErrorForAllEntityTypes(String endpoint) + throws Exception { + // given + submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALID); + submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); + + // when + webApp + .perform( + post("/submissionEnvelopes/{id}/" + endpoint, submissionEnvelope.getId()) + .contentType(MediaType.APPLICATION_JSON) + .content("{\"content\": {}}")) + .andExpect(status().isForbidden()); + } + + @ParameterizedTest + @EnumSource( + value = SubmissionState.class, + names = {"EXPORTING", "PROCESSING", "ARCHIVED", "SUBMITTED"}) + public void testAdditionToNonEditableSubmissionThrowsErrorInAllStates(SubmissionState state) + throws Exception { + // given + submissionEnvelope.enactStateTransition(state); + submissionEnvelope = submissionEnvelopeRepository.save(submissionEnvelope); + + // when + webApp + .perform( + post("/submissionEnvelopes/{id}/biomaterials", submissionEnvelope.getId()) + .contentType(MediaType.APPLICATION_JSON) + .content("{\"content\": {}}")) + .andExpect(status().isForbidden()); + } + + @ParameterizedTest + @ValueSource( + strings = {"/submissionEnvelopes/{id}/projects", "/submissionEnvelopes/{id}/relatedProjects"}) + @WithMockUser + public void testProjectsAreReturnedWhenTheyIncludeTheSubmissionInTheirEnvelopes(String endpoint) + throws Exception { + webApp + .perform( + // when + get(endpoint, submissionEnvelope.getId()) + .contentType(MediaType.APPLICATION_JSON)) // then + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.projects", hasSize(1))) + .andExpect( + jsonPath( + "$._embedded.projects[0].uuid.uuid", is(project.getUuid().getUuid().toString()))); + } +} diff --git a/src/integration/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionLinkMapControllerTest.java b/src/integration/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionLinkMapControllerTest.java new file mode 100644 index 000000000..06d6b6d5c --- /dev/null +++ b/src/integration/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionLinkMapControllerTest.java @@ -0,0 +1,148 @@ +package uk.ac.ebi.subs.ingest.submission.web; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.HashSet; +import java.util.List; + +import org.junit.Ignore; +import org.junit.jupiter.api.AfterEach; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.data.mongodb.repository.MongoRepository; + +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@Ignore( + "ignoring because $toString mongo aggregation operator is not supported by the in memory mongo version we use. See dcp-936") +@SpringBootTest +@AutoConfigureDataMongo() +public class SubmissionLinkMapControllerTest { + @Autowired private SubmissionLinkMapController controller; + + @Autowired BiomaterialRepository biomaterialRepository; + + @Autowired FileRepository fileRepository; + + @Autowired ProcessRepository processRepository; + + @Autowired ProtocolRepository protocolRepository; + + @Autowired SubmissionEnvelopeRepository submissionEnvelopeRepository; + + @MockBean private MigrationConfiguration migrationConfiguration; + + @MockBean private MessageRouter messageRouter; + + @AfterEach + private void tearDown() { + List.of( + biomaterialRepository, + fileRepository, + processRepository, + protocolRepository, + submissionEnvelopeRepository) + .forEach(MongoRepository::deleteAll); + } + + /** + * @Ignore("ignoring because $toString mongo aggregation operator is not supported by the in + * memory mongo version we use. See dcp-936") + */ + public void testSubmissionLinkMap() { + // given: + SubmissionEnvelope submissionEnvelope = + submissionEnvelopeRepository.save(new SubmissionEnvelope()); + + Biomaterial donor = biomaterialRepository.save(new Biomaterial(null)); + Biomaterial specimen = biomaterialRepository.save(new Biomaterial(null)); + Biomaterial cellSuspension = biomaterialRepository.save(new Biomaterial(null)); + + Process donorSpecimen = processRepository.save(new Process(null)); + Process cellSuspensionSequenceFile = processRepository.save(new Process(null)); + Process sequenceFileAnalysisFile = processRepository.save(new Process(null)); + + Protocol collectionProtocol = protocolRepository.save(new Protocol(null)); + Protocol sequencingProtocol = protocolRepository.save(new Protocol(null)); + Protocol analysisProtocol = protocolRepository.save(new Protocol(null)); + + File sequencingFile = fileRepository.save(new File(null, "sequenceFile")); + File analysisFile = fileRepository.save(new File(null, "analysisFile")); + + List.of( + donor, + specimen, + cellSuspension, + donorSpecimen, + cellSuspensionSequenceFile, + sequenceFileAnalysisFile, + collectionProtocol, + sequencingProtocol, + analysisProtocol, + sequencingFile, + analysisFile) + .forEach(entity -> entity.setSubmissionEnvelope(submissionEnvelope)); + + specimen.addAsDerivedByProcess(donorSpecimen); + sequencingFile.addAsDerivedByProcess(cellSuspensionSequenceFile); + analysisFile.addAsDerivedByProcess(sequenceFileAnalysisFile); + + donor.addAsInputToProcess(donorSpecimen); + cellSuspension.addAsInputToProcess(cellSuspensionSequenceFile); + sequencingFile.addAsInputToProcess(sequenceFileAnalysisFile); + + donorSpecimen.addProtocol(collectionProtocol); + cellSuspensionSequenceFile.addProtocol(sequencingProtocol); + sequenceFileAnalysisFile.addProtocol(analysisProtocol); + + submissionEnvelopeRepository.save(submissionEnvelope); + biomaterialRepository.saveAll(List.of(donor, specimen, cellSuspension)); + protocolRepository.saveAll(List.of(collectionProtocol, sequencingProtocol, analysisProtocol)); + processRepository.saveAll( + List.of(donorSpecimen, cellSuspensionSequenceFile, sequenceFileAnalysisFile)); + fileRepository.saveAll(List.of(sequencingFile, analysisFile)); + + // when: + SubmissionLinkMapController.SubmissionLinkingMap submissionLinkMap = + controller.getSubmissionLinkMap(submissionEnvelope); + + // then: + assertThat(submissionLinkMap).isNotNull(); + assertThat(submissionLinkMap.processes.get(donorSpecimen.getId()).protocols) + .isEqualTo(new HashSet<>(List.of(collectionProtocol.getId()))); + assertThat(submissionLinkMap.processes.get(donorSpecimen.getId()).inputBiomaterials) + .isEqualTo(new HashSet<>(List.of(donor.getId()))); + assertThat(submissionLinkMap.processes.get(cellSuspensionSequenceFile.getId()).protocols) + .isEqualTo(new HashSet<>(List.of(sequencingProtocol.getId()))); + assertThat( + submissionLinkMap.processes.get(cellSuspensionSequenceFile.getId()).inputBiomaterials) + .isEqualTo(new HashSet<>(List.of(cellSuspension.getId()))); + assertThat(submissionLinkMap.processes.get(cellSuspensionSequenceFile.getId()).inputFiles) + .isEmpty(); + assertThat(submissionLinkMap.processes.get(sequenceFileAnalysisFile.getId()).protocols) + .isEqualTo(new HashSet<>(List.of(analysisProtocol.getId()))); + assertThat(submissionLinkMap.processes.get(sequenceFileAnalysisFile.getId()).inputBiomaterials) + .isEmpty(); + assertThat(submissionLinkMap.processes.get(sequenceFileAnalysisFile.getId()).inputFiles) + .isEqualTo(new HashSet<>(List.of(sequencingFile.getId()))); + assertThat(submissionLinkMap.biomaterials.get(donor.getId()).inputToProcesses) + .isEqualTo(new HashSet<>(List.of(donorSpecimen.getId()))); + assertThat(submissionLinkMap.biomaterials.get(cellSuspension.getId()).inputToProcesses) + .isEqualTo(new HashSet<>(List.of(cellSuspensionSequenceFile.getId()))); + assertThat(submissionLinkMap.files.get(sequencingFile.getId()).inputToProcesses) + .isEqualTo(new HashSet<>(List.of(sequenceFileAnalysisFile.getId()))); + } +} diff --git a/src/integration/resources/application.properties b/src/integration/resources/application.properties index 6c070ef6a..76aa277f2 100644 --- a/src/integration/resources/application.properties +++ b/src/integration/resources/application.properties @@ -4,7 +4,7 @@ SVC_AUTH_AUDIENCE='' USR_AUTH_AUDIENCE='' AUTH_ISSUER=http://domain.tld/issuer GCP_JWK_PROVIDER_BASE_URL=http://domain.tld - -# display queries -logging.level.org.springframework.data.mongodb.core.MongoTemplate=DEBUG -logging.level.org.springframework.data.mongodb.repository.query=DEBUG \ No newline at end of file +AWS_COGNITO_DOMAIN=https://morphic-dev.auth.eu-west-2.amazoncognito.com/oauth2 +logging.level.org.springframework.security=DEBUG +management.endpoints.web.base-path=/ +management.endpoints.web.exposure.include=health,info,jolokia,prometheus diff --git a/src/main/java/org/humancellatlas/ingest/archiving/Error.java b/src/main/java/org/humancellatlas/ingest/archiving/Error.java deleted file mode 100644 index 5812e931f..000000000 --- a/src/main/java/org/humancellatlas/ingest/archiving/Error.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.humancellatlas.ingest.archiving; - -import lombok.Data; - -@Data -public class Error { - private String errorCode; - private String message; - private Object details; -} diff --git a/src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveEntity.java b/src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveEntity.java deleted file mode 100644 index 1203f2b3b..000000000 --- a/src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveEntity.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.humancellatlas.ingest.archiving.entity; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Getter; -import lombok.Setter; -import org.humancellatlas.ingest.archiving.Error; -import org.humancellatlas.ingest.archiving.submission.ArchiveSubmission; -import org.springframework.data.annotation.CreatedDate; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.DBRef; -import org.springframework.data.mongodb.core.mapping.Document; -import org.springframework.hateoas.Identifiable; - -import java.net.URI; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -@Getter -@Setter -@Document -public class ArchiveEntity implements Identifiable { - @DBRef(lazy = true) - ArchiveSubmission archiveSubmission; - - @Id - @JsonIgnore - private String id; - - @CreatedDate - private Instant created; - - private ArchiveEntityType type; - - private String alias; - - @Indexed(unique = true) - private String dspUuid; - - private URI dspUrl; - - private String accession; - - private Object conversion; - - private Set metadataUuids; - - private Set accessionedMetadataUuids; - - private List errors = new ArrayList<>(); - -} diff --git a/src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveEntityRepository.java b/src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveEntityRepository.java deleted file mode 100644 index f14958dfb..000000000 --- a/src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveEntityRepository.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.humancellatlas.ingest.archiving.entity; - -import org.humancellatlas.ingest.archiving.submission.ArchiveSubmission; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.data.rest.core.annotation.RestResource; -import org.springframework.web.bind.annotation.CrossOrigin; - -@CrossOrigin -public interface ArchiveEntityRepository extends MongoRepository { - Page findByArchiveSubmission(ArchiveSubmission archiveSubmission, - Pageable pageable); - - Page findByAlias(String alias, - Pageable pageable); - - ArchiveEntity findByArchiveSubmissionAndAlias(ArchiveSubmission archiveSubmission, String alias); - - ArchiveEntity findByDspUuid(String dspUuid); - - Page findByArchiveSubmissionAndType(ArchiveSubmission archiveSubmission, - ArchiveEntityType archiveEntityType, - Pageable pageable); - @RestResource(exported = false) - Long deleteByArchiveSubmission(ArchiveSubmission archiveSubmission); - -} diff --git a/src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveEntityType.java b/src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveEntityType.java deleted file mode 100644 index 0ab3cab3c..000000000 --- a/src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveEntityType.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.humancellatlas.ingest.archiving.entity; - -import com.fasterxml.jackson.databind.annotation.JsonSerialize; - -@JsonSerialize(using = ArchiveEntityTypeSerializer.class) -public enum ArchiveEntityType { - SAMPLE("sample"), - PROJECT("project"), - STUDY("study"), - SEQUENCING_EXPERIMENT("sequencingExperiment"), - SEQUENCING_RUN("sequencingRun"); - - protected String type; - - ArchiveEntityType(String type) { - this.type = type; - } -} - diff --git a/src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveJob.java b/src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveJob.java deleted file mode 100644 index da0b285c1..000000000 --- a/src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveJob.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.humancellatlas.ingest.archiving.entity; - -import com.fasterxml.jackson.annotation.JsonInclude; -import lombok.Data; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.Document; - -import java.time.Instant; -import java.util.Map; - -@Data -@Document -@JsonInclude(JsonInclude.Include.NON_NULL) -public class ArchiveJob { - - protected @Id String id; - private String submissionUuid; - private Instant createdDate; - private Instant responseDate; - private ArchiveJobStatus overallStatus; - private Map resultsFromArchives; - - public enum ArchiveJobStatus { - PENDING("Pending"), - RUNNING("Running"), - FAILED("Failed"), - COMPLETED("Completed"); - - final String status; - - ArchiveJobStatus(String status) { - this.status = status; - } - } -} diff --git a/src/main/java/org/humancellatlas/ingest/archiving/submission/web/ArchiveJobController.java b/src/main/java/org/humancellatlas/ingest/archiving/submission/web/ArchiveJobController.java deleted file mode 100644 index 57ba86da0..000000000 --- a/src/main/java/org/humancellatlas/ingest/archiving/submission/web/ArchiveJobController.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.humancellatlas.ingest.archiving.submission.web; - -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.archiving.entity.ArchiveJob; -import org.humancellatlas.ingest.archiving.entity.ArchiveJobRepository; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.data.rest.webmvc.BasePathAwareController; -import org.springframework.data.rest.webmvc.PersistentEntityResource; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; - -import java.net.URI; -import java.time.Instant; -import java.util.Optional; - -@RepositoryRestController -@ExposesResourceFor(ArchiveJob.class) -@BasePathAwareController -@RequiredArgsConstructor -public class ArchiveJobController { - - private static final Logger LOGGER = LoggerFactory.getLogger(ArchiveJobController.class); - - private final ArchiveJobRepository archiveJobRepository; - - @PostMapping("/archiveJobs") - ResponseEntity createArchiveJob(@RequestBody ArchiveJob archiveJob, - PersistentEntityResourceAssembler resourceAssembler) { - initResource(archiveJob); - - final ArchiveJob persistedArchiveJob = archiveJobRepository.save(archiveJob); - final PersistentEntityResource entityResource = resourceAssembler.toFullResource(persistedArchiveJob); - return ResponseEntity.created(URI.create(entityResource.getId().getHref())) - .contentType(MediaType.APPLICATION_JSON) - .body(entityResource); - } - - private void initResource(ArchiveJob archiveJob) { - archiveJob.setCreatedDate(Instant.now()); - archiveJob.setOverallStatus(ArchiveJob.ArchiveJobStatus.PENDING); - } - - @GetMapping("/archiveJobs/{id}") - ResponseEntity getArchiveJob(@PathVariable String id, - PersistentEntityResourceAssembler resourceAssembler) { - Optional archiveJob = archiveJobRepository.findById(id); - - if (archiveJob.isEmpty()) { - return ResponseEntity.notFound().build(); - } - - return ResponseEntity.ok().body(resourceAssembler.toFullResource(archiveJob.get())); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/archiving/submission/web/ArchiveSubmissionController.java b/src/main/java/org/humancellatlas/ingest/archiving/submission/web/ArchiveSubmissionController.java deleted file mode 100644 index efaf32165..000000000 --- a/src/main/java/org/humancellatlas/ingest/archiving/submission/web/ArchiveSubmissionController.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.humancellatlas.ingest.archiving.submission.web; - -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.archiving.entity.ArchiveEntity; -import org.humancellatlas.ingest.archiving.entity.ArchiveEntityRepository; -import org.humancellatlas.ingest.archiving.submission.ArchiveSubmission; -import org.humancellatlas.ingest.archiving.submission.ArchiveSubmissionRepository; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.rest.webmvc.PersistentEntityResource; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.data.web.PagedResourcesAssembler; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.hateoas.Resource; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; - -@RepositoryRestController -@RequiredArgsConstructor -@ExposesResourceFor(ArchiveSubmission.class) -@Getter -public class ArchiveSubmissionController { - private final @NonNull ArchiveSubmissionRepository archiveSubmissionRepository; - private final @NonNull ArchiveEntityRepository archiveEntityRepository; - - private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; - - @RequestMapping(path = "archiveSubmissions/{sub_id}/entities", method = RequestMethod.POST) - ResponseEntity> addEntity(@PathVariable("sub_id") ArchiveSubmission archiveSubmission, - @RequestBody ArchiveEntity entity, - PersistentEntityResourceAssembler assembler) { - entity.setArchiveSubmission(archiveSubmission); - entity = archiveEntityRepository.save(entity); - PersistentEntityResource resource = assembler.toFullResource(entity); - return ResponseEntity.accepted().body(resource); - } - - @RequestMapping(path = "archiveSubmissions/{sub_id}/entities", method = RequestMethod.GET) - ResponseEntity addEntity(@PathVariable("sub_id") ArchiveSubmission archiveSubmission, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - Page archiveEntities = archiveEntityRepository.findByArchiveSubmission(archiveSubmission, pageable); - return ResponseEntity.ok(pagedResourcesAssembler.toResource(archiveEntities, resourceAssembler)); - } - - @RequestMapping(path = "archiveSubmissions/{sub_id}", method = RequestMethod.DELETE) - ResponseEntity deleteSubmission(@PathVariable("sub_id") ArchiveSubmission archiveSubmission) { - archiveEntityRepository.deleteByArchiveSubmission(archiveSubmission); - archiveSubmissionRepository.delete(archiveSubmission); - return ResponseEntity.accepted().build(); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/archiving/submission/web/ArchiveSubmissionResourceProcessor.java b/src/main/java/org/humancellatlas/ingest/archiving/submission/web/ArchiveSubmissionResourceProcessor.java deleted file mode 100644 index 802b1eb3e..000000000 --- a/src/main/java/org/humancellatlas/ingest/archiving/submission/web/ArchiveSubmissionResourceProcessor.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.humancellatlas.ingest.archiving.submission.web; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.archiving.submission.ArchiveSubmission; -import org.springframework.hateoas.EntityLinks; -import org.springframework.hateoas.Link; -import org.springframework.hateoas.Resource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.stereotype.Component; - -@Component -@RequiredArgsConstructor -public class ArchiveSubmissionResourceProcessor implements ResourceProcessor> { - private final @NonNull EntityLinks entityLinks; - - private Link getEntitiesLink(ArchiveSubmission archiveSubmission) { - return entityLinks.linkForSingleResource(archiveSubmission) - .slash("/entities") - .withRel("entities"); - } - - @Override - public Resource process(Resource resource) { - ArchiveSubmission archiveSubmission = resource.getContent(); - resource.add(getEntitiesLink(archiveSubmission)); - return resource; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/audit/AuditEntry.java b/src/main/java/org/humancellatlas/ingest/audit/AuditEntry.java deleted file mode 100644 index 3d6b2f392..000000000 --- a/src/main/java/org/humancellatlas/ingest/audit/AuditEntry.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.humancellatlas.ingest.audit; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Getter; -import lombok.NonNull; -import org.humancellatlas.ingest.core.AbstractEntity; -import org.springframework.data.annotation.CreatedBy; -import org.springframework.data.annotation.CreatedDate; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.DBRef; - - -import java.time.Instant; - -@Getter -public class AuditEntry { - protected @Id @JsonIgnore String id; - @NonNull private final AuditType auditType; - private final Object before; - private final Object after; - private @CreatedDate Instant date; - // todo: @CreatedBy isn't working, need to figure out why - private @CreatedBy String user; - @DBRef(lazy = true) @JsonIgnore final @NonNull private AbstractEntity entity; - - public AuditEntry(AuditType auditType, Object before, Object after, AbstractEntity entity) { - this.auditType = auditType; - this.before = before; - this.after = after; - this.entity = entity; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/audit/AuditEntryService.java b/src/main/java/org/humancellatlas/ingest/audit/AuditEntryService.java deleted file mode 100644 index a9cbe250f..000000000 --- a/src/main/java/org/humancellatlas/ingest/audit/AuditEntryService.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.humancellatlas.ingest.audit; - -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.AbstractEntity; -import org.springframework.stereotype.Service; - -import java.util.List; - - -@Service -@RequiredArgsConstructor -public class AuditEntryService { - - private final AuditEntryRepository auditEntryRepository; - - public void addAuditEntry(AuditEntry auditEntry) { - auditEntryRepository.save(auditEntry); - } - - public List getAuditEntriesForAbstractEntity(AbstractEntity entity) { - return auditEntryRepository.findByEntityEqualsOrderByDateDesc(entity); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/audit/AuditType.java b/src/main/java/org/humancellatlas/ingest/audit/AuditType.java deleted file mode 100644 index e3a050221..000000000 --- a/src/main/java/org/humancellatlas/ingest/audit/AuditType.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.humancellatlas.ingest.audit; - -import com.fasterxml.jackson.annotation.JsonValue; - -public enum AuditType { - STATUS_UPDATED("Status updated"); - - protected String value; - - AuditType(String value) { - this.value = value; - } - - @JsonValue - public String getValue() { - return this.value; - } - - } diff --git a/src/main/java/org/humancellatlas/ingest/biomaterial/Biomaterial.java b/src/main/java/org/humancellatlas/ingest/biomaterial/Biomaterial.java deleted file mode 100644 index 3e537937d..000000000 --- a/src/main/java/org/humancellatlas/ingest/biomaterial/Biomaterial.java +++ /dev/null @@ -1,107 +0,0 @@ -package org.humancellatlas.ingest.biomaterial; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import org.humancellatlas.ingest.core.EntityType; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.project.Project; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.DBRef; -import org.springframework.data.mongodb.core.mapping.Document; -import org.springframework.data.rest.core.annotation.RestResource; -import org.springframework.web.bind.annotation.CrossOrigin; - -import java.util.HashSet; -import java.util.Set; - -import static com.fasterxml.jackson.annotation.JsonProperty.Access.READ_ONLY; - -/** - * Created by rolando on 16/02/2018. - */ -@CrossOrigin -@Getter -@Document -@EqualsAndHashCode(callSuper = true, exclude = {"project", "projects", "inputToProcesses", "derivedByProcesses"}) -@NoArgsConstructor -public class Biomaterial extends MetadataDocument { - - @Indexed - private @Setter - @DBRef(lazy = true) - Project project; - - @RestResource - @DBRef(lazy = true) - private Set projects = new HashSet<>(); - - @Indexed - @RestResource - @DBRef(lazy = true) - private Set inputToProcesses = new HashSet<>(); - - @Indexed - @RestResource - @DBRef(lazy = true) - private Set derivedByProcesses = new HashSet<>(); - - @JsonCreator - public Biomaterial(@JsonProperty("content") Object content) { - super(EntityType.BIOMATERIAL, content); - } - - /** - * Adds to the collection of processes that this biomaterial serves as an input to - * - * @param process the process to add - * @return a reference to this biomaterial - */ - public Biomaterial addAsInputToProcess(Process process) { - this.inputToProcesses.add(process); - return this; - } - - /** - * Adds to the collection of processes that this biomaterial was derived by - * - * @param process the process to add - * @return a reference to this biomaterial - */ - public Biomaterial addAsDerivedByProcess(Process process) { - this.derivedByProcesses.add(process); - return this; - } - - @JsonProperty(access = READ_ONLY) - public boolean isLinked() { - return !inputToProcesses.isEmpty() || !derivedByProcesses.isEmpty(); - } - - /** - * Removes a process to the collection of processes that this biomaterial serves as an input to - * - * @param process the process to add - * @return a reference to this biomaterial - */ - public Biomaterial removeAsInputToProcess(Process process) { - this.inputToProcesses.remove(process); - return this; - } - - /** - * Removes a process to the collection of processes that this biomaterial was derived by - * - * @param process the process to add - * @return a reference to this biomaterial - */ - public Biomaterial removeAsDerivedByProcess(Process process) { - this.derivedByProcesses.remove(process); - return this; - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/biomaterial/BiomaterialRepository.java b/src/main/java/org/humancellatlas/ingest/biomaterial/BiomaterialRepository.java deleted file mode 100644 index 660553bd1..000000000 --- a/src/main/java/org/humancellatlas/ingest/biomaterial/BiomaterialRepository.java +++ /dev/null @@ -1,79 +0,0 @@ -package org.humancellatlas.ingest.biomaterial; - -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.data.mongodb.repository.Query; -import org.springframework.data.repository.query.Param; -import org.springframework.data.rest.core.annotation.RestResource; -import org.springframework.web.bind.annotation.CrossOrigin; - -import java.util.Collection; -import java.util.Optional; -import java.util.UUID; -import java.util.stream.Stream; - - -@CrossOrigin -public interface BiomaterialRepository extends MongoRepository { - - @RestResource(rel = "findAllByUuid", path = "findAllByUuid") - Page findByUuid(@Param("uuid") Uuid uuid, Pageable pageable); - - @RestResource(rel = "findByUuid", path = "findByUuid") - Optional findByUuidUuidAndIsUpdateFalse(@Param("uuid") UUID uuid); - - Page findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope, Pageable pageable); - - Page findByProject(Project project, Pageable pageable); - - @RestResource(exported = false) - Stream findByProject(Project project); - - @RestResource(exported = false) - Stream findByProjectsContaining(Project project); - - @RestResource(exported = false) - Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - @RestResource(exported = false) - Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - @RestResource(exported = false) - Long deleteBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - @RestResource(rel = "findBySubmissionAndValidationState") - public Page findBySubmissionEnvelopeAndValidationState(@Param("envelopeUri") SubmissionEnvelope submissionEnvelope, - @Param("state") ValidationState state, - Pageable pageable); - - @Query(value = "{'submissionEnvelope.id': ?0, graphValidationErrors: { $exists: true, $not: {$size: 0} } }") - @RestResource(rel = "findBySubmissionIdWithGraphValidationErrors") - public Page findBySubmissionIdWithGraphValidationErrors( - @Param("envelopeId") String envelopeId, - Pageable pageable - ); - - @RestResource(exported = false) - Stream findByInputToProcessesContains(Process process); - - Page findByInputToProcessesContaining(Process process, Pageable pageable); - - @RestResource(exported = false) - Stream findByDerivedByProcessesContains(Process process); - - Page findByDerivedByProcessesContaining(Process process, Pageable pageable); - - long countBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - long countBySubmissionEnvelopeAndValidationState(SubmissionEnvelope submissionEnvelope, ValidationState validationState); - - @Query(value = "{'submissionEnvelope.id': ?0, graphValidationErrors: { $exists: true, $not: {$size: 0} } }", count = true) - long countBySubmissionEnvelopeAndCountWithGraphValidationErrors(String submissionEnvelopeId); - -} diff --git a/src/main/java/org/humancellatlas/ingest/biomaterial/BiomaterialService.java b/src/main/java/org/humancellatlas/ingest/biomaterial/BiomaterialService.java deleted file mode 100644 index 7b89e6b7e..000000000 --- a/src/main/java/org/humancellatlas/ingest/biomaterial/BiomaterialService.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.humancellatlas.ingest.biomaterial; - -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.service.MetadataCrudService; -import org.humancellatlas.ingest.core.service.MetadataUpdateService; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; - -/** - * Created by rolando on 19/02/2018. - */ -@Service -@RequiredArgsConstructor -@Getter -public class BiomaterialService { - private final @NonNull SubmissionEnvelopeRepository submissionEnvelopeRepository; - private final @NonNull BiomaterialRepository biomaterialRepository; - private final @NonNull ProcessRepository processRepository; - private final @NonNull ProjectRepository projectRepository; - private final @NonNull MetadataUpdateService metadataUpdateService; - private final @NonNull MetadataCrudService metadataCrudService; - - private final Logger log = LoggerFactory.getLogger(getClass()); - - protected Logger getLog() { - return log; - } - - public Biomaterial addBiomaterialToSubmissionEnvelope(SubmissionEnvelope submissionEnvelope, Biomaterial biomaterial) { - if (!biomaterial.getIsUpdate()) { - projectRepository - .findBySubmissionEnvelopesContains(submissionEnvelope) - .findFirst().ifPresent(project -> { - biomaterial.setProject(project); - biomaterial.getProjects().add(project); - }); - return metadataCrudService.addToSubmissionEnvelopeAndSave(biomaterial, submissionEnvelope); - } else { - return metadataUpdateService.acceptUpdate(biomaterial, submissionEnvelope); - } - } -} diff --git a/src/main/java/org/humancellatlas/ingest/biomaterial/web/BiomaterialController.java b/src/main/java/org/humancellatlas/ingest/biomaterial/web/BiomaterialController.java deleted file mode 100644 index d41c8a3c8..000000000 --- a/src/main/java/org/humancellatlas/ingest/biomaterial/web/BiomaterialController.java +++ /dev/null @@ -1,156 +0,0 @@ -package org.humancellatlas.ingest.biomaterial.web; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.biomaterial.BiomaterialService; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.core.service.*; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.security.CheckAllowed; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.exception.NotAllowedDuringSubmissionStateException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.rest.webmvc.PersistentEntityResource; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.data.web.PagedResourcesAssembler; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.hateoas.Resource; -import org.springframework.hateoas.Resources; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpMethod; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; - -import java.lang.reflect.InvocationTargetException; -import java.net.URISyntaxException; -import java.util.List; -import java.util.Optional; -import java.util.UUID; - -import static org.springframework.data.rest.webmvc.RestMediaTypes.TEXT_URI_LIST_VALUE; -import static org.springframework.web.bind.annotation.RequestMethod.POST; -import static org.springframework.web.bind.annotation.RequestMethod.PUT; - -/** - * Created by rolando on 16/02/2018. - */ -@RepositoryRestController -@RequiredArgsConstructor -@ExposesResourceFor(Biomaterial.class) -@Getter -public class BiomaterialController { - - private final @NonNull ProcessRepository processRepository; - - private final @NonNull BiomaterialService biomaterialService; - - private final @NonNull BiomaterialRepository biomaterialRepository; - - private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; - - private final @NonNull MetadataCrudService metadataCrudService; - - private final @NonNull MetadataUpdateService metadataUpdateService; - - private @Autowired - ValidationStateChangeService validationStateChangeService; - - private @Autowired - UriToEntityConversionService uriToEntityConversionService; - - private @Autowired - MetadataLinkingService metadataLinkingService; - - @CheckAllowed(value = "#submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @PostMapping(path = "submissionEnvelopes/{sub_id}/biomaterials") - ResponseEntity> addBiomaterialToEnvelope(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - @RequestBody Biomaterial biomaterial, - @RequestParam("updatingUuid") Optional updatingUuid, - PersistentEntityResourceAssembler assembler) { - updatingUuid.ifPresent(uuid -> { - biomaterial.setUuid(new Uuid(uuid.toString())); - biomaterial.setIsUpdate(true); - }); - Biomaterial entity = getBiomaterialService().addBiomaterialToSubmissionEnvelope(submissionEnvelope, biomaterial); - PersistentEntityResource resource = assembler.toFullResource(entity); - return ResponseEntity.accepted().body(resource); - } - - @CheckAllowed(value = "#submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @RequestMapping(path = "submissionEnvelopes/{sub_id}/biomaterials/{id}", method = RequestMethod.PUT) - ResponseEntity> linkBiomaterialToEnvelope(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - @PathVariable("id") Biomaterial biomaterial, - PersistentEntityResourceAssembler assembler) { - Biomaterial entity = getBiomaterialService().addBiomaterialToSubmissionEnvelope(submissionEnvelope, biomaterial); - PersistentEntityResource resource = assembler.toFullResource(entity); - return ResponseEntity.accepted().body(resource); - } - - @CheckAllowed(value = "#biomaterial.submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @PatchMapping(path = "/biomaterials/{id}") - HttpEntity patchBiomaterial(@PathVariable("id") Biomaterial biomaterial, - @RequestBody final ObjectNode patch, - PersistentEntityResourceAssembler assembler) { - List allowedFields = List.of("content", "validationErrors", "graphValidationErrors"); - ObjectNode validPatch = patch.retain(allowedFields); - Biomaterial updatedBiomaterial = metadataUpdateService.update(biomaterial, validPatch); - PersistentEntityResource resource = assembler.toFullResource(updatedBiomaterial); - return ResponseEntity.accepted().body(resource); - } - - @CheckAllowed(value = "#biomaterial.submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @RequestMapping(path = "/biomaterials/{id}/inputToProcesses", method = {PUT, POST}, consumes = {TEXT_URI_LIST_VALUE}) - HttpEntity linkBiomaterialAsInputToProcesses(@PathVariable("id") Biomaterial biomaterial, - @RequestBody Resources incoming, - HttpMethod requestMethod, - PersistentEntityResourceAssembler assembler) throws URISyntaxException, InvocationTargetException, NoSuchMethodException, IllegalAccessException { - - List processes = uriToEntityConversionService.convertLinks(incoming.getLinks(), Process.class); - metadataLinkingService.updateLinks(biomaterial, processes, "inputToProcesses", requestMethod.equals(HttpMethod.PUT)); - - return ResponseEntity.ok().build(); - } - - @CheckAllowed(value = "#biomaterial.submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @RequestMapping(path = "/biomaterials/{id}/derivedByProcesses", method = {PUT, POST}, consumes = {TEXT_URI_LIST_VALUE}) - HttpEntity linkBiomaterialAsDerivedByProcesses(@PathVariable("id") Biomaterial biomaterial, - @RequestBody Resources incoming, - HttpMethod requestMethod, - PersistentEntityResourceAssembler assembler) throws URISyntaxException, InvocationTargetException, NoSuchMethodException, IllegalAccessException { - - List processes = uriToEntityConversionService.convertLinks(incoming.getLinks(), Process.class); - metadataLinkingService.updateLinks(biomaterial, processes, "derivedByProcesses", requestMethod.equals(HttpMethod.PUT)); - return ResponseEntity.ok().build(); - } - - @CheckAllowed(value = "#biomaterial.submissionEnvelope.isEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @DeleteMapping(path = "/biomaterials/{id}/inputToProcesses/{processId}") - HttpEntity unlinkBiomaterialAsInputToProcesses(@PathVariable("id") Biomaterial biomaterial, - @PathVariable("processId") Process process, - PersistentEntityResourceAssembler assembler) throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { - metadataLinkingService.removeLink(biomaterial, process, "inputToProcesses"); - return ResponseEntity.noContent().build(); - } - - @CheckAllowed(value = "#biomaterial.submissionEnvelope.isEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @DeleteMapping(path = "/biomaterials/{id}/derivedByProcesses/{processId}") - HttpEntity unlinkBiomaterialAsDerivedProcesses(@PathVariable("id") Biomaterial biomaterial, - @PathVariable("processId") Process process, - PersistentEntityResourceAssembler assembler) throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { - metadataLinkingService.removeLink(biomaterial, process, "derivedByProcesses"); - return ResponseEntity.noContent().build(); - } - - @DeleteMapping(path = "/biomaterials/{id}") - @CheckAllowed(value = "#biomaterial.submissionEnvelope.isEditable()", exception = NotAllowedDuringSubmissionStateException.class) - ResponseEntity deleteBiomaterial(@PathVariable("id") Biomaterial biomaterial) { - metadataCrudService.deleteDocument(biomaterial); - return ResponseEntity.noContent().build(); - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/bundle/BundleManifest.java b/src/main/java/org/humancellatlas/ingest/bundle/BundleManifest.java deleted file mode 100644 index c30111baa..000000000 --- a/src/main/java/org/humancellatlas/ingest/bundle/BundleManifest.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.humancellatlas.ingest.bundle; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.AllArgsConstructor; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.Document; -import org.springframework.hateoas.Identifiable; - -import java.util.Collection; -import java.util.List; -import java.util.Map; - -/** - * Created by rolando on 05/09/2017. - */ -@AllArgsConstructor -@Getter -@Document -@EqualsAndHashCode -public class BundleManifest implements Identifiable { - private @Id @JsonIgnore String id; - - @Indexed - private final String bundleUuid; - @Indexed - private final String bundleVersion; - - private final String envelopeUuid; - - private final List dataFiles; - private final Map> fileBiomaterialMap; - private final Map> fileProcessMap; - private final Map> fileProjectMap; - private final Map> fileProtocolMap; - private final Map> fileFilesMap; - -} diff --git a/src/main/java/org/humancellatlas/ingest/bundle/BundleManifestRepository.java b/src/main/java/org/humancellatlas/ingest/bundle/BundleManifestRepository.java deleted file mode 100644 index cc84b9a81..000000000 --- a/src/main/java/org/humancellatlas/ingest/bundle/BundleManifestRepository.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.humancellatlas.ingest.bundle; - -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.data.repository.query.Param; -import org.springframework.data.rest.core.annotation.RestResource; -import org.springframework.web.bind.annotation.CrossOrigin; - -import java.util.Optional; -import java.util.stream.Stream; - -/** - * Created by rolando on 05/09/2017. - */ -@CrossOrigin -public interface BundleManifestRepository extends MongoRepository, BundleManifestRepositoryCustom { - Page findByBundleUuid(@Param("uuid") String uuid, Pageable pageable); - - Optional findTopByBundleUuidOrderByBundleVersionDesc(String uuid); - - Page findByEnvelopeUuid(String uuid, Pageable pageable); - - Page findAll(Pageable pageable); - - Long deleteByEnvelopeUuid (String uuid); - - @RestResource(exported = false) - Stream findByEnvelopeUuid(String envelopeUuid); -} diff --git a/src/main/java/org/humancellatlas/ingest/bundle/BundleManifestRepositoryCustom.java b/src/main/java/org/humancellatlas/ingest/bundle/BundleManifestRepositoryCustom.java deleted file mode 100644 index 853f76162..000000000 --- a/src/main/java/org/humancellatlas/ingest/bundle/BundleManifestRepositoryCustom.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.humancellatlas.ingest.bundle; - -import org.humancellatlas.ingest.project.Project; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; - -public interface BundleManifestRepositoryCustom { - Page findBundleManifestsByProjectAndBundleType(Project project, BundleType bundleType, Pageable pageable); -} diff --git a/src/main/java/org/humancellatlas/ingest/bundle/BundleManifestRepositoryImpl.java b/src/main/java/org/humancellatlas/ingest/bundle/BundleManifestRepositoryImpl.java deleted file mode 100644 index 532e78e0a..000000000 --- a/src/main/java/org/humancellatlas/ingest/bundle/BundleManifestRepositoryImpl.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.humancellatlas.ingest.bundle; - -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageImpl; -import org.springframework.data.domain.Pageable; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; - -import java.util.List; - -public class BundleManifestRepositoryImpl implements BundleManifestRepositoryCustom { - - private final MongoTemplate mongoTemplate; - - @Autowired - public BundleManifestRepositoryImpl(final MongoTemplate mongoTemplate){ - this.mongoTemplate = mongoTemplate; - } - - @Override - public Page findBundleManifestsByProjectAndBundleType(Project project, BundleType bundleType, Pageable pageable) { - SubmissionEnvelope submissionEnvelope = project.getSubmissionEnvelope(); - String submissionUuid = submissionEnvelope.getUuid().getUuid().toString(); - String projectUuid = project.getUuid().getUuid().toString(); - - Query query = new Query(); - query.addCriteria(Criteria.where("fileProjectMap." + projectUuid).exists(true)); - - if (bundleType != null){ - if(bundleType.equals(BundleType.PRIMARY)) { - query.addCriteria(Criteria.where("envelopeUuid").is(submissionUuid)); - } - else if (bundleType.equals(BundleType.ANALYSIS)){ - // TODO This might not be the best criteria to query analysis bundles. Might need to remodel bundle manifest. - query.addCriteria(Criteria.where("envelopeUuid").ne(submissionUuid)); - } - } - - query.with(pageable); - - List result = mongoTemplate.find(query, BundleManifest.class); - long count = mongoTemplate.count(query, BundleManifest.class); - Page bundleManifestPage = new PageImpl<>(result, pageable, count); - return bundleManifestPage; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/bundle/BundleManifestSearchProcessor.java b/src/main/java/org/humancellatlas/ingest/bundle/BundleManifestSearchProcessor.java deleted file mode 100644 index ca799ebba..000000000 --- a/src/main/java/org/humancellatlas/ingest/bundle/BundleManifestSearchProcessor.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.humancellatlas.ingest.bundle; - -import org.humancellatlas.ingest.bundle.web.BundleManifestController; -import org.springframework.data.rest.webmvc.RepositorySearchesResource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.stereotype.Component; - -import static org.springframework.hateoas.mvc.ControllerLinkBuilder.linkTo; -import static org.springframework.hateoas.mvc.ControllerLinkBuilder.methodOn; - -@Component -public class BundleManifestSearchProcessor implements ResourceProcessor { - @Override - public RepositorySearchesResource process(RepositorySearchesResource resource) { - if(resource.getDomainType().equals(BundleManifest.class)) { - resource.add(linkTo(methodOn(BundleManifestController.class) - .findBundleManifestsByProjectUuidAndBundleType(null, null, null, null)) - .withRel("findBundleManifestsByProjectUuidAndBundleType")); - } - - return resource; - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/bundle/BundleManifestService.java b/src/main/java/org/humancellatlas/ingest/bundle/BundleManifestService.java deleted file mode 100644 index 639ee9be8..000000000 --- a/src/main/java/org/humancellatlas/ingest/bundle/BundleManifestService.java +++ /dev/null @@ -1,119 +0,0 @@ -package org.humancellatlas.ingest.bundle; - - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.EntityType; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Pageable; -import org.springframework.stereotype.Service; - -import java.text.DecimalFormat; -import java.util.*; - -@Service -@RequiredArgsConstructor -public class BundleManifestService { - private final @NonNull - BundleManifestRepository bundleManifestRepository; - private final Logger log = LoggerFactory.getLogger(getClass()); - - public Map> bundleManifestsForDocuments(Collection documents) { - - Map> hits = new HashMap<>(); - - long fileStartTime = System.currentTimeMillis(); - Iterator iterator = allManifestsIterator(); - - while(iterator.hasNext()) { - BundleManifest bundleManifest = iterator.next(); - documents.forEach(document -> { - String documentUuid = document.getUuid().getUuid().toString(); - String bundleUuid = bundleManifest.getBundleUuid(); - EntityType documentType = document.getType(); - entityMapFromManifest(documentType, bundleManifest).ifPresent(entityMap -> { - if(entityMap.containsKey(documentUuid)){ - if(hits.containsKey(bundleUuid)) { - hits.get(bundleUuid).add(document); - } else { - hits.put(bundleUuid, new HashSet<>(Collections.singletonList(document))); - } - } - }); - }); - } - - long fileEndTime = System.currentTimeMillis(); - float fileQueryTime = ((float)(fileEndTime - fileStartTime)) / 1000; - String fileQt = new DecimalFormat("#,###.##").format(fileQueryTime); - log.info("Finding bundles to update took {}s", fileQt); - log.info("documentsToUpdate: {}, bundlesToUpdate:{}", documents.size(), hits.keySet().size()); - return hits; - } - - private Iterator allManifestsIterator() { - Iterator manifestsIterator = new Iterator() { - Pageable pageable = new PageRequest(0, 5000); - Page pagedBundleManifests = null; - Queue bundleManifests; - - private void fetch(Pageable pageable){ - pagedBundleManifests = bundleManifestRepository.findAll(pageable); - bundleManifests = new LinkedList<>(pagedBundleManifests.getContent()); - } - - @Override - public boolean hasNext() { - return (pagedBundleManifests == null || bundleManifests.size() > 0 || pagedBundleManifests.hasNext()); - } - - @Override - public BundleManifest next() { - BundleManifest bundleManifest = null; - - if (pagedBundleManifests == null){ - fetch(pageable); - - } - - if(bundleManifests.size() == 0 && pagedBundleManifests.hasNext()) { - fetch(pagedBundleManifests.nextPageable()); - } - - if(bundleManifests.size() > 0) { - bundleManifest = bundleManifests.remove(); - } - - return bundleManifest; - } - - }; - - return manifestsIterator; - } - - - private Optional>> entityMapFromManifest(EntityType entityType, BundleManifest bundleManifest) { - if(entityType.equals(EntityType.BIOMATERIAL)) { - return Optional.ofNullable(bundleManifest.getFileBiomaterialMap()); - } else if(entityType.equals(EntityType.FILE)) { - return Optional.ofNullable(bundleManifest.getFileFilesMap()); - } else if(entityType.equals(EntityType.PROTOCOL)) { - return Optional.ofNullable(bundleManifest.getFileProtocolMap()); - } else if(entityType.equals(EntityType.PROCESS)) { - return Optional.ofNullable(bundleManifest.getFileProcessMap()); - } else if(entityType.equals(EntityType.PROJECT)) { - return Optional.ofNullable(bundleManifest.getFileProjectMap()); - } else { - throw new RuntimeException(String.format("Bundle manifest %s contains no entity map for entity type %s", - bundleManifest.getId(), - entityType.toString())); - } - } - - -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/bundle/BundleType.java b/src/main/java/org/humancellatlas/ingest/bundle/BundleType.java deleted file mode 100644 index 8d4b88a82..000000000 --- a/src/main/java/org/humancellatlas/ingest/bundle/BundleType.java +++ /dev/null @@ -1,6 +0,0 @@ -package org.humancellatlas.ingest.bundle; - -public enum BundleType { - PRIMARY, - ANALYSIS -} diff --git a/src/main/java/org/humancellatlas/ingest/bundle/web/BundleManifestController.java b/src/main/java/org/humancellatlas/ingest/bundle/web/BundleManifestController.java deleted file mode 100644 index f646d3863..000000000 --- a/src/main/java/org/humancellatlas/ingest/bundle/web/BundleManifestController.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.humancellatlas.ingest.bundle.web; - -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.bundle.BundleManifest; -import org.humancellatlas.ingest.bundle.BundleType; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.project.ProjectService; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.data.web.PagedResourcesAssembler; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RequestParam; - -import java.util.Optional; - -@RepositoryRestController -@RequiredArgsConstructor -@ExposesResourceFor(BundleManifest.class) -@Getter -public class BundleManifestController { - private final @NonNull - ProjectService projectService; - - private final @NonNull - PagedResourcesAssembler pagedResourcesAssembler; - - @RequestMapping(path = "/projects/search/findBundleManifestsByProjectUuidAndBundleType", method = RequestMethod.GET) - public ResponseEntity findBundleManifestsByProjectUuidAndBundleType(@RequestParam("projectUuid") Uuid projectUuid, - @RequestParam("bundleType") Optional bundleType, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - - Page bundleManifests = this.projectService.findBundleManifestsByProjectUuidAndBundleType(projectUuid, bundleType.orElse(null), pageable); - return ResponseEntity.ok(pagedResourcesAssembler.toResource(bundleManifests, resourceAssembler)); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/config/ConfigurationService.java b/src/main/java/org/humancellatlas/ingest/config/ConfigurationService.java deleted file mode 100644 index d91002ec6..000000000 --- a/src/main/java/org/humancellatlas/ingest/config/ConfigurationService.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.humancellatlas.ingest.config; - -import lombok.Getter; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -/** - * Created by rolando on 05/09/2018. - */ -@Component -public class ConfigurationService implements InitializingBean { - @Value("${STATE_TRACKER_SCHEME:http}") - private String stateTrackerSchemeString; - @Value("${STATE_TRACKER_HOST:localhost}") - private String stateTrackerHostString; - @Value("${STATE_TRACKER_PORT:8999}") - private String stateTrackerPortString; - @Value("${STATE_TRACKER_DOCUMENT_STATES_PATH:machine-reports}") - private String documentStatesPathString; - @Value("${STATE_TRACKER_DOCUMENT_STATES_UPDATE_PATH:state-updates/metadata-documents}") - private String documentStatesUpdatePathString; - @Value("${STATE_TRACKER_DOCUMENT_PARAM:metadataDocumentId}") - private String documentIdParamNameString; - @Value("${STATE_TRACKER_DOCUMENT_PARAM:envelopeId}") - private String envelopeIdParamNameString; - - @Getter - private String stateTrackerScheme; - @Getter - private String stateTrackerHost; - @Getter - private int stateTrackerPort; - @Getter - private String documentStatesPath; - @Getter - private String documentStatesUpdatePath; - @Getter - private String documentIdParamName; - @Getter - private String envelopeIdParamName; - - private void init(){ - this.stateTrackerScheme = this.stateTrackerSchemeString; - this.stateTrackerHost = this.stateTrackerHostString; - this.stateTrackerPort = Integer.parseInt(this.stateTrackerPortString); - this.documentStatesPath = this.documentStatesPathString; - this.documentStatesUpdatePath = this.documentStatesUpdatePathString; - this.documentIdParamName = this.documentIdParamNameString; - this.envelopeIdParamName = this.envelopeIdParamNameString; - } - - @Override - public void afterPropertiesSet() throws Exception { - init(); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/config/MigrationConfiguration.java b/src/main/java/org/humancellatlas/ingest/config/MigrationConfiguration.java deleted file mode 100644 index e3b687707..000000000 --- a/src/main/java/org/humancellatlas/ingest/config/MigrationConfiguration.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.humancellatlas.ingest.config; - -import com.github.mongobee.Mongobee; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class MigrationConfiguration { - @Value("${spring.data.mongodb.uri}") - private String mongoURI; - - @Bean - public Mongobee Configure() { - Mongobee runner = new Mongobee(mongoURI); - runner.setChangeLogsScanPackage("org.humancellatlas.ingest.migrations"); - return runner; - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/config/MongoConfiguration.java b/src/main/java/org/humancellatlas/ingest/config/MongoConfiguration.java deleted file mode 100644 index 6d603a9a3..000000000 --- a/src/main/java/org/humancellatlas/ingest/config/MongoConfiguration.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.humancellatlas.ingest.config; - -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.config.EnableMongoAuditing; - -@Configuration -@EnableMongoAuditing(auditorAwareRef = "userAuditing") -public class MongoConfiguration {} diff --git a/src/main/java/org/humancellatlas/ingest/core/AbstractEntity.java b/src/main/java/org/humancellatlas/ingest/core/AbstractEntity.java deleted file mode 100644 index 60682e936..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/AbstractEntity.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.humancellatlas.ingest.core; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; -import org.springframework.data.annotation.*; -import org.springframework.hateoas.Identifiable; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 30/08/17 - */ -@Getter -@ToString -@JsonIgnoreProperties(value = {"type"}, allowGetters = true) -@EqualsAndHashCode -public abstract class AbstractEntity implements Identifiable { - protected @Id @JsonIgnore String id; - - private @Version Long version; - - private @CreatedDate Instant submissionDate; - - private @LastModifiedDate Instant updateDate; - - private @CreatedBy String user; - - private @LastModifiedBy String lastModifiedUser; - - private EntityType type; - - private @Setter Uuid uuid; - - private @Setter List events = new ArrayList<>(); - - protected AbstractEntity(EntityType type) { - this.type = type; - } - - protected AbstractEntity() {} - -} diff --git a/src/main/java/org/humancellatlas/ingest/core/Accession.java b/src/main/java/org/humancellatlas/ingest/core/Accession.java deleted file mode 100644 index ba9ae219e..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/Accession.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.humancellatlas.ingest.core; - -import lombok.Data; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 30/08/17 - */ -@Data -public class Accession { - private String number; - - protected Accession() { - this.number = null; - } - - public Accession(String number) { - if (!isValid(number)) { - throw new IllegalArgumentException(String.format("Accession number '%s' is not a valid format ", number)); - } - this.number = number; - } - - public static boolean isValid(String number) { - return !number.isEmpty(); // todo might want to regex this, maybe this service doesn't care - } -} diff --git a/src/main/java/org/humancellatlas/ingest/core/Checksums.java b/src/main/java/org/humancellatlas/ingest/core/Checksums.java deleted file mode 100644 index 3d022b2a9..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/Checksums.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.humancellatlas.ingest.core; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -/** - * Created by rolando on 06/09/2017. - */ -@Data -public class Checksums { - private String sha1; - private String sha256; - private String crc32c; - @JsonProperty("s3_etag") - private String s3Etag; - - public Checksums(String sha1, String sha256, String crc32c, String s3Etag) { - this.sha1 = sha1; - this.sha256 = sha256; - this.crc32c = crc32c; - this.s3Etag = s3Etag; - } - - protected Checksums() { - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/core/EntityType.java b/src/main/java/org/humancellatlas/ingest/core/EntityType.java deleted file mode 100644 index b81f41ccc..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/EntityType.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.humancellatlas.ingest.core; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 30/08/17 - */ -public enum EntityType { - SUBMISSION, - PROJECT, - BIOMATERIAL, - PROCESS, - PROTOCOL, - FILE -} diff --git a/src/main/java/org/humancellatlas/ingest/core/MetadataDocument.java b/src/main/java/org/humancellatlas/ingest/core/MetadataDocument.java deleted file mode 100644 index 5810bc4b4..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/MetadataDocument.java +++ /dev/null @@ -1,135 +0,0 @@ -package org.humancellatlas.ingest.core; - -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.DBRef; -import org.springframework.data.mongodb.core.mapping.Field; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 31/08/17 - */ -@Getter -@EqualsAndHashCode(callSuper = true, exclude = "contentLastModified") -public abstract class MetadataDocument extends AbstractEntity { - - @Setter - private Instant firstDcpVersion; - - private Instant dcpVersion; - - private Instant contentLastModified; - - private Object content; - - // This property holds the reference to the submissionEnvelope this metadata document was part of. - // A metadata document is part of one submissionEnvelope. - // The other end of this relationship can be defined as a set of metadataDocuments in a submissionEnvelope. - @Indexed - @Setter - @DBRef(lazy = true) - private SubmissionEnvelope submissionEnvelope; - - private @Setter - Accession accession; - - private @Setter - ValidationState validationState; - - private @Setter - List validationErrors; - - private @Setter - List graphValidationErrors; - - private @Setter - @Field - Boolean isUpdate = false; - - - private static final Logger log = LoggerFactory.getLogger(SubmissionEnvelope.class); - - protected static Logger getLog() { - return log; - } - - protected MetadataDocument() { - } - - protected MetadataDocument(EntityType type, - Object content) { - super(type); - this.content = content; - this.contentLastModified = Instant.now(); - this.validationState = ValidationState.DRAFT; - } - - public static List allowedStateTransitions(ValidationState fromState) { - List allowedStates = new ArrayList<>(); - switch (fromState) { - case DRAFT: - allowedStates.add(ValidationState.VALIDATING); - break; - case VALIDATING: - allowedStates.add(ValidationState.VALID); - allowedStates.add(ValidationState.INVALID); - break; - case VALID: - allowedStates.add(ValidationState.PROCESSING); - allowedStates.add(ValidationState.DRAFT); - break; - case INVALID: - allowedStates.add(ValidationState.DRAFT); - break; - case PROCESSING: - allowedStates.add(ValidationState.COMPLETE); - allowedStates.add(ValidationState.DRAFT); - break; - default: - getLog().warn(String.format("There are no legal state transitions for '%s' state", fromState.name())); - break; - } - return allowedStates; - } - - public List allowedStateTransitions() { - return allowedStateTransitions(getValidationState()); - } - - - public MetadataDocument enactStateTransition(ValidationState targetState) { - this.validationState = targetState; - return this; - } - - public void setContent(Object content) { - if (this.content == null || !this.content.equals(content)) { - this.content = content; - this.contentLastModified = Instant.now(); - this.setDcpVersion(this.contentLastModified); - } - } - - public MetadataDocument setDcpVersion(Instant dcpVersion) { - // DCP version should never be set to null - if (dcpVersion != null) { - if (this.dcpVersion == null) { - this.firstDcpVersion = dcpVersion; - } - this.dcpVersion = dcpVersion; - } - return this; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/core/MetadataDocumentMessageBuilder.java b/src/main/java/org/humancellatlas/ingest/core/MetadataDocumentMessageBuilder.java deleted file mode 100644 index 32797aadb..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/MetadataDocumentMessageBuilder.java +++ /dev/null @@ -1,98 +0,0 @@ -package org.humancellatlas.ingest.core; - -import org.humancellatlas.ingest.core.web.LinkGenerator; -import org.humancellatlas.ingest.messaging.model.MetadataDocumentMessage; -import org.humancellatlas.ingest.state.ValidationState; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.hateoas.Identifiable; - -import java.time.Instant; - - -public class MetadataDocumentMessageBuilder { - - private LinkGenerator linkGenerator; - - private Class documentType; - private String metadataDocId; - private String metadataDocUuid; - private String envelopeId; - private String envelopeUuid; - private Instant metadataDocVersion; - private ValidationState validationState; - - private final Logger log = LoggerFactory.getLogger(getClass()); - - private MetadataDocumentMessageBuilder(LinkGenerator linkGenerator) { - this.linkGenerator = linkGenerator; - } - - public static MetadataDocumentMessageBuilder using(LinkGenerator linkGenerator) { - return new MetadataDocumentMessageBuilder(linkGenerator); - } - - protected Logger getLog() { - return log; - } - - public MetadataDocumentMessageBuilder messageFor(MetadataDocument metadataDocument) { - MetadataDocumentMessageBuilder builder = withDocumentType(metadataDocument.getClass()) - .withId(metadataDocument.getId()); - Uuid metadataDocumentUuid = metadataDocument.getUuid(); - if (metadataDocumentUuid != null && metadataDocumentUuid.getUuid() != null) { - builder = builder.withUuid(metadataDocument.getUuid().getUuid().toString()); - } - builder = builder.withVersion(metadataDocument.getDcpVersion()); - - return builder; - } - - private MetadataDocumentMessageBuilder withDocumentType( - Class documentClass) { - this.documentType = documentClass; - return this; - } - - private MetadataDocumentMessageBuilder withId(String metadataDocId) { - this.metadataDocId = metadataDocId; - - return this; - } - - private MetadataDocumentMessageBuilder withUuid(String metadataDocUuid) { - this.metadataDocUuid = metadataDocUuid; - - return this; - } - - private MetadataDocumentMessageBuilder withVersion(Instant metadataDocVersion) { - this.metadataDocVersion = metadataDocVersion; - - return this; - } - - public MetadataDocumentMessageBuilder withValidationState(ValidationState validationState) { - this.validationState = validationState; - - return this; - } - - public MetadataDocumentMessageBuilder withEnvelopeId(String envelopeId) { - this.envelopeId = envelopeId; - - return this; - } - - public MetadataDocumentMessageBuilder withEnvelopeUuid(String envelopeUuid) { - this.envelopeUuid = envelopeUuid; - - return this; - } - - public MetadataDocumentMessage build() { - String callbackLink = linkGenerator.createCallback(documentType, metadataDocId); - return new MetadataDocumentMessage(documentType.getSimpleName().toLowerCase(), - metadataDocId, metadataDocUuid, validationState, callbackLink, envelopeId); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/core/SubmissionDate.java b/src/main/java/org/humancellatlas/ingest/core/SubmissionDate.java deleted file mode 100644 index 4733a47e0..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/SubmissionDate.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.humancellatlas.ingest.core; - -import lombok.Data; - -import java.util.Date; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 30/08/17 - */ -@Data -public class SubmissionDate { - private Date date; - - protected SubmissionDate() { - this.date = null; - } - - public SubmissionDate(Date date) { - if (!isValid(date)) { - throw new IllegalArgumentException(String.format("Submission date '%s' is in the future!", date)); - } - this.date = date; - } - - public static boolean isValid(Date date) { - return !date.after(new Date()); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/core/UpdateDate.java b/src/main/java/org/humancellatlas/ingest/core/UpdateDate.java deleted file mode 100644 index bbefffc5b..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/UpdateDate.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.humancellatlas.ingest.core; - -import lombok.Data; - -import java.util.Date; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 30/08/17 - */ -@Data -public class UpdateDate { - private Date date; - - protected UpdateDate() { - this.date = null; - } - - public UpdateDate(Date date) { - if (!isValid(date)) { - throw new IllegalArgumentException(String.format("Update date '%s' is in the future!", date)); - } - this.date = date; - } - - public static boolean isValid(Date date) { - return !date.after(new Date()); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/core/Uuid.java b/src/main/java/org/humancellatlas/ingest/core/Uuid.java deleted file mode 100644 index 557246b91..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/Uuid.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.humancellatlas.ingest.core; - -import com.fasterxml.jackson.annotation.JsonCreator; -import lombok.Data; - -import java.util.UUID; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 31/08/17 - */ -@Data -public class Uuid { - private UUID uuid; - - @JsonCreator - public Uuid(String name) { - // throws IllegalArgumentException if not valid - this.uuid = UUID.fromString(name); - } - - public Uuid() { - } - - public static Uuid newUuid() { - Uuid uuid = new Uuid(); - uuid.setUuid(UUID.randomUUID()); - return uuid; - } - - @Override - public String toString() { - return uuid.toString(); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/core/ValidationChecksum.java b/src/main/java/org/humancellatlas/ingest/core/ValidationChecksum.java deleted file mode 100644 index 9a04a8a5c..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/ValidationChecksum.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.humancellatlas.ingest.core; - -import lombok.Data; - -/** - * Created by rolando on 11/09/2017. - */ -@Data -public class ValidationChecksum { - private String md5 = ""; -} diff --git a/src/main/java/org/humancellatlas/ingest/core/ValidationEvent.java b/src/main/java/org/humancellatlas/ingest/core/ValidationEvent.java deleted file mode 100644 index f5edc07fc..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/ValidationEvent.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.humancellatlas.ingest.core; - -import lombok.Getter; -import org.springframework.context.ApplicationEvent; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 12/09/17 - */ -@Getter -public class ValidationEvent extends ApplicationEvent { - private String message; - - public ValidationEvent(Object source, String message) { - super(source); - this.message = message; - } - public String getMessage() { - return message; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/core/exception/CoreEntityNotFoundException.java b/src/main/java/org/humancellatlas/ingest/core/exception/CoreEntityNotFoundException.java deleted file mode 100644 index 559da17a9..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/exception/CoreEntityNotFoundException.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.humancellatlas.ingest.core.exception; - -/** - * Created by rolando on 07/09/2017. - */ -public class CoreEntityNotFoundException extends Exception { - public CoreEntityNotFoundException(){ - - } - - public CoreEntityNotFoundException(String message) { - super(message); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/core/exception/MultipleOpenSubmissionsException.java b/src/main/java/org/humancellatlas/ingest/core/exception/MultipleOpenSubmissionsException.java deleted file mode 100644 index 6cae59c2e..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/exception/MultipleOpenSubmissionsException.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.humancellatlas.ingest.core.exception; - -public class MultipleOpenSubmissionsException extends RuntimeException { - - public MultipleOpenSubmissionsException() { - super(); - } - - public MultipleOpenSubmissionsException(String message) { - super(message); - } - - public MultipleOpenSubmissionsException(String message, Throwable cause) { - super(message, cause); - } - - public MultipleOpenSubmissionsException(Throwable cause) { - super(cause); - } - - protected MultipleOpenSubmissionsException(String message, - Throwable cause, - boolean enableSuppression, - boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/core/exception/RedundantUpdateException.java b/src/main/java/org/humancellatlas/ingest/core/exception/RedundantUpdateException.java deleted file mode 100644 index f6338cd06..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/exception/RedundantUpdateException.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.humancellatlas.ingest.core.exception; - -public class RedundantUpdateException extends RuntimeException { - public RedundantUpdateException(){ - - } - - public RedundantUpdateException (String message) { - super(message); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/core/exception/StateTransitionNotAllowed.java b/src/main/java/org/humancellatlas/ingest/core/exception/StateTransitionNotAllowed.java deleted file mode 100644 index 59501c27e..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/exception/StateTransitionNotAllowed.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.humancellatlas.ingest.core.exception; - -/** - * Created by rolando on 10/03/2018. - */ -public class StateTransitionNotAllowed extends RuntimeException { - public StateTransitionNotAllowed(){ - - } - - public StateTransitionNotAllowed (String message) { - super(message); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/core/service/MetadataCrudService.java b/src/main/java/org/humancellatlas/ingest/core/service/MetadataCrudService.java deleted file mode 100644 index c091588ed..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/service/MetadataCrudService.java +++ /dev/null @@ -1,75 +0,0 @@ -package org.humancellatlas.ingest.core.service; - -import lombok.AllArgsConstructor; -import lombok.NonNull; -import org.humancellatlas.ingest.core.EntityType; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.core.service.strategy.MetadataCrudStrategy; -import org.humancellatlas.ingest.core.service.strategy.impl.*; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.stereotype.Service; - -import java.util.Collection; - -@Service -@AllArgsConstructor -public class MetadataCrudService { - private final @NonNull BiomaterialCrudStrategy biomaterialCrudStrategy; - private final @NonNull ProcessCrudStrategy processCrudStrategy; - private final @NonNull ProtocolCrudStrategy protocolCrudStrategy; - private final @NonNull ProjectCrudStrategy projectCrudStrategy; - private final @NonNull FileCrudStrategy fileCrudStrategy; - - private MetadataCrudStrategy crudStrategyForMetadataType(EntityType metadataType) { - switch (metadataType) { - case BIOMATERIAL: - return biomaterialCrudStrategy; - case PROCESS: - return processCrudStrategy; - case PROTOCOL: - return protocolCrudStrategy; - case PROJECT: - return projectCrudStrategy; - case FILE: - return fileCrudStrategy; - default: - throw new RuntimeException(String.format("No such metadata type: %s", metadataType)); - } - } - - public T save(T metadataDocument) { - MetadataCrudStrategy crudStrategy = crudStrategyForMetadataType(metadataDocument.getType()); - return (T) crudStrategy.saveMetadataDocument(metadataDocument); - } - - public T setValidationState(EntityType entityType, String entityId, ValidationState validationState) { - MetadataCrudStrategy crudStrategy = crudStrategyForMetadataType(entityType); - T document = (T) crudStrategy.findMetadataDocument(entityId); - document.setValidationState(validationState); - return (T) crudStrategy.saveMetadataDocument(document); - - } - - public T addToSubmissionEnvelopeAndSave(T metadataDocument, SubmissionEnvelope submissionEnvelope) { - metadataDocument.setSubmissionEnvelope(submissionEnvelope); - return (T) (crudStrategyForMetadataType(metadataDocument.getType()).saveMetadataDocument(metadataDocument)); - } - - public T findOriginalByUuid(String uuid, EntityType entityType) { - return (T) crudStrategyForMetadataType(entityType).findOriginalByUuid(uuid); - } - - public Collection findAllBySubmission(SubmissionEnvelope submissionEnvelope, EntityType entityType) { - return crudStrategyForMetadataType(entityType).findAllBySubmissionEnvelope(submissionEnvelope); - } - - public void removeLinksToDocument(T metadataDocument) { - crudStrategyForMetadataType(metadataDocument.getType()).removeLinksToDocument(metadataDocument); - } - - public void deleteDocument(T metadataDocument) { - crudStrategyForMetadataType(metadataDocument.getType()).deleteDocument(metadataDocument); - - } -} diff --git a/src/main/java/org/humancellatlas/ingest/core/service/MetadataDifferService.java b/src/main/java/org/humancellatlas/ingest/core/service/MetadataDifferService.java deleted file mode 100644 index 8cd38c623..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/service/MetadataDifferService.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.humancellatlas.ingest.core.service; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.zjsonpatch.JsonDiff; -import lombok.AllArgsConstructor; -import org.humancellatlas.ingest.core.JsonPatch; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.springframework.stereotype.Service; - -@Service -@AllArgsConstructor -public class MetadataDifferService { - - public boolean anyDifference(MetadataDocument source, MetadataDocument target) { - ObjectMapper objectMapper = new ObjectMapper(); - - JsonNode sourceContent = objectMapper.valueToTree(source.getContent()); - JsonNode targetContent = objectMapper.valueToTree(target.getContent()); - - return ! sourceContent.equals(targetContent); - } - - public JsonPatch generatePatch(T originalDocument, T updateDocument) { - ObjectMapper objectMapper = new ObjectMapper(); - - JsonNode sourceContent = objectMapper.valueToTree(originalDocument.getContent()); - JsonNode targetContent = objectMapper.valueToTree(updateDocument.getContent()); - - return this.generatePatch(sourceContent, targetContent); - } - - public JsonPatch generatePatch(JsonNode source, JsonNode target) { - return new JsonPatch(JsonDiff.asJson(source, target)); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/core/service/MetadataLinkingService.java b/src/main/java/org/humancellatlas/ingest/core/service/MetadataLinkingService.java deleted file mode 100644 index 6e58f3b66..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/service/MetadataLinkingService.java +++ /dev/null @@ -1,106 +0,0 @@ -package org.humancellatlas.ingest.core.service; - -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.state.SubmissionState; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.OptimisticLockingFailureException; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.retry.support.RetryTemplate; -import org.springframework.stereotype.Service; -import reactor.util.StringUtils; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.*; - -@Service -public class MetadataLinkingService { - - private static final long RETRY_BACKOFF_MS = 100; - private static final int RETRY_MAX_ATTEMPTS = 5; - - private ValidationStateChangeService validationStateChangeService; - - private MongoTemplate mongoTemplate; - - @Autowired - public MetadataLinkingService(ValidationStateChangeService validationStateChangeService, MongoTemplate mongoTemplate) { - this.validationStateChangeService = validationStateChangeService; - this.mongoTemplate = mongoTemplate; - } - - public T updateLinks(T targetEntity, List entitiesToLink, String linkProperty, Boolean replace) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { - if (replace) { - replaceLinks(targetEntity, entitiesToLink, linkProperty); - } else { - addLinks(targetEntity, entitiesToLink, linkProperty); - } - return targetEntity; - } - - public T addLinks(T targetEntity, List entitiesToLink, String linkProperty) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { - Method method = getGetterMethod(targetEntity, entitiesToLink.get(0).getClass(), linkProperty); - Set linkedEntities = (Set) invoke(targetEntity, method); - entitiesToLink.forEach(doc -> { - linkedEntities.add(doc); - }); - mongoTemplate.save(targetEntity); - - setValidationStateToDraftIfGraphValid(targetEntity); - - return targetEntity; - } - - public T replaceLinks(T targetEntity, List entitiesToLink, String linkProperty) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { - Method method = getGetterMethod(targetEntity, entitiesToLink.get(0).getClass(), linkProperty); - Set linkedEntities = (Set) invoke(targetEntity, method); - linkedEntities.clear(); - linkedEntities.addAll(entitiesToLink); - mongoTemplate.save(targetEntity); - - setValidationStateToDraftIfGraphValid(targetEntity); - - return targetEntity; - } - - public T removeLink(T targetEntity, S entityToUnlink, String linkProperty) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { - Method method = getGetterMethod(targetEntity, entityToUnlink.getClass(), linkProperty); - Set linkedEntities = (Set) invoke(targetEntity, method); - linkedEntities.remove(entityToUnlink); - mongoTemplate.save(targetEntity); - - setValidationStateToDraftIfGraphValid(targetEntity); - - return targetEntity; - } - - private Method getGetterMethod(T metadataDocument, Class parameterType, String linkProperty) throws NoSuchMethodException { - return metadataDocument.getClass().getMethod("get" + StringUtils.capitalize(linkProperty)); - } - - private Object invoke(T metadataDocument, Method method) throws IllegalAccessException, InvocationTargetException { - return method.invoke(metadataDocument); - } - - private void setValidationStateToDraftIfGraphValid(MetadataDocument... entities) { - Arrays.stream(entities).forEach(entity -> { - SubmissionEnvelope submission = entity.getSubmissionEnvelope(); - if (submission != null && submission.getSubmissionState().equals(SubmissionState.GRAPH_VALID)) { - setToDraft(entity); - } - }); - } - - private void setToDraft(MetadataDocument entity) { - RetryTemplate retry = RetryTemplate.builder() - .maxAttempts(RETRY_MAX_ATTEMPTS) - .fixedBackoff(RETRY_BACKOFF_MS) - .retryOn(OptimisticLockingFailureException.class) - .build(); - retry.execute(context -> { - return validationStateChangeService.changeValidationState(entity.getType(), entity.getId(), ValidationState.DRAFT); - }); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/core/service/MetadataUpdateService.java b/src/main/java/org/humancellatlas/ingest/core/service/MetadataUpdateService.java deleted file mode 100644 index dc62910b3..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/service/MetadataUpdateService.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.humancellatlas.ingest.core.service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.AllArgsConstructor; -import lombok.NonNull; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.core.exception.RedundantUpdateException; -import org.humancellatlas.ingest.patch.JsonPatcher; -import org.humancellatlas.ingest.patch.PatchService; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.stereotype.Service; - -import java.util.Optional; - -@AllArgsConstructor -@Service -public class MetadataUpdateService { - private final @NonNull MetadataDifferService metadataDifferService; - private final @NonNull MetadataCrudService metadataCrudService; - private final @NonNull PatchService patchService; - - private final @NonNull ValidationStateChangeService validationStateChangeService; - private final @NonNull JsonPatcher jsonPatcher; - - public T update(T metadataDocument, ObjectNode patch) { - ObjectMapper mapper = new ObjectMapper(); - - Boolean contentChanged = Optional.ofNullable(patch.get("content")) - .map(content -> !content.equals(mapper.valueToTree(metadataDocument.getContent()))) - .orElse(false); - - T patchedMetadata = jsonPatcher.merge(patch, metadataDocument); - T doc = metadataCrudService.save(patchedMetadata); - - if (contentChanged) { - validationStateChangeService.changeValidationState(doc.getType(), doc.getId(), ValidationState.DRAFT); - } - - return doc; - } - - public T acceptUpdate(T updateDocument, SubmissionEnvelope submissionEnvelope) { - T originalDocument = metadataCrudService.findOriginalByUuid(updateDocument.getUuid().getUuid().toString(), updateDocument.getType()); - - if (metadataDifferService.anyDifference(originalDocument, updateDocument)) { - T savedUpdateDocument = metadataCrudService.addToSubmissionEnvelopeAndSave(updateDocument, submissionEnvelope); - patchService.storePatch(originalDocument, savedUpdateDocument, submissionEnvelope); - return savedUpdateDocument; - } else { - throw new RedundantUpdateException(String.format("Attempted to update %s document at %s with contents of %s but there is no diff", - updateDocument.getType(), - originalDocument.getId(), - updateDocument.getId())); - } - } - -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/core/service/UriToEntityConversionService.java b/src/main/java/org/humancellatlas/ingest/core/service/UriToEntityConversionService.java deleted file mode 100644 index 1b0615c82..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/service/UriToEntityConversionService.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.humancellatlas.ingest.core.service; - -import org.humancellatlas.ingest.core.MetadataDocument; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.convert.TypeDescriptor; -import org.springframework.data.mapping.context.PersistentEntities; -import org.springframework.data.repository.support.Repositories; -import org.springframework.data.repository.support.RepositoryInvokerFactory; -import org.springframework.data.rest.core.UriToEntityConverter; -import org.springframework.hateoas.Link; -import org.springframework.stereotype.Service; - -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; - -@Service -public class UriToEntityConversionService { - - private UriToEntityConverter converter; - - @Autowired - public UriToEntityConversionService(PersistentEntities entities, RepositoryInvokerFactory invokerFactory, - Repositories repositories) { - converter = new UriToEntityConverter(entities, invokerFactory, repositories); - } - - public T convert(Object source, TypeDescriptor sourceType, TypeDescriptor targetType) { - return (T) converter.convert(source, sourceType, targetType); - } - - public T convertLink(Link link, TypeDescriptor targetType) throws URISyntaxException { - URI uri = new URI(link.getHref()); - return (T) convert(uri, TypeDescriptor.valueOf(URI.class), targetType); - } - - public List convertLinks(List links, Class clazz) throws URISyntaxException { - List list = new ArrayList<>(); - for (Link link : links) list.add((T) convertLink(link, TypeDescriptor.valueOf(clazz))); - return list; - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/core/service/ValidationStateChangeService.java b/src/main/java/org/humancellatlas/ingest/core/service/ValidationStateChangeService.java deleted file mode 100644 index d7889dbc5..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/service/ValidationStateChangeService.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.humancellatlas.ingest.core.service; - -import lombok.AllArgsConstructor; -import lombok.NonNull; -import org.humancellatlas.ingest.core.EntityType; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.state.ValidationStateEventPublisher; -import org.springframework.stereotype.Service; - -@Service -@AllArgsConstructor -public class ValidationStateChangeService { - private final @NonNull MetadataCrudService metadataCrudService; - - private final @NonNull ValidationStateEventPublisher validationStateEventPublisher; - - public T changeValidationState(EntityType metadataType, - String metadataId, - ValidationState validationState) { - T metadataDocument = metadataCrudService.setValidationState(metadataType, metadataId, validationState); - validationStateEventPublisher.publishValidationStateChangeEventFor(metadataDocument); - return metadataDocument; - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/core/service/strategy/MetadataCrudStrategy.java b/src/main/java/org/humancellatlas/ingest/core/service/strategy/MetadataCrudStrategy.java deleted file mode 100644 index 08680bf4b..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/service/strategy/MetadataCrudStrategy.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.humancellatlas.ingest.core.service.strategy; - -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; - -import java.util.Collection; -import java.util.stream.Stream; - -public interface MetadataCrudStrategy { - T saveMetadataDocument(T document); - - T findMetadataDocument(String id); - - T findOriginalByUuid(String uuid); - - Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - void removeLinksToDocument(T document); - - void deleteDocument(T document); -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/core/service/strategy/impl/BiomaterialCrudStrategy.java b/src/main/java/org/humancellatlas/ingest/core/service/strategy/impl/BiomaterialCrudStrategy.java deleted file mode 100644 index 2d09f1761..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/service/strategy/impl/BiomaterialCrudStrategy.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.humancellatlas.ingest.core.service.strategy.impl; - -import lombok.AllArgsConstructor; -import lombok.NonNull; -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.core.service.strategy.MetadataCrudStrategy; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.rest.webmvc.ResourceNotFoundException; -import org.springframework.stereotype.Component; - -import java.util.Collection; -import java.util.UUID; -import java.util.stream.Stream; - - -@AllArgsConstructor -@Component -public class BiomaterialCrudStrategy implements MetadataCrudStrategy { - private final @NonNull BiomaterialRepository biomaterialRepository; - private final @NonNull MessageRouter messageRouter; - - @Override - public Biomaterial saveMetadataDocument(Biomaterial document) { - return biomaterialRepository.save(document); - } - - @Override - public Biomaterial findMetadataDocument(String id) { - return biomaterialRepository.findById(id) - .orElseThrow(() -> { - throw new ResourceNotFoundException(); - }); - } - - @Override - public Biomaterial findOriginalByUuid(String uuid) { - return biomaterialRepository.findByUuidUuidAndIsUpdateFalse(UUID.fromString(uuid)) - .orElseThrow(() -> { - throw new ResourceNotFoundException(); - }); - } - - @Override - public Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { - return biomaterialRepository.findBySubmissionEnvelope(submissionEnvelope); - } - - @Override - public Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { - return biomaterialRepository.findAllBySubmissionEnvelope(submissionEnvelope); - } - - @Override - public void removeLinksToDocument(Biomaterial document) { - messageRouter.routeStateTrackingDeleteMessageFor(document); - } - - @Override - public void deleteDocument(Biomaterial document) { - removeLinksToDocument(document); - biomaterialRepository.delete(document); - } - -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/core/service/strategy/impl/FileCrudStrategy.java b/src/main/java/org/humancellatlas/ingest/core/service/strategy/impl/FileCrudStrategy.java deleted file mode 100644 index 52e0125fc..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/service/strategy/impl/FileCrudStrategy.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.humancellatlas.ingest.core.service.strategy.impl; - -import lombok.AllArgsConstructor; -import lombok.NonNull; -import org.humancellatlas.ingest.core.service.strategy.MetadataCrudStrategy; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.rest.webmvc.ResourceNotFoundException; -import org.springframework.stereotype.Component; - -import java.util.Collection; -import java.util.UUID; -import java.util.stream.Stream; - -@Component -@AllArgsConstructor -public class FileCrudStrategy implements MetadataCrudStrategy { - private final @NonNull FileRepository fileRepository; - private final @NonNull ProjectRepository projectRepository; - private final @NonNull MessageRouter messageRouter; - - @Override - public File saveMetadataDocument(File document) { - return fileRepository.save(document); - } - - @Override - public File findMetadataDocument(String id) { - return fileRepository.findById(id) - .orElseThrow(() -> { - throw new ResourceNotFoundException(); - }); - } - - @Override - public File findOriginalByUuid(String uuid) { - return fileRepository.findByUuidUuidAndIsUpdateFalse(UUID.fromString(uuid)) - .orElseThrow(() -> { - throw new ResourceNotFoundException(); - }); - } - - @Override - public Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { - return fileRepository.findBySubmissionEnvelope(submissionEnvelope); - } - - @Override - public Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { - return fileRepository.findAllBySubmissionEnvelope(submissionEnvelope); - } - - @Override - public void removeLinksToDocument(File document) { - messageRouter.routeStateTrackingDeleteMessageFor(document); - projectRepository.findBySupplementaryFilesContains(document).forEach(project -> { - project.getSupplementaryFiles().remove(document); - projectRepository.save(project); - }); - } - - @Override - public void deleteDocument(File document) { - removeLinksToDocument(document); - fileRepository.delete(document); - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/core/service/strategy/impl/ProcessCrudStrategy.java b/src/main/java/org/humancellatlas/ingest/core/service/strategy/impl/ProcessCrudStrategy.java deleted file mode 100644 index e4bc608b4..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/service/strategy/impl/ProcessCrudStrategy.java +++ /dev/null @@ -1,85 +0,0 @@ -package org.humancellatlas.ingest.core.service.strategy.impl; - -import lombok.AllArgsConstructor; -import lombok.NonNull; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.core.service.strategy.MetadataCrudStrategy; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.rest.webmvc.ResourceNotFoundException; -import org.springframework.stereotype.Component; - -import java.util.Collection; -import java.util.UUID; -import java.util.stream.Stream; - -@Component -@AllArgsConstructor -public class ProcessCrudStrategy implements MetadataCrudStrategy { - private final @NonNull ProcessRepository processRepository; - private final @NonNull FileRepository fileRepository; - private final @NonNull BiomaterialRepository biomaterialRepository; - private final @NonNull MessageRouter messageRouter; - - @Override - public Process saveMetadataDocument(Process document) { - return processRepository.save(document); - } - - @Override - public Process findMetadataDocument(String id) { - return processRepository.findById(id) - .orElseThrow(() -> { - throw new ResourceNotFoundException(); - }); - } - - @Override - public Process findOriginalByUuid(String uuid) { - return processRepository.findByUuidUuidAndIsUpdateFalse(UUID.fromString(uuid)) - .orElseThrow(() -> { - throw new ResourceNotFoundException(); - }); - } - - @Override - public Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { - return processRepository.findBySubmissionEnvelope(submissionEnvelope); - } - - @Override - public Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { - return processRepository.findAllBySubmissionEnvelope(submissionEnvelope); - } - - @Override - public void removeLinksToDocument(Process document) { - messageRouter.routeStateTrackingDeleteMessageFor(document); - fileRepository.findByInputToProcessesContains(document).forEach(file -> { - file.getInputToProcesses().remove(document); - fileRepository.save(file); - }); - fileRepository.findByDerivedByProcessesContains(document).forEach(file -> { - file.getDerivedByProcesses().remove(document); - fileRepository.save(file); - }); - biomaterialRepository.findByInputToProcessesContains(document).forEach(biomaterial -> { - biomaterial.getInputToProcesses().remove(document); - biomaterialRepository.save(biomaterial); - }); - biomaterialRepository.findByDerivedByProcessesContains(document).forEach(biomaterial -> { - biomaterial.getDerivedByProcesses().remove(document); - biomaterialRepository.save(biomaterial); - }); - - } - - @Override - public void deleteDocument(Process document) { - removeLinksToDocument(document); - processRepository.delete(document); - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/core/service/strategy/impl/ProjectCrudStrategy.java b/src/main/java/org/humancellatlas/ingest/core/service/strategy/impl/ProjectCrudStrategy.java deleted file mode 100644 index 6fc5a4099..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/service/strategy/impl/ProjectCrudStrategy.java +++ /dev/null @@ -1,97 +0,0 @@ -package org.humancellatlas.ingest.core.service.strategy.impl; - -import lombok.AllArgsConstructor; -import lombok.NonNull; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.core.service.strategy.MetadataCrudStrategy; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.rest.webmvc.ResourceNotFoundException; -import org.springframework.stereotype.Component; - -import java.util.Collection; -import java.util.UUID; -import java.util.stream.Stream; - -@Component -@AllArgsConstructor -public class ProjectCrudStrategy implements MetadataCrudStrategy { - private final @NonNull ProjectRepository projectRepository; - private final @NonNull ProtocolRepository protocolRepository; - private final @NonNull ProcessRepository processRepository; - private final @NonNull FileRepository fileRepository; - private final @NonNull BiomaterialRepository biomaterialRepository; - private final @NonNull MessageRouter messageRouter; - - @Override - public Project saveMetadataDocument(Project document) { - return projectRepository.save(document); - } - - @Override - public Project findMetadataDocument(String id) { - return projectRepository.findById(id) - .orElseThrow(() -> { - throw new ResourceNotFoundException(); - }); - } - - @Override - public Project findOriginalByUuid(String uuid) { - return projectRepository.findByUuidUuidAndIsUpdateFalse(UUID.fromString(uuid)) - .orElseThrow(() -> { - throw new ResourceNotFoundException(); - }); - } - - @Override - public Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { - return projectRepository.findBySubmissionEnvelope(submissionEnvelope); - } - - @Override - public Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { - return projectRepository.findAllBySubmissionEnvelope(submissionEnvelope); - } - - @Override - public void removeLinksToDocument(Project document) { - messageRouter.routeStateTrackingDeleteMessageFor(document); - biomaterialRepository.findByProject(document).forEach(biomaterial -> { - biomaterial.setProject(null); - biomaterialRepository.save(biomaterial); - }); - biomaterialRepository.findByProjectsContaining(document).forEach(biomaterial -> { - biomaterial.getProjects().remove(document); - biomaterialRepository.save(biomaterial); - }); - fileRepository.findByProject(document).forEach(file -> { - file.setProject(null); - fileRepository.save(file); - }); - processRepository.findByProject(document).forEach(process -> { - process.setProject(null); - processRepository.save(process); - }); - processRepository.findByProjectsContaining(document).forEach(process -> { - process.getProjects().remove(document); - processRepository.save(process); - }); - protocolRepository.findByProject(document).forEach(protocol -> { - protocol.setProject(null); - protocolRepository.save(protocol); - }); - - } - - @Override - public void deleteDocument(Project document) { - removeLinksToDocument(document); - projectRepository.delete(document); - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/core/service/strategy/impl/ProtocolCrudStrategy.java b/src/main/java/org/humancellatlas/ingest/core/service/strategy/impl/ProtocolCrudStrategy.java deleted file mode 100644 index 160543cdb..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/service/strategy/impl/ProtocolCrudStrategy.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.humancellatlas.ingest.core.service.strategy.impl; - -import lombok.AllArgsConstructor; -import lombok.NonNull; -import org.humancellatlas.ingest.core.service.strategy.MetadataCrudStrategy; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.rest.webmvc.ResourceNotFoundException; -import org.springframework.stereotype.Component; - -import java.util.Collection; -import java.util.UUID; -import java.util.stream.Stream; - -@Component -@AllArgsConstructor -public class ProtocolCrudStrategy implements MetadataCrudStrategy { - private final @NonNull ProtocolRepository protocolRepository; - private final @NonNull ProcessRepository processRepository; - private final @NonNull MessageRouter messageRouter; - - @Override - public Protocol saveMetadataDocument(Protocol document) { - return protocolRepository.save(document); - } - - @Override - public Protocol findMetadataDocument(String id) { - return protocolRepository.findById(id) - .orElseThrow(() -> { - throw new ResourceNotFoundException(); - }); - } - - @Override - public Protocol findOriginalByUuid(String uuid) { - return protocolRepository.findByUuidUuidAndIsUpdateFalse(UUID.fromString(uuid)) - .orElseThrow(() -> { - throw new ResourceNotFoundException(); - }); - } - - @Override - public Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { - return protocolRepository.findBySubmissionEnvelope(submissionEnvelope); - } - - @Override - public Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { - return protocolRepository.findAllBySubmissionEnvelope(submissionEnvelope); - } - - @Override - public void removeLinksToDocument(Protocol document) { - messageRouter.routeStateTrackingDeleteMessageFor(document); - processRepository.findByProtocolsContains(document).forEach(process -> { - process.getProtocols().remove(document); - processRepository.save(process); - }); - } - - @Override - public void deleteDocument(Protocol document) { - removeLinksToDocument(document); - protocolRepository.delete(document); - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/core/web/GlobalStateExceptionHandler.java b/src/main/java/org/humancellatlas/ingest/core/web/GlobalStateExceptionHandler.java deleted file mode 100644 index d48d94e57..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/web/GlobalStateExceptionHandler.java +++ /dev/null @@ -1,132 +0,0 @@ -package org.humancellatlas.ingest.core.web; - -import org.humancellatlas.ingest.core.exception.MultipleOpenSubmissionsException; -import org.humancellatlas.ingest.core.exception.RedundantUpdateException; -import org.humancellatlas.ingest.core.exception.StateTransitionNotAllowed; -import org.humancellatlas.ingest.security.exception.NotAllowedException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.dao.OptimisticLockingFailureException; -import org.springframework.data.repository.support.QueryMethodParameterConversionException; -import org.springframework.data.rest.webmvc.ResourceNotFoundException; -import org.springframework.http.HttpStatus; -import org.springframework.http.converter.HttpMessageNotReadableException; -import org.springframework.web.bind.annotation.ControllerAdvice; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.bind.annotation.ResponseBody; -import org.springframework.web.bind.annotation.ResponseStatus; - -import javax.servlet.http.HttpServletRequest; -import java.io.IOException; -import java.util.stream.Collectors; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 27/09/17 - */ -@ControllerAdvice -public class GlobalStateExceptionHandler { - private final Logger log = LoggerFactory.getLogger(getClass()); - - protected Logger getLog() { - return log; - } - - @ResponseStatus(HttpStatus.CONFLICT) - @ExceptionHandler(IllegalStateException.class) - public @ResponseBody - ExceptionInfo handleIllegalStateException(HttpServletRequest request, Exception e) { - getLog().warn(String.format("Attempted an illegal state transition at '%s';" + - "this will generate a CONFLICT RESPONSE", - request.getRequestURL().toString())); - getLog().debug("Handling IllegalStateException and returning CONFLICT response", e); - return new ExceptionInfo(request.getRequestURL().toString(), e.getLocalizedMessage()); - } - - @ResponseStatus(HttpStatus.CONFLICT) - @ExceptionHandler(OptimisticLockingFailureException.class) - public @ResponseBody - ExceptionInfo handleOptimisticLock(HttpServletRequest request, Exception e) { - getLog().warn(String.format("Attempt a failed save, likely due to multiple requests, at '%s'; " + - "this will generate a CONFLICT RESPONSE", - request.getRequestURL().toString())); - getLog().debug("Handling OptimisticLockingFailureException and returning CONFLICT response", e); - return new ExceptionInfo(request.getRequestURL().toString(), e.getLocalizedMessage()); - } - - @ResponseStatus(HttpStatus.BAD_REQUEST) - @ExceptionHandler({ - IllegalArgumentException.class, - HttpMessageNotReadableException.class, - RedundantUpdateException.class, - MultipleOpenSubmissionsException.class, - QueryMethodParameterConversionException.class - }) - public @ResponseBody - ExceptionInfo handleIllegalArgument(HttpServletRequest request, Exception e) { - getLog().warn(String.format("Caught an illegal argument at '%s %s'. error: %s; " + - "this will generate a BAD_REQUEST RESPONSE", - request.getMethod(), - request.getRequestURL().toString(), - e.getLocalizedMessage() - )); - getLog().error("Handling IllegalArgumentException and returning BAD_REQUEST response", e); - return new ExceptionInfo(request.getRequestURL().toString(), e.getLocalizedMessage()); - } - - @ResponseStatus(HttpStatus.NOT_FOUND) - @ExceptionHandler(ResourceNotFoundException.class) - public @ResponseBody - ExceptionInfo handleResourceNotFound(HttpServletRequest request, Exception e) { - getLog().warn(String.format("Caught a resource not found exception argument at '%s'; " + - "this will generate a NOT_FOUND RESPONSE. Error message: %s", - request.getRequestURL().toString(), e.getLocalizedMessage())); - getLog().warn("Handling ResourceNotFoundException and returning NOT_FOUND response"); - return new ExceptionInfo(request.getRequestURL().toString(), e.getLocalizedMessage()); - } - - @ResponseStatus(HttpStatus.BAD_REQUEST) - @ExceptionHandler(StateTransitionNotAllowed.class) - public @ResponseBody - ExceptionInfo handleStateTransitionNotAllowed(HttpServletRequest request, Exception e) { - getLog().warn(String.format("Caught a state transition not allowed exception at '%s'; " + - "this will generate a BAD_REQUEST RESPONSE", - request.getRequestURL().toString())); - getLog().debug("Handling StateTransitionNotAllowed and returning BAD_REQUEST response", e); - return new ExceptionInfo(request.getRequestURL().toString(), e.getLocalizedMessage()); - } - - - @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) - @ExceptionHandler(RuntimeException.class) - public @ResponseBody - ExceptionInfo handleRuntimeException(HttpServletRequest request, Exception e) { - getLog().error(String.format("Runtime exception encountered on %s request to resource %s ", request.getMethod(), - request.getRequestURL().toString()), e); - getLog().error("Handling RuntimeException and returning INTERNAL_SERVER_ERROR response"); - return new ExceptionInfo(request.getRequestURL().toString(), "Unexpected server error"); - } - - @ResponseStatus(HttpStatus.FORBIDDEN) - @ExceptionHandler(NotAllowedException.class) - public @ResponseBody - ExceptionInfo handNotAllowedException(HttpServletRequest request, Exception e) { - getLog().error(String.format("Not allowed exception encountered on %s request to resource %s ", request.getMethod(), - request.getRequestURL().toString()), e); - getLog().debug("Handling NotAllowedException and returning FORBIDDEN response", e); - return new ExceptionInfo(request.getRequestURL().toString(), e.getLocalizedMessage()); - } - - @ResponseStatus(HttpStatus.METHOD_NOT_ALLOWED) - @ExceptionHandler(UnsupportedOperationException.class) - public @ResponseBody - ExceptionInfo handleUnsupportedOperationException(HttpServletRequest request, Exception e) { - getLog().error(String.format("UnsupportedOperationException encountered on %s request to resource %s ", request.getMethod(), - request.getRequestURL().toString()), e); - getLog().debug("Handling UnsupportedOperationException and returning METHOD_NOT_ALLOWED response", e); - return new ExceptionInfo(request.getRequestURL().toString(), e.getLocalizedMessage()); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/core/web/LinkGenerator.java b/src/main/java/org/humancellatlas/ingest/core/web/LinkGenerator.java deleted file mode 100644 index 7bfae5b55..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/web/LinkGenerator.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.humancellatlas.ingest.core.web; - -public interface LinkGenerator { - - String createCallback(Class documentType, String documentId); - -} diff --git a/src/main/java/org/humancellatlas/ingest/core/web/Links.java b/src/main/java/org/humancellatlas/ingest/core/web/Links.java deleted file mode 100644 index ffa0d1342..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/web/Links.java +++ /dev/null @@ -1,158 +0,0 @@ -package org.humancellatlas.ingest.core.web; - -/** - * Enumerates the relations that are available in this application to provide some stability across different - * implementations. These should not change without serious motivation, as will likely cause backwards-compatibility - * breaking changes for clients. - * - * @author Tony Burdett - * @date 05/09/17 - */ -public class Links { - // Links to request state changes for submission envelopes - - public static final String UPDATE_SUBMISSION_URL = "/updateSubmissions"; - public static final String UPDATE_SUBMISSION_REL = "updateSubmissions"; - - public static final String SUBMIT_URL = "/submissionEvent"; - public static final String SUBMIT_REL = "submit"; - - public static final String DRAFT_REL = "draft"; - public static final String DRAFT_URL = "/draftEvent"; - public static final String METADATA_VALIDATING_REL = "validating"; - public static final String METADATA_VALIDATING_URL = "/validatingEvent"; - public static final String METADATA_VALID_REL = "valid"; - public static final String METADATA_VALID_URL = "/validEvent"; - public static final String GRAPH_VALIDATION_REQUESTED_REL = "graphValidationRequested"; - public static final String GRAPH_VALIDATION_REQUESTED_URL = "/graphValidationRequestedEvent"; - public static final String GRAPH_VALIDATING_REL = "graphValidating"; - public static final String GRAPH_VALIDATING_URL = "/graphValidatingEvent"; - public static final String GRAPH_VALID_REL = "graphValid"; - public static final String GRAPH_VALID_URL = "/graphValidEvent"; - public static final String GRAPH_INVALID_REL = "graphInvalid"; - public static final String GRAPH_INVALID_URL = "/graphInvalidEvent"; - public static final String INVALID_REL = "invalid"; - public static final String INVALID_URL = "/invalidEvent"; - public static final String PROCESSING_REL ="processing"; - public static final String PROCESSING_URL ="/processingEvent"; - public static final String ARCHIVING_REL = "archiving"; - public static final String ARCHIVING_URL = "/archivingEvent"; - public static final String ARCHIVED_REL = "archived"; - public static final String ARCHIVED_URL = "/archivedEvent"; - public static final String EXPORT_REL = "export"; - public static final String EXPORT_URL = "/exportEvent"; - public static final String EXPORTED_REL = "exported"; - public static final String EXPORTED_URL = "/exportedEvent"; - public static final String CLEANUP_REL = "cleanup"; - public static final String CLEANUP_URL = "/cleanupEvent"; - public static final String COMPLETE_REL = "complete"; - public static final String COMPLETE_URL = "/completionEvent"; - - // links to commit state changes - public static final String COMMIT_SUBMIT_URL = "/commitSubmissionEvent"; - public static final String COMMIT_SUBMIT_REL = "commitSubmit"; - - public static final String COMMIT_DRAFT_REL = "commitDraft"; - public static final String COMMIT_DRAFT_URL = "/commitDraftEvent"; - public static final String COMMIT_METADATA_VALIDATING_REL = "commitValidating"; - public static final String COMMIT_METADATA_VALIDATING_URL = "/commitValidatingEvent"; - public static final String COMMIT_METADATA_VALID_REL = "commitValid"; - public static final String COMMIT_METADATA_VALID_URL = "/commitValidEvent"; - public static final String COMMIT_METADATA_INVALID_REL = "commitInvalid"; - public static final String COMMIT_METADATA_INVALID_URL = "/commitInvalidEvent"; - public static final String COMMIT_GRAPH_VALIDATION_REQUESTED_REL = "commitGraphValidationRequested"; - public static final String COMMIT_GRAPH_VALIDATION_REQUESTED_URL = "/commitGraphValidationRequestedEvent"; - public static final String COMMIT_GRAPH_VALIDATING_REL = "commitGraphValidating"; - public static final String COMMIT_GRAPH_VALIDATING_URL = "/commitGraphValidatingEvent"; - public static final String COMMIT_GRAPH_VALID_REL = "commitGraphValid"; - public static final String COMMIT_GRAPH_VALID_URL = "/commitGraphValidEvent"; - public static final String COMMIT_GRAPH_INVALID_REL = "commitGraphInvalid"; - public static final String COMMIT_GRAPH_INVALID_URL = "/commitGraphInvalidEvent"; - public static final String COMMIT_PROCESSING_REL ="commitProcessing"; - public static final String COMMIT_PROCESSING_URL ="/commitProcessingEvent"; - public static final String COMMIT_ARCHIVING_REL = "commitArchiving"; - public static final String COMMIT_ARCHIVING_URL = "/commitArchivingEvent"; - public static final String COMMIT_ARCHIVED_REL = "commitArchived"; - public static final String COMMIT_ARCHIVED_URL = "/commitArchivedEvent"; - public static final String COMMIT_EXPORTING_REL = "commitExporting"; - public static final String COMMIT_EXPORTING_URL = "/commitExportingEvent"; - public static final String COMMIT_EXPORTED_REL = "commitExported"; - public static final String COMMIT_EXPORTED_URL = "/commitExportedEvent"; - public static final String COMMIT_CLEANUP_REL = "commitCleanup"; - public static final String COMMIT_CLEANUP_URL = "/commitCleanupEvent"; - public static final String COMMIT_COMPLETE_REL = "commitComplete"; - public static final String COMMIT_COMPLETE_URL = "/commitCompleteEvent"; - - // Links to entities for submission envelopes - public static final String BIOMATERIALS_URL = "/biomaterials"; - public static final String BIOMATERIALS_REL = "biomaterials"; - public static final String PROCESSES_URL = "/processes"; - public static final String PROCESSES_REL = "processes"; - public static final String FILES_URL = "/files"; - public static final String FILES_REL = "files"; - public static final String PROJECTS_URL = "/projects"; - public static final String PROJECTS_REL = "projects"; - public static final String PROTOCOLS_URL = "/protocols"; - public static final String PROTOCOLS_REL = "protocols"; - public static final String BUNDLE_MANIFESTS_URL = "/bundleManifests"; - public static final String BUNDLE_MANIFESTS_REL = "bundleManifests"; - public static final String SUBMISSION_MANIFEST_URL = "/submissionManifest"; - public static final String SUBMISSION_MANIFEST_REL = "submissionManifest"; - public static final String SUBMISSION_ERRORS_URL = "/submissionErrors"; - public static final String SUBMISSION_ERRORS_REL = "submissionEnvelopeErrors"; - public static final String SUBMISSION_SUMMARY_URL = "/summary"; - public static final String SUBMISSION_SUMMARY_REL = "summary"; - public static final String SUBMISSION_CONTENT_LAST_UPDATED_URL = "/contentLastUpdated"; - public static final String SUBMISSION_CONTENT_LAST_UPDATED_REL = "contentLastUpdated"; - public static final String SUBMISSION_LINKING_MAP_URL = "/linkingMap"; - public static final String SUBMISSION_LINKING_MAP_REL = "linkingMap"; - public static final String SUBMISSION_RELATED_PROJECTS_URL = "/relatedProjects"; - public static final String SUBMISSION_RELATED_PROJECTS_REL = "relatedProjects"; - - public static final String SUBMISSION_DOCUMENTS_SM_URL = "/documentSmReport"; - public static final String SUBMISSION_DOCUMENTS_SM_REL = "documentSmReport"; - - // Links to entities for projects - public static final String AUDIT_LOGS_URL = "/auditLogs"; - public static final String AUDIT_LOGS_REL = "auditLogs"; - - // Links for analyses - public static final String BUNDLE_REF_URL = "/bundleReferences"; - public static final String BUNDLE_REF_REL = "inputBundleReferences"; - public static final String BUNDLE_REF_OLD_EVIL_REL = "add-input-bundles"; - public static final String FILE_REF_URL = "/fileReference"; - public static final String FILE_REF_REL = "inputFileReferences"; - public static final String FILE_REF_OLD_EVIL_REL = "add-file-reference"; - - // Links from Processes - public static final String INPUT_BIOMATERIALS_URL = "/inputBiomaterials"; - public static final String INPUT_BIOMATERIALS_REL = "inputBiomaterials"; - public static final String INPUT_FILES_URL = "/inputFiles"; - public static final String INPUT_FILES_REL = "inputFiles"; - - public static final String DERIVED_BY_BIOMATERIALS_URL = "/derivedBiomaterials"; - public static final String DERIVED_BY_BIOMATERIALS_REL = "derivedBiomaterials"; - public static final String DERIVED_BY_FILES_URL = "/derivedFiles"; - public static final String DERIVED_BY_FILES_REL = "derivedFiles"; - - // Links from Files - public static final String FILE_VALIDATION_JOB_URL = "/validationJob"; - public static final String FILE_VALIDATION_JOB_REL = "validationJob"; - - // Links from StagingJobs - public static final String COMPLETE_STAGING_JOB_URL = "/complete"; - public static final String COMPLETE_STAGING_JOB_REL = "completeStagingJob"; - - // Links to ExportJobs - public static final String EXPORT_JOBS_URL = "/exportJobs"; - public static final String EXPORT_JOBS_REL = "exportJobs"; - - public static final String EXPORT_JOB_ENTITIES_URL = "/entities"; - public static final String EXPORT_JOB_ENTITIES_REL = "exportEntities"; - public static final String EXPORT_JOB_ENTITIES_BY_STATUS_REL = "exportEntitiesByStatus"; - - public static final String EXPORT_JOB_FIND_URL = "/find"; - public static final String EXPORT_JOB_FIND_REL = "find"; - - -} diff --git a/src/main/java/org/humancellatlas/ingest/core/web/MetadataController.java b/src/main/java/org/humancellatlas/ingest/core/web/MetadataController.java deleted file mode 100644 index 53bfefee8..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/web/MetadataController.java +++ /dev/null @@ -1,104 +0,0 @@ -package org.humancellatlas.ingest.core.web; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; - -import java.util.List; -import java.util.Optional; - -import org.humancellatlas.ingest.core.EntityType; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.core.service.ValidationStateChangeService; -import org.humancellatlas.ingest.query.MetadataQueryService; -import org.humancellatlas.ingest.query.MetadataCriteria; -import org.humancellatlas.ingest.state.ValidationState; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.data.rest.webmvc.ResourceNotFoundException; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.hateoas.PagedResources; -import org.springframework.http.HttpEntity; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.web.PagedResourcesAssembler; -import org.springframework.hateoas.Resource; - -@RequiredArgsConstructor -@RepositoryRestController -@ExposesResourceFor(MetadataDocument.class) -public class MetadataController { - private final @NonNull ValidationStateChangeService validationStateChangeService; - private final @NonNull MetadataQueryService metadataQueryService; - private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; - - @PutMapping("/{metadataType}/{id}" + Links.DRAFT_URL) - HttpEntity draftEvent(@PathVariable("metadataType") String metadataType, - @PathVariable("id") String metadataId, - PersistentEntityResourceAssembler assembler) { - MetadataDocument metadataDocument = validationStateChangeService.changeValidationState(entityTypeForCollection(metadataType), - metadataId, - ValidationState.DRAFT); - return ResponseEntity.accepted().body(assembler.toFullResource(metadataDocument)); - } - - @PutMapping("/{metadataType}/{id}" + Links.METADATA_VALIDATING_URL) - HttpEntity validatingEvent(@PathVariable("metadataType") String metadataType, - @PathVariable("id") String metadataId, - PersistentEntityResourceAssembler assembler) { - MetadataDocument metadataDocument = validationStateChangeService.changeValidationState(entityTypeForCollection(metadataType), - metadataId, - ValidationState.VALIDATING); - return ResponseEntity.accepted().body(assembler.toFullResource(metadataDocument)); - } - - @PutMapping("/{metadataType}/{id}" + Links.METADATA_VALID_URL) - HttpEntity validEvent(@PathVariable("metadataType") String metadataType, - @PathVariable("id") String metadataId, - PersistentEntityResourceAssembler assembler) { - MetadataDocument metadataDocument = validationStateChangeService.changeValidationState(entityTypeForCollection(metadataType), - metadataId, - ValidationState.VALID); - return ResponseEntity.accepted().body(assembler.toFullResource(metadataDocument)); - } - - @PutMapping("/{metadataType}/{id}" + Links.INVALID_URL) - HttpEntity invalidEvent(@PathVariable("metadataType") String metadataType, - @PathVariable("id") String metadataId, - PersistentEntityResourceAssembler assembler) { - MetadataDocument metadataDocument = validationStateChangeService.changeValidationState(entityTypeForCollection(metadataType), - metadataId, - ValidationState.INVALID); - return ResponseEntity.accepted().body(assembler.toFullResource(metadataDocument)); - } - - @PostMapping("/{metadataType}/query") - ResponseEntity>> query( - @PathVariable("metadataType") String metadataType, - @RequestBody List criteriaList, - @RequestParam("operator") Optional operator, - Pageable pageable, - final PersistentEntityResourceAssembler assembler) { - Boolean andCriteria = operator.map("and"::equalsIgnoreCase).orElse(false); - Page docs = metadataQueryService.findByCriteria(entityTypeForCollection(metadataType), criteriaList, andCriteria, pageable); - return ResponseEntity.ok(pagedResourcesAssembler.toResource(docs, assembler)); - } - - private EntityType entityTypeForCollection(String collection) { - switch (collection) { - case "biomaterials": - return EntityType.BIOMATERIAL; - case "protocols": - return EntityType.PROTOCOL; - case "projects": - return EntityType.PROJECT; - case "processes": - return EntityType.PROCESS; - case "files": - return EntityType.FILE; - default: - throw new ResourceNotFoundException(); - } - } -} diff --git a/src/main/java/org/humancellatlas/ingest/core/web/MetadataDocumentResourceProcessor.java b/src/main/java/org/humancellatlas/ingest/core/web/MetadataDocumentResourceProcessor.java deleted file mode 100644 index d8c7d5ada..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/web/MetadataDocumentResourceProcessor.java +++ /dev/null @@ -1,112 +0,0 @@ -package org.humancellatlas.ingest.core.web; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.patch.Patch; -import org.humancellatlas.ingest.state.ValidationState; -import org.springframework.data.rest.webmvc.support.RepositoryEntityLinks; -import org.springframework.hateoas.EntityLinks; -import org.springframework.hateoas.Link; -import org.springframework.hateoas.Resource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.stereotype.Component; - -import java.util.Optional; - -@Component -@RequiredArgsConstructor -public class MetadataDocumentResourceProcessor implements ResourceProcessor> { - - private final @NonNull EntityLinks entityLinks; - - @NonNull - private final RepositoryEntityLinks repositoryEntityLinks; - - private Optional getStateTransitionLink(MetadataDocument metadataDocument, - ValidationState targetState) { - Optional transitionResourceName = getSubresourceNameForValidationState(targetState); - if (transitionResourceName.isPresent()) { - Optional rel = getRelNameForValidationState(targetState); - if (rel.isPresent()) { - return Optional.of(entityLinks - .linkForSingleResource(metadataDocument) - .slash(transitionResourceName.get()) - .withRel(rel.get())); - } else { - String messageTemplate = "Unexpected link/rel mismatch exception " + - "(link = '%s', rel = '%s')"; - throw new RuntimeException(String.format(messageTemplate, - transitionResourceName.toString(), rel.toString())); - } - } else { - return Optional.empty(); - } - } - - private Optional getRelNameForValidationState(ValidationState validationState) { - switch (validationState) { - case DRAFT: - return Optional.of(Links.DRAFT_REL); - case VALIDATING: - return Optional.of(Links.METADATA_VALIDATING_REL); - case VALID: - return Optional.of(Links.METADATA_VALID_REL); - case INVALID: - return Optional.of(Links.INVALID_REL); - case PROCESSING: - return Optional.of(Links.PROCESSING_REL); - case COMPLETE: - return Optional.of(Links.COMPLETE_REL); - default: - // default returns no links (not expecting external user interaction) - return Optional.empty(); - } - } - - private Optional getSubresourceNameForValidationState(ValidationState validationState) { - switch (validationState) { - case DRAFT: - return Optional.of(Links.DRAFT_URL); - case VALIDATING: - return Optional.of(Links.METADATA_VALIDATING_URL); - case VALID: - return Optional.of(Links.METADATA_VALID_URL); - case INVALID: - return Optional.of(Links.INVALID_URL); - case PROCESSING: - return Optional.of(Links.PROCESSING_URL); - case COMPLETE: - return Optional.of(Links.COMPLETE_URL); - default: - // default returns no links (not expecting external user interaction) - return Optional.empty(); - } - } - - @Override public Resource process(Resource resource) { - MetadataDocument metadataDocument = resource.getContent(); - addStateLinks(resource, metadataDocument); - if (metadataDocument.getIsUpdate()) { - addPatchLink(resource, metadataDocument.getId()); - } - return resource; - } - - private void addStateLinks(Resource resource, - MetadataDocument metadataDocument) { - metadataDocument.allowedStateTransitions().stream() - .map(validationState -> getStateTransitionLink(metadataDocument, validationState)) - .filter(Optional::isPresent) - .map(Optional::get) - .forEach(resource::add); - } - - private void addPatchLink(Resource resource, String documentId) { - Link link = repositoryEntityLinks - .linkToSearchResource(Patch.class, "WithUpdateDocument") - .withRel("patch") - .expand(documentId); - resource.add(link); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/core/web/SpringLinkGenerator.java b/src/main/java/org/humancellatlas/ingest/core/web/SpringLinkGenerator.java deleted file mode 100644 index 262330155..000000000 --- a/src/main/java/org/humancellatlas/ingest/core/web/SpringLinkGenerator.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.humancellatlas.ingest.core.web; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.rest.core.mapping.ResourceMappings; -import org.springframework.data.rest.core.mapping.ResourceMetadata; -import org.springframework.stereotype.Component; - -@Component -public class SpringLinkGenerator implements LinkGenerator { - - @Autowired - private ResourceMappings resourceMappings; - - @Override - public String createCallback(Class documentType, String documentId) { - ResourceMetadata metadata = resourceMappings.getMetadataFor(documentType); - return String.format("/%s/%s", metadata.getRel(), documentId); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/errors/IngestError.java b/src/main/java/org/humancellatlas/ingest/errors/IngestError.java deleted file mode 100644 index f9e57ffb9..000000000 --- a/src/main/java/org/humancellatlas/ingest/errors/IngestError.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.humancellatlas.ingest.errors; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.zalando.problem.Problem; - -import java.net.URI; - -@Data -@NoArgsConstructor -@JsonIgnoreProperties({"parameters","status"}) -public class IngestError implements Problem { - private URI type; - private String title; - private String detail; - private URI instance; - - IngestError(Problem problem) { - this.type = problem.getType(); - this.title = problem.getTitle(); - this.detail = problem.getDetail(); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/errors/SubmissionErrorService.java b/src/main/java/org/humancellatlas/ingest/errors/SubmissionErrorService.java deleted file mode 100644 index be7536308..000000000 --- a/src/main/java/org/humancellatlas/ingest/errors/SubmissionErrorService.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.humancellatlas.ingest.errors; - -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.stereotype.Service; -import org.zalando.problem.Problem; - -@Service -public class SubmissionErrorService { - @Autowired - private SubmissionErrorRepository submissionErrorRepository; - - public Page getErrorsFromEnvelope(SubmissionEnvelope submissionEnvelope, Pageable pageable) { - return submissionErrorRepository.findBySubmissionEnvelope(submissionEnvelope, pageable); - } - - public SubmissionError addErrorToEnvelope(SubmissionEnvelope submissionEnvelope, Problem submissionProblem) { - SubmissionError submissionError = new SubmissionError(submissionEnvelope, submissionProblem); - submissionErrorRepository.insert(submissionError); - return submissionError; - } - - public void deleteSubmissionEnvelopeErrors(SubmissionEnvelope submissionEnvelope) { - submissionErrorRepository.deleteBySubmissionEnvelope(submissionEnvelope); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/errors/web/SubmissionErrorController.java b/src/main/java/org/humancellatlas/ingest/errors/web/SubmissionErrorController.java deleted file mode 100644 index 593119b1f..000000000 --- a/src/main/java/org/humancellatlas/ingest/errors/web/SubmissionErrorController.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.humancellatlas.ingest.errors.web; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.errors.IngestError; -import org.humancellatlas.ingest.errors.SubmissionError; -import org.humancellatlas.ingest.errors.SubmissionErrorService; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.domain.Pageable; -import org.springframework.data.rest.webmvc.PersistentEntityResource; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.data.web.PagedResourcesAssembler; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.hateoas.PagedResources; -import org.springframework.hateoas.Resource; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.DeleteMapping; - -import java.net.URI; - -@RepositoryRestController -@ExposesResourceFor(SubmissionError.class) -@RequiredArgsConstructor -public class SubmissionErrorController { - private final @NonNull SubmissionErrorService submissionErrorService; - private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; - - @DeleteMapping(path = "submissionEnvelopes/{sub_id}" + Links.SUBMISSION_ERRORS_URL) - public ResponseEntity deleteSubmissionEnvelopeErrors( - @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope) { - submissionErrorService.deleteSubmissionEnvelopeErrors(submissionEnvelope); - return ResponseEntity.noContent().build(); - } - - @GetMapping(path = "submissionEnvelopes/{sub_id}" + Links.SUBMISSION_ERRORS_URL) - public ResponseEntity>> getSubmissionEnvelopeErrors( - @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - return ResponseEntity.ok( - pagedResourcesAssembler.toResource( - submissionErrorService.getErrorsFromEnvelope(submissionEnvelope, pageable), - resourceAssembler - ) - ); - } - - @PostMapping(path = "submissionEnvelopes/{sub_id}" + Links.SUBMISSION_ERRORS_URL) - public ResponseEntity addErrorToEnvelope( - @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - @RequestBody IngestError ingestError, - final PersistentEntityResourceAssembler resourceAssembler) { - SubmissionError submissionError = submissionErrorService.addErrorToEnvelope(submissionEnvelope, ingestError); - PersistentEntityResource submissionErrorResource = resourceAssembler.toFullResource(submissionError); - return ResponseEntity.created(URI.create(submissionErrorResource.getId().getHref())).body(submissionErrorResource); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/errors/web/SubmissionErrorResourceProcessor.java b/src/main/java/org/humancellatlas/ingest/errors/web/SubmissionErrorResourceProcessor.java deleted file mode 100644 index 1d897aecc..000000000 --- a/src/main/java/org/humancellatlas/ingest/errors/web/SubmissionErrorResourceProcessor.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.humancellatlas.ingest.errors.web; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.errors.SubmissionError; -import org.springframework.hateoas.EntityLinks; -import org.springframework.hateoas.Resource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.stereotype.Component; - -import java.net.URI; - -@Component -@RequiredArgsConstructor -public class SubmissionErrorResourceProcessor implements ResourceProcessor> { - private final @NonNull EntityLinks entityLinks; - - @Override - public Resource process(Resource resource) { - resource.getContent().setInstance(URI.create(resource.getId().getHref())); - resource.add( - entityLinks.linkForSingleResource(resource.getContent().getSubmissionEnvelope()) - .slash(Links.SUBMISSION_ERRORS_URL) - .withRel(Links.SUBMISSION_ERRORS_REL) - ); - return resource; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/export/ExportError.java b/src/main/java/org/humancellatlas/ingest/export/ExportError.java deleted file mode 100644 index 8fa52eda5..000000000 --- a/src/main/java/org/humancellatlas/ingest/export/ExportError.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.humancellatlas.ingest.export; - -import lombok.Data; -import lombok.NonNull; - -@Data -public class ExportError { - private final String errorCode; - @NonNull - private final String message; - private final Object details; -} diff --git a/src/main/java/org/humancellatlas/ingest/export/ExportState.java b/src/main/java/org/humancellatlas/ingest/export/ExportState.java deleted file mode 100644 index b419d4d07..000000000 --- a/src/main/java/org/humancellatlas/ingest/export/ExportState.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.humancellatlas.ingest.export; - -public enum ExportState { - EXPORTING, - FAILED, - EXPORTED, - DEPRECATED -} diff --git a/src/main/java/org/humancellatlas/ingest/export/destination/ExportDestination.java b/src/main/java/org/humancellatlas/ingest/export/destination/ExportDestination.java deleted file mode 100644 index c5af099e3..000000000 --- a/src/main/java/org/humancellatlas/ingest/export/destination/ExportDestination.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.humancellatlas.ingest.export.destination; - -import lombok.Data; - -import java.util.Map; - -@Data -public class ExportDestination { - private final ExportDestinationName name; - private final String version; - private final Map context; -} - diff --git a/src/main/java/org/humancellatlas/ingest/export/destination/ExportDestinationName.java b/src/main/java/org/humancellatlas/ingest/export/destination/ExportDestinationName.java deleted file mode 100644 index efd0c69fb..000000000 --- a/src/main/java/org/humancellatlas/ingest/export/destination/ExportDestinationName.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.humancellatlas.ingest.export.destination; - -public enum ExportDestinationName { - DCP, - DSP, - ENA, - BIO_SAMPLES, - BIO_STUDIES -} diff --git a/src/main/java/org/humancellatlas/ingest/export/entity/ExportEntityService.java b/src/main/java/org/humancellatlas/ingest/export/entity/ExportEntityService.java deleted file mode 100644 index 453b61fc6..000000000 --- a/src/main/java/org/humancellatlas/ingest/export/entity/ExportEntityService.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.humancellatlas.ingest.export.entity; - -import lombok.AllArgsConstructor; -import org.humancellatlas.ingest.export.entity.web.ExportEntityRequest; -import org.humancellatlas.ingest.export.job.ExportJob; -import org.springframework.stereotype.Component; - -@Component -@AllArgsConstructor -public class ExportEntityService { - private final ExportEntityRepository exportEntityRepository; - - public ExportEntity createExportEntity(ExportJob exportJob, ExportEntityRequest exportEntityRequest) { - ExportEntity newExportEntity = ExportEntity.builder() - .exportJob(exportJob) - .status(exportEntityRequest.getStatus()) - .context(exportEntityRequest.getContext()) - .errors(exportEntityRequest.getErrors()) - .build(); - return exportEntityRepository.insert(newExportEntity); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/export/entity/web/ExportEntityController.java b/src/main/java/org/humancellatlas/ingest/export/entity/web/ExportEntityController.java deleted file mode 100644 index 4d32887bc..000000000 --- a/src/main/java/org/humancellatlas/ingest/export/entity/web/ExportEntityController.java +++ /dev/null @@ -1,73 +0,0 @@ -package org.humancellatlas.ingest.export.entity.web; - -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.export.ExportState; -import org.humancellatlas.ingest.export.entity.ExportEntity; -import org.humancellatlas.ingest.export.entity.ExportEntityRepository; -import org.humancellatlas.ingest.export.entity.ExportEntityService; -import org.humancellatlas.ingest.export.job.ExportJob; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.rest.webmvc.PersistentEntityResource; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.data.web.PagedResourcesAssembler; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestParam; - -import java.net.URI; - -@RepositoryRestController -@RequiredArgsConstructor -@ExposesResourceFor(ExportEntity.class) -public class ExportEntityController { - private final ExportEntityService exportEntityService; - private final ExportEntityRepository exportEntityRepository; - private final PagedResourcesAssembler pagedAssembler; - - @GetMapping(path = Links.EXPORT_JOBS_URL + "/{id}" + Links.EXPORT_JOB_ENTITIES_URL) - public ResponseEntity getExportJobEntities(@PathVariable("id") ExportJob exportJob, - @RequestParam(name = "status", required = false) ExportState exportState, - Pageable pageable, - PersistentEntityResourceAssembler assembler) { - if (exportJob == null) { - return ResponseEntity.notFound().build(); - } - Page entityPage; - if (exportState == null) { - entityPage = exportEntityRepository.findByExportJob(exportJob, pageable); - } else { - entityPage = exportEntityRepository.findByExportJobAndStatus(exportJob, exportState, pageable); - } - return ResponseEntity.ok(pagedAssembler.toResource(entityPage ,assembler)); - } - - @GetMapping(path = Links.EXPORT_JOBS_URL + "/{job_id}" + Links.EXPORT_JOB_ENTITIES_URL + "/{entity_id}") - ResponseEntity getExportJobEntity(@PathVariable("job_id") ExportJob exportJob, - @PathVariable("entity_id") ExportEntity exportEntity, - PersistentEntityResourceAssembler assembler) { - if (exportJob == null || exportEntity == null) { - return ResponseEntity.notFound().build(); - } - PersistentEntityResource newExportEntityResource = assembler.toFullResource(exportEntity); - return ResponseEntity.ok(newExportEntityResource); - } - - @PostMapping(path = Links.EXPORT_JOBS_URL + "/{id}" + Links.EXPORT_JOB_ENTITIES_URL) - ResponseEntity createExportEntity(@PathVariable("id") ExportJob exportJob, - @RequestBody ExportEntityRequest exportEntityRequest, - PersistentEntityResourceAssembler resourceAssembler){ - if (exportJob == null) { - return ResponseEntity.notFound().build(); - } - ExportEntity newExportEntity = exportEntityService.createExportEntity(exportJob, exportEntityRequest); - PersistentEntityResource newExportEntityResource = resourceAssembler.toFullResource(newExportEntity); - return ResponseEntity.created(URI.create(newExportEntityResource.getId().getHref())).body(newExportEntityResource); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/export/entity/web/ExportEntityRequest.java b/src/main/java/org/humancellatlas/ingest/export/entity/web/ExportEntityRequest.java deleted file mode 100644 index 56c9631cc..000000000 --- a/src/main/java/org/humancellatlas/ingest/export/entity/web/ExportEntityRequest.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.humancellatlas.ingest.export.entity.web; - -import lombok.Data; -import lombok.NonNull; -import org.humancellatlas.ingest.export.ExportError; -import org.humancellatlas.ingest.export.ExportState; - -import java.util.List; -import java.util.Map; - -@Data -public class ExportEntityRequest { - @NonNull - ExportState status; - - @NonNull - Map context; - - @NonNull - List errors; -} diff --git a/src/main/java/org/humancellatlas/ingest/export/entity/web/ExportEntityResourceProcessor.java b/src/main/java/org/humancellatlas/ingest/export/entity/web/ExportEntityResourceProcessor.java deleted file mode 100644 index 2ed3e0ab1..000000000 --- a/src/main/java/org/humancellatlas/ingest/export/entity/web/ExportEntityResourceProcessor.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.humancellatlas.ingest.export.entity.web; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.export.entity.ExportEntity; -import org.humancellatlas.ingest.export.job.ExportJob; -import org.springframework.hateoas.EntityLinks; -import org.springframework.hateoas.Link; -import org.springframework.hateoas.Resource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.stereotype.Component; - -@Component -@RequiredArgsConstructor -public class ExportEntityResourceProcessor implements ResourceProcessor> { - private final @NonNull EntityLinks entityLinks; - - private Link getSelfLink(ExportEntity exportEntity) { - return entityLinks - .linkForSingleResource(exportEntity.getExportJob()) - .slash(Links.EXPORT_JOB_ENTITIES_URL) - .slash(exportEntity.getId()) - .withSelfRel(); - } - - private Link getExportJobLink(ExportJob exportJob) { - return entityLinks.linkForSingleResource(exportJob).withRel("exportJob"); - } - - @Override - public Resource process(Resource resource) { - ExportEntity exportEntity = resource.getContent(); - resource.removeLinks(); - resource.add(getSelfLink(exportEntity)); - resource.add(getSelfLink(exportEntity).withRel("exportEntity")); - resource.add(getExportJobLink(exportEntity.getExportJob())); - return resource; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/export/job/ExportJob.java b/src/main/java/org/humancellatlas/ingest/export/job/ExportJob.java deleted file mode 100644 index e9df048df..000000000 --- a/src/main/java/org/humancellatlas/ingest/export/job/ExportJob.java +++ /dev/null @@ -1,85 +0,0 @@ -package org.humancellatlas.ingest.export.job; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Builder; -import lombok.Data; -import org.humancellatlas.ingest.core.web.LinkGenerator; -import org.humancellatlas.ingest.export.ExportError; -import org.humancellatlas.ingest.export.ExportState; -import org.humancellatlas.ingest.export.destination.ExportDestination; -import org.humancellatlas.ingest.messaging.model.ExportSubmissionMessage; -import org.humancellatlas.ingest.messaging.model.SpreadsheetGenerationMessage; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.annotation.CreatedDate; -import org.springframework.data.annotation.Id; -import org.springframework.data.annotation.LastModifiedDate; -import org.springframework.data.mongodb.core.index.CompoundIndex; -import org.springframework.data.mongodb.core.index.CompoundIndexes; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.DBRef; -import org.springframework.data.mongodb.core.mapping.Document; -import org.springframework.data.rest.core.annotation.RestResource; -import org.springframework.hateoas.Identifiable; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -@Data -@Builder -@Document -@CompoundIndexes({ - @CompoundIndex(name = "exportDestinationName", def = "{ 'destination.name': 1 }"), - @CompoundIndex(name = "exportDestinationVersion", def = "{ 'destination.version': 1 }") -}) -public class ExportJob implements Identifiable { - @Id - @JsonIgnore - private String id; - - @CreatedDate - @Builder.Default - private Instant createdDate = Instant.now(); - - @Indexed - @DBRef(lazy = true) - @RestResource(exported = false) - @JsonIgnore - private final SubmissionEnvelope submission; - - private final ExportDestination destination; - - @Indexed - @Builder.Default - private ExportState status = ExportState.EXPORTING; - - @LastModifiedDate - private Instant updatedDate; - - private Map context; - - @Builder.Default - private List errors = new ArrayList<>(); - - public ExportSubmissionMessage toExportSubmissionMessage(LinkGenerator linkGenerator, Map context) { - String callbackLink = linkGenerator.createCallback(getClass(), getId()); - return new ExportSubmissionMessage( - getId(), - submission.getUuid().getUuid().toString(), - destination.getContext().get("projectUuid").toString(), - callbackLink, - context - ); - } - - public SpreadsheetGenerationMessage toGenerateSubmissionMessage(LinkGenerator linkGenerator, Map context) { - // TODO: unify with toExportSubmissionMessage - String callbackLink = linkGenerator.createCallback(getClass(), getId()); - return new SpreadsheetGenerationMessage(getId(), - submission.getUuid().getUuid().toString(), - destination.getContext().get("projectUuid").toString(), - callbackLink, - context); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/export/job/ExportJobService.java b/src/main/java/org/humancellatlas/ingest/export/job/ExportJobService.java deleted file mode 100644 index 567ceeeae..000000000 --- a/src/main/java/org/humancellatlas/ingest/export/job/ExportJobService.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.humancellatlas.ingest.export.job; - -import lombok.AllArgsConstructor; -import lombok.NonNull; -import org.humancellatlas.ingest.export.ExportState; -import org.humancellatlas.ingest.export.destination.ExportDestination; -import org.humancellatlas.ingest.export.destination.ExportDestinationName; -import org.humancellatlas.ingest.export.job.web.ExportJobRequest; -import org.humancellatlas.ingest.exporter.Exporter; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.data.domain.Example; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.stereotype.Component; - -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.function.Consumer; - -@Component -@AllArgsConstructor -public class ExportJobService { - private final ExportJobRepository exportJobRepository; - private final SubmissionEnvelopeRepository submissionEnvelopeRepository; - private final Exporter exporter; - private final @NonNull ExecutorService executorService = Executors.newFixedThreadPool(5); - private final @NonNull Logger log = LoggerFactory.getLogger(getClass()); - - public ExportJob createExportJob(SubmissionEnvelope submissionEnvelope, ExportJobRequest exportJobRequest) { - ExportJob newExportJob = ExportJob.builder() - .submission(submissionEnvelope) - .destination(exportJobRequest.getDestination()) - .context(exportJobRequest.getContext()) - .build(); - return exportJobRepository.insert(newExportJob); - } - - public Page find(UUID submissionUuid, - ExportState exportState, - ExportDestinationName destinationName, - String version, - Pageable pageable) { - SubmissionEnvelope submissionEnvelope = submissionEnvelopeRepository.findByUuidUuid(submissionUuid); - ExportJob exportJobProbe = ExportJob.builder() - .submission(submissionEnvelope) - .status(exportState) - .destination(new ExportDestination(destinationName, version, null)) - .build(); - return this.exportJobRepository.findAll(Example.of(exportJobProbe), pageable); - } - - public ExportJob updateContext(ExportJob exportJob, Map context) { - exportJob.getContext().putAll(context); - var savedJob = exportJobRepository.save(exportJob); - if (context.getOrDefault("dataFileTransfer", "").equals("COMPLETE")) { - submit(exporter::generateSpreadsheet, exportJob, "spreadsheetGeneration"); - } else if (context.getOrDefault("spreadsheetGeneration", "").equals("COMPLETE")) { - submit(exporter::exportMetadata, exportJob, "exportMetadata"); - } - return savedJob; - } - - private void submit(Consumer exportAction, ExportJob exportJob, String actionName) { - String submissionUuid = exportJob.getSubmission().getUuid().getUuid().toString(); - executorService.submit(() -> { - try { - log.info("submitting export action {} for export job {} for submission {}", - actionName, - exportJob.getId(), - submissionUuid); - exportAction.accept(exportJob); - } catch (Exception e) { - log.error(String.format("Uncaught Exception sending message %s for Export Job %s for submission %s", - actionName, exportJob.getId(), submissionUuid), e); - } - }); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/export/job/web/ExportJobController.java b/src/main/java/org/humancellatlas/ingest/export/job/web/ExportJobController.java deleted file mode 100644 index 0bce88de1..000000000 --- a/src/main/java/org/humancellatlas/ingest/export/job/web/ExportJobController.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.humancellatlas.ingest.export.job.web; - -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.export.ExportState; -import org.humancellatlas.ingest.export.destination.ExportDestinationName; -import org.humancellatlas.ingest.export.job.ExportJob; -import org.humancellatlas.ingest.export.job.ExportJobRepository; -import org.humancellatlas.ingest.export.job.ExportJobService; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.rest.webmvc.PersistentEntityResource; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.data.web.PagedResourcesAssembler; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.hateoas.PagedResources; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; - -import java.net.URI; -import java.util.Map; -import java.util.UUID; - -@RepositoryRestController -@RequiredArgsConstructor -@ExposesResourceFor(ExportJob.class) -public class ExportJobController { - private final ExportJobService exportJobService; - private final ExportJobRepository exportJobRepository; - private final PagedResourcesAssembler pagedResourcesAssembler; - - @GetMapping(path = "/submissionEnvelopes/{id}" + Links.EXPORT_JOBS_URL) - ResponseEntity getExportJobsForSubmission(@PathVariable("id") SubmissionEnvelope submission, - Pageable pageable, - PersistentEntityResourceAssembler resourceAssembler) { - if (submission == null) { - return ResponseEntity.notFound().build(); - } - return ResponseEntity.ok(pagedResourcesAssembler.toResource( - exportJobRepository.findBySubmission(submission, pageable), - resourceAssembler - )); - } - - @PostMapping(path = "/submissionEnvelopes/{id}" + Links.EXPORT_JOBS_URL) - ResponseEntity createExportJob(@PathVariable("id") SubmissionEnvelope submission, - @RequestBody ExportJobRequest exportJobRequest, - PersistentEntityResourceAssembler resourceAssembler) { - if (submission == null) { - return ResponseEntity.notFound().build(); - } - ExportJob newExportJob = exportJobService.createExportJob(submission, exportJobRequest); - PersistentEntityResource newExportJobResource = resourceAssembler.toFullResource(newExportJob); - return ResponseEntity.created(URI.create(newExportJobResource.getId().getHref())).body(newExportJobResource); - } - - @GetMapping(path = Links.EXPORT_JOBS_URL + "/search" + Links.EXPORT_JOB_FIND_URL) - ResponseEntity> findExportJobs( - @RequestParam("submissionUuid") UUID submissionUuid, - @RequestParam("status") ExportState exportState, - @RequestParam("destination") ExportDestinationName exportDestinationName, - @RequestParam("version") String destinationVersion, - Pageable pageable, - PersistentEntityResourceAssembler resourceAssembler - ) { - String version = destinationVersion.isEmpty() ? null : destinationVersion; - Page searchResults = exportJobService.find(submissionUuid, exportState, exportDestinationName, version, pageable); - return ResponseEntity.ok(pagedResourcesAssembler.toResource(searchResults, resourceAssembler)); - } - - @PatchMapping(Links.EXPORT_JOBS_URL + "/{id}/context") - ResponseEntity patchExportJobContext( - @PathVariable("id") ExportJob exportJob, - @RequestBody Map context, - PersistentEntityResourceAssembler assembler - ) { - ExportJob updatedExportJob = exportJobService.updateContext(exportJob, context); - PersistentEntityResource resource = assembler.toFullResource(updatedExportJob); - return ResponseEntity.accepted().body(resource); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/export/job/web/ExportJobRequest.java b/src/main/java/org/humancellatlas/ingest/export/job/web/ExportJobRequest.java deleted file mode 100644 index 903e8e9f9..000000000 --- a/src/main/java/org/humancellatlas/ingest/export/job/web/ExportJobRequest.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.humancellatlas.ingest.export.job.web; - -import lombok.Data; -import lombok.NonNull; -import org.humancellatlas.ingest.export.destination.ExportDestination; - -import java.util.Map; - -@Data -public class ExportJobRequest { - @NonNull - ExportDestination destination; - - @NonNull - Map context; -} diff --git a/src/main/java/org/humancellatlas/ingest/export/job/web/ExportJobResourceProcessor.java b/src/main/java/org/humancellatlas/ingest/export/job/web/ExportJobResourceProcessor.java deleted file mode 100644 index 865805390..000000000 --- a/src/main/java/org/humancellatlas/ingest/export/job/web/ExportJobResourceProcessor.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.humancellatlas.ingest.export.job.web; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.export.job.ExportJob; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.hateoas.EntityLinks; -import org.springframework.hateoas.Link; -import org.springframework.hateoas.Resource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.stereotype.Component; - -@Component -@RequiredArgsConstructor -public class ExportJobResourceProcessor implements ResourceProcessor> { - private final @NonNull EntityLinks entityLinks; - - private Link getEntitiesLink(ExportJob exportJob) { - return entityLinks.linkForSingleResource(exportJob) - .slash(Links.EXPORT_JOB_ENTITIES_URL) - .withRel(Links.EXPORT_JOB_ENTITIES_REL); - } - - private Link getEntitiesStatusLink(ExportJob exportJob) { - return entityLinks.linkForSingleResource(exportJob) - .slash(Links.EXPORT_JOB_ENTITIES_URL + "?status={status}") - .withRel(Links.EXPORT_JOB_ENTITIES_BY_STATUS_REL); - } - - private Link getSubmissionLink(SubmissionEnvelope submission) { - return entityLinks.linkForSingleResource(submission).withRel("submission"); - } - - @Override - public Resource process(Resource resource) { - ExportJob exportJob = resource.getContent(); - resource.add(getEntitiesLink(exportJob)); - resource.add(getEntitiesStatusLink(exportJob)); - resource.add(getSubmissionLink(exportJob.getSubmission())); - return resource; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/export/job/web/ExportJobSearchProcessor.java b/src/main/java/org/humancellatlas/ingest/export/job/web/ExportJobSearchProcessor.java deleted file mode 100644 index a6820f477..000000000 --- a/src/main/java/org/humancellatlas/ingest/export/job/web/ExportJobSearchProcessor.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.humancellatlas.ingest.export.job.web; - -import static org.springframework.hateoas.mvc.ControllerLinkBuilder.linkTo; -import static org.springframework.hateoas.mvc.ControllerLinkBuilder.methodOn; - -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.export.job.ExportJob; -import org.springframework.data.rest.webmvc.RepositorySearchesResource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.stereotype.Component; - -@Component -public class ExportJobSearchProcessor implements ResourceProcessor { - - @Override - public RepositorySearchesResource process(RepositorySearchesResource searchesResource) { - if(searchesResource.getDomainType().equals(ExportJob.class)) { - searchesResource.add(linkTo(methodOn(ExportJobController.class).findExportJobs(null, null, null, null, null, null)) - .withRel(Links.EXPORT_JOB_FIND_REL)); - } - - return searchesResource; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/exporter/DefaultExporter.java b/src/main/java/org/humancellatlas/ingest/exporter/DefaultExporter.java deleted file mode 100644 index 955d39cac..000000000 --- a/src/main/java/org/humancellatlas/ingest/exporter/DefaultExporter.java +++ /dev/null @@ -1,142 +0,0 @@ -package org.humancellatlas.ingest.exporter; - -import org.apache.commons.collections4.ListUtils; -import org.humancellatlas.ingest.export.destination.ExportDestination; -import org.humancellatlas.ingest.export.job.ExportJob; -import org.humancellatlas.ingest.export.job.ExportJobRepository; -import org.humancellatlas.ingest.export.job.web.ExportJobRequest; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.process.ProcessService; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.json.simple.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import static org.humancellatlas.ingest.export.destination.ExportDestinationName.DCP; - -@Component -public class DefaultExporter implements Exporter { - - private final Logger log = LoggerFactory.getLogger(getClass()); - - @Autowired - private ProcessService processService; - - @Autowired - private ProcessRepository processRepository; - - @Autowired - private ExportJobRepository exportJobRepository; - - @Autowired - private ProjectRepository projectRepository; - - @Autowired - private MessageRouter messageRouter; - - /** - * Divides a set of process IDs into lists of size partitionSize - * - * @param processIds - * @param partitionSize - * @return A collection of partitionSize sized lists of processes - */ - private static List> partitionProcessIds(Collection processIds, int partitionSize) { - return ListUtils.partition(new ArrayList<>(processIds), partitionSize); - } - - @Override - public void exportManifests(SubmissionEnvelope envelope) { - Collection assayingProcessIds = processService.findAssays(envelope); - - log.info(String.format("Found %s assays processes for envelope with ID %s", - assayingProcessIds.size(), - envelope.getId())); - - int totalCount = assayingProcessIds.size(); - ExperimentProcess.IndexCounter counter = new ExperimentProcess.IndexCounter(totalCount); - - int partitionSize = 500; - partitionProcessIds(assayingProcessIds, partitionSize) - .stream() - .flatMap(processIdBatch -> processService.getProcesses(processIdBatch)) - .map(process -> ExperimentProcess.from(process, counter)) - .forEach(messageRouter::sendManifestForExport); - } - - @Override - public void exportData(SubmissionEnvelope envelope) { - Project project = projectRepository.findBySubmissionEnvelopesContains(envelope).findFirst().orElseThrow(); - var destinationContext = new JSONObject(); - destinationContext.put("projectUuid", project.getUuid().getUuid().toString()); - - var exportJobContext = new JSONObject(); - exportJobContext.put("dataFileTransfer", false); - ExportJob exportJob = createDcpExportJob(envelope, destinationContext, exportJobContext); - - var messageContext = new JSONObject(); - messageRouter.sendSubmissionForDataExport(exportJob, messageContext); - } - - @Override - public void generateSpreadsheet(SubmissionEnvelope submissionEnvelope) { - Project project = projectRepository.findBySubmissionEnvelopesContains(submissionEnvelope).findFirst().orElseThrow(); - var destinationContext = new JSONObject(); - destinationContext.put("projectUuid", project.getUuid().getUuid().toString()); - - var exportJob = createDcpExportJob(submissionEnvelope, destinationContext, new JSONObject()); - generateSpreadsheet(exportJob); - } - - @Override - public void exportMetadata(ExportJob exportJob) { - var submission = exportJob.getSubmission(); - Collection assayingProcessIds = processService.findAssays(submission); - exportJob.getContext().put("totalAssayCount", assayingProcessIds.size()); - exportJobRepository.save(exportJob); - updateDcpVersionAndSendMessageForEachProcess(assayingProcessIds, exportJob); - } - - private ExportJob createDcpExportJob(SubmissionEnvelope submissionEnvelope, JSONObject destinationContext, JSONObject exportJobContext) { - ExportDestination exportDestination = new ExportDestination(DCP, "v2", destinationContext); - ExportJobRequest exportJobRequest = new ExportJobRequest(exportDestination, exportJobContext); - ExportJob newExportJob = ExportJob.builder() - .submission(submissionEnvelope) - .destination(exportJobRequest.getDestination()) - .context(exportJobRequest.getContext()) - .build(); - return exportJobRepository.insert(newExportJob); - } - - private void updateDcpVersionAndSendMessageForEachProcess(Collection assayingProcessIds, ExportJob exportJob) { - int totalCount = assayingProcessIds.size(); - ExperimentProcess.IndexCounter counter = new ExperimentProcess.IndexCounter(totalCount); - - int partitionSize = 500; - partitionProcessIds(assayingProcessIds, partitionSize) - .stream() - .flatMap(processIdBatch -> processService.getProcesses(processIdBatch)) - .map(process -> (Process) process.setDcpVersion(exportJob.getCreatedDate())) - .map(process -> processRepository.save(process)) - .map(process -> ExperimentProcess.from(process, counter)) - .forEach(exportData -> messageRouter.sendExperimentForExport(exportData, exportJob, null)); - } - - @Override - public void generateSpreadsheet(ExportJob exportJob) { - exportJob.getContext().put("spreadsheetGeneration", false); - exportJobRepository.save(exportJob); - var messageContext = new JSONObject(); - messageRouter.sendGenerateSpreadsheet(exportJob, messageContext); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/exporter/ExperimentProcess.java b/src/main/java/org/humancellatlas/ingest/exporter/ExperimentProcess.java deleted file mode 100644 index 0ac2e4308..000000000 --- a/src/main/java/org/humancellatlas/ingest/exporter/ExperimentProcess.java +++ /dev/null @@ -1,102 +0,0 @@ -package org.humancellatlas.ingest.exporter; - -import org.humancellatlas.ingest.core.web.LinkGenerator; -import org.humancellatlas.ingest.export.job.ExportJob; -import org.humancellatlas.ingest.messaging.model.ExportEntityMessage; -import org.humancellatlas.ingest.messaging.model.ManifestMessage; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.project.Project; -import org.json.simple.JSONObject; - -import java.time.Instant; -import java.util.Map; -import java.util.UUID; - - -public class ExperimentProcess { - - private final int index; - private final int totalCount; - - private final Process process; - - private final SubmissionEnvelope submissionEnvelope; - - private final Project project; - - public ExperimentProcess(int index, int totalCount, Process process, SubmissionEnvelope submission, Project project) { - this.index = index; - this.totalCount = totalCount; - this.process = process; - this.submissionEnvelope = submission; - this.project = project; - } - - public static ExperimentProcess from(Process process, IndexCounter counter) { - return new ExperimentProcess(counter.next(), counter.totalCount, process, process.getSubmissionEnvelope(), process.getProject()); - } - - public static class IndexCounter { - int base; - int totalCount; - - IndexCounter(int totalCount) { - this.base = 0; - this.totalCount = totalCount; - } - - int next() { - return base++; - } - } - - public Integer getIndex() { - return index; - } - - public Integer getTotalCount() { - return totalCount; - } - - public Process getProcess() { - return process; - } - - public SubmissionEnvelope getSubmissionEnvelope() { - return submissionEnvelope; - } - - public ExportEntityMessage toExportEntityMessage(LinkGenerator linkGenerator, ExportJob exportJob, Map context) { - String callbackLink = linkGenerator.createCallback(process.getClass(), process.getId()); - return new ExportEntityMessage( - exportJob.getId(), - process.getId(), - process.getUuid().toString(), - callbackLink, - process.getClass().getSimpleName().toLowerCase(), - submissionEnvelope.getId(), - submissionEnvelope.getUuid().toString(), - project.getId(), - process.getProject().getUuid().toString(), - index, - totalCount, - context); - } - - public ManifestMessage toManifestMessage(LinkGenerator linkGenerator) { - String callbackLink = linkGenerator.createCallback(process.getClass(), process.getId()); - return new ManifestMessage( - UUID.randomUUID(), - Instant.now().toString(), - process.getId(), - process.getUuid().toString(), - callbackLink, - process.getClass().getSimpleName(), - submissionEnvelope.getId(), - submissionEnvelope.getUuid().toString(), - index, - totalCount); - - } -} diff --git a/src/main/java/org/humancellatlas/ingest/exporter/Exporter.java b/src/main/java/org/humancellatlas/ingest/exporter/Exporter.java deleted file mode 100644 index 9ae06a8f4..000000000 --- a/src/main/java/org/humancellatlas/ingest/exporter/Exporter.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.humancellatlas.ingest.exporter; - -import org.humancellatlas.ingest.export.job.ExportJob; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; - -public interface Exporter { - - void exportManifests(SubmissionEnvelope submissionEnvelope); - - void exportMetadata(ExportJob exportJob); - - void generateSpreadsheet(SubmissionEnvelope envelope); - - void exportData(SubmissionEnvelope submissionEnvelope); - - void generateSpreadsheet(ExportJob exportJob); -} diff --git a/src/main/java/org/humancellatlas/ingest/file/File.java b/src/main/java/org/humancellatlas/ingest/file/File.java deleted file mode 100644 index e4567c2a4..000000000 --- a/src/main/java/org/humancellatlas/ingest/file/File.java +++ /dev/null @@ -1,142 +0,0 @@ -package org.humancellatlas.ingest.file; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; -import org.humancellatlas.ingest.core.Checksums; -import org.humancellatlas.ingest.core.EntityType; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.mongodb.core.index.CompoundIndex; -import org.springframework.data.mongodb.core.index.CompoundIndexes; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.DBRef; -import org.springframework.data.mongodb.core.mapping.Document; -import org.springframework.data.rest.core.annotation.RestResource; - -import javax.validation.constraints.NotNull; -import java.util.*; - -import static com.fasterxml.jackson.annotation.JsonProperty.Access.READ_ONLY; - -@Getter -@Setter -@Document -@CompoundIndexes({ - @CompoundIndex(name = "validationId", def = "{ 'validationJob.validationId': 1 }") -}) -@EqualsAndHashCode(callSuper = true, exclude = {"project", "inputToProcesses", "derivedByProcesses"}) -public class File extends MetadataDocument { - - @Indexed - private @Setter - @DBRef(lazy = true) - Project project; - - @Indexed - @RestResource - @DBRef(lazy = true) - private Set inputToProcesses = new HashSet<>(); - - @Indexed - @RestResource - @DBRef(lazy = true) - private Set derivedByProcesses = new HashSet<>(); - - @Indexed - private String fileName; - private String cloudUrl; - - private Checksums checksums; - private Checksums lastExportedChecksums; - - private ValidationJob validationJob; - private FileArchiveResult fileArchiveResult; - private UUID validationId; - @NotNull - private UUID dataFileUuid; - private Long size; - private String fileContentType; - - public File() { - super(EntityType.FILE, null); - initFile(); - } - - @JsonCreator - public File(@JsonProperty("content") Object content, - @JsonProperty("fileName") String fileName) { - super(EntityType.FILE, content); - this.setFileName(fileName); - initFile(); - } - - private void initFile() { - setDataFileUuid(UUID.randomUUID()); - } - - /** - * Adds to the collection of processes that this file serves as an input to - * - * @param process the process to add - * @return a reference to this file - */ - public File addAsInputToProcess(Process process) { - this.inputToProcesses.add(process); - - return this; - } - - /** - * Adds to the collection of processes that this file was derived by - * - * @param process the process to add - * @return a reference to this file - */ - public File addAsDerivedByProcess(Process process) { - - // XXX why we implementing this check here but not above?? - String processId = process.getId(); - boolean processInList = derivedByProcesses.stream() - .map(Process::getId) - .anyMatch(id -> id.equals(processId)); - if (!processInList) { - this.derivedByProcesses.add(process); - } - return this; - } - - public void addToAnalysis(Process analysis) { - //TODO check if this File and the Analysis belong to the same Submission? - SubmissionEnvelope submissionEnvelope = analysis.getSubmissionEnvelope(); - super.setSubmissionEnvelope(submissionEnvelope); - addAsDerivedByProcess(analysis); - } - - @JsonProperty(access = READ_ONLY) - public boolean isLinked() { - return !inputToProcesses.isEmpty() || !derivedByProcesses.isEmpty(); - } - - /** - * Removes a process to the collection of processes that this file was derived by - * - * @param process the process to add - * @return a reference to this file - */ - public File removeAsDerivedByProcess(Process process) { - this.derivedByProcesses.remove(process); - return this; - } - - public File removeAsInputToProcess(Process process) { - this.inputToProcesses.remove(process); - return this; - } - - -} diff --git a/src/main/java/org/humancellatlas/ingest/file/FileAlreadyExistsException.java b/src/main/java/org/humancellatlas/ingest/file/FileAlreadyExistsException.java deleted file mode 100644 index 961f1f5be..000000000 --- a/src/main/java/org/humancellatlas/ingest/file/FileAlreadyExistsException.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.humancellatlas.ingest.file; - -import lombok.Getter; -import lombok.Setter; - -/** - * Created by rolando on 04/06/2018. - */ -public class FileAlreadyExistsException extends RuntimeException { - @Getter - @Setter - private String fileName; - - public FileAlreadyExistsException(){ - - } - - public FileAlreadyExistsException(String message) { - super(message); - } - - public FileAlreadyExistsException(String message, String fileName) { - super(message); - this.fileName = fileName; - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/file/FileArchiveResult.java b/src/main/java/org/humancellatlas/ingest/file/FileArchiveResult.java deleted file mode 100644 index 6cf4da22e..000000000 --- a/src/main/java/org/humancellatlas/ingest/file/FileArchiveResult.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.humancellatlas.ingest.file; - -import java.time.Instant; -import lombok.Data; - -@Data -public class FileArchiveResult { - private Instant lastArchived; - private Boolean compressed; - private String md5; - private String enaUploadPath; - private String error; - - protected FileArchiveResult() {} - -} diff --git a/src/main/java/org/humancellatlas/ingest/file/FileRepository.java b/src/main/java/org/humancellatlas/ingest/file/FileRepository.java deleted file mode 100644 index 7ee2d6d16..000000000 --- a/src/main/java/org/humancellatlas/ingest/file/FileRepository.java +++ /dev/null @@ -1,96 +0,0 @@ -package org.humancellatlas.ingest.file; - - -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.data.mongodb.repository.Query; -import org.springframework.data.repository.query.Param; -import org.springframework.data.rest.core.annotation.RestResource; -import org.springframework.web.bind.annotation.CrossOrigin; - -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.stream.Stream; - -/** - * Created by rolando on 06/09/2017. - */ -@CrossOrigin -public interface FileRepository extends MongoRepository { - - @RestResource(rel = "findAllByUuid", path = "findAllByUuid") - Page findByUuid(@Param("uuid") Uuid uuid, Pageable pageable); - - @RestResource(rel = "findByUuid", path = "findByUuid") - Optional findByUuidUuidAndIsUpdateFalse(@Param("uuid") UUID uuid); - - Page findByProject(Project project, Pageable pageable); - - @RestResource(exported = false) - Stream findByProject(Project project); - - @RestResource(rel = "findBySubmissionEnvelope") - Page findBySubmissionEnvelope(@Param("envelopeUri") SubmissionEnvelope submissionEnvelope, Pageable pageable); - - @RestResource(exported = false) - Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - List findBySubmissionEnvelopeAndFileName(SubmissionEnvelope submissionEnvelope, String fileName); - - - @RestResource(rel = "findBySubmissionAndValidationState") - public Page findBySubmissionEnvelopeAndValidationState(@Param("envelopeUri") SubmissionEnvelope submissionEnvelope, - @Param("state") ValidationState state, - Pageable pageable); - - @Query(value = "{'submissionEnvelope.id': ?0, graphValidationErrors: { $exists: true, $not: {$size: 0} } }") - @RestResource(rel = "findBySubmissionIdWithGraphValidationErrors") - public Page findBySubmissionIdWithGraphValidationErrors( - @Param("envelopeId") String envelopeId, - Pageable pageable - ); - - @RestResource(exported = false) - Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - @RestResource(exported = false) - Long deleteBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - @RestResource(rel = "findByValidationId") - File findByValidationJobValidationId(@Param("validationId") UUID id); - - @RestResource(exported = false) - Stream findByInputToProcessesContains(Process process); - - Page findByInputToProcessesContaining(Process process, Pageable pageable); - - @RestResource(exported = false) - Stream findByDerivedByProcessesContains(Process process); - - Page findByDerivedByProcessesContaining(Process process, Pageable pageable); - - long countBySubmissionEnvelopeAndValidationState(SubmissionEnvelope submissionEnvelope, ValidationState validationState); - - long countBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - @Query(value = "{'submissionEnvelope.id': ?0, validationErrors: {$elemMatch: {errorType: ?1} }}", count = true) - long countBySubmissionEnvelopeIdAndErrorType(@Param("id") String submissionEnvelopeId, @Param("errorType") String errorType); - - @Query(value = "{'submissionEnvelope.id': ?0, validationErrors: {$elemMatch: {errorType: ?1} }}") - Page findBySubmissionEnvelopeIdAndErrorType(@Param("id") String submissionEnvelopeId, @Param("errorType") String errorType, Pageable pageable); - - @Query(value = "{'submissionEnvelope.id': ?0, validationErrors: {$not: {$elemMatch: {errorType: ?1} }}}", count = true) - long countBySubmissionEnvelopeIdAndNotErrorType(@Param("id") String submissionEnvelopeId, @Param("errorType") String errorType); - - @Query(value = "{'submissionEnvelope.id': ?0, graphValidationErrors: { $exists: true, $not: {$size: 0} } }", count = true) - long countBySubmissionEnvelopeAndCountWithGraphValidationErrors(String submissionEnvelopeId); -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/file/FileService.java b/src/main/java/org/humancellatlas/ingest/file/FileService.java deleted file mode 100644 index f8c20cd94..000000000 --- a/src/main/java/org/humancellatlas/ingest/file/FileService.java +++ /dev/null @@ -1,144 +0,0 @@ -package org.humancellatlas.ingest.file; - -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.core.Checksums; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.core.exception.CoreEntityNotFoundException; -import org.humancellatlas.ingest.core.service.MetadataCrudService; -import org.humancellatlas.ingest.core.service.MetadataUpdateService; -import org.humancellatlas.ingest.file.web.FileMessage; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.state.MetadataDocumentEventHandler; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.dao.OptimisticLockingFailureException; -import org.springframework.retry.annotation.Backoff; -import org.springframework.retry.annotation.Retryable; -import org.springframework.stereotype.Service; -import org.springframework.validation.annotation.Validated; - -import javax.validation.Valid; -import java.util.List; -import java.util.Optional; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 06/09/17 - */ -@Service -@RequiredArgsConstructor -@Getter -@Validated -public class FileService { - private final @NonNull - SubmissionEnvelopeRepository submissionEnvelopeRepository; - private final @NonNull - FileRepository fileRepository; - private final @NonNull - BiomaterialRepository biomaterialRepository; - private final @NonNull - ProcessRepository processRepository; - private final @NonNull - ProjectRepository projectRepository; - private final @NonNull - MetadataDocumentEventHandler metadataDocumentEventHandler; - private final @NonNull - MetadataCrudService metadataCrudService; - private final @NonNull - MetadataUpdateService metadataUpdateService; - - private final Logger log = LoggerFactory.getLogger(getClass()); - - public File addFileToSubmissionEnvelope(SubmissionEnvelope submissionEnvelope, - @Valid File file) { - if (!fileRepository.findBySubmissionEnvelopeAndFileName(submissionEnvelope, file.getFileName()).isEmpty()) { - throw new FileAlreadyExistsException(String.format("File with name %s already exists in envelope %s", file.getFileName(), submissionEnvelope.getId())); - } else { - projectRepository.findBySubmissionEnvelopesContains(submissionEnvelope).findFirst().ifPresent(file::setProject); - File createdFile = metadataCrudService.addToSubmissionEnvelopeAndSave(file, submissionEnvelope); - metadataDocumentEventHandler.handleMetadataDocumentCreate(createdFile); - return createdFile; - } - - } - - public File addFileValidationJob(File file, ValidationJob validationJob) { - if (file.getChecksums().getSha1().equals(validationJob.getChecksums().getSha1())) { - file.setValidationJob(validationJob); - return fileRepository.save(file); - } else { - throw new IllegalStateException(String.format("Failed to create validation job for file with ID %s : checksums mismatch", file.getId())); - } - } - - - public void createFileFromFileMessage(FileMessage fileMessage) throws CoreEntityNotFoundException { - String envelopeUuid = fileMessage.getStagingAreaId(); - SubmissionEnvelope envelope = findEnvelope(envelopeUuid); - try { - addFileToSubmissionEnvelope(envelope, new File(null, fileMessage.getFileName())); - } catch (FileAlreadyExistsException e) { - log.info(String.format("File listener attempted to create a File resource with name %s but it already existed for envelope %s", - fileMessage.getFileName(), - envelope.getId())); - } - - } - - @Retryable( - value = OptimisticLockingFailureException.class, - maxAttempts = 5, - backoff = @Backoff(delay = 500, maxDelay = 60000, multiplier = 2)) - public File updateFileFromFileMessage(FileMessage fileMessage) throws CoreEntityNotFoundException { - String envelopeUuid = fileMessage.getStagingAreaId(); - SubmissionEnvelope envelope = findEnvelope(envelopeUuid); - return findAndUpdateFile(fileMessage, envelope); - } - - private File findAndUpdateFile(FileMessage fileMessage, SubmissionEnvelope envelope) { - String fileName = fileMessage.getFileName(); - File file = findFile(fileName, envelope); - - String newFileUrl = fileMessage.getCloudUrl(); - Checksums checksums = fileMessage.getChecksums(); - Long size = fileMessage.getSize(); - String contentType = fileMessage.getContentType(); - - log.info(String.format("Updating file with cloudUrl %s and submission UUID %s", newFileUrl, envelope.getUuid())); - - file.setCloudUrl(newFileUrl); - file.setChecksums(checksums); - file.setSize(size); - file.setFileContentType(contentType); - file.enactStateTransition(ValidationState.DRAFT); - File updatedFile = fileRepository.save(file); - - log.info(String.format("File validation state is %s for file with cloudUrl %s and submission UUID %s ", updatedFile.getValidationState(), file.getCloudUrl(), envelope.getUuid())); - - return updatedFile; - } - - private SubmissionEnvelope findEnvelope(String envelopeUuid) throws CoreEntityNotFoundException { - return Optional.ofNullable(submissionEnvelopeRepository.findByUuid(new Uuid(envelopeUuid))) - .orElseThrow(() -> new CoreEntityNotFoundException(String.format("Couldn't find envelope with with uuid %s", envelopeUuid))); - } - - private File findFile(String fileName, SubmissionEnvelope envelope) { - List filesInEnvelope = fileRepository.findBySubmissionEnvelopeAndFileName(envelope, fileName); - - if (filesInEnvelope.size() != 1) { - throw new RuntimeException(String.format("Expected 1 file with name %s, but found %s", fileName, filesInEnvelope.size())); - } - return filesInEnvelope.get(0); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/file/ValidationErrorType.java b/src/main/java/org/humancellatlas/ingest/file/ValidationErrorType.java deleted file mode 100644 index 338505631..000000000 --- a/src/main/java/org/humancellatlas/ingest/file/ValidationErrorType.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.humancellatlas.ingest.file; - -public enum ValidationErrorType { - METADATA_ERROR, - FILE_NOT_UPLOADED, - FILE_ERROR -} diff --git a/src/main/java/org/humancellatlas/ingest/file/ValidationJob.java b/src/main/java/org/humancellatlas/ingest/file/ValidationJob.java deleted file mode 100644 index 0683f2a79..000000000 --- a/src/main/java/org/humancellatlas/ingest/file/ValidationJob.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.humancellatlas.ingest.file; - -import lombok.Data; - -import org.humancellatlas.ingest.core.Checksums; -import java.util.UUID; - -@Data -public class ValidationJob { - private UUID validationId; - private Checksums checksums; - private boolean jobCompleted; - private ValidationReport validationReport; - - protected ValidationJob() {} - -} diff --git a/src/main/java/org/humancellatlas/ingest/file/ValidationReport.java b/src/main/java/org/humancellatlas/ingest/file/ValidationReport.java deleted file mode 100644 index 8b586ecbf..000000000 --- a/src/main/java/org/humancellatlas/ingest/file/ValidationReport.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.humancellatlas.ingest.file; - -import lombok.Data; - -import org.humancellatlas.ingest.core.Checksums; -import org.humancellatlas.ingest.state.ValidationState; - -import java.util.List; -import java.util.UUID; - -@Data -public class ValidationReport { - private ValidationState validationState; - private List validationErrors; - - protected ValidationReport() {} - -} diff --git a/src/main/java/org/humancellatlas/ingest/file/web/FileController.java b/src/main/java/org/humancellatlas/ingest/file/web/FileController.java deleted file mode 100644 index 075576487..000000000 --- a/src/main/java/org/humancellatlas/ingest/file/web/FileController.java +++ /dev/null @@ -1,188 +0,0 @@ -package org.humancellatlas.ingest.file.web; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.service.*; -import org.humancellatlas.ingest.file.*; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.security.CheckAllowed; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.exception.NotAllowedDuringSubmissionStateException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.rest.webmvc.PersistentEntityResource; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.data.web.PagedResourcesAssembler; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.hateoas.MediaTypes; -import org.springframework.hateoas.Resource; -import org.springframework.hateoas.Resources; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.*; - -import javax.validation.ConstraintViolationException; -import javax.validation.Valid; -import java.lang.reflect.InvocationTargetException; -import java.net.URISyntaxException; -import java.util.List; - -import static org.springframework.data.rest.webmvc.RestMediaTypes.TEXT_URI_LIST_VALUE; -import static org.springframework.web.bind.annotation.RequestMethod.POST; -import static org.springframework.web.bind.annotation.RequestMethod.PUT; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 06/09/17 - */ -@RepositoryRestController -@ExposesResourceFor(File.class) -@RequiredArgsConstructor -@Getter -@Validated -public class FileController { - - @NonNull - private final FileService fileService; - - @NonNull - private final FileRepository fileRepository; - - @NonNull - private final ProcessRepository processRepository; - - @NonNull - private final PagedResourcesAssembler pagedResourcesAssembler; - - @NonNull - private final MetadataCrudService metadataCrudService; - - @NonNull - private final MetadataUpdateService metadataUpdateService; - - private @Autowired - ValidationStateChangeService validationStateChangeService; - - private @Autowired - UriToEntityConversionService uriToEntityConversionService; - - private @Autowired - MetadataLinkingService metadataLinkingService; - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - @ExceptionHandler(ConstraintViolationException.class) - @ResponseStatus(HttpStatus.BAD_REQUEST) - ResponseEntity handleConstraintViolationException(ConstraintViolationException e) { - return new ResponseEntity<>("not valid due to validation error: " + e.getMessage(), HttpStatus.BAD_REQUEST); - } - - @CheckAllowed(value = "#submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @RequestMapping(path = "/submissionEnvelopes/{sub_id}/files", - method = RequestMethod.POST, - produces = MediaTypes.HAL_JSON_VALUE) - ResponseEntity> createFile(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - @RequestBody @Valid File file, - final PersistentEntityResourceAssembler assembler) { - try { - File createdFile = fileService.addFileToSubmissionEnvelope(submissionEnvelope, file); - logFileDetails(submissionEnvelope, createdFile); - return ResponseEntity.accepted().body(assembler.toFullResource(createdFile)); - } catch (FileAlreadyExistsException e) { - throw new IllegalStateException(e); - } - } - - private void logFileDetails(SubmissionEnvelope submissionEnvelope, File createdFile) { - logger.info("submission uuid {}: created File: id {} uuid {} name {} dataFileUuid {}", - submissionEnvelope.getUuid(), - createdFile.getId(), - createdFile.getUuid(), - createdFile.getFileName(), - createdFile.getDataFileUuid()); - } - - @RequestMapping(path = "/files/{id}/validationJob", - method = RequestMethod.PUT, - produces = MediaTypes.HAL_JSON_VALUE) - ResponseEntity> addFileValidationJob(@PathVariable("id") File file, - @RequestBody ValidationJob validationJob, - final PersistentEntityResourceAssembler assembler) { - File entity = getFileService().addFileValidationJob(file, validationJob); - PersistentEntityResource resource = assembler.toFullResource(entity); - return ResponseEntity.accepted().body(resource); - } - - @CheckAllowed(value = "#file.submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @PatchMapping(path = "/files/{id}") - HttpEntity patchFile(@PathVariable("id") File file, - @RequestBody final ObjectNode patch, - PersistentEntityResourceAssembler assembler) { - List allowedFields = List.of("content", "fileName", "validationJob", "validationErrors", "graphValidationErrors", "fileArchiveResult"); - ObjectNode validPatch = patch.retain(allowedFields); - File updatedFile = metadataUpdateService.update(file, validPatch); - PersistentEntityResource resource = assembler.toFullResource(updatedFile); - return ResponseEntity.accepted().body(resource); - } - - @CheckAllowed(value = "#file.submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @RequestMapping(path = "/files/{id}/inputToProcesses", method = {PUT, POST}, consumes = {TEXT_URI_LIST_VALUE}) - HttpEntity linkFileAsInputToProcesses(@PathVariable("id") File file, - @RequestBody Resources incoming, - HttpMethod requestMethod, - PersistentEntityResourceAssembler assembler) throws URISyntaxException, InvocationTargetException, NoSuchMethodException, IllegalAccessException { - - List processes = uriToEntityConversionService.convertLinks(incoming.getLinks(), Process.class); - metadataLinkingService.updateLinks(file, processes, "inputToProcesses", requestMethod.equals(HttpMethod.PUT)); - - return ResponseEntity.ok().build(); - } - - @CheckAllowed(value = "#file.submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @RequestMapping(path = "/files/{id}/derivedByProcesses", method = {PUT, POST}, consumes = {TEXT_URI_LIST_VALUE}) - HttpEntity linkFileAsDerivedByProcesses(@PathVariable("id") File file, - @RequestBody Resources incoming, - HttpMethod requestMethod, - PersistentEntityResourceAssembler assembler) throws URISyntaxException, InvocationTargetException, NoSuchMethodException, IllegalAccessException { - - List processes = uriToEntityConversionService.convertLinks(incoming.getLinks(), Process.class); - metadataLinkingService.updateLinks(file, processes, "derivedByProcesses", requestMethod.equals(HttpMethod.PUT)); - - return ResponseEntity.ok().build(); - } - - @CheckAllowed(value = "#file.submissionEnvelope.isEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @DeleteMapping(path = "/files/{id}/inputToProcesses/{processId}") - HttpEntity unlinkFileAsInputToProcesses(@PathVariable("id") File file, - @PathVariable("processId") Process process, - PersistentEntityResourceAssembler assembler) throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { - metadataLinkingService.removeLink(file, process, "inputToProcesses"); - return ResponseEntity.noContent().build(); - } - - @CheckAllowed(value = "#file.submissionEnvelope.isEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @DeleteMapping(path = "/files/{id}/derivedByProcesses/{processId}") - HttpEntity unlinkFileAsDerivedByProcesses(@PathVariable("id") File file, - @PathVariable("processId") Process process, - PersistentEntityResourceAssembler assembler) throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { - metadataLinkingService.removeLink(file, process, "derivedByProcesses"); - return ResponseEntity.noContent().build(); - } - - @CheckAllowed(value = "#file.submissionEnvelope.isEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @DeleteMapping(path = "/files/{id}") - ResponseEntity deleteFile(@PathVariable("id") File file) { - metadataCrudService.deleteDocument(file); - return ResponseEntity.noContent().build(); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/file/web/FileListener.java b/src/main/java/org/humancellatlas/ingest/file/web/FileListener.java deleted file mode 100644 index 081621612..000000000 --- a/src/main/java/org/humancellatlas/ingest/file/web/FileListener.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.humancellatlas.ingest.file.web; - -import lombok.AllArgsConstructor; -import lombok.NonNull; -import org.humancellatlas.ingest.core.exception.CoreEntityNotFoundException; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.file.FileAlreadyExistsException; -import org.humancellatlas.ingest.file.FileService; -import org.humancellatlas.ingest.messaging.Constants; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.amqp.AmqpRejectAndDontRequeueException; -import org.springframework.amqp.ImmediateRequeueAmqpException; -import org.springframework.amqp.rabbit.annotation.RabbitListener; -import org.springframework.dao.OptimisticLockingFailureException; -import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; - -import java.util.Optional; -import java.util.UUID; - -/** - * Created by rolando on 07/09/2017. - */ -@Component -@AllArgsConstructor -public class FileListener { - private final @NonNull FileService fileService; - private final Logger log = LoggerFactory.getLogger(getClass()); - - @RabbitListener(queues = Constants.Queues.FILE_STAGED_QUEUE) - public void handleFileStagedEvent(FileMessage fileMessage) { - if(!StringUtils.isEmpty(fileMessage.getContentType()) - && fileMessage.getMediaType().isPresent() - && fileMessage.getMediaType().get().equals(FileMediaTypes.HCA_DATA_FILE)){ - try { - fileService.createFileFromFileMessage(fileMessage); - fileService.updateFileFromFileMessage(fileMessage); - } catch (CoreEntityNotFoundException e) { - log.warn(e.getMessage()); - throw new AmqpRejectAndDontRequeueException(e.getMessage()); - } catch (OptimisticLockingFailureException e){ - log.warn("Putting file back on queue: " + e.getMessage()); - throw new ImmediateRequeueAmqpException(e); - } catch (RuntimeException e) { - log.error(e.getMessage()); - throw new AmqpRejectAndDontRequeueException(e.getMessage()); - } - } - } -} diff --git a/src/main/java/org/humancellatlas/ingest/file/web/FileMediaTypes.java b/src/main/java/org/humancellatlas/ingest/file/web/FileMediaTypes.java deleted file mode 100644 index e182b2386..000000000 --- a/src/main/java/org/humancellatlas/ingest/file/web/FileMediaTypes.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.humancellatlas.ingest.file.web; - -/** - * Created by rolando on 10/10/2017. - */ -public class FileMediaTypes { - public static final String HCA_DATA_FILE = "data"; - public static final String HCA_SAMPLE = "metadata/sample"; - public static final String HCA_PROJECT = "metadata/project"; - public static final String HCA_PROTOCOL = "metadata/protocol"; - public static final String HCA_ASSAY = "metadata/assay"; - public static final String HCA_ANALYSIS = "metadata/analysis"; -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/file/web/FileMessage.java b/src/main/java/org/humancellatlas/ingest/file/web/FileMessage.java deleted file mode 100644 index edde4b3f5..000000000 --- a/src/main/java/org/humancellatlas/ingest/file/web/FileMessage.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.humancellatlas.ingest.file.web; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import org.apache.http.entity.ContentType; -import org.humancellatlas.ingest.core.Checksums; -import java.util.Optional; - -/** - * Created by rolando on 07/09/2017. - */ -@Getter -@AllArgsConstructor -@NoArgsConstructor -public class FileMessage { - @JsonProperty("url") - private String cloudUrl; - @JsonProperty("name") - private String fileName; - @JsonProperty("upload_area_id") - private String stagingAreaId; - @JsonProperty("content_type") - private String contentType; - private Checksums checksums; - private long size; - - /** - * given existence of substring "dcp-type={type}" in this.contentType, extracts {type} - * - * @return the DCP media-type of the file uploaded that triggered this event - */ - @JsonIgnore - public Optional getMediaType(){ - return Optional.ofNullable(ContentType.parse(this.getContentType()).getParameter("dcp-type")); - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/file/web/FileResourceProcessor.java b/src/main/java/org/humancellatlas/ingest/file/web/FileResourceProcessor.java deleted file mode 100644 index 7dc591f42..000000000 --- a/src/main/java/org/humancellatlas/ingest/file/web/FileResourceProcessor.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.humancellatlas.ingest.file.web; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.file.File; -import org.springframework.hateoas.EntityLinks; -import org.springframework.hateoas.Link; -import org.springframework.hateoas.Resource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.stereotype.Component; - - -@Component -@RequiredArgsConstructor -public class FileResourceProcessor implements ResourceProcessor> { - private final @NonNull EntityLinks entityLinks; - - private Link getCreateValidationJobLink(File file) { - return entityLinks.linkForSingleResource(file) - .slash(Links.FILE_VALIDATION_JOB_URL) - .withRel(Links.FILE_VALIDATION_JOB_REL); - } - - @Override public Resource process(Resource resource) { - File fileDocument = resource.getContent(); - resource.add(getCreateValidationJobLink(fileDocument)); - return resource; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/Constants.java b/src/main/java/org/humancellatlas/ingest/messaging/Constants.java deleted file mode 100644 index eb981c1de..000000000 --- a/src/main/java/org/humancellatlas/ingest/messaging/Constants.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.humancellatlas.ingest.messaging; - -public class Constants { - public class Queues { - public static final String FILE_STAGED_QUEUE = "ingest.file.create.staged"; - public static final String FILE_VALIDATION_QUEUE = "ingest.file.validation.queue"; - public static final String METADATA_VALIDATION_QUEUE = "ingest.metadata.validation.queue"; - public static final String NOTIFICATIONS_QUEUE = "ingest.notifications.queue"; - public static final String GRAPH_VALIDATION_QUEUE = "ingest.validation.graph.queue"; - } - - public class Exchanges { - public static final String VALIDATION_EXCHANGE = "ingest.validation.exchange"; - public static final String FILE_STAGED_EXCHANGE = "ingest.file.staged.exchange"; - public static final String STATE_TRACKING_EXCHANGE = "ingest.state-tracking.exchange"; - public static final String EXPORTER_EXCHANGE = "ingest.exporter.exchange"; - public static final String UPLOAD_AREA_EXCHANGE = "ingest.upload.area.exchange"; - public static final String NOTIFICATIONS_EXCHANGE = "ingest.notifications.exchange"; - public static final String SPREADSHEET_EXCHANGE = "ingest.spreadsheet.exchange"; - } - - public class Routing { - public static final String ENVELOPE_STATE_UPDATE = "ingest.state-tracking.envelope.state.update"; - public static final String ENVELOPE_CREATE = "ingest.state-tracking.envelope.create"; - - public static final String MANIFEST_SUBMITTED = "ingest.exporter.manifest.submitted"; - public static final String EXPERIMENT_SUBMITTED = "ingest.exporter.experiment.submitted"; - - public static final String SUBMISSION_SUBMITTED = "ingest.exporter.submission.submitted"; - - public static final String UPLOAD_AREA_CREATE = "ingest.upload.area.create"; - public static final String UPLOAD_AREA_CLEANUP = "ingest.upload.area.cleanup"; - - public static final String NOTIFICATION_NEW = "ingest.notifications.new"; - public static final String SPREADSHEET_GENERATION = "ingest.exporter.spreadsheet.requested"; - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/MessageRouter.java b/src/main/java/org/humancellatlas/ingest/messaging/MessageRouter.java deleted file mode 100644 index 34dd85311..000000000 --- a/src/main/java/org/humancellatlas/ingest/messaging/MessageRouter.java +++ /dev/null @@ -1,216 +0,0 @@ -package org.humancellatlas.ingest.messaging; - -import lombok.NoArgsConstructor; -import org.humancellatlas.ingest.config.ConfigurationService; -import org.humancellatlas.ingest.core.EntityType; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.core.MetadataDocumentMessageBuilder; -import org.humancellatlas.ingest.core.web.LinkGenerator; -import org.humancellatlas.ingest.export.job.ExportJob; -import org.humancellatlas.ingest.exporter.ExperimentProcess; -import org.humancellatlas.ingest.messaging.model.MetadataDocumentMessage; -import org.humancellatlas.ingest.messaging.model.SubmissionEnvelopeMessage; -import org.humancellatlas.ingest.messaging.model.SubmissionEnvelopeStateUpdateMessage; -import org.humancellatlas.ingest.state.SubmissionState; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeMessageBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.web.util.UriComponentsBuilder; - -import java.net.URI; -import java.util.Map; - -import static org.humancellatlas.ingest.messaging.Constants.Exchanges.EXPORTER_EXCHANGE; -import static org.humancellatlas.ingest.messaging.Constants.Exchanges.SPREADSHEET_EXCHANGE; -import static org.humancellatlas.ingest.messaging.Constants.Routing.EXPERIMENT_SUBMITTED; -import static org.humancellatlas.ingest.messaging.Constants.Routing.MANIFEST_SUBMITTED; -import static org.humancellatlas.ingest.messaging.Constants.Routing.SPREADSHEET_GENERATION; -import static org.humancellatlas.ingest.messaging.Constants.Routing.SUBMISSION_SUBMITTED; - - -@Component -@NoArgsConstructor -public class MessageRouter { - - @Autowired - private MessageSender messageSender; - - @Autowired - private ConfigurationService configurationService; - - @Autowired - private LinkGenerator linkGenerator; - - private final Logger log = LoggerFactory.getLogger(getClass()); - - /* messages to validator */ - public boolean routeValidationMessageFor(MetadataDocument document) { - if (document.getValidationState().equals(ValidationState.DRAFT)) { - this.messageSender.queueValidationMessage(Constants.Exchanges.VALIDATION_EXCHANGE, - Constants.Queues.METADATA_VALIDATION_QUEUE, - messageFor(document), - document.getUpdateDate().toEpochMilli()); - return true; - } else { - return false; - } - } - - /* messages to graph validator */ - public boolean routeGraphValidationMessageFor(SubmissionEnvelope envelope) { - if (envelope.allowedSubmissionStateTransitions().contains(SubmissionState.GRAPH_VALIDATING)) { - this.messageSender.queueGraphValidationMessage( - Constants.Exchanges.VALIDATION_EXCHANGE, - Constants.Queues.GRAPH_VALIDATION_QUEUE, - messageFor(envelope), - System.currentTimeMillis() - ); - return true; - } - return false; - } - - /* messages to state tracker */ - public boolean routeStateTrackingUpdateMessageFor(MetadataDocument document) { - // allow projects to be created first before submission envelope - if (document.getSubmissionEnvelope() != null || document.getType() != EntityType.PROJECT) { - URI documentUpdateUri = UriComponentsBuilder.newInstance() - .scheme(configurationService.getStateTrackerScheme()) - .host(configurationService.getStateTrackerHost()) - .port(configurationService.getStateTrackerPort()) - .pathSegment(configurationService.getDocumentStatesUpdatePath()) - .build().toUri(); - - this.messageSender.queueDocumentStateUpdateMessage(documentUpdateUri, - documentStateUpdateMessage(document), - document.getUpdateDate().toEpochMilli()); - } - return true; - } - - public boolean routeStateTrackingDeleteMessageFor(MetadataDocument document) { - if (document.getSubmissionEnvelope() != null) { - URI documentDeleteUri = UriComponentsBuilder.newInstance() - .scheme(configurationService.getStateTrackerScheme()) - .host(configurationService.getStateTrackerHost()) - .port(configurationService.getStateTrackerPort()) - .pathSegment(configurationService.getDocumentStatesUpdatePath()) - .queryParam(configurationService.getDocumentIdParamName(), document.getId()) - .queryParam(configurationService.getEnvelopeIdParamName(), document.getSubmissionEnvelope().getId()) - .build().toUri(); - this.messageSender.queueDocumentStateDeleteMessage( - documentDeleteUri, - document.getUpdateDate().toEpochMilli() - ); - return true; - } else { - log.warn(String.format("The metadata document '%s' is not linked to a submission envelope", document.getId())); - return false; - } - } - - public boolean routeStateTrackingUpdateMessageForEnvelopeEvent(SubmissionEnvelope envelope, SubmissionState state) { - // TODO: call this when a user requests a state change on an envelope - this.messageSender.queueStateTrackingMessage(Constants.Exchanges.STATE_TRACKING_EXCHANGE, - Constants.Routing.ENVELOPE_STATE_UPDATE, - messageFor(envelope, state), - envelope.getUpdateDate().toEpochMilli()); - return true; - } - - public boolean routeStateTrackingNewSubmissionEnvelope(SubmissionEnvelope envelope) { - this.messageSender.queueStateTrackingMessage(Constants.Exchanges.STATE_TRACKING_EXCHANGE, - Constants.Routing.ENVELOPE_CREATE, - messageFor(envelope), - envelope.getUpdateDate().toEpochMilli()); - return true; - } - - /* messages to the exporter */ - - public void sendManifestForExport(ExperimentProcess experimentProcess) { - messageSender.queueNewExportMessage(EXPORTER_EXCHANGE, MANIFEST_SUBMITTED, - experimentProcess.toManifestMessage(linkGenerator), - System.currentTimeMillis()); - } - - public void sendExperimentForExport(ExperimentProcess experimentProcess, ExportJob exportJob, Map context) { - messageSender.queueNewExportMessage(EXPORTER_EXCHANGE, EXPERIMENT_SUBMITTED, - experimentProcess.toExportEntityMessage(linkGenerator, exportJob, context), - System.currentTimeMillis()); - } - - public void sendSubmissionForDataExport(ExportJob exportJob, Map context) { - messageSender.queueNewExportMessage( - EXPORTER_EXCHANGE, - SUBMISSION_SUBMITTED, - exportJob.toExportSubmissionMessage(linkGenerator, context), - System.currentTimeMillis() - ); - } - /* messages to the upload/staging area manager */ - - public boolean routeRequestUploadAreaCredentials(SubmissionEnvelope envelope) { - this.messageSender.queueUploadManagerMessage(Constants.Exchanges.UPLOAD_AREA_EXCHANGE, - Constants.Routing.UPLOAD_AREA_CREATE, - messageFor(envelope), - envelope.getUpdateDate().toEpochMilli()); - return true; - } - - public boolean routeRequestUploadAreaCleanup(SubmissionEnvelope envelope) { - this.messageSender.queueUploadManagerMessage(Constants.Exchanges.UPLOAD_AREA_EXCHANGE, - Constants.Routing.UPLOAD_AREA_CLEANUP, - messageFor(envelope), - envelope.getUpdateDate().toEpochMilli()); - return true; - } - - public void sendGenerateSpreadsheet(ExportJob exportJob, Map context) { - this.messageSender.queueSpreadsheetGenerationMessage( - EXPORTER_EXCHANGE, - SPREADSHEET_GENERATION, - exportJob.toGenerateSubmissionMessage(linkGenerator, context), - System.currentTimeMillis() - ); - } - - private MetadataDocumentMessage messageFor(MetadataDocument document) { - return MetadataDocumentMessageBuilder.using(linkGenerator) - .messageFor(document) - .build(); - } - - private SubmissionEnvelopeMessage messageFor(SubmissionEnvelope envelope) { - return SubmissionEnvelopeMessageBuilder.using(linkGenerator) - .messageFor(envelope) - .build(); - } - - private MetadataDocumentMessage documentStateUpdateMessage(MetadataDocument document) { - if (document.getSubmissionEnvelope() == null) { - throw new RuntimeException("The metadata document should have a link to a submission envelope."); - } - - String envelopeId = document.getSubmissionEnvelope().getId(); - - return MetadataDocumentMessageBuilder.using(linkGenerator) - .messageFor(document) - .withEnvelopeId(envelopeId) - .withValidationState(document.getValidationState()) - .build(); - } - - - private SubmissionEnvelopeStateUpdateMessage messageFor(SubmissionEnvelope envelope, SubmissionState state) { - SubmissionEnvelopeStateUpdateMessage message = SubmissionEnvelopeStateUpdateMessage.fromSubmissionEnvelopeMessage(messageFor(envelope)); - message.setRequestedState(state); - return message; - } - - -} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/MessageSender.java b/src/main/java/org/humancellatlas/ingest/messaging/MessageSender.java deleted file mode 100644 index 39d838349..000000000 --- a/src/main/java/org/humancellatlas/ingest/messaging/MessageSender.java +++ /dev/null @@ -1,242 +0,0 @@ -package org.humancellatlas.ingest.messaging; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.Data; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.NonNull; -import org.humancellatlas.ingest.config.ConfigurationService; -import org.humancellatlas.ingest.messaging.model.MessageProtocol; -import org.humancellatlas.ingest.messaging.model.MetadataDocumentMessage; -import org.humancellatlas.ingest.messaging.model.SpreadsheetGenerationMessage; -import org.humancellatlas.ingest.messaging.model.SubmissionEnvelopeMessage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.amqp.rabbit.core.RabbitMessagingTemplate; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.stereotype.Service; -import org.springframework.web.client.RestTemplate; - -import javax.annotation.Nullable; -import javax.annotation.PostConstruct; -import java.net.URI; -import java.util.*; -import java.util.concurrent.*; -import java.util.stream.Stream; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; - -@Service -@Getter -@NoArgsConstructor -public class MessageSender { - - private static final Logger LOGGER = LoggerFactory.getLogger(MessageSender.class); - - private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(5); - - private @Autowired @NonNull RabbitMessagingTemplate rabbitMessagingTemplate; - private @Autowired @NonNull ConfigurationService configurationService; - - - public void queueValidationMessage(String exchange, String routingKey, - MetadataDocumentMessage payload, long intendedSendTime){ - MessageBuffer.VALIDATION.queueAmqpMessage(exchange, routingKey, payload, intendedSendTime); - } - - public void queueGraphValidationMessage(String exchange, String routingKey, Object payload, - long intendedSendTime) { - MessageBuffer.GRAPH_VALIDATION.queueAmqpMessage(exchange, routingKey, payload, intendedSendTime); - } - - public void queueNewExportMessage(String exchange, String routingKey, Object payload, long intendedSendTime){ - MessageBuffer.EXPORT.queueAmqpMessage(exchange, routingKey, payload, intendedSendTime); - } - - public void queueStateTrackingMessage(String exchange, String routingKey, Object payload, long intendedSendTime){ - MessageBuffer.STATE_TRACKING.queueAmqpMessage(exchange, routingKey, payload, intendedSendTime); - } - - public void queueDocumentStateUpdateMessage(URI uri, Object payload, long intendedSendTime) { - MessageBuffer.STATE_TRACKING.queueHttpMessage(uri, HttpMethod.POST, payload, intendedSendTime); - } - - public void queueDocumentStateDeleteMessage(URI uri, long intendedSendTime) { - MessageBuffer.STATE_TRACKING.queueHttpMessage(uri, HttpMethod.DELETE, null, intendedSendTime); - } - - public void queueUploadManagerMessage(String exchange, String routingKey, - SubmissionEnvelopeMessage payload, long intendedSendTime) { - MessageBuffer.UPLOAD_MANAGER.queueAmqpMessage(exchange, routingKey, payload ,intendedSendTime); - } - - public void queueSpreadsheetGenerationMessage(String exchange, String routingKey, SpreadsheetGenerationMessage payload, long intendedSendTime) { - MessageBuffer.SPREADSHEET_GENERATION.queueAmqpMessage(exchange, routingKey, payload ,intendedSendTime); - } - - @PostConstruct - private void initiateSending(){ - List amqpMessageBuffers = Arrays.asList( - MessageBuffer.ACCESSIONER, - MessageBuffer.EXPORT, - MessageBuffer.UPLOAD_MANAGER, - MessageBuffer.VALIDATION, - MessageBuffer.STATE_TRACKING, - MessageBuffer.GRAPH_VALIDATION, - MessageBuffer.SPREADSHEET_GENERATION); - - amqpMessageBuffers - .forEach(buffer -> scheduler.scheduleWithFixedDelay(new AmqpHttpMixinBufferSender(buffer, new RestTemplate(), rabbitMessagingTemplate), - 0, - buffer.getDelayMillis(), - TimeUnit.MILLISECONDS)); - } - - @Data - static class QueuedMessage implements Delayed { - private final MessageProtocol messageProtocol; - private final Object payload; - private String exchange; - private String routingKey; - private URI uri; - private HttpMethod method; - - private final long intendedStartTime; - - public QueuedMessage(String exchange, String routingKey, Object payload, long intendedStartTime) { - this.messageProtocol = MessageProtocol.AMQP; - this.exchange = exchange; - this.routingKey = routingKey; - this.payload = payload; - this.intendedStartTime = intendedStartTime; - } - - public QueuedMessage(URI uri, HttpMethod method, @Nullable Object payload, long intendedStartTime) { - this.messageProtocol = MessageProtocol.HTTP; - this.method = method; - this.uri = uri; - this.payload = payload; - this.intendedStartTime = intendedStartTime; - } - - @Override - public long getDelay(TimeUnit unit) { - long delay = intendedStartTime - System.currentTimeMillis(); - return unit.convert(delay, MILLISECONDS); - } - - - @Override - public int compareTo(Delayed other) { - long otherDelay = other.getDelay(MILLISECONDS); - return Math.toIntExact(getDelay(TimeUnit.MILLISECONDS) - otherDelay); - } - } - - private enum MessageBuffer { - - VALIDATION(SECONDS.toMillis(3)), - EXPORT(SECONDS.toMillis(5)), - UPLOAD_MANAGER(SECONDS.toMillis(1)), - ACCESSIONER(SECONDS.toMillis(2)), - STATE_TRACKING(500L), - GRAPH_VALIDATION(SECONDS.toMillis(5)), - SPREADSHEET_GENERATION(SECONDS.toMillis(5)); - - @Getter - private final Long delayMillis; - - private final BlockingQueue messageQueue = new DelayQueue<>(); - - private final Logger log = LoggerFactory.getLogger(getClass()); - - protected Logger getLog() { - return log; - } - - MessageBuffer(Long delayMillis) { - this.delayMillis = delayMillis; - } - - //TODO each enum should already know exchange and routing key - //Why are these part of the contract when they're already defined in Constants? - void queueAmqpMessage(String exchange, String routingKey, Object payload, long intendedStartTime) { - QueuedMessage message = new QueuedMessage(exchange, routingKey, payload, intendedStartTime + delayMillis); - try { - messageQueue.add(message); - } catch (IllegalStateException e) { - LOGGER.error(String.format("Failed to queue message: %s", convertToString(message)), e); - throw new RuntimeException(e); - } - } - - void queueHttpMessage(URI uri, HttpMethod method, @Nullable Object payload, long intendedStartTime) { - QueuedMessage message = new QueuedMessage(uri, method, payload, intendedStartTime + delayMillis); - try { - messageQueue.add(message); - } catch (IllegalStateException e) { - LOGGER.error(String.format("Failed to queue message: %s", convertToString(message)), e); - throw new RuntimeException(e); - } - } - - public Stream takeAll() { - Queue drainedQueue = new PriorityQueue<>(Comparator.comparing(QueuedMessage::getIntendedStartTime)); - this.messageQueue.drainTo(drainedQueue); - return Stream.generate(drainedQueue::remove) - .limit(drainedQueue.size()); - } - - private String convertToString(Object object) { - try { - return new ObjectMapper().writeValueAsString(object); - } catch (JsonProcessingException e) { - LOGGER.debug(String.format("An error in converting message object to string occurred: %s", e.getMessage())); - return ""; - } - } - } - - private static class AmqpHttpMixinBufferSender implements Runnable { - private final MessageBuffer buffer; - private final RestTemplate restTemplate; - private final RabbitMessagingTemplate messagingTemplate; - private final Logger log = LoggerFactory.getLogger(MessageSender.AmqpHttpMixinBufferSender.class); - - private AmqpHttpMixinBufferSender(MessageBuffer buffer, RestTemplate restTemplate, RabbitMessagingTemplate rabbitMessagingTemplate) { - this.buffer = buffer; - this.restTemplate = restTemplate; - this.messagingTemplate = rabbitMessagingTemplate; - } - - @Override - public void run() { - HttpHeaders headers = uriListHeaders(); - buffer.takeAll().forEach(message -> { - if(message.getMessageProtocol().equals(MessageProtocol.AMQP)) { - messagingTemplate.convertAndSend(message.exchange, message.routingKey, message.payload); - } else { - try { - restTemplate.exchange(message.getUri(), message.method, new HttpEntity<>(message.getPayload(), headers), Object.class); - } catch (Exception e) { - log.error(String.format("error sending HTTP %s message to uri %s with payload %s", - message.method, - message.uri, - message.payload), e); - } - } - }); - } - - private HttpHeaders uriListHeaders() { - HttpHeaders headers = new HttpHeaders(); - headers.add("Content-type", "application/json"); - return headers; - } - } -} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/MessageService.java b/src/main/java/org/humancellatlas/ingest/messaging/MessageService.java deleted file mode 100644 index 095775cfc..000000000 --- a/src/main/java/org/humancellatlas/ingest/messaging/MessageService.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.humancellatlas.ingest.messaging; - -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.springframework.amqp.rabbit.core.RabbitMessagingTemplate; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.messaging.MessagingException; -import org.springframework.messaging.converter.MessageConversionException; -import org.springframework.stereotype.Service; - -@Service -@RequiredArgsConstructor -@Getter -public class MessageService { - @Autowired - @NonNull - private RabbitMessagingTemplate messagingTemplate; - - public void publish(Message message){ - try { - messagingTemplate.convertAndSend(message.getExchange(), message.getRoutingKey(), message.getPayload()); - } catch(MessageConversionException e){ - throw new IllegalArgumentException(String.format("Unable to convert payload '%s'", message.getPayload())); - } catch(MessagingException e){ - throw new RuntimeException(String.format( - "There was a problem sending message '%s' to exchange '%s', with routing key '%s'.", - message.getPayload(), message.getExchange(), message.getRoutingKey())); - } - return; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/QueueConfig.java b/src/main/java/org/humancellatlas/ingest/messaging/QueueConfig.java deleted file mode 100644 index 6a72476e1..000000000 --- a/src/main/java/org/humancellatlas/ingest/messaging/QueueConfig.java +++ /dev/null @@ -1,130 +0,0 @@ -package org.humancellatlas.ingest.messaging; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import org.humancellatlas.ingest.messaging.Constants.Exchanges; -import org.humancellatlas.ingest.messaging.Constants.Queues; -import org.humancellatlas.ingest.messaging.Constants.Routing; -import org.springframework.amqp.core.*; -import org.springframework.amqp.rabbit.annotation.RabbitListenerConfigurer; -import org.springframework.amqp.rabbit.core.RabbitMessagingTemplate; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistrar; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.messaging.converter.MappingJackson2MessageConverter; -import org.springframework.messaging.converter.MessageConverter; -import org.springframework.messaging.handler.annotation.support.DefaultMessageHandlerMethodFactory; - - -@Configuration -public class QueueConfig implements RabbitListenerConfigurer { - @Bean - Queue queueFileStaged() { - return new Queue(Constants.Queues.FILE_STAGED_QUEUE, false); - } - - @Bean - FanoutExchange fileStagedExchange() { - return new FanoutExchange(Constants.Exchanges.FILE_STAGED_EXCHANGE); - } - - @Bean - Queue queueMetadataValidation() { - return new Queue(Constants.Queues.METADATA_VALIDATION_QUEUE, false); - } - - @Bean - DirectExchange validationExchange() { - return new DirectExchange(Constants.Exchanges.VALIDATION_EXCHANGE); - } - - @Bean - Queue queueGraphValidation() { return new Queue(Constants.Queues.GRAPH_VALIDATION_QUEUE); } - - @Bean - TopicExchange stateTrackingExchange() { - return new TopicExchange(Constants.Exchanges.STATE_TRACKING_EXCHANGE); - } - - @Bean - Queue queueNotifications() { - return new Queue(Queues.NOTIFICATIONS_QUEUE, true); - } - - @Bean - TopicExchange notificationExchange() { - return new TopicExchange(Exchanges.NOTIFICATIONS_EXCHANGE); - } - - @Bean - TopicExchange exporterExchange() { - return new TopicExchange(Constants.Exchanges.EXPORTER_EXCHANGE); - } - - @Bean - TopicExchange uploadAreaExchange() { - return new TopicExchange(Constants.Exchanges.UPLOAD_AREA_EXCHANGE); - } - - - /* bindings */ - - @Bean - Binding bindingFileStaged(Queue queueFileStaged, FanoutExchange fileStagedExchange) { - return BindingBuilder.bind(queueFileStaged).to(fileStagedExchange); - } - - @Bean - Binding bindingValidation(Queue queueMetadataValidation, DirectExchange validationExchange) { - return BindingBuilder.bind(queueMetadataValidation).to(validationExchange).with(Constants.Queues.METADATA_VALIDATION_QUEUE); - } - - @Bean - Binding bindingGraphValidation(Queue queueGraphValidation, DirectExchange validationExchange) { - return BindingBuilder.bind(queueGraphValidation).to(validationExchange).with(Constants.Queues.GRAPH_VALIDATION_QUEUE); - } - - @Bean - Binding bindingNewNotificationQueue(Queue queueNotifications, TopicExchange notificationExchange) { - return BindingBuilder.bind(queueNotifications).to(notificationExchange).with(Routing.NOTIFICATION_NEW); - } - - - /* rabbit config */ - - @Bean - public MessageConverter messageConverter() { - return jackson2Converter(); - } - - @Bean - public MappingJackson2MessageConverter jackson2Converter() { - ObjectMapper mapper = new ObjectMapper(); - - mapper.registerModule(new JavaTimeModule()); - mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); - - return new MappingJackson2MessageConverter(); - } - - @Bean - public DefaultMessageHandlerMethodFactory myHandlerMethodFactory() { - DefaultMessageHandlerMethodFactory factory = new DefaultMessageHandlerMethodFactory(); - factory.setMessageConverter(jackson2Converter()); - return factory; - } - - @Bean - public RabbitMessagingTemplate rabbitMessagingTemplate(RabbitTemplate rabbitTemplate) { - RabbitMessagingTemplate rmt = new RabbitMessagingTemplate(rabbitTemplate); - rmt.setMessageConverter(this.jackson2Converter()); - return rmt; - } - - @Override - public void configureRabbitListeners(RabbitListenerEndpointRegistrar registrar) { - registrar.setMessageHandlerMethodFactory(myHandlerMethodFactory()); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/ValidationMessage.java b/src/main/java/org/humancellatlas/ingest/messaging/ValidationMessage.java deleted file mode 100644 index 72a51dfde..000000000 --- a/src/main/java/org/humancellatlas/ingest/messaging/ValidationMessage.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.humancellatlas.ingest.messaging; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.humancellatlas.ingest.core.EntityType; -import org.humancellatlas.ingest.core.Uuid; - -/** - * Created by rolando on 11/09/2017. - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class ValidationMessage { - private EntityType entityType; - private Uuid uuid; - private Object content; -} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/model/ExportEntityMessage.java b/src/main/java/org/humancellatlas/ingest/messaging/model/ExportEntityMessage.java deleted file mode 100644 index d86f01db9..000000000 --- a/src/main/java/org/humancellatlas/ingest/messaging/model/ExportEntityMessage.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.humancellatlas.ingest.messaging.model; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -import java.util.Map; -import java.util.UUID; - -@Getter -@AllArgsConstructor -public class ExportEntityMessage { - private final String exportJobId; - private final String documentId; - private final String documentUuid; - private final String callbackLink; - private final String documentType; - private final String envelopeId; - private final String envelopeUuid; - private final String projectId; - private final String projectUuid; - private final int index; - private final int total; - private final Map context; -} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/model/ExportSubmissionMessage.java b/src/main/java/org/humancellatlas/ingest/messaging/model/ExportSubmissionMessage.java deleted file mode 100644 index a309893ac..000000000 --- a/src/main/java/org/humancellatlas/ingest/messaging/model/ExportSubmissionMessage.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.humancellatlas.ingest.messaging.model; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -import java.util.Map; - -@Getter -@AllArgsConstructor -public class ExportSubmissionMessage { - private final String exportJobId; - private final String submissionUuid; - private final String projectUuid; - private final String callbackLink; - private final Map context; -} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/model/ManifestMessage.java b/src/main/java/org/humancellatlas/ingest/messaging/model/ManifestMessage.java deleted file mode 100644 index aecad51f4..000000000 --- a/src/main/java/org/humancellatlas/ingest/messaging/model/ManifestMessage.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.humancellatlas.ingest.messaging.model; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -import java.util.UUID; - -@Getter -@AllArgsConstructor -public class ManifestMessage { - private final UUID bundleUuid; - private final String versionTimestamp; - - private final String documentId; - private final String documentUuid; - private final String callbackLink; - private final String documentType; - private final String envelopeId; - private final String envelopeUuid; - private final int index; - private final int total; -} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/model/MessageProtocol.java b/src/main/java/org/humancellatlas/ingest/messaging/model/MessageProtocol.java deleted file mode 100644 index 2ed7138e8..000000000 --- a/src/main/java/org/humancellatlas/ingest/messaging/model/MessageProtocol.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.humancellatlas.ingest.messaging.model; - -public enum MessageProtocol { - AMQP, HTTP -} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/model/MetadataDocumentMessage.java b/src/main/java/org/humancellatlas/ingest/messaging/model/MetadataDocumentMessage.java deleted file mode 100644 index d29d12c2e..000000000 --- a/src/main/java/org/humancellatlas/ingest/messaging/model/MetadataDocumentMessage.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.humancellatlas.ingest.messaging.model; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import org.humancellatlas.ingest.state.ValidationState; - - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 12/09/17 - */ -@Getter -@AllArgsConstructor -public class MetadataDocumentMessage { - private final String documentType; - private final String documentId; - private final String documentUuid; - private final ValidationState validationState; - private final String callbackLink; - private final String envelopeId; -} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/model/SpreadsheetGenerationMessage.java b/src/main/java/org/humancellatlas/ingest/messaging/model/SpreadsheetGenerationMessage.java deleted file mode 100644 index 8e61e541b..000000000 --- a/src/main/java/org/humancellatlas/ingest/messaging/model/SpreadsheetGenerationMessage.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.humancellatlas.ingest.messaging.model; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -import java.util.Map; - -@AllArgsConstructor -@Getter -public class SpreadsheetGenerationMessage { - private final String exportJobId; - private final String submissionUuid; - private final String projectUuid; - private final String callbackLink; - private final Map context; -} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/model/SubmissionEnvelopeStateUpdateMessage.java b/src/main/java/org/humancellatlas/ingest/messaging/model/SubmissionEnvelopeStateUpdateMessage.java deleted file mode 100644 index 45ad58506..000000000 --- a/src/main/java/org/humancellatlas/ingest/messaging/model/SubmissionEnvelopeStateUpdateMessage.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.humancellatlas.ingest.messaging.model; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Getter; -import lombok.Setter; -import org.humancellatlas.ingest.state.SubmissionState; - - -public class SubmissionEnvelopeStateUpdateMessage extends SubmissionEnvelopeMessage { - @Getter @Setter private SubmissionState requestedState; - - public SubmissionEnvelopeStateUpdateMessage(String documentType, String documentId, String documentUuid, String callbackLink) { - super(documentType, documentId, documentUuid, callbackLink); - } - - @JsonIgnore - public static SubmissionEnvelopeStateUpdateMessage fromSubmissionEnvelopeMessage(SubmissionEnvelopeMessage message) { - return new SubmissionEnvelopeStateUpdateMessage( - message.getDocumentType(), - message.getDocumentId(), - message.getDocumentUuid(), - message.getCallbackLink()); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/web/MessagingController.java b/src/main/java/org/humancellatlas/ingest/messaging/web/MessagingController.java deleted file mode 100644 index e6be3e290..000000000 --- a/src/main/java/org/humancellatlas/ingest/messaging/web/MessagingController.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.humancellatlas.ingest.messaging.web; -import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.messaging.Constants; -import org.humancellatlas.ingest.messaging.Message; -import org.humancellatlas.ingest.messaging.MessageService; -import org.springframework.hateoas.MediaTypes; -import org.springframework.hateoas.Resource; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RestController; - -@RestController -@RequiredArgsConstructor -@Getter -public class MessagingController { - @NonNull - private final MessageService messageService; - - @PostMapping(path = "/messaging/fileUploadInfo", - consumes = MediaType.APPLICATION_JSON_VALUE, - produces = MediaTypes.HAL_JSON_VALUE) - ResponseEntity> publishFileUploadInfo(@RequestBody ObjectNode uploadInfo){ - Message uploadInfoMessage = new Message(Constants.Exchanges.FILE_STAGED_EXCHANGE, Constants.Queues.FILE_STAGED_QUEUE, uploadInfo); - getMessageService().publish(uploadInfoMessage); - return new ResponseEntity<>(HttpStatus.OK); - } - - @PostMapping(path = "/messaging/fileValidationResult", - consumes = MediaType.APPLICATION_JSON_VALUE, - produces = MediaTypes.HAL_JSON_VALUE) - ResponseEntity> publishFileValidationResult(@RequestBody ObjectNode validationResult){ - Message uploadInfoMessage = new Message(Constants.Exchanges.VALIDATION_EXCHANGE, Constants.Queues.FILE_VALIDATION_QUEUE, validationResult); - getMessageService().publish(uploadInfoMessage); - return new ResponseEntity<>(HttpStatus.OK); - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/migrations/MongoChangeLog.java b/src/main/java/org/humancellatlas/ingest/migrations/MongoChangeLog.java deleted file mode 100644 index 7ffc1329c..000000000 --- a/src/main/java/org/humancellatlas/ingest/migrations/MongoChangeLog.java +++ /dev/null @@ -1,211 +0,0 @@ -package org.humancellatlas.ingest.migrations; - -import com.github.mongobee.changeset.ChangeLog; -import com.github.mongobee.changeset.ChangeSet; -import com.mongodb.MongoCommandException; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoDatabase; -import com.mongodb.client.model.IndexOptions; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.function.Consumer; - -import static com.mongodb.client.model.Filters.eq; - -@ChangeLog -public class MongoChangeLog { - - private static final Logger LOGGER = LoggerFactory.getLogger(MongoChangeLog.class); - - private static final Integer MONGO_INDEX_NOT_FOUND = 27; - - @ChangeSet(order = "2019-10-30", id = "featureCompatibilityVersion 3.4", author = "alexie.staffer@ebi.ac.uk") - public void featureCompatibilityThreeFour(MongoDatabase db) { - if (MongoVersionHelper.featureCompatibilityLessThan(db, "3.4")) - db.runCommand(new Document("setFeatureCompatibilityVersion", "3.4")); - } - - @ChangeSet(order = "2019-10-31", id = "featureCompatibilityVersion 3.6", author = "alexie.staffer@ebi.ac.uk") - public void featureCompatibilityThreeSix(MongoDatabase db) { - if (MongoVersionHelper.featureCompatibilityLessThan(db, "3.6")) - db.runCommand(new Document("setFeatureCompatibilityVersion", "3.6")); - } - - @ChangeSet(order = "2019-11-01", id = "featureCompatibilityVersion 4.0", author = "alexie.staffer@ebi.ac.uk") - public void featureCompatibilityFourZero(MongoDatabase db) { - if (MongoVersionHelper.featureCompatibilityLessThan(db, "4.0")) { - db.runCommand(new Document("setFeatureCompatibilityVersion", "4.0")); - db.runCommand(new Document("setFreeMonitoring", 1).append("action", "disable")); - } - } - - @ChangeSet(order = "2019-11-02", id = "featureCompatibilityVersion 4.2", author = "alexie.staffer@ebi.ac.uk") - public void featureCompatibilityFourTwo(MongoDatabase db) { - if (MongoVersionHelper.featureCompatibilityLessThan(db, "4.2")) - db.runCommand(new Document("setFeatureCompatibilityVersion", "4.2")); - } - - @ChangeSet(order = "2019-11-03", id = "singletonSubmissionEnvelope Biomaterial", author = "alexie.staffer@ebi.ac.uk") - public void singletonSubmissionEnvelopeBiomaterial(MongoDatabase db) { - Document filter = Document.parse("{submissionEnvelopes: {$exists: 1}}"); - List update = new ArrayList<>(); - update.add(new Document("$set", Document.parse("{ submissionEnvelope: { $arrayElemAt: [ \"$submissionEnvelopes\", 0 ] } }"))); - update.add(new Document("$unset", "submissionEnvelopes")); - - db.getCollection("biomaterial").updateMany(filter, update); - } - - @ChangeSet(order = "2019-11-04", id = "singletonSubmissionEnvelope Process", author = "alexie.staffer@ebi.ac.uk") - public void singletonSubmissionEnvelopeProcess(MongoDatabase db) { - Document filter = Document.parse("{submissionEnvelopes: {$exists: 1}}"); - List update = new ArrayList<>(); - update.add(new Document("$set", Document.parse("{ submissionEnvelope: { $arrayElemAt: [ \"$submissionEnvelopes\", 0 ] } }"))); - update.add(new Document("$unset", "submissionEnvelopes")); - - db.getCollection("process").updateMany(filter, update); - } - - @ChangeSet(order = "2019-11-05", id = "singletonSubmissionEnvelope Protocol", author = "alexie.staffer@ebi.ac.uk") - public void singletonSubmissionEnvelopeProtocol(MongoDatabase db) { - Document filter = Document.parse("{submissionEnvelopes: {$exists: 1}}"); - List update = new ArrayList<>(); - update.add(new Document("$set", Document.parse("{ submissionEnvelope: { $arrayElemAt: [ \"$submissionEnvelopes\", 0 ] } }"))); - update.add(new Document("$unset", "submissionEnvelopes")); - - db.getCollection("protocol").updateMany(filter, update); - } - - @ChangeSet(order = "2019-11-06", id = "singletonSubmissionEnvelope File", author = "alexie.staffer@ebi.ac.uk") - public void singletonSubmissionEnvelopeFile(MongoDatabase db) { - Document filter = Document.parse("{submissionEnvelopes: {$exists: 1}}"); - List update = new ArrayList<>(); - update.add(new Document("$set", Document.parse("{ submissionEnvelope: { $arrayElemAt: [ \"$submissionEnvelopes\", 0 ] } }"))); - update.add(new Document("$unset", "submissionEnvelopes")); - - db.getCollection("file").updateMany(filter, update); - } - - @ChangeSet(order = "2019-11-07", id = "singletonSubmissionEnvelope Project", author = "alexie.staffer@ebi.ac.uk") - public void singletonSubmissionEnvelopeProject(MongoDatabase db) { - Document filter = Document.parse("{submissionEnvelopes: {$exists: 1}}"); - List update = new ArrayList<>(); - update.add(new Document("$set", Document.parse("{ submissionEnvelope: { $arrayElemAt: [ \"$submissionEnvelopes\", 0 ] } }"))); - - db.getCollection("project").updateMany(filter, update); - } - - @ChangeSet(order = "2020-08-11", id = "Drop Alias Index on archiveEntity", author = "karoly@ebi.ac.uk") - public void dropAliasIndexOnArchiveEntity(MongoDatabase db) { - try { - db.getCollection("archiveEntity").dropIndex("alias"); - // If the collection does not exist this code will still succeed, - // Which is good because we may change the collection name soon. - } catch (MongoCommandException e) { - if (!MONGO_INDEX_NOT_FOUND.equals(e.getErrorCode())) throw e; - LOGGER.info(e.getErrorMessage()); - } - } - - @ChangeSet(order = "2020-09-15", id = "singelton project.dataAccess.type", author = "alexie.staffer@ebi.ac.uk") - public void singeltonProjectDataAccessType(MongoDatabase db) { - Document filter = Document.parse("{'dataAccess.type': {$type: 'array'}}"); - List update = new ArrayList<>(); - update.add(new Document("$set", Document.parse("{ 'dataAccess.type': { $arrayElemAt: [ '$dataAccess.type', 0 ] } }"))); - db.getCollection("project").updateMany(filter, update); - } - - @ChangeSet(order = "2021-05-20", id = "set default publications info", author = "alexie.staffer@ebi.ac.uk") - public void setDefaultPublicationsInfo(MongoDatabase db) { - Document filter = Document.parse("{ }"); - List update = new ArrayList<>(); - update.add(new Document("$set", Document.parse("{ 'publicationsInfo': [] }"))); - db.getCollection("project").updateMany(filter, update); - } - - @ChangeSet(order = "2021-05-27", id = "Set default isInCatalogue", author = "alexie.staffer@ebi.ac.uk") - public void setDefaultIsInCatalogue(MongoDatabase db) { - Document filter = Document.parse("{ }"); - List update = new ArrayList<>(); - update.add(new Document("$unset", "publishedToCatalogue")); - update.add(new Document("$set", Document.parse("{ 'isInCatalogue': false }"))); - db.getCollection("project").updateMany(filter, update); - } - - @ChangeSet(order = "2021-07-16", id = "Add index to project", author = "jcbwndsr@ebi.ac.uk") - public void addIndexToProject(MongoDatabase db) { - Document indexQuery = Document.parse("{" + - "'content.project_core.project_title': 'text'," + - "'content.project_core.project_short_name': 'text'," + - "'content.project_core.project_description': 'text'," + - "'content.publications.authors': 'text'," + - "'content.publications.title': 'text'," + - "'content.publications.doi': 'text'," + - "'content.contributors.name': 'text'," + - "'content.insdc_project_accessions': 'text'," + - "'content.ega_accessions': 'text'," + - "'content.dbgap_accessions': 'text'," + - "'content.geo_series_accessions': 'text'," + - "'content.array_express_accessions': 'text'," + - "'content.insdc_study_accessions': 'text'," + - "'content.biostudies_accessions': 'text'," + - "'technology.ontologies.ontology': 'text'," + - "'technology.ontologies.ontology_label': 'text'," + - "'organ.ontologies.ontology': 'text'," + - "'organ.ontologies.ontology_label': 'text'," + - "}"); - db.getCollection("project").createIndex(indexQuery); - } - - @ChangeSet(order = "2021-11-22", id = "Set empty graphValidationErrors", author = "jcbwndsr@ebi.ac.uk") - public void setGraphValidationErrors(MongoDatabase db) { - Document filter = Document.parse("{ }"); - List update = new ArrayList<>(); - update.add(new Document("$set", Document.parse("{ 'graphValidationErrors': [] }"))); - db.getCollection("biomaterial").updateMany(filter, update); - db.getCollection("process").updateMany(filter, update); - db.getCollection("protocol").updateMany(filter, update); - db.getCollection("file").updateMany(filter, update); - } - - @ChangeSet(order = "2021-12-07", id = "Rename submission states", author = "jcbwndsr@ebi.ac.uk") - public void renameSubmissionStates(MongoDatabase db) { - Document filter = Document.parse("{ 'submissionState': 'VALID' }"); - List update = new ArrayList<>(); - update.add(new Document("$set", Document.parse("{ 'submissionState': 'METADATA_VALID' }"))); - db.getCollection("submissionEnvelope").updateMany(filter, update); - - filter = Document.parse("{ 'submissionState': 'VALIDATING' }"); - update = new ArrayList<>(); - update.add(new Document("$set", Document.parse("{ 'submissionState': 'METADATA_VALIDATING' }"))); - db.getCollection("submissionEnvelope").updateMany(filter, update); - - filter = Document.parse("{ 'submissionState': 'INVALID' }"); - update = new ArrayList<>(); - update.add(new Document("$set", Document.parse("{ 'submissionState': 'METADATA_INVALID' }"))); - db.getCollection("submissionEnvelope").updateMany(filter, update); - } - - @ChangeSet(order = "2022-05-06", - id = "add missing dataFileUuid for File documents with a unique uuid. dcp-764", - author = "amnon@ebi.ac.uk") - public void addMissingDataFileUuidToFiles(MongoDatabase db) { - MongoCollection files = db.getCollection("file"); - files.find(eq("dataFileUuid", null)) - .forEach((Consumer) (Document file) -> - files - .updateOne(eq("_id", file.get("_id")), - Updates.set("dataFileUuid", UUID.randomUUID()))); - } - - public void addSubmissionEnvelopeIndexToProcess(MongoDatabase db) { - db.getCollection("process").createIndex( - Document.parse("{ \"submissionEnvelope\": 1 }") - ); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/migrations/MongoVersionHelper.java b/src/main/java/org/humancellatlas/ingest/migrations/MongoVersionHelper.java deleted file mode 100644 index de6cb3c6f..000000000 --- a/src/main/java/org/humancellatlas/ingest/migrations/MongoVersionHelper.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.humancellatlas.ingest.migrations; - -import com.mongodb.client.MongoDatabase; -import com.mongodb.connection.ServerVersion; -import org.bson.Document; - -import java.util.ArrayList; -import java.util.List; - -class MongoVersionHelper { - private static ServerVersion getVersionFromString(String version) { - List numberList = new ArrayList<>(); - for(String number : version.split("\\.")) { - numberList.add(Integer.parseInt(number)); - } - while (numberList.size() < 3) { - numberList.add(0); - } - return new ServerVersion(numberList); - } - private static ServerVersion getMajorMinor(ServerVersion version) { - List list = new ArrayList<>(); - list.add(version.getVersionList().get(0)); - list.add(version.getVersionList().get(1)); - list.add(0); - return new ServerVersion(list); - } - - private static String getMajorMinorString(ServerVersion version) { - String major = version.getVersionList().get(0).toString(); - String minor = version.getVersionList().get(1).toString(); - return major + "." + minor; - } - - static ServerVersion getFeatureCompatibilityVersion(MongoDatabase db) { - Document response = db.runCommand(new Document("getParameter", 1).append("featureCompatibilityVersion", 1)); - if (response.containsKey("ok") && response.containsKey("featureCompatibilityVersion")) { - if (getServerVersion(db).compareTo(getVersionFromString("3.6")) < 0) - return getVersionFromString(response.getString("featureCompatibilityVersion")); - else { - Document featureCompatibilityVersion = response.get("featureCompatibilityVersion", Document.class); - if (featureCompatibilityVersion.containsKey("version")) - return getVersionFromString(featureCompatibilityVersion.getString("version")); - } - } - throw new UnsupportedOperationException("Could not retrieve featureCompatibilityVersion."); - } - - static ServerVersion getServerVersion(MongoDatabase db) { - Document server_doc = db.runCommand(new Document("buildinfo", 1)); - if (server_doc.containsKey("ok") && server_doc.containsKey("version")) { - return getVersionFromString(server_doc.getString("version")); - } - throw new UnsupportedOperationException("Could not retrieve server version."); - } - - static Boolean featureCompatibilityLessThan(MongoDatabase db, String version) { - return (getFeatureCompatibilityVersion(db).compareTo(getVersionFromString(version)) < 0); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/notifications/NotificationConfiguration.java b/src/main/java/org/humancellatlas/ingest/notifications/NotificationConfiguration.java deleted file mode 100644 index 7bbebaf3a..000000000 --- a/src/main/java/org/humancellatlas/ingest/notifications/NotificationConfiguration.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.humancellatlas.ingest.notifications; - -import java.util.Collection; -import java.util.Collections; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import org.humancellatlas.ingest.notifications.NotificationConfiguration.NotificationProperties; -import org.humancellatlas.ingest.notifications.NotificationConfiguration.NotificationProperties.AmqpProperties; -import org.humancellatlas.ingest.notifications.NotificationConfiguration.NotificationProperties.SmtpProperties; -import org.humancellatlas.ingest.notifications.processors.NotificationProcessor; -import org.humancellatlas.ingest.notifications.processors.impl.email.EmailNotificationProcessor; -import org.humancellatlas.ingest.notifications.processors.impl.email.SMTPConfig; -import org.humancellatlas.ingest.notifications.sources.NotificationSource; -import org.humancellatlas.ingest.notifications.sources.impl.rabbit.AmqpConfig; -import org.humancellatlas.ingest.notifications.sources.impl.rabbit.RabbitNotificationSource; -import org.springframework.amqp.rabbit.core.RabbitMessagingTemplate; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -@EnableConfigurationProperties({NotificationProperties.class, SmtpProperties.class, AmqpProperties.class}) -public class NotificationConfiguration { - - @Bean - public SMTPConfig smtpConfig(SmtpProperties smtpEnvVars) { - return SMTPConfig.builder() - .host(smtpEnvVars.getHost()) - .port(Integer.parseInt(smtpEnvVars.getPort())) - .username(smtpEnvVars.getUsername()) - .password(smtpEnvVars.getPassword()) - .build(); - } - - @Bean - public AmqpConfig amqpConfig(AmqpProperties amqpEnvVars) { - return AmqpConfig.builder() - .sendExchange(amqpEnvVars.getSendExchange()) - .sendRoutingKey(amqpEnvVars.getSendRoutingKey()) - .build(); - } - - @Bean - public Collection notificationProcessors(SMTPConfig smtpConfig) { - EmailNotificationProcessor emailNotificationProcessor = new EmailNotificationProcessor(smtpConfig); - return Collections.singletonList(emailNotificationProcessor); - } - - @Bean - public NotificationSource notificationSource(RabbitMessagingTemplate rabbitMessagingTemplate, - AmqpConfig amqpConfig) { - return new RabbitNotificationSource(rabbitMessagingTemplate, amqpConfig); - } - - @ConfigurationProperties(prefix = "notifications") - class NotificationProperties { - - @ConfigurationProperties(prefix = "notifications.smtp") - @NoArgsConstructor - @Getter - @Setter - class SmtpProperties { - - private String host = "localhost"; - private String port = "587"; - private String username = "provide username"; - private String password = "provide password"; - - } - - @ConfigurationProperties(prefix = "notifications.amqp") - @NoArgsConstructor - @Getter - @Setter - class AmqpProperties { - - private String sendExchange = "provide notifications send exchange"; - private String sendRoutingKey = "provide notifications routing key"; - } - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/notifications/NotificationCoordinator.java b/src/main/java/org/humancellatlas/ingest/notifications/NotificationCoordinator.java deleted file mode 100644 index bba76b1f0..000000000 --- a/src/main/java/org/humancellatlas/ingest/notifications/NotificationCoordinator.java +++ /dev/null @@ -1,110 +0,0 @@ -package org.humancellatlas.ingest.notifications; - -import java.util.Collection; -import java.util.Collections; -import java.util.stream.Stream; -import lombok.AccessLevel; -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.notifications.exception.ProcessingException; -import org.humancellatlas.ingest.notifications.model.Notification; -import org.humancellatlas.ingest.notifications.model.NotificationState; -import org.humancellatlas.ingest.notifications.processors.NotificationProcessor; -import org.humancellatlas.ingest.notifications.sources.NotificationSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Component; - -@RequiredArgsConstructor -@Component -public class NotificationCoordinator { - - private final @NonNull Collection notificationProcessors; - private final @NonNull NotificationSource notificationSource; - private final @NonNull NotificationService notificationService; - - private final Logger log = LoggerFactory.getLogger(getClass()); - - public void queue() { - this.notificationService.getUnhandledNotifications() - .forEach(notification -> { - this.notificationService - .changeState(notification, NotificationState.QUEUED); - this.notificationSource - .supply(Collections.singletonList(notification)); - }); - } - - public void process() { - this.notificationSource - .stream() - .forEach(notification -> { - Notification processingNotification = this.notificationService.changeState(notification, NotificationState.PROCESSING); - - this.processNotification(processingNotification) - .filter(report -> !report.isSuccessful()) - .findAny() - .ifPresentOrElse(failedReport -> this.notificationService.changeState(processingNotification, NotificationState.FAILED), - () -> this.notificationService.changeState(processingNotification, NotificationState.PROCESSED)); - }); - } - - public void cleanup() { - this.notificationService.getHandledNotifications() - .forEach(notificationService::deleteNotification); - } - - private Stream processNotification(Notification notification) { - Stream processers = this.notificationProcessors.stream(); - - return processers.filter(notificationProcessor -> notificationProcessor.isEligible(notification)) - .map(notificationProcessor -> { - try { - notificationProcessor.handle(notification); - return NotificationProcessReport.successReport(notification); - } catch (ProcessingException e) { - log.warn(String.format("Notification processor failed for %s on notification with ID %s", notificationProcessor.getClass(), - notification.getId()), - e); - return NotificationProcessReport.failureReport(notification); - } - }); - } - - @Scheduled(fixedDelay = 20000) - private void scheduledQueue() { - this.queue(); - } - - @Scheduled(fixedDelay = 60000) - private void scheduledProcess() { - this.process(); - } - - @Scheduled(fixedDelay = 300000) - private void scheduledCleanup() { - this.cleanup(); - } - - @Getter - @RequiredArgsConstructor(access = AccessLevel.PRIVATE) - private static class NotificationProcessReport { - - private final Notification notification; - private final NotificationState result; - - public static NotificationProcessReport successReport(Notification notification) { - return new NotificationProcessReport(notification, NotificationState.PROCESSED); - } - - public static NotificationProcessReport failureReport(Notification notification) { - return new NotificationProcessReport(notification, NotificationState.FAILED); - } - - public boolean isSuccessful() { - return this.result.equals(NotificationState.PROCESSED); - } - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/notifications/NotificationRepository.java b/src/main/java/org/humancellatlas/ingest/notifications/NotificationRepository.java deleted file mode 100644 index dede1fc67..000000000 --- a/src/main/java/org/humancellatlas/ingest/notifications/NotificationRepository.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.humancellatlas.ingest.notifications; - -import java.util.Optional; -import java.util.stream.Stream; -import org.humancellatlas.ingest.notifications.model.Checksum; -import org.humancellatlas.ingest.notifications.model.Notification; -import org.humancellatlas.ingest.notifications.model.NotificationState; -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.data.rest.core.annotation.RestResource; - -public interface NotificationRepository extends MongoRepository { - - @RestResource(exported = false) - S save(S notification); - - @RestResource(exported = false) - void delete(Notification notification); - - @RestResource(exported = false) - Stream findByStateOrderByNotifyAtDesc(NotificationState state); - - @RestResource(rel = "findByChecksumValue") - Optional findByChecksum_Value(String checksumValue); - - @RestResource(exported = false) - Optional findByChecksum(Checksum checksumValue); - -} diff --git a/src/main/java/org/humancellatlas/ingest/notifications/NotificationService.java b/src/main/java/org/humancellatlas/ingest/notifications/NotificationService.java deleted file mode 100644 index 9d817b1ec..000000000 --- a/src/main/java/org/humancellatlas/ingest/notifications/NotificationService.java +++ /dev/null @@ -1,74 +0,0 @@ -package org.humancellatlas.ingest.notifications; - -import java.util.Optional; -import java.util.stream.Stream; -import lombok.AllArgsConstructor; -import org.humancellatlas.ingest.notifications.exception.DuplicateNotification; -import org.humancellatlas.ingest.notifications.model.Checksum; -import org.humancellatlas.ingest.notifications.model.Notification; -import org.humancellatlas.ingest.notifications.model.NotificationRequest; -import org.humancellatlas.ingest.notifications.model.NotificationState; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.dao.DuplicateKeyException; -import org.springframework.stereotype.Component; - - -@Component -@AllArgsConstructor -public class NotificationService { - - private final NotificationRepository notificationRepository; - private final Logger log = LoggerFactory.getLogger(getClass()); - - public Notification createNotification(NotificationRequest notificationRequest) { - try { - Notification notification = Notification.buildNew() - .content(notificationRequest.getContent()) - .metadata(notificationRequest.getMetadata()) - .checksum(notificationRequest.getChecksum()) - .build(); - - return this.notificationRepository.save(notification); - } catch (DuplicateKeyException e) { - String checksumValue = notificationRequest.getChecksum().getValue(); - String id = this.notificationRepository.findByChecksum_Value(checksumValue) - .orElseThrow(() -> { - throw new RuntimeException(e); - }) - .getId(); - - throw new DuplicateNotification( - String.format("Notification checksum value already exists in notification %s", id)); - } - } - - public Optional retrieveForChecksum(Checksum checksum) { - return this.notificationRepository.findByChecksum(checksum); - } - - public Notification changeState(Notification notification, NotificationState toState) { - if (notification.getState().isLegalTransition(toState)) { - notification.setState(toState); - return this.notificationRepository.save(notification); - } else { - throw new IllegalStateException( - String.format("Cannot transition notification with ID %s from state %s to %s", - notification.getId(), - notification.getState(), - toState)); - } - } - - public Stream getUnhandledNotifications() { - return notificationRepository.findByStateOrderByNotifyAtDesc(NotificationState.PENDING); - } - - public Stream getHandledNotifications() { - return notificationRepository.findByStateOrderByNotifyAtDesc(NotificationState.PROCESSED); - } - - public void deleteNotification(Notification notification) { - notificationRepository.delete(notification); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/notifications/sources/NotificationSource.java b/src/main/java/org/humancellatlas/ingest/notifications/sources/NotificationSource.java deleted file mode 100644 index 80f3e3a16..000000000 --- a/src/main/java/org/humancellatlas/ingest/notifications/sources/NotificationSource.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.humancellatlas.ingest.notifications.sources; - -import java.util.List; -import java.util.stream.Stream; -import org.humancellatlas.ingest.notifications.model.Notification; - -public interface NotificationSource { - - Stream stream(); - - void supply(List notifications); -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/notifications/sources/impl/inmemory/InmemoryNotificationSource.java b/src/main/java/org/humancellatlas/ingest/notifications/sources/impl/inmemory/InmemoryNotificationSource.java deleted file mode 100644 index 89507c7d8..000000000 --- a/src/main/java/org/humancellatlas/ingest/notifications/sources/impl/inmemory/InmemoryNotificationSource.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.humancellatlas.ingest.notifications.sources.impl.inmemory; - -import java.util.List; -import java.util.NoSuchElementException; -import java.util.Objects; -import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.stream.Stream; -import org.humancellatlas.ingest.notifications.model.Notification; -import org.humancellatlas.ingest.notifications.sources.NotificationSource; - -public class InmemoryNotificationSource implements NotificationSource { - - private final Queue queue = new ConcurrentLinkedQueue<>(); - - @Override - public Stream stream() { - return Stream.generate(() -> { - try { - return queue.remove(); - } catch (NoSuchElementException e) { - return null; - } - }).takeWhile(Objects::nonNull); - } - - // ignore IDE suggestion to replace this.queue::add with addAll(); addAll isn't thread safe for - // this particular queue implementation (ConcurrentLinkedQueue) - @Override - public void supply(List notifications) { - notifications.forEach(this.queue::add); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/notifications/sources/impl/rabbit/AmqpConfig.java b/src/main/java/org/humancellatlas/ingest/notifications/sources/impl/rabbit/AmqpConfig.java deleted file mode 100644 index daf9c5377..000000000 --- a/src/main/java/org/humancellatlas/ingest/notifications/sources/impl/rabbit/AmqpConfig.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.humancellatlas.ingest.notifications.sources.impl.rabbit; - -import lombok.Builder; -import lombok.Data; - -@Data -@Builder -public class AmqpConfig { - - private final String sendExchange; - private final String sendRoutingKey; -} diff --git a/src/main/java/org/humancellatlas/ingest/notifications/sources/impl/rabbit/RabbitNotificationSource.java b/src/main/java/org/humancellatlas/ingest/notifications/sources/impl/rabbit/RabbitNotificationSource.java deleted file mode 100644 index 091cfe8f4..000000000 --- a/src/main/java/org/humancellatlas/ingest/notifications/sources/impl/rabbit/RabbitNotificationSource.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.humancellatlas.ingest.notifications.sources.impl.rabbit; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.stream.Stream; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.messaging.Constants.Queues; -import org.humancellatlas.ingest.notifications.model.Notification; -import org.humancellatlas.ingest.notifications.sources.NotificationSource; -import org.humancellatlas.ingest.notifications.sources.impl.inmemory.InmemoryNotificationSource; -import org.springframework.amqp.rabbit.annotation.RabbitListener; -import org.springframework.amqp.rabbit.core.RabbitMessagingTemplate; -import org.springframework.stereotype.Component; - -@Component -@RequiredArgsConstructor -public class RabbitNotificationSource implements NotificationSource { - - private final InmemoryNotificationSource inmemoryNotificationSource = new InmemoryNotificationSource(); - private final RabbitMessagingTemplate rabbitMessagingTemplate; - private final AmqpConfig amqpConfig; - - private static String jsonString(Notification notification) { - try { - return new ObjectMapper().registerModules(new JavaTimeModule()) - .writeValueAsString(notification); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static Notification fromJsonString(String notification) { - try { - return new ObjectMapper().registerModules(new JavaTimeModule()) - .readValue(notification, Notification.class); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @RabbitListener(queues = Queues.NOTIFICATIONS_QUEUE) - private void listen(String notification) { - this.inmemoryNotificationSource - .supply(Collections.singletonList(fromJsonString(notification))); - } - - @Override - public Stream stream() { - return this.inmemoryNotificationSource.stream(); - } - - @Override - public void supply(List notifications) { - notifications.forEach(notification -> { - this.rabbitMessagingTemplate.convertAndSend(amqpConfig.getSendExchange(), - amqpConfig.getSendRoutingKey(), - jsonString(notification)); - }); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/notifications/web/NotificationController.java b/src/main/java/org/humancellatlas/ingest/notifications/web/NotificationController.java deleted file mode 100644 index 27ed37a45..000000000 --- a/src/main/java/org/humancellatlas/ingest/notifications/web/NotificationController.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.humancellatlas.ingest.notifications.web; - -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.notifications.NotificationService; -import org.humancellatlas.ingest.notifications.model.Notification; -import org.humancellatlas.ingest.notifications.model.NotificationRequest; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; - -@RepositoryRestController -@ExposesResourceFor(Notification.class) -@RequiredArgsConstructor -public class NotificationController {} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/patch/JsonPatcher.java b/src/main/java/org/humancellatlas/ingest/patch/JsonPatcher.java deleted file mode 100644 index 004af0ce3..000000000 --- a/src/main/java/org/humancellatlas/ingest/patch/JsonPatcher.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.humancellatlas.ingest.patch; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.mapping.context.PersistentEntities; -import org.springframework.data.rest.webmvc.json.DomainObjectReader; -import org.springframework.data.rest.webmvc.mapping.Associations; -import org.springframework.stereotype.Component; - -/** - * Utility class mainly for applying patches to domain objects. - */ -@Component -public class JsonPatcher { - - private final DomainObjectReader domainObjectReader; - - private final ObjectMapper objectMapper; - - @Autowired - public JsonPatcher(PersistentEntities persistentEntities, Associations associations, ObjectMapper objectMapper) { - this.domainObjectReader = new DomainObjectReader(persistentEntities, associations); - this.objectMapper = objectMapper; - } - - /* - Almost the same exact implementation used in {@link org.springframework.data.rest.webmvc.config.JsonPatchHandler} - to merge JSON documents for Spring Data REST. It's copied here because the patch code that Spring uses was made - internal to the framework. - */ - public T merge(ObjectNode patch, T target) { - return domainObjectReader.merge(patch, target, objectMapper); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/patch/Patch.java b/src/main/java/org/humancellatlas/ingest/patch/Patch.java deleted file mode 100644 index c33bfdf83..000000000 --- a/src/main/java/org/humancellatlas/ingest/patch/Patch.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.humancellatlas.ingest.patch; - -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.humancellatlas.ingest.core.AbstractEntity; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.mongodb.core.mapping.DBRef; -import org.springframework.data.mongodb.core.mapping.Document; - -import java.util.Map; - -@Data -@AllArgsConstructor -@Document -@EqualsAndHashCode(callSuper=true) -public class Patch extends AbstractEntity { - private Map jsonPatch; - private @DBRef SubmissionEnvelope submissionEnvelope; - @JsonTypeInfo(use=JsonTypeInfo.Id.CLASS, property="@class") - private @DBRef T originalDocument; - @JsonTypeInfo(use=JsonTypeInfo.Id.CLASS, property="@class") - private @DBRef T updateDocument; - -} diff --git a/src/main/java/org/humancellatlas/ingest/patch/PatchRepository.java b/src/main/java/org/humancellatlas/ingest/patch/PatchRepository.java deleted file mode 100644 index 79ed5a3ea..000000000 --- a/src/main/java/org/humancellatlas/ingest/patch/PatchRepository.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.humancellatlas.ingest.patch; - -import org.bson.types.ObjectId; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.data.mongodb.repository.Query; -import org.springframework.data.rest.core.annotation.RestResource; -import org.springframework.web.bind.annotation.CrossOrigin; - -@CrossOrigin -public interface PatchRepository extends MongoRepository { - - @RestResource(path="updatedocument", rel="WithUpdateDocument") - @Query("{ 'updateDocument.$id': ?0 }") - Patch findByUpdateDocumentId(ObjectId id); - - @RestResource(path="submissionEnvelope", rel="WithSubmissionEnvelope") - @Query("{ 'submissionEnvelope.id': ?0 }") - Page> findBySubmissionEnvelopeId(String id, Pageable pageable); - - @RestResource(exported = false) - Long deleteBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); -} diff --git a/src/main/java/org/humancellatlas/ingest/patch/PatchService.java b/src/main/java/org/humancellatlas/ingest/patch/PatchService.java deleted file mode 100644 index 025d33e20..000000000 --- a/src/main/java/org/humancellatlas/ingest/patch/PatchService.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.humancellatlas.ingest.patch; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.util.Converter; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NonNull; -import org.humancellatlas.ingest.core.JsonPatch; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.core.service.MetadataDifferService; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.stereotype.Service; - -import java.util.Map; - -@Service -@AllArgsConstructor -@Getter -public class PatchService { - private final @NonNull PatchRepository patchRepository; - private final @NonNull MetadataDifferService metadataDifferService; - - public Patch storePatch(T originalDocument, T updateDocument, SubmissionEnvelope submissionEnvelope) { - JsonPatch patch = metadataDifferService.generatePatch(originalDocument, updateDocument); - Patch patchDocument = new Patch<>(new ObjectMapper().convertValue(patch, Map.class), - submissionEnvelope, - originalDocument, - updateDocument); - Patch savedPatch = patchRepository.save(patchDocument); - return savedPatch; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/patch/web/PatchResourceProcessor.java b/src/main/java/org/humancellatlas/ingest/patch/web/PatchResourceProcessor.java deleted file mode 100644 index 480e8f316..000000000 --- a/src/main/java/org/humancellatlas/ingest/patch/web/PatchResourceProcessor.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.humancellatlas.ingest.patch.web; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.patch.Patch; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.rest.core.annotation.RestResource; -import org.springframework.hateoas.EntityLinks; -import org.springframework.hateoas.Link; -import org.springframework.hateoas.Resource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.stereotype.Component; - -@Component -@RequiredArgsConstructor -public class PatchResourceProcessor implements ResourceProcessor>> { - private final @NonNull EntityLinks entityLinks; - - - @Override - public Resource> process(Resource> resource) { - Link originalDocumentLink = entityLinks.linkForSingleResource(resource.getContent().getOriginalDocument()) - .withRel("originalDocument"); - - Link updateDocumentLink = entityLinks.linkForSingleResource(resource.getContent().getUpdateDocument()) - .withRel("updateDocument"); - - resource.add(originalDocumentLink); - resource.add(updateDocumentLink); - - return resource; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/process/Process.java b/src/main/java/org/humancellatlas/ingest/process/Process.java deleted file mode 100644 index cb2488f1f..000000000 --- a/src/main/java/org/humancellatlas/ingest/process/Process.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.humancellatlas.ingest.process; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; -import org.humancellatlas.ingest.bundle.BundleManifest; -import org.humancellatlas.ingest.core.EntityType; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.protocol.Protocol; -import org.springframework.data.annotation.PersistenceConstructor; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.DBRef; -import org.springframework.data.rest.core.annotation.RestResource; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * Created by rolando on 16/02/2018. - */ -@Getter -@EqualsAndHashCode(callSuper = true, exclude = {"project", "projects", "protocols", "inputBundleManifests", "chainedProcesses"}) -public class Process extends MetadataDocument { - - @Indexed - private @Setter - @DBRef(lazy = true) - Project project; - - @RestResource - @DBRef(lazy = true) - private Set projects = new HashSet<>(); - - @RestResource - @DBRef(lazy = true) - private Set protocols = new HashSet<>(); - - @RestResource - @DBRef(lazy = true) - @Indexed - private Set inputBundleManifests = new HashSet<>(); - - private @DBRef - Set chainedProcesses = new HashSet<>(); - - @JsonCreator - public Process(@JsonProperty("content") Object content) { - super(EntityType.PROCESS, content); - } - - public Process addInputBundleManifest(BundleManifest bundleManifest) { - this.inputBundleManifests.add(bundleManifest); - return this; - } - - public Process addProtocol(Protocol protocol) { - protocols.add(protocol); - return this; - } - - public Process removeProtocol(Protocol protocol) { - protocols.remove(protocol); - return this; - } - -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/process/ProcessRepository.java b/src/main/java/org/humancellatlas/ingest/process/ProcessRepository.java deleted file mode 100644 index 08740132b..000000000 --- a/src/main/java/org/humancellatlas/ingest/process/ProcessRepository.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.humancellatlas.ingest.process; - -import org.humancellatlas.ingest.bundle.BundleManifest; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.data.mongodb.repository.Query; -import org.springframework.data.repository.query.Param; -import org.springframework.data.rest.core.annotation.RestResource; -import org.springframework.web.bind.annotation.CrossOrigin; - -import java.util.Collection; -import java.util.Optional; -import java.util.UUID; -import java.util.stream.Stream; - -@CrossOrigin -public interface ProcessRepository extends MongoRepository { - - @RestResource(rel = "findAllByUuid", path = "findAllByUuid") - Page findByUuid(@Param("uuid") Uuid uuid, Pageable pageable); - - @RestResource(rel = "findByUuid", path = "findByUuid") - Optional findByUuidUuidAndIsUpdateFalse(@Param("uuid") UUID uuid); - - Page findByProject(Project project, Pageable pageable); - - @RestResource(exported = false) - Stream findByProject(Project project); - - @RestResource(exported = false) - Stream findByProjectsContaining(Project project); - - Page findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope, Pageable pageable); - - @RestResource(exported = false) - Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - @RestResource(rel = "findBySubmissionAndValidationState") - public Page findBySubmissionEnvelopeAndValidationState(@Param("envelopeUri") SubmissionEnvelope submissionEnvelope, - @Param("state") ValidationState state, - Pageable pageable); - - @Query(value = "{'submissionEnvelope.id': ?0, graphValidationErrors: { $exists: true, $not: {$size: 0} } }") - @RestResource(rel = "findBySubmissionIdWithGraphValidationErrors") - public Page findBySubmissionIdWithGraphValidationErrors( - @Param("envelopeId") String envelopeId, - Pageable pageable - ); - - @RestResource(exported = false) - Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - @RestResource(exported = false) - Long deleteBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - @RestResource(exported = false) - Page findByInputBundleManifestsContaining(BundleManifest bundleManifest, Pageable pageable); - - @RestResource(exported = false) - public Stream findAllByIdIn(Collection ids); - - @RestResource(exported = false) - Stream findByProtocolsContains(Protocol protocol); - - Stream findByInputBundleManifestsContains(BundleManifest bundleManifest); - - @RestResource(exported = false) - Optional findFirstByProtocolsContains(Protocol protocol); - - long countBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - long countBySubmissionEnvelopeAndValidationState(SubmissionEnvelope submissionEnvelope, ValidationState validationState); - - @Query(value = "{'submissionEnvelope.id': ?0, graphValidationErrors: { $exists: true, $not: {$size: 0} } }", count = true) - long countBySubmissionEnvelopeAndCountWithGraphValidationErrors(String submissionEnvelopeId); - -} diff --git a/src/main/java/org/humancellatlas/ingest/process/ProcessService.java b/src/main/java/org/humancellatlas/ingest/process/ProcessService.java deleted file mode 100644 index 683d69f90..000000000 --- a/src/main/java/org/humancellatlas/ingest/process/ProcessService.java +++ /dev/null @@ -1,228 +0,0 @@ -package org.humancellatlas.ingest.process; - -import lombok.Getter; -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.bundle.BundleManifest; -import org.humancellatlas.ingest.bundle.BundleManifestRepository; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.core.service.MetadataCrudService; -import org.humancellatlas.ingest.core.service.MetadataUpdateService; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.state.MetadataDocumentEventHandler; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.rest.webmvc.ResourceNotFoundException; -import org.springframework.stereotype.Service; - -import java.text.DecimalFormat; -import java.util.*; -import java.util.stream.Stream; - -@Service -@Getter -public class ProcessService { - - @Autowired - private SubmissionEnvelopeRepository submissionEnvelopeRepository; - @Autowired - private ProcessRepository processRepository; - @Autowired - private FileRepository fileRepository; - @Autowired - private BiomaterialRepository biomaterialRepository; - @Autowired - private BundleManifestRepository bundleManifestRepository; - @Autowired - private ProjectRepository projectRepository; - @Autowired - private MetadataCrudService metadataCrudService; - @Autowired - private MetadataUpdateService metadataUpdateService; - - - @Autowired - MetadataDocumentEventHandler metadataDocumentEventHandler; - - private final Logger log = LoggerFactory.getLogger(getClass()); - - protected Logger getLog() { - return log; - } - - public Page findInputBiomaterialsForProcess(Process process, Pageable pageable) { - return biomaterialRepository.findByInputToProcessesContaining(process, pageable); - } - - public Page findInputFilesForProcess(Process process, Pageable pageable) { - return fileRepository.findByInputToProcessesContaining(process, pageable); - } - - public Page findOutputBiomaterialsForProcess(Process process, Pageable pageable) { - return biomaterialRepository.findByDerivedByProcessesContaining(process, pageable); - } - - public Page findOutputFilesForProcess(Process process, Pageable pageable) { - return fileRepository.findByDerivedByProcessesContaining(process, pageable); - } - - public Process addProcessToSubmissionEnvelope(SubmissionEnvelope submissionEnvelope, Process process) { - if(!process.getIsUpdate()) { - projectRepository - .findBySubmissionEnvelopesContains(submissionEnvelope) - .findFirst().ifPresent(project -> { - process.setProject(project); - process.getProjects().add(project); - }); - return metadataCrudService.addToSubmissionEnvelopeAndSave(process, submissionEnvelope); - } else { - return metadataUpdateService.acceptUpdate(process, submissionEnvelope); - } - } - - // TODO Refactor this to use FileService - // Implement logic to have the option to only create and createOrUpdate - public Process addOutputFileToAnalysisProcess(final Process analysis, final File file) { - SubmissionEnvelope submissionEnvelope = analysis.getSubmissionEnvelope(); - File targetFile = determineTargetFile(submissionEnvelope, file); - targetFile.addToAnalysis(analysis); - targetFile.setUuid(Uuid.newUuid()); - getFileRepository().save(targetFile); - metadataDocumentEventHandler.handleMetadataDocumentCreate(targetFile); - - return analysis; - } - - public Process addInputFileUuidToProcess(final Process process, final UUID inputFileUuid) { - return fileRepository.findByUuidUuidAndIsUpdateFalse(inputFileUuid) - .map(inputFile -> addInputFileToProcess(process, inputFile)) - .orElseThrow(() -> { - throw new ResourceNotFoundException(); - }); - } - - public Process addInputFileToProcess(final Process process, final File inputFile) { - fileRepository.save(inputFile.addAsInputToProcess(process)); - return process; - } - - private File determineTargetFile(SubmissionEnvelope submissionEnvelope, File file) { - List persistentFiles = fileRepository - .findBySubmissionEnvelopeAndFileName(submissionEnvelope, file.getFileName()); - - File targetFile = persistentFiles.stream().findFirst().orElse(file); - return targetFile; - } - - public Process addInputBundleManifest(final Process analysisProcess, BundleReference bundleReference) { - for (String bundleUuid : bundleReference.getBundleUuids()) { - Optional maybeBundleManifest = getBundleManifestRepository().findTopByBundleUuidOrderByBundleVersionDesc(bundleUuid); - maybeBundleManifest.ifPresentOrElse(analysisProcess::addInputBundleManifest, () -> { - throw new ResourceNotFoundException(String.format("Could not find bundle with UUID %s", bundleUuid)); - }); - } - - return getProcessRepository().save(analysisProcess); - } - - public Page findProcessesByInputBundleUuid(UUID bundleUuid, Pageable pageable) { - Optional maybeBundleManifest = bundleManifestRepository.findTopByBundleUuidOrderByBundleVersionDesc(bundleUuid.toString()); - - return maybeBundleManifest.map(bundleManifest -> processRepository.findByInputBundleManifestsContaining(maybeBundleManifest.get(), pageable)) - .orElseThrow(() -> { - throw new ResourceNotFoundException(String.format("Bundle with UUID %s not found", bundleUuid.toString())); - }); - - } - - /** - * - * Find all assay process IDs in a submission - * - * @param submissionEnvelope - * @return A collection of IDs of every assay process in a submission - */ - public Set findAssays(SubmissionEnvelope submissionEnvelope) { - Set results = new LinkedHashSet<>(); - long fileStartTime = System.currentTimeMillis(); - - long fileEndTime = System.currentTimeMillis(); - float fileQueryTime = ((float)(fileEndTime - fileStartTime)) / 1000; - String fileQt = new DecimalFormat("#,###.##").format(fileQueryTime); - getLog().info("Retrieving assays: file query time: {} s", fileQt); - long allBioStartTime = System.currentTimeMillis(); - - fileRepository.findBySubmissionEnvelope(submissionEnvelope) - .forEach(derivedFile -> { - for (Process derivedByProcess : derivedFile.getDerivedByProcesses()) { - biomaterialRepository.findByInputToProcessesContains(derivedByProcess).findAny() - .ifPresent(__ -> results.add(derivedByProcess.getId())); - } - }); - - long allBioEndTime = System.currentTimeMillis(); - float allBioQueryTime = ((float)(allBioEndTime - allBioStartTime)) / 1000; - String allBioQt = new DecimalFormat("#,###.##").format(allBioQueryTime); - getLog().info("Retrieving assays: biomaterial query time: {} s", allBioQt); - return results; - } - - /** - * - * Find all analysis process IDs in a submission - * - * @param submissionEnvelope - * @return A collection of IDs of every analysis process in a submission - */ - public Set findAnalyses(SubmissionEnvelope submissionEnvelope) { - Set results = new LinkedHashSet<>(); - fileRepository.findBySubmissionEnvelope(submissionEnvelope) - .forEach(derivedFile -> { - for (Process derivedByProcess : derivedFile.getDerivedByProcesses()) { - fileRepository.findByInputToProcessesContains(derivedByProcess).findAny() - .ifPresent(__ -> results.add(derivedByProcess.getId())); - } - }); - return results; - } - - public Process resolveBundleReferencesForProcess(Process analysis, BundleReference bundleReference) { - for (String bundleUuid : bundleReference.getBundleUuids()) { - Optional maybeBundleManifest = getBundleManifestRepository().findTopByBundleUuidOrderByBundleVersionDesc(bundleUuid); - - maybeBundleManifest.ifPresentOrElse(bundleManifest -> { - getLog().info("Adding bundle manifest link to process '" + analysis.getId() + "'"); - analysis.addInputBundleManifest(bundleManifest); - Process savedAnalysis = getProcessRepository().save(analysis); - - // add the input files - for (String fileUuid : bundleManifest.getFileFilesMap().keySet()) { - fileRepository.findByUuidUuidAndIsUpdateFalse(UUID.fromString(fileUuid)) - .ifPresentOrElse(analysisInputFile -> { - analysisInputFile.addAsInputToProcess(savedAnalysis); - fileRepository.save(analysisInputFile); - }, () -> { - throw new ResourceNotFoundException(String.format("Could not find file with UUID %s", fileUuid)); - }); - - - } - }, () -> { - throw new ResourceNotFoundException(String.format("Could not find bundle with UUID %s", bundleUuid)); - }); - } - - return getProcessRepository().save(analysis); - } - - public Stream getProcesses(Collection processIds) { - return processRepository.findAllByIdIn(processIds); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/process/web/ProcessController.java b/src/main/java/org/humancellatlas/ingest/process/web/ProcessController.java deleted file mode 100644 index 69ba07870..000000000 --- a/src/main/java/org/humancellatlas/ingest/process/web/ProcessController.java +++ /dev/null @@ -1,229 +0,0 @@ -package org.humancellatlas.ingest.process.web; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.core.service.*; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.*; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.security.CheckAllowed; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.exception.NotAllowedDuringSubmissionStateException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.rest.webmvc.PersistentEntityResource; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.data.web.PagedResourcesAssembler; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.hateoas.Resource; -import org.springframework.hateoas.Resources; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; - -import java.lang.reflect.InvocationTargetException; -import java.net.URISyntaxException; -import java.util.List; -import java.util.Optional; -import java.util.UUID; - -import static org.springframework.data.rest.webmvc.RestMediaTypes.TEXT_URI_LIST_VALUE; -import static org.springframework.web.bind.annotation.RequestMethod.POST; -import static org.springframework.web.bind.annotation.RequestMethod.PUT; - -/** - * Created by rolando on 16/02/2018. - */ -@RepositoryRestController -@RequiredArgsConstructor -@ExposesResourceFor(Process.class) -@Getter -public class ProcessController { - private final @NonNull ProcessService processService; - private final @NonNull ProcessRepository processRepository; - private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; - private final @NonNull MetadataCrudService metadataCrudService; - private final @NonNull MetadataUpdateService metadataUpdateService; - - private @Autowired - ValidationStateChangeService validationStateChangeService; - - private @Autowired - UriToEntityConversionService uriToEntityConversionService; - - private @Autowired - MetadataLinkingService metadataLinkingService; - - @RequestMapping(path = "processes/{proc_id}/inputBiomaterials", method = RequestMethod.GET) - ResponseEntity getProcessInputBiomaterials(@PathVariable("proc_id") Process process, - Pageable pageable, - PersistentEntityResourceAssembler assembler) { - Page inputBiomaterials = getProcessService().findInputBiomaterialsForProcess(process, pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(inputBiomaterials, assembler)); - } - - @RequestMapping(path = "processes/{proc_id}/inputFiles", method = RequestMethod.GET) - ResponseEntity getProcessInputFiles(@PathVariable("proc_id") Process process, - Pageable pageable, - PersistentEntityResourceAssembler assembler) { - Page inputFiles = getProcessService().findInputFilesForProcess(process, pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(inputFiles, assembler)); - } - - @RequestMapping(path = "processes/{proc_id}/derivedBiomaterials", method = RequestMethod.GET) - ResponseEntity getProcessOutputBiomaterials(@PathVariable("proc_id") Process process, - Pageable pageable, - PersistentEntityResourceAssembler assembler) { - Page outputBiomaterials = getProcessService().findOutputBiomaterialsForProcess(process, pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(outputBiomaterials, assembler)); - } - - @RequestMapping(path = "processes/{proc_id}/derivedFiles", method = RequestMethod.GET) - ResponseEntity getProcessOutputFiles(@PathVariable("proc_id") Process process, - Pageable pageable, - PersistentEntityResourceAssembler assembler) { - Page outputFiles = getProcessService().findOutputFilesForProcess(process, pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(outputFiles, assembler)); - } - - - @CheckAllowed(value = "#submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @PostMapping(path = "submissionEnvelopes/{sub_id}/processes") - ResponseEntity> addProcessToEnvelope(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - @RequestBody Process process, - @RequestParam("updatingUuid") Optional updatingUuid, - PersistentEntityResourceAssembler assembler) { - updatingUuid.ifPresent(uuid -> { - process.setUuid(new Uuid(uuid.toString())); - process.setIsUpdate(true); - }); - Process entity = getProcessService().addProcessToSubmissionEnvelope(submissionEnvelope, process); - PersistentEntityResource resource = assembler.toFullResource(entity); - return ResponseEntity.accepted().body(resource); - } - - @CheckAllowed(value = "#submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @RequestMapping(path = "submissionEnvelopes/{sub_id}/processes/{id}", method = RequestMethod.PUT) - ResponseEntity> linkProcessToEnvelope(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - @PathVariable("id") Process process, - PersistentEntityResourceAssembler assembler) { - Process entity = getProcessService().addProcessToSubmissionEnvelope(submissionEnvelope, process); - PersistentEntityResource resource = assembler.toFullResource(entity); - return ResponseEntity.accepted().body(resource); - } - - @Deprecated - @RequestMapping(path = "/processes/{analysis_id}/" + Links.BUNDLE_REF_URL) - ResponseEntity> addBundleReference() { - return ResponseEntity.status(HttpStatus.METHOD_NOT_ALLOWED).build(); - } - - @CheckAllowed(value = "#analysis.submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @RequestMapping(path = "/processes/{analysis_id}/" + Links.BUNDLE_REF_URL, - method = RequestMethod.PUT) - ResponseEntity> oldAddBundleReference(@PathVariable("analysis_id") Process analysis, - @RequestBody BundleReference bundleReference, - final PersistentEntityResourceAssembler assembler) { - Process entity = getProcessService().resolveBundleReferencesForProcess(analysis, bundleReference); - PersistentEntityResource resource = assembler.toFullResource(entity); - return ResponseEntity.accepted().body(resource); - } - - @CheckAllowed(value = "#analysis.submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @RequestMapping(path = "/processes/{analysis_id}/" + Links.BUNDLE_REF_URL, - method = RequestMethod.POST) - ResponseEntity> addBundleReference(@PathVariable("analysis_id") Process analysis, - @RequestBody BundleReference bundleReference, - final PersistentEntityResourceAssembler assembler) { - Process entity = getProcessService().addInputBundleManifest(analysis, bundleReference); - PersistentEntityResource resource = assembler.toFullResource(entity); - return ResponseEntity.accepted().body(resource); - } - - @RequestMapping(path = "/processes/{analysis_id}/" + Links.FILE_REF_URL) - ResponseEntity> addOutputFileReference() { - return ResponseEntity.status(HttpStatus.METHOD_NOT_ALLOWED).build(); - } - - @CheckAllowed(value = "#analysis.submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @RequestMapping(path = "/processes/{analysis_id}/" + Links.FILE_REF_URL, - method = RequestMethod.PUT) - ResponseEntity> addOutputFileReference(@PathVariable("analysis_id") Process analysis, - @RequestBody File file, - final PersistentEntityResourceAssembler assembler) { - Process result = processService.addOutputFileToAnalysisProcess(analysis, file); - PersistentEntityResource resource = assembler.toFullResource(result); - return ResponseEntity.accepted().body(resource); - } - - @CheckAllowed(value = "#analysis.submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @RequestMapping(path = "/processes/{analysis_id}/" + Links.INPUT_FILES_URL, - method = RequestMethod.POST) - ResponseEntity> addInputFileReference(@PathVariable("analysis_id") Process analysis, - @RequestBody InputFileReference inputFileReference, - final PersistentEntityResourceAssembler assembler) { - Process result = processService.addInputFileUuidToProcess(analysis, inputFileReference.getInputFileUuid()); - PersistentEntityResource resource = assembler.toFullResource(result); - return ResponseEntity.accepted().body(resource); - } - - @RequestMapping(path = "/processes/search/findByInputBundleUuid", method = RequestMethod.GET) - ResponseEntity findProcesessByInputBundleUuid(@RequestParam String bundleUuid, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - Page processes = processService.findProcessesByInputBundleUuid(UUID.fromString(bundleUuid), pageable); - return ResponseEntity.ok(pagedResourcesAssembler.toResource(processes, resourceAssembler)); - } - - @CheckAllowed(value = "#process.submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @PatchMapping(path = "/processes/{id}") - HttpEntity patchProcess(@PathVariable("id") Process process, - @RequestBody final ObjectNode patch, - PersistentEntityResourceAssembler assembler) { - List allowedFields = List.of("content", "validationErrors", "graphValidationErrors"); - ObjectNode validPatch = patch.retain(allowedFields); - Process updatedProcess = metadataUpdateService.update(process, validPatch); - PersistentEntityResource resource = assembler.toFullResource(updatedProcess); - return ResponseEntity.accepted().body(resource); - } - - @CheckAllowed(value = "#process.submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @RequestMapping(path = "/processes/{id}/protocols", method = {PUT, POST}, consumes = {TEXT_URI_LIST_VALUE}) - HttpEntity linkProtocolsToProcess(@PathVariable("id") Process process, - @RequestBody Resources incoming, - HttpMethod requestMethod, - PersistentEntityResourceAssembler assembler) throws URISyntaxException, InvocationTargetException, NoSuchMethodException, IllegalAccessException { - - List protocols = uriToEntityConversionService.convertLinks(incoming.getLinks(), Protocol.class); - metadataLinkingService.updateLinks(process, protocols, "protocols", requestMethod.equals(HttpMethod.PUT)); - return ResponseEntity.ok().build(); - } - - @CheckAllowed(value = "#process.submissionEnvelope.isEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @DeleteMapping(path = "/processes/{id}/protocols/{protocolId}") - HttpEntity unlinkProtocolFromProcess(@PathVariable("id") Process process, - @PathVariable("protocolId") Protocol protocol, - PersistentEntityResourceAssembler assembler) throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { - - metadataLinkingService.removeLink(process, protocol, "protocols"); - return ResponseEntity.noContent().build(); - } - - @CheckAllowed(value = "#process.submissionEnvelope.isEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @DeleteMapping(path = "/processes/{id}") - ResponseEntity deleteProcess(@PathVariable("id") Process process) { - metadataCrudService.deleteDocument(process); - return ResponseEntity.noContent().build(); - } -} - diff --git a/src/main/java/org/humancellatlas/ingest/process/web/ProcessResourceProcessor.java b/src/main/java/org/humancellatlas/ingest/process/web/ProcessResourceProcessor.java deleted file mode 100644 index fefc7a2e8..000000000 --- a/src/main/java/org/humancellatlas/ingest/process/web/ProcessResourceProcessor.java +++ /dev/null @@ -1,75 +0,0 @@ -package org.humancellatlas.ingest.process.web; - - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.process.Process; -import org.springframework.hateoas.EntityLinks; -import org.springframework.hateoas.Link; -import org.springframework.hateoas.Resource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.stereotype.Component; - -@Component -@RequiredArgsConstructor -public class ProcessResourceProcessor implements ResourceProcessor> { - private final @NonNull - EntityLinks entityLinks; - - private Link getInputBiomaterialsLink(Process process) { - return entityLinks.linkForSingleResource(process) - .slash(Links.INPUT_BIOMATERIALS_URL) - .withRel(Links.INPUT_BIOMATERIALS_REL); - } - - private Link getDerivedBiomaterialsLink(Process process) { - return entityLinks.linkForSingleResource(process) - .slash(Links.DERIVED_BY_BIOMATERIALS_URL) - .withRel(Links.DERIVED_BY_BIOMATERIALS_REL); - } - - private Link getInputFilesLink(Process process) { - return entityLinks.linkForSingleResource(process) - .slash(Links.INPUT_FILES_URL) - .withRel(Links.INPUT_FILES_REL); - } - - private Link getDerivedFilesLink(Process process) { - return entityLinks.linkForSingleResource(process) - .slash(Links.DERIVED_BY_FILES_URL) - .withRel(Links.DERIVED_BY_FILES_REL); - } - - private Link getBundleReferencesLink(Process process) { - return entityLinks.linkForSingleResource(process).slash(Links.BUNDLE_REF_URL).withRel(Links.BUNDLE_REF_REL); - } - - private Link getFileReferencesLink(Process process) { - return entityLinks.linkForSingleResource(process).slash(Links.FILE_REF_URL).withRel(Links.FILE_REF_REL); - } - - @Deprecated - private Link getOldEvilBundleReferencesLink(Process process) { - return entityLinks.linkForSingleResource(process).slash(Links.BUNDLE_REF_URL).withRel(Links.BUNDLE_REF_OLD_EVIL_REL); - } - - @Deprecated - private Link getOldEvilFileReferencesLink(Process process) { - return entityLinks.linkForSingleResource(process).slash(Links.FILE_REF_URL).withRel(Links.FILE_REF_OLD_EVIL_REL); - } - - @Override - public Resource process(Resource resource) { - Process process = resource.getContent(); - resource.add(getInputBiomaterialsLink(process), - getDerivedBiomaterialsLink(process), - getInputFilesLink(process), - getDerivedFilesLink(process), - getBundleReferencesLink(process), - getFileReferencesLink(process), - getOldEvilBundleReferencesLink(process), - getOldEvilFileReferencesLink(process)); - return resource; - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/process/web/ProcessSearchProcessor.java b/src/main/java/org/humancellatlas/ingest/process/web/ProcessSearchProcessor.java deleted file mode 100644 index 3922b455c..000000000 --- a/src/main/java/org/humancellatlas/ingest/process/web/ProcessSearchProcessor.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.humancellatlas.ingest.process.web; - -import org.humancellatlas.ingest.process.Process; -import org.springframework.data.rest.webmvc.RepositorySearchesResource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.stereotype.Component; - -import static org.springframework.hateoas.mvc.ControllerLinkBuilder.linkTo; -import static org.springframework.hateoas.mvc.ControllerLinkBuilder.methodOn; - -/** - * Created by rolando on 29/06/2018. - */ -@Component -public class ProcessSearchProcessor implements ResourceProcessor { - - @Override - public RepositorySearchesResource process(RepositorySearchesResource resource) { - if(resource.getDomainType().equals(Process.class)) { - resource.add(linkTo(methodOn(ProcessController.class).findProcesessByInputBundleUuid(null, null, null)).withRel("findByInputBundleUuid")); - } - - return resource; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/project/DataAccessTypes.java b/src/main/java/org/humancellatlas/ingest/project/DataAccessTypes.java deleted file mode 100644 index 2122cf1a9..000000000 --- a/src/main/java/org/humancellatlas/ingest/project/DataAccessTypes.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.humancellatlas.ingest.project; - -import lombok.Getter; - -public enum DataAccessTypes { - OPEN("All fully open"), - MANAGED("All managed access"), - MIXTURE("A mixture of open and managed"), - COMPLICATED("It's complicated"); - - @Getter - final String label; - - DataAccessTypes(String label) { - this.label = label; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/project/Project.java b/src/main/java/org/humancellatlas/ingest/project/Project.java deleted file mode 100644 index eefff52fb..000000000 --- a/src/main/java/org/humancellatlas/ingest/project/Project.java +++ /dev/null @@ -1,128 +0,0 @@ -package org.humancellatlas.ingest.project; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; -import org.humancellatlas.ingest.core.EntityType; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.mongodb.core.mapping.DBRef; -import org.springframework.data.rest.core.annotation.RestResource; - -import javax.validation.constraints.NotNull; -import java.time.Instant; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 30/08/17 - */ -@Getter -@EqualsAndHashCode(callSuper = true, exclude = {"supplementaryFiles", "submissionEnvelopes"}) -public class Project extends MetadataDocument { - @RestResource - @JsonIgnore - @DBRef(lazy = true) - private Set supplementaryFiles = new HashSet<>(); - - // A project may have 1 or more submissions related to it. - @JsonIgnore - private @DBRef(lazy = true) - Set submissionEnvelopes = new HashSet<>(); - - @Setter - private Instant releaseDate; - - @Setter - private Instant accessionDate; - - @Setter - private Object technology; - - @Setter - private Object organ; - - @Setter - private Integer cellCount; - - @Setter - private Object dataAccess; - - @Setter - private Object identifyingOrganisms; - - @Setter - private String primaryWrangler; - - @Setter - private String secondaryWrangler; - - @Setter - private WranglingState wranglingState; - - @Setter - private Integer wranglingPriority; - - @Setter - private String wranglingNotes; - - @Setter - private Boolean isInCatalogue; - - @Setter - private Instant cataloguedDate; - - @Setter - private List publicationsInfo; - - @Setter - private Integer dcpReleaseNumber; - - @Setter - private List projectLabels; - - @Setter - private List projectNetworks; - - - @JsonCreator - public Project(@JsonProperty("content") Object content) { - super(EntityType.PROJECT, content); - } - - public void addToSubmissionEnvelopes(@NotNull SubmissionEnvelope submissionEnvelope) { - this.submissionEnvelopes.add(submissionEnvelope); - } - - //ToDo: Find a better way of ensuring that DBRefs to deleted objects aren't returned. - @JsonIgnore - public List getOpenSubmissionEnvelopes() { - return this.submissionEnvelopes.stream() - .filter(Objects::nonNull) - .filter(env -> env.getSubmissionState() != null) - .filter(SubmissionEnvelope::isOpen) - .collect(Collectors.toList()); - } - - public Boolean getHasOpenSubmission() { - return !getOpenSubmissionEnvelopes().isEmpty(); - } - - @JsonIgnore - public Boolean isEditable() { - return this.submissionEnvelopes.stream() - .filter(Objects::nonNull) - .allMatch(SubmissionEnvelope::isEditable); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/project/ProjectChangeListener.java b/src/main/java/org/humancellatlas/ingest/project/ProjectChangeListener.java deleted file mode 100644 index af3d389f6..000000000 --- a/src/main/java/org/humancellatlas/ingest/project/ProjectChangeListener.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.humancellatlas.ingest.project; - -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.exception.MultipleOpenSubmissionsException; -import org.springframework.data.mongodb.core.mapping.event.AbstractMongoEventListener; -import org.springframework.data.mongodb.core.mapping.event.AfterSaveEvent; -import org.springframework.data.mongodb.core.mapping.event.BeforeSaveEvent; -import org.springframework.stereotype.Component; - - -@Component -@RequiredArgsConstructor -@Getter -public class ProjectChangeListener extends AbstractMongoEventListener { - private final ProjectEventHandler projectEventHandler; - - @Override - public void onBeforeSave(BeforeSaveEvent event) { - Project project = event.getSource(); - if (project.getOpenSubmissionEnvelopes().size() > 1) - throw new MultipleOpenSubmissionsException("A project can't have multiple open submissions."); - } - - @Override - public void onAfterSave(AfterSaveEvent event) { - //do nothing - } -} diff --git a/src/main/java/org/humancellatlas/ingest/project/ProjectEventHandler.java b/src/main/java/org/humancellatlas/ingest/project/ProjectEventHandler.java deleted file mode 100644 index 6c3a20bc1..000000000 --- a/src/main/java/org/humancellatlas/ingest/project/ProjectEventHandler.java +++ /dev/null @@ -1,143 +0,0 @@ -package org.humancellatlas.ingest.project; - -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.notifications.NotificationService; -import org.humancellatlas.ingest.notifications.exception.DuplicateNotification; -import org.humancellatlas.ingest.notifications.model.Checksum; -import org.humancellatlas.ingest.notifications.model.Notification; -import org.humancellatlas.ingest.notifications.model.NotificationRequest; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.user.IdentityService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; -import org.springframework.util.DigestUtils; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Optional; - -import static org.springframework.util.DigestUtils.md5DigestAsHex; - -@Component -@RequiredArgsConstructor -public class ProjectEventHandler { - - private final NotificationService notificationService; - private final Environment environment; - private final IdentityService identityService; - - private final Logger log = LoggerFactory.getLogger(getClass()); - - private static String objectToString(Object object) { - try { - return new ObjectMapper().writeValueAsString(object); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static String objectToPrettyString(Object object) { - try { - return new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(object); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public Notification registeredProject(Project project) { - String message = "A new project [" + project.getUuid() + "] was registered."; - String header = "project-registered"; - Checksum checksum = new Checksum(header, md5DigestAsHex((header + ":" + project.getUuid()).getBytes())); - return notifyWranglersByEmail(message, checksum); - } - - public Notification editedProjectMetadata(Project project) { - String notificationContent = String.format("Project %s was updated:\n\nNew content:\n\n%s", - project.getUuid().getUuid().toString(), - objectToPrettyString(project.getContent())); - Checksum checksum = editedProjectChecksum(project); - - return notifyWranglersByEmail(notificationContent, checksum); - } - - public Notification deletedProject(Project project) { - String notificationContent = String.format("Project %s was deleted:\n\n%s", - project.getUuid().getUuid().toString(), - objectToPrettyString(project.getContent())); - - Checksum checksum = deletedProjectChecksum(project); - - return notifyWranglersByEmail(notificationContent, checksum); - } - - public Optional validatedProject(Project project) { - if (project.getValidationState().equals(ValidationState.VALID)) { - String notificationContent = String.format("Project %s has been validated:\n\n%s", - project.getUuid().getUuid().toString(), - objectToPrettyString(project.getContent())); - - Checksum checksum = validProjectChecksum(project); - - return Optional.of(notifyWranglersByEmail(notificationContent, checksum)); - } else { - return Optional.empty(); - } - } - - private Notification notifyWranglersByEmail(String notificationContent, - Checksum notificationChecksum) { - var notificationMetadata = new HashMap(); - var emailMetadata = new HashMap(); - emailMetadata.put("to", this.emailNotificationsFromAddress()); - emailMetadata.put("from", identityService.wranglerEmail()); - emailMetadata.put("subject", "HCA DCP project update"); - emailMetadata.put("body", notificationContent); - notificationMetadata.put("email", emailMetadata); - - NotificationRequest notificationRequest = new NotificationRequest(notificationContent, - notificationMetadata, - notificationChecksum); - try { - return this.notificationService.createNotification(notificationRequest); - } catch (DuplicateNotification e) { - return this.notificationService.retrieveForChecksum(notificationChecksum) - .orElseThrow(() -> { - log.error( - "Duplicate notification for non-existent checksum"); - throw new RuntimeException(e); - }); - } - } - - private Checksum editedProjectChecksum(Project project) { - String checksumInput = String - .format("%s:%s", "project-edited", project.getUuid().getUuid()); - return new Checksum("project-edited", - md5DigestAsHex(checksumInput.getBytes())); - } - - private Checksum deletedProjectChecksum(Project project) { - String checksumInput = String - .format("%s:%s", "project-deleted", project.getUuid().getUuid()); - return new Checksum("project-deleted", - md5DigestAsHex(checksumInput.getBytes())); - } - - private Checksum validProjectChecksum(Project project) { - String checksumInput = String.format("%s:%s:%s", - "project-validated", - project.getUuid().getUuid(), - objectToString(project.getContent())); - return new Checksum("project-validated", - md5DigestAsHex(checksumInput.getBytes())); - } - - private String emailNotificationsFromAddress() { - return environment.getProperty("PROJECT_NOTIFICATIONS_FROM_ADDRESS", - "hca-notifications-test@ebi.ac.uk"); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/project/ProjectLinkChangeListener.java b/src/main/java/org/humancellatlas/ingest/project/ProjectLinkChangeListener.java deleted file mode 100644 index 21b5679b1..000000000 --- a/src/main/java/org/humancellatlas/ingest/project/ProjectLinkChangeListener.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.humancellatlas.ingest.project; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.rest.core.annotation.*; -import org.springframework.stereotype.Component; - -import java.util.Collection; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Stream; - -@Component -@RepositoryEventHandler -@RequiredArgsConstructor -public class ProjectLinkChangeListener { - - @Autowired - ProjectService projectService; - private final @NonNull Logger log = LoggerFactory.getLogger(getClass()); - - /** - * The `linked` parameter, due to a bug in Spring, is passed to the handler as a {@link java.lang.reflect.Proxy} - * object. This is a proxy to a {@link Collection}. Since we need to respond to associations of - * {@link SubmissionEnvelope}s and not other properties of {@link Project}, we need to filter the contents - * of the `linked` Collection. - * - * @link Stack Overflow ticket - */ - @HandleBeforeLinkSave - public void beforeLinkSave(Project project, Object linked) { - Stream.of(linked) - .map(Collection.class::cast) - .flatMap(Collection::stream) - .filter(Objects::nonNull) - .filter(SubmissionEnvelope.class::isInstance) - .findAny() - .ifPresent(o -> { - projectService.updateWranglingState(project, WranglingState.IN_PROGRESS); - }); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/project/ProjectQueryBuilder.java b/src/main/java/org/humancellatlas/ingest/project/ProjectQueryBuilder.java deleted file mode 100644 index b5cb93713..000000000 --- a/src/main/java/org/humancellatlas/ingest/project/ProjectQueryBuilder.java +++ /dev/null @@ -1,131 +0,0 @@ -package org.humancellatlas.ingest.project; - -import org.humancellatlas.ingest.project.web.SearchFilter; -import org.humancellatlas.ingest.project.web.SearchType; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.CriteriaDefinition; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.data.mongodb.core.query.TextCriteria; - -import java.util.*; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -public class ProjectQueryBuilder { - - public static Query buildProjectsQuery(SearchFilter searchFilter) { - List criteriaList = new ArrayList<>(); - criteriaList.add(Criteria.where("isUpdate").is(false)); - addIsCriterionForAttribute(criteriaList, "wranglingState", searchFilter.getWranglingState()); - addIsCriterionForAttribute(criteriaList, "primaryWrangler", searchFilter.getPrimaryWrangler()); - addIsCriterionForAttribute(criteriaList, "wranglingPriority", searchFilter.getWranglingPriority()); - addIsCriterionForAttribute(criteriaList, "isInCatalogue", searchFilter.getHcaCatalogue()); - addLTECriterionForAttribute(criteriaList, "cellCount", searchFilter.getMaxCellCount()); - addGTECriterionForAttribute(criteriaList, "cellCount", searchFilter.getMinCellCount()); - addInCriterionForAttribute(criteriaList, "identifyingOrganisms", searchFilter.getIdentifyingOrganism()); - addInCriterionForAttribute(criteriaList, "dcpReleaseNumber", searchFilter.getDcpReleaseNumber()); - addInCriterionForAttribute(criteriaList, "projectLabels", searchFilter.getProjectLabels()); - addInCriterionForAttribute(criteriaList, "projectNetworks", searchFilter.getProjectNetworks()); - - if(searchFilter.getDataAccess() != null){ - addIsCriterionForAttribute(criteriaList, "dataAccess.type", searchFilter.getDataAccess().getLabel()); - } - - Optional.ofNullable(searchFilter.getHasOfficialHcaPublication()) - .map(value -> - Criteria.where("content.publications") - .elemMatch(Criteria.where("official_hca_publication").is(value)) - ).ifPresent(criteriaList::add); - - Optional.ofNullable(searchFilter.getOrganOntology()) - .map(value -> - Criteria.where("organ.ontologies").elemMatch(Criteria.where("ontology").is(value)) - ).ifPresent(criteriaList::add); - - Criteria queryCriteria = new Criteria().andOperator(criteriaList.toArray(new Criteria[criteriaList.size()])); - Query query = new Query().addCriteria(queryCriteria); - addKeywordSearchCriteria(searchFilter) - .ifPresent(query::addCriteria); - return query; - } - - private static Optional addKeywordSearchCriteria(SearchFilter searchFilter) { - try { - return buildUuidCriteria(searchFilter); - } catch (IllegalArgumentException e) { - return buildTextCriteria(searchFilter); - } - } - - private static Optional buildTextCriteria(SearchFilter searchFilter) { - return Optional.ofNullable(searchFilter.getSearch()) - .map(search -> ProjectQueryBuilder.formatSearchString(searchFilter)) - .map(search -> - TextCriteria.forDefaultLanguage().matching(String.valueOf(search))); - } - - private static Optional buildUuidCriteria(SearchFilter searchFilter) { - return Optional.ofNullable(searchFilter.getSearch()) - .map(UUID::fromString) - .map(uuid -> Criteria.where("uuid.uuid").is(uuid)); - } - - private static void addIsCriterionForAttribute(List criteria_list, - String attributeName, - Object attributeValue) { - Optional.ofNullable(attributeValue) - .map(value -> Criteria.where(attributeName).is(value)) - .ifPresent(criteria_list::add); - } - - private static void addLTECriterionForAttribute(List criteria_list, - String attributeName, - Integer attributeValue) { - Optional.ofNullable(attributeValue) - .map(value -> Criteria.where(attributeName).lte(value)) - .ifPresent(criteria_list::add); - } - - private static void addGTECriterionForAttribute(List criteria_list, - String attributeName, - Integer attributeValue) { - Optional.ofNullable(attributeValue) - .map(value -> Criteria.where(attributeName).gte(value)) - .ifPresent(criteria_list::add); - } - - private static void addInCriterionForAttribute(List criteria_list, - String attributeName, - Object attributeValue) { - Optional.ofNullable(attributeValue) - .map(value -> Criteria.where(attributeName).in(value)) - .ifPresent(criteria_list::add); - } - - private static final Map> keywordFormatterMap = Map.of( - SearchType.ExactMatch, (SearchFilter searchFilter)->encloseInQuotes(searchFilter.getSearch()), - SearchType.AllKeywords, (SearchFilter searchFilter)->Stream.of(splitBySpace(searchFilter)) - .map(ProjectQueryBuilder::encloseInQuotes) - .collect(Collectors.joining(" ")) - ); - - protected static String formatSearchString(SearchFilter searchFilter) { - if(searchFilter.getSearch()!=null && searchFilter.getSearch().contains("\"")) { - return searchFilter.getSearch(); - } - return Optional.ofNullable(searchFilter) - .map(SearchFilter::getSearchType) - .map(keywordFormatterMap::get) - .map(formatterFunction->formatterFunction.apply(searchFilter)) - .orElse(searchFilter.getSearch()); - } - - private static String[] splitBySpace(SearchFilter searchFilter) { - return searchFilter.getSearch().split(" +"); - } - - private static String encloseInQuotes(String search) { - return "\"" + search + "\""; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/project/ProjectRepository.java b/src/main/java/org/humancellatlas/ingest/project/ProjectRepository.java deleted file mode 100644 index b7f105ea4..000000000 --- a/src/main/java/org/humancellatlas/ingest/project/ProjectRepository.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.humancellatlas.ingest.project; - -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.query.MetadataCriteria; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.data.repository.query.Param; -import org.springframework.data.rest.core.annotation.RestResource; -import org.springframework.web.bind.annotation.CrossOrigin; - -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.stream.Stream; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 31/08/17 - */ -@CrossOrigin -public interface ProjectRepository extends MongoRepository { - - @RestResource(rel = "findAllByUuid", path = "findAllByUuid") - Page findByUuid(@Param("uuid") Uuid uuid, Pageable pageable); - - @RestResource(rel = "findByUuid", path = "findByUuid") - Optional findByUuidUuidAndIsUpdateFalse(@Param("uuid") UUID uuid); - - @RestResource(path = "findByUser", rel = "findByUser") - Page findByUser(@Param(value = "user") String user, Pageable pageable); - - @RestResource(rel = "findByUserAndPrimaryWrangler") - Page findByUserOrPrimaryWrangler(@Param(value = "user") String user, - @Param(value = "primaryWrangler") String primaryWrangler, - Pageable pageable); - - Page findBySubmissionEnvelopesContaining(SubmissionEnvelope submissionEnvelope, Pageable pageable); - - @RestResource(exported = false) - Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - @RestResource(rel = "findBySubmissionAndValidationState") - public Page findBySubmissionEnvelopeAndValidationState(@Param("envelopeUri") SubmissionEnvelope submissionEnvelope, - @Param("state") ValidationState state, - Pageable pageable); - - long countByUser(String user); - - @RestResource(exported = false) - Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - @RestResource(exported = false) - Stream findBySupplementaryFilesContains(File file); - - @RestResource(exported = false) - Stream findBySubmissionEnvelopesContains(SubmissionEnvelope submissionEnvelope); - - @RestResource(exported = false) - Stream findByUuid(Uuid uuid); - - @RestResource(rel = "catalogue", path = "catalogue") - Page findByIsInCatalogueTrue(Pageable pageable); - -} diff --git a/src/main/java/org/humancellatlas/ingest/project/ProjectService.java b/src/main/java/org/humancellatlas/ingest/project/ProjectService.java deleted file mode 100644 index 3351867cc..000000000 --- a/src/main/java/org/humancellatlas/ingest/project/ProjectService.java +++ /dev/null @@ -1,222 +0,0 @@ -package org.humancellatlas.ingest.project; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.audit.AuditEntry; -import org.humancellatlas.ingest.audit.AuditEntryService; -import org.humancellatlas.ingest.audit.AuditType; -import org.humancellatlas.ingest.bundle.BundleManifest; -import org.humancellatlas.ingest.bundle.BundleManifestRepository; -import org.humancellatlas.ingest.bundle.BundleType; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.core.service.MetadataCrudService; -import org.humancellatlas.ingest.core.service.MetadataUpdateService; -import org.humancellatlas.ingest.project.exception.NonEmptyProject; -import org.humancellatlas.ingest.project.web.SearchFilter; -import org.humancellatlas.ingest.schemas.SchemaService; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageImpl; -import org.springframework.data.domain.Pageable; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.data.rest.webmvc.ResourceNotFoundException; -import org.springframework.stereotype.Service; - -import java.time.Instant; -import java.util.*; - -import static java.lang.String.format; -import static java.util.Objects.isNull; -import static java.util.stream.Collectors.toSet; - - -@Service -@RequiredArgsConstructor -@Getter -public class ProjectService { - @Autowired - private final MongoTemplate mongoTemplate; - - //Helper class for capturing copies of a Project and all Submission Envelopes related to them. - private static class ProjectBag { - - private final Set projects; - private final Set submissionEnvelopes; - - public ProjectBag(Set projects, Set submissionEnvelopes) { - this.projects = projects; - this.submissionEnvelopes = submissionEnvelopes; - } - - } - - private final @NonNull SubmissionEnvelopeRepository submissionEnvelopeRepository; - private final @NonNull ProjectRepository projectRepository; - private final @NonNull MetadataCrudService metadataCrudService; - private final @NonNull MetadataUpdateService metadataUpdateService; - private final @NonNull SchemaService schemaService; - private final @NonNull BundleManifestRepository bundleManifestRepository; - private final @NonNull AuditEntryService auditEntryService; - - private final @NonNull ProjectEventHandler projectEventHandler; - - private final Logger log = LoggerFactory.getLogger(getClass()); - - protected Logger getLog() { - return log; - } - - public Project register(final Project project) { - project.setCataloguedDate(null); - if (!isNull(project.getIsInCatalogue()) && project.getIsInCatalogue()) { - project.setCataloguedDate(Instant.now()); - } - Project persistentProject = projectRepository.save(project); - projectEventHandler.registeredProject(persistentProject); - return persistentProject; - } - - public Project createSuggestedProject(final ObjectNode suggestion) { - Map content = createBaseContentForProject(); - Project suggestedProject = new Project(content); - suggestedProject.setWranglingState(WranglingState.NEW_SUGGESTION); - var notes = String.format( - "DOI: %s \nName: %s \nEmail: %s \nComments: %s", - suggestion.get("doi"), - suggestion.get("name"), - suggestion.get("email"), - suggestion.get("comments") - ); - suggestedProject.setWranglingNotes(notes); - return this.register(suggestedProject); - } - - public Project update(final Project project, ObjectNode patch, Boolean sendNotification) { - if (patch.has("isInCatalogue") - && patch.get("isInCatalogue").asBoolean() - && project.getCataloguedDate() == null) { - project.setCataloguedDate(Instant.now()); - } - - updateWranglingState(project, patch); - Project updatedProject = metadataUpdateService.update(project, patch); - - if (sendNotification) { - projectEventHandler.editedProjectMetadata(updatedProject); - } - - return updatedProject; - } - - private void updateWranglingState(Project project, ObjectNode patch) { - Optional.ofNullable(patch.get("wranglingState")) - .map(JsonNode::asText) - .map(WranglingState::getName) - .ifPresent(newWranglingState->updateWranglingState(project, newWranglingState)); - } - - public void updateWranglingState(Project project, - @NonNull WranglingState newWranglingState) { - WranglingState currentWranglingState = project.getWranglingState(); - if(currentWranglingState != newWranglingState) { - log.info("setting project {} from {} to {}", project.getId(), currentWranglingState, newWranglingState); - project.setWranglingState(newWranglingState); - projectRepository.save(project); - AuditEntry wranglingStateUpdate = new AuditEntry(AuditType.STATUS_UPDATED, currentWranglingState, newWranglingState, project); - auditEntryService.addAuditEntry(wranglingStateUpdate); - } - } - - public Project addProjectToSubmissionEnvelope(SubmissionEnvelope submissionEnvelope, Project project) { - if (!project.getIsUpdate()) { - return metadataCrudService.addToSubmissionEnvelopeAndSave(project, submissionEnvelope); - } else { - return metadataUpdateService.acceptUpdate(project, submissionEnvelope); - } - } - - public Project linkProjectSubmissionEnvelope(SubmissionEnvelope submissionEnvelope, Project project) { - final String projectId = project.getId(); - project.addToSubmissionEnvelopes(submissionEnvelope); - projectRepository.save(project); - - projectRepository.findByUuidUuidAndIsUpdateFalse(project.getUuid().getUuid()).ifPresent(projectByUuid -> { - if (!projectByUuid.getId().equals(projectId)) { - projectByUuid.addToSubmissionEnvelopes(submissionEnvelope); - projectRepository.save(projectByUuid); - } - }); - return project; - } - - public Page findBundleManifestsByProjectUuidAndBundleType(Uuid projectUuid, BundleType bundleType, - Pageable pageable) { - return this.projectRepository - .findByUuidUuidAndIsUpdateFalse(projectUuid.getUuid()) - .map(project -> bundleManifestRepository.findBundleManifestsByProjectAndBundleType(project, - bundleType, pageable)) - .orElseThrow(() -> { - throw new ResourceNotFoundException(format("Project with UUID %s not found", - projectUuid.getUuid().toString())); - }); - } - - public Set getSubmissionEnvelopes(Project project) { - return gather(project).submissionEnvelopes; - } - - public void delete(Project project) throws NonEmptyProject { - ProjectBag projectBag = gather(project); - if (projectBag.submissionEnvelopes.isEmpty()) { - projectBag.projects.forEach(_project -> { - metadataCrudService.deleteDocument(_project); - projectEventHandler.deletedProject(_project); - }); - } else { - throw new NonEmptyProject(); - } - } - - private Map createBaseContentForProject() { - Map content = new HashMap<>(); - final String entityType = "project"; - final String highLevelEntity = "type"; - content.put("describedBy", schemaService.getLatestSchemaByEntityType(highLevelEntity, entityType).getSchemaUri()); - content.put("schema_type", entityType); - return content; - } - - private ProjectBag gather(Project project) { - Set envelopes = new HashSet<>(); - Set projects = this.projectRepository.findByUuid(project.getUuid()).collect(toSet()); - projects.forEach(copy -> { - envelopes.addAll(copy.getSubmissionEnvelopes()); - envelopes.add(copy.getSubmissionEnvelope()); - }); - - //ToDo: Find a better way of ensuring that DBRefs to deleted objects aren't returned. - envelopes.removeIf(env -> env == null || env.getSubmissionState() == null); - return new ProjectBag(projects, envelopes); - } - - public Page filterProjects(SearchFilter searchFilter, Pageable pageable) { - Query query = ProjectQueryBuilder.buildProjectsQuery(searchFilter); - log.debug("Project Search query: " + query); - - List projects = mongoTemplate.find(query.with(pageable), Project.class); - long count = mongoTemplate.count(query, Project.class); - return new PageImpl<>(projects, pageable, count); - } - - public List getProjectAuditEntries(Project project) { - return auditEntryService.getAuditEntriesForAbstractEntity(project); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/project/WranglingState.java b/src/main/java/org/humancellatlas/ingest/project/WranglingState.java deleted file mode 100644 index d70ea6bac..000000000 --- a/src/main/java/org/humancellatlas/ingest/project/WranglingState.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.humancellatlas.ingest.project; - -import com.fasterxml.jackson.annotation.JsonValue; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; - -@JsonSerialize(using = WranglingStateSerializer.class) -public enum WranglingState { - NEW("New"), - ELIGIBLE("Eligible"), - NOT_ELIGIBLE("Not eligible"), - IN_PROGRESS("In progress"), - STALLED("Stalled"), - SUBMITTED("Submitted"), - PUBLISHED_IN_DCP("Published in DCP"), - DELETED("Deleted"), - NEW_SUGGESTION("New Suggestion"); - - protected String value; - - WranglingState(String value) { - this.value = value; - } - - @JsonValue - public String getValue() { - return this.value; - } - - public static WranglingState getName(String value) { - for (WranglingState wranglingState : values()) { - if (wranglingState.value.equals(value)) { - return wranglingState; - } - } - return null; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/project/WranglingStateSerializer.java b/src/main/java/org/humancellatlas/ingest/project/WranglingStateSerializer.java deleted file mode 100644 index 86894c678..000000000 --- a/src/main/java/org/humancellatlas/ingest/project/WranglingStateSerializer.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.humancellatlas.ingest.project; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import org.humancellatlas.ingest.project.WranglingState; - -import java.io.IOException; - -public class WranglingStateSerializer extends JsonSerializer { - @Override - public void serialize(WranglingState value, JsonGenerator generator, SerializerProvider serializers) throws IOException { - generator.writeString(value.value); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/project/exception/NonEmptyProject.java b/src/main/java/org/humancellatlas/ingest/project/exception/NonEmptyProject.java deleted file mode 100644 index fc7e94c4f..000000000 --- a/src/main/java/org/humancellatlas/ingest/project/exception/NonEmptyProject.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.humancellatlas.ingest.project.exception; - -public class NonEmptyProject extends Exception { - - public NonEmptyProject() { - super("Operation cannot be carried out on non-empty Project."); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/project/exception/NotAllowedWithSubmissionInStateException.java b/src/main/java/org/humancellatlas/ingest/project/exception/NotAllowedWithSubmissionInStateException.java deleted file mode 100644 index 3af2f0281..000000000 --- a/src/main/java/org/humancellatlas/ingest/project/exception/NotAllowedWithSubmissionInStateException.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.humancellatlas.ingest.project.exception; - -import org.humancellatlas.ingest.security.exception.NotAllowedException; - -public class NotAllowedWithSubmissionInStateException extends NotAllowedException { - public NotAllowedWithSubmissionInStateException() { - super("Operation not allowed while the project has a submission in a non-editable state."); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/project/web/ProjectController.java b/src/main/java/org/humancellatlas/ingest/project/web/ProjectController.java deleted file mode 100644 index 716835b33..000000000 --- a/src/main/java/org/humancellatlas/ingest/project/web/ProjectController.java +++ /dev/null @@ -1,235 +0,0 @@ -package org.humancellatlas.ingest.project.web; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.bundle.BundleManifest; -import org.humancellatlas.ingest.bundle.BundleType; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.core.service.MetadataUpdateService; -import org.humancellatlas.ingest.core.service.ValidationStateChangeService; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectEventHandler; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.project.ProjectService; -import org.humancellatlas.ingest.project.exception.NonEmptyProject; -import org.humancellatlas.ingest.project.exception.NotAllowedWithSubmissionInStateException; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.humancellatlas.ingest.security.CheckAllowed; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.exception.NotAllowedDuringSubmissionStateException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageImpl; -import org.springframework.data.domain.Pageable; -import org.springframework.data.rest.webmvc.PersistentEntityResource; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.data.rest.webmvc.ResourceNotFoundException; -import org.springframework.data.web.PagedResourcesAssembler; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.hateoas.PagedResources; -import org.springframework.hateoas.Resource; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.security.access.annotation.Secured; -import org.springframework.web.bind.annotation.*; - -import java.util.*; -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 05/09/17 - */ -@RepositoryRestController -@ExposesResourceFor(Project.class) -@RequiredArgsConstructor -@Getter -public class ProjectController { - - private static final Logger LOGGER = LoggerFactory.getLogger(ProjectController.class); - - private final @NonNull ProjectService projectService; - - private final @NonNull ValidationStateChangeService validationStateChangeService; - - private final @NonNull ProjectEventHandler projectEventHandler; - - private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; - - private final @NonNull ProjectRepository projectRepository; - private final @NonNull BiomaterialRepository biomaterialRepository; - private final @NonNull ProcessRepository processRepository; - private final @NonNull ProtocolRepository protocolRepository; - private final @NonNull FileRepository fileRepository; - - private final @NonNull MetadataUpdateService metadataUpdateService; - - @PostMapping("/projects") - ResponseEntity> register(@RequestBody final Project project, - final PersistentEntityResourceAssembler assembler) { - Project result = projectService.register(project); - return ResponseEntity.ok().body(assembler.toFullResource(result)); - } - - @PostMapping("/projects/suggestion") - ResponseEntity> suggest(@RequestBody final ObjectNode suggestion, - final PersistentEntityResourceAssembler assembler) { - Project suggestedProject = projectService.createSuggestedProject(suggestion); - return ResponseEntity.ok().body(assembler.toFullResource(suggestedProject)); - } - - @CheckAllowed(value = "#project.isEditable()", exception = NotAllowedWithSubmissionInStateException.class) - @PatchMapping("/projects/{id}") - ResponseEntity> update(@PathVariable("id") final Project project, - @RequestParam(value = "partial", defaultValue = "false") Boolean partial, - @RequestBody final ObjectNode patch, final PersistentEntityResourceAssembler assembler) { - - List allowedFields = List.of( - "accessionDate", - "cellCount", - "content", - "dataAccess", - "identifyingOrganisms", - "isInCatalogue", - "organ", - "primaryWrangler", - "publicationsInfo", - "releaseDate", - "secondaryWrangler", - "technology", - "validationErrors", - "wranglingState", - "wranglingPriority", - "wranglingNotes", - "dcpReleaseNumber", - "projectLabels", - "projectNetworks" - ); - - ObjectNode validPatch = patch.retain(allowedFields); - Project updatedProject = projectService.update(project, validPatch, !partial); - return ResponseEntity.ok().body(assembler.toFullResource(updatedProject)); - } - - @CheckAllowed(value = "#submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @PostMapping(path = "submissionEnvelopes/{sub_id}/projects") - ResponseEntity> addProjectToEnvelope( - @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - @RequestBody Project project, - @RequestParam("updatingUuid") Optional updatingUuid, - PersistentEntityResourceAssembler assembler) { - updatingUuid.ifPresent(uuid -> { - project.setUuid(new Uuid(uuid.toString())); - project.setIsUpdate(true); - }); - Project entity = getProjectService().addProjectToSubmissionEnvelope(submissionEnvelope, project); - PersistentEntityResource resource = assembler.toFullResource(entity); - return ResponseEntity.accepted().body(resource); - } - - @GetMapping(path = "/projects/{id}/bundleManifests") - ResponseEntity>> getBundleManifests( - @PathVariable("id") Project project, - @RequestParam("bundleType") Optional bundleType, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - Page bundleManifests = projectService.getBundleManifestRepository().findBundleManifestsByProjectAndBundleType(project, bundleType.orElse(null), pageable); - return ResponseEntity.ok(pagedResourcesAssembler.toResource(bundleManifests, resourceAssembler)); - } - - @GetMapping(path = "/projects/{id}/submissionEnvelopes") - ResponseEntity>> getProjectSubmissionEnvelopes( - @PathVariable("id") Project project, Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - var envelopes = projectService.getSubmissionEnvelopes(project); - var resultPage = new PageImpl<>(new ArrayList<>(envelopes), pageable, envelopes.size()); - return ResponseEntity.ok(pagedResourcesAssembler.toResource(resultPage, resourceAssembler)); - } - - @RequestMapping(path = "/projects/{project_id}/biomaterials", method = RequestMethod.GET) - ResponseEntity getBiomaterials(@PathVariable("project_id") Project project, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - Page biomaterials = getBiomaterialRepository().findByProject(project, pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(biomaterials, resourceAssembler)); - } - - @RequestMapping(path = "/projects/{project_id}/processes", method = RequestMethod.GET) - ResponseEntity getProcesses(@PathVariable("project_id") Project project, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - Page processes = getProcessRepository().findByProject(project, pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(processes, resourceAssembler)); - } - - @RequestMapping(path = "/projects/{project_id}/protocols", method = RequestMethod.GET) - ResponseEntity getProtocols(@PathVariable("project_id") Project project, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - Page protocols = getProtocolRepository().findByProject(project, pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(protocols, resourceAssembler)); - } - - @RequestMapping(path = "/projects/{project_id}/files", method = RequestMethod.GET) - ResponseEntity getFiles(@PathVariable("project_id") Project project, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - Page files = getFileRepository().findByProject(project, pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(files, resourceAssembler)); - } - - @CheckAllowed(value = "#submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @PutMapping(path = "projects/{proj_id}/submissionEnvelopes/{sub_id}") - ResponseEntity> linkSubmissionToProject( - @PathVariable("proj_id") Project project, - @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - PersistentEntityResourceAssembler assembler) { - Project savedProject = getProjectService().linkProjectSubmissionEnvelope(submissionEnvelope, project); - PersistentEntityResource projectResource = assembler.toFullResource(savedProject); - return ResponseEntity.accepted().body(projectResource); - } - - @CheckAllowed(value = "#project.isEditable()", exception = NotAllowedWithSubmissionInStateException.class) - @DeleteMapping(path = "projects/{id}") - public ResponseEntity delete(@PathVariable("id") Project project) { - try { - projectService.delete(project); - return ResponseEntity.noContent().build(); - } catch (NonEmptyProject nonEmptyProject) { - String message = nonEmptyProject.getMessage(); - LOGGER.debug(message); - Map errorResponse = Map.of("message", message); - return ResponseEntity.status(HttpStatus.CONFLICT).body(errorResponse); - } - } - - @GetMapping(path = "projects/filter") - @Secured({"ROLE_WRANGLER", "ROLE_SERVICE"}) - public ResponseEntity>> filterProjects( - @ModelAttribute SearchFilter searchFilter, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - var projects = projectService.filterProjects(searchFilter, pageable); - return ResponseEntity.ok(pagedResourcesAssembler.toResource(projects, resourceAssembler)); - } - - @GetMapping(path="projects/{id}/auditLogs") - public ResponseEntity getProjectAuditLogs(@PathVariable("id") Project project) { - if (project == null) { - return ResponseEntity.notFound().build(); - } - - return ResponseEntity.ok(projectService.getProjectAuditEntries(project)); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/project/web/ProjectResourceProcessor.java b/src/main/java/org/humancellatlas/ingest/project/web/ProjectResourceProcessor.java deleted file mode 100644 index 1b1bc0e85..000000000 --- a/src/main/java/org/humancellatlas/ingest/project/web/ProjectResourceProcessor.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.humancellatlas.ingest.project.web; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.project.Project; -import org.springframework.hateoas.EntityLinks; -import org.springframework.hateoas.Link; -import org.springframework.hateoas.Resource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.stereotype.Component; - -@Component -@RequiredArgsConstructor -public class ProjectResourceProcessor implements ResourceProcessor>{ - private final @NonNull EntityLinks entityLinks; - - @Override - public Resource process(Resource resource) { - Project project = resource.getContent(); - resource.add(getBundleManifestsLink(project)); - resource.add(getAuditLogsLink(project)); - - return resource; - } - - private Link getBundleManifestsLink(Project project) { - return entityLinks.linkForSingleResource(project) - .slash(Links.BUNDLE_MANIFESTS_URL) - .withRel(Links.BUNDLE_MANIFESTS_REL); - } - - private Link getAuditLogsLink(Project project) { - return entityLinks.linkForSingleResource(project) - .slash(Links.AUDIT_LOGS_URL) - .withRel(Links.AUDIT_LOGS_REL); - } - - -} diff --git a/src/main/java/org/humancellatlas/ingest/project/web/SearchFilter.java b/src/main/java/org/humancellatlas/ingest/project/web/SearchFilter.java deleted file mode 100644 index fe619170b..000000000 --- a/src/main/java/org/humancellatlas/ingest/project/web/SearchFilter.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.humancellatlas.ingest.project.web; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Getter; -import lombok.ToString; -import org.humancellatlas.ingest.project.DataAccessTypes; - -import java.util.List; - -@AllArgsConstructor -@Builder -@ToString -public class SearchFilter { - @Getter String search; - @Getter String wranglingState; - @Getter String primaryWrangler; - @Getter Integer wranglingPriority; - @Getter Boolean hasOfficialHcaPublication; - @Getter String identifyingOrganism; - @Getter String organOntology; - @Getter Integer minCellCount; - @Getter Integer maxCellCount; - @Getter Integer dcpReleaseNumber; - @Getter DataAccessTypes dataAccess; - @Getter String projectLabels; - @Getter String projectNetworks; - @Getter Boolean hcaCatalogue; - - @Builder.Default - @Getter SearchType searchType = SearchType.AllKeywords; -} diff --git a/src/main/java/org/humancellatlas/ingest/project/web/SearchType.java b/src/main/java/org/humancellatlas/ingest/project/web/SearchType.java deleted file mode 100644 index eb8a432a8..000000000 --- a/src/main/java/org/humancellatlas/ingest/project/web/SearchType.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.humancellatlas.ingest.project.web; - -public enum SearchType { - AnyKeyword, - AllKeywords, - ExactMatch -} diff --git a/src/main/java/org/humancellatlas/ingest/protocol/Protocol.java b/src/main/java/org/humancellatlas/ingest/protocol/Protocol.java deleted file mode 100644 index 8b783688e..000000000 --- a/src/main/java/org/humancellatlas/ingest/protocol/Protocol.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.humancellatlas.ingest.protocol; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import org.humancellatlas.ingest.core.EntityType; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.project.Project; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.DBRef; - -@Getter -@EqualsAndHashCode(callSuper = true, exclude = {"project"}) -@NoArgsConstructor -public class Protocol extends MetadataDocument { - @Indexed - private @Setter - @DBRef(lazy = true) - Project project; - - private boolean linked = false; - - @JsonCreator - public Protocol(@JsonProperty("content") Object content) { - super(EntityType.PROTOCOL, content); - } - - public boolean isLinked() { - return linked; - } - - /* TODO - This method was originally made as simple as possible to only support the orphaned entity use case. However, - this can be enhanced further to add a full-fledged component that enables back linking to all Processes that refer - to this Protocol. In that case, the isLinked implementation will need to be changed to check if the list of - processes is empty or not. This approach was not initially chosen because it would have required data migration. - */ - public void markAsLinked() { - this.linked = true; - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/protocol/ProtocolRepository.java b/src/main/java/org/humancellatlas/ingest/protocol/ProtocolRepository.java deleted file mode 100644 index cbd2a4870..000000000 --- a/src/main/java/org/humancellatlas/ingest/protocol/ProtocolRepository.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.humancellatlas.ingest.protocol; - -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.data.mongodb.repository.Query; -import org.springframework.data.repository.query.Param; -import org.springframework.data.rest.core.annotation.RestResource; -import org.springframework.web.bind.annotation.CrossOrigin; - -import java.util.Collection; -import java.util.Optional; -import java.util.UUID; -import java.util.stream.Stream; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 31/08/17 - */ -@CrossOrigin -public interface ProtocolRepository extends MongoRepository { - - public Page findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope, Pageable pageable); - - public Page findByProject(Project project, Pageable pageable); - - @RestResource(exported = false) - Stream findByProject(Project project); - - @RestResource(exported = false) - public Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - @RestResource(exported = false) - Long deleteBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - @RestResource(rel = "findBySubmissionAndValidationState") - public Page findBySubmissionEnvelopeAndValidationState(@Param("envelopeUri") SubmissionEnvelope submissionEnvelope, - @Param("state") ValidationState state, - Pageable pageable); - - @Query(value = "{'submissionEnvelope.id': ?0, graphValidationErrors: { $exists: true, $not: {$size: 0} } }") - @RestResource(rel = "findBySubmissionIdWithGraphValidationErrors") - public Page findBySubmissionIdWithGraphValidationErrors( - @Param("envelopeId") String envelopeId, - Pageable pageable - ); - - @RestResource(exported = false) - Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - @RestResource(rel = "findAllByUuid", path = "findAllByUuid") - Page findByUuid(@Param("uuid") Uuid uuid, Pageable pageable); - - @RestResource(rel = "findByUuid", path = "findByUuid") - Optional findByUuidUuidAndIsUpdateFalse(@Param("uuid") UUID uuid); - - long countBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); - - long countBySubmissionEnvelopeAndValidationState(SubmissionEnvelope submissionEnvelope, ValidationState validationState); - - @Query(value = "{'submissionEnvelope.id': ?0, graphValidationErrors: { $exists: true, $not: {$size: 0} } }", count = true) - long countBySubmissionEnvelopeAndCountWithGraphValidationErrors(String submissionEnvelopeId); - - -} diff --git a/src/main/java/org/humancellatlas/ingest/protocol/ProtocolService.java b/src/main/java/org/humancellatlas/ingest/protocol/ProtocolService.java deleted file mode 100644 index fc5c4ac5a..000000000 --- a/src/main/java/org/humancellatlas/ingest/protocol/ProtocolService.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.humancellatlas.ingest.protocol; - -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.service.MetadataCrudService; -import org.humancellatlas.ingest.core.service.MetadataUpdateService; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.stereotype.Service; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 05/09/17 - */ -@Service -@RequiredArgsConstructor -@Getter -public class ProtocolService { - - private final @NonNull MetadataCrudService metadataCrudService; - private final @NonNull MetadataUpdateService metadataUpdateService; - - private final @NonNull SubmissionEnvelopeRepository submissionEnvelopeRepository; - private final @NonNull ProjectRepository projectRepository; - private final @NonNull ProtocolRepository protocolRepository; - private final @NonNull ProcessRepository processRepository; - - - private final Logger log = LoggerFactory.getLogger(getClass()); - - protected Logger getLog() { - return log; - } - - public Protocol addProtocolToSubmissionEnvelope(SubmissionEnvelope submissionEnvelope, Protocol protocol) { - if (!protocol.getIsUpdate()) { - projectRepository.findBySubmissionEnvelopesContains(submissionEnvelope).findFirst().ifPresent(protocol::setProject); - return metadataCrudService.addToSubmissionEnvelopeAndSave(protocol, submissionEnvelope); - } else { - return metadataUpdateService.acceptUpdate(protocol, submissionEnvelope); - } - } - - public Page retrieve(SubmissionEnvelope submission, Pageable pageable) { - Page protocols = protocolRepository.findBySubmissionEnvelope(submission, pageable); - protocols.forEach(protocol -> { - processRepository.findFirstByProtocolsContains(protocol).ifPresent(it -> protocol.markAsLinked()); - }); - return protocols; - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/protocol/web/ProtocolController.java b/src/main/java/org/humancellatlas/ingest/protocol/web/ProtocolController.java deleted file mode 100644 index 2f9f2c293..000000000 --- a/src/main/java/org/humancellatlas/ingest/protocol/web/ProtocolController.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.humancellatlas.ingest.protocol.web; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.core.service.MetadataCrudService; -import org.humancellatlas.ingest.core.service.MetadataUpdateService; -import org.humancellatlas.ingest.patch.JsonPatcher; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.humancellatlas.ingest.protocol.ProtocolService; -import org.humancellatlas.ingest.security.CheckAllowed; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.exception.NotAllowedDuringSubmissionStateException; -import org.springframework.data.rest.webmvc.PersistentEntityResource; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.data.web.PagedResourcesAssembler; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.hateoas.Resource; -import org.springframework.http.HttpEntity; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; - -import java.util.List; -import java.util.Optional; -import java.util.UUID; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 05/09/17 - */ -@RepositoryRestController -@ExposesResourceFor(Protocol.class) -@RequiredArgsConstructor -@Getter -public class ProtocolController { - private final @NonNull ProtocolService protocolService; - private final @NonNull ProtocolRepository protocolRepository; - - private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; - - private final @NonNull JsonPatcher jsonPatcher; - - private final @NonNull MetadataCrudService metadataCrudService; - private final @NonNull MetadataUpdateService metadataUpdateService; - - @CheckAllowed(value = "#submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @RequestMapping(path = "/submissionEnvelopes/{sub_id}/protocols", method = RequestMethod.POST) - ResponseEntity> addProtocolToEnvelope(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - @RequestBody Protocol protocol, - @RequestParam("updatingUuid") Optional updatingUuid, - PersistentEntityResourceAssembler assembler) { - updatingUuid.ifPresent(uuid -> { - protocol.setUuid(new Uuid(uuid.toString())); - protocol.setIsUpdate(true); - }); - Protocol entity = getProtocolService().addProtocolToSubmissionEnvelope(submissionEnvelope, protocol); - PersistentEntityResource resource = assembler.toFullResource(entity); - return ResponseEntity.accepted().body(resource); - } - - @CheckAllowed(value = "#submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @RequestMapping(path = "/submissionEnvelopes/{sub_id}/protocols/{protocol_id}", method = RequestMethod.PUT) - ResponseEntity> linkProtocolToEnvelope(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - @PathVariable("id") Protocol protocol, - PersistentEntityResourceAssembler assembler) { - Protocol entity = getProtocolService().addProtocolToSubmissionEnvelope(submissionEnvelope, protocol); - PersistentEntityResource resource = assembler.toFullResource(entity); - return ResponseEntity.accepted().body(resource); - } - - @CheckAllowed(value = "#protocol.submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @RequestMapping(path = "/protocols/{id}", method = RequestMethod.PATCH) - HttpEntity patchProtocol(@PathVariable("id") Protocol protocol, - @RequestBody final ObjectNode patch, - PersistentEntityResourceAssembler assembler) { - List allowedFields = List.of("content", "validationErrors", "graphValidationErrors"); - ObjectNode validPatch = patch.retain(allowedFields); - Protocol updatedProtocol = metadataUpdateService.update(protocol, validPatch); - PersistentEntityResource resource = assembler.toFullResource(updatedProtocol); - return ResponseEntity.accepted().body(resource); - } - - @CheckAllowed(value = "#protocol.submissionEnvelope.isEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @DeleteMapping(path = "/protocols/{id}") - ResponseEntity deleteProtocol(@PathVariable("id") Protocol protocol) { - metadataCrudService.deleteDocument(protocol); - return ResponseEntity.noContent().build(); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/query/MetadataQueryService.java b/src/main/java/org/humancellatlas/ingest/query/MetadataQueryService.java deleted file mode 100644 index 7b93676b0..000000000 --- a/src/main/java/org/humancellatlas/ingest/query/MetadataQueryService.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.humancellatlas.ingest.query; - -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.core.EntityType; -import org.springframework.stereotype.Service; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageImpl; -import org.springframework.data.domain.Pageable; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.data.rest.webmvc.ResourceNotFoundException; - -import java.util.List; - -/** - * Created by prabhat on 02/11/2020. - */ -@Service -public class MetadataQueryService { - @Autowired - private MongoTemplate mongoTemplate; - - @Autowired - private QueryBuilder queryBuilder; - - public Page findByCriteria(EntityType metadataType, List criteriaList, Boolean andCriteria, Pageable pageable) { - Query query = queryBuilder.build(criteriaList, andCriteria); - List result = mongoTemplate.find(query.with(pageable), getEntityClass(metadataType)); - long count = mongoTemplate.count(query, getEntityClass(metadataType)); - - return new PageImpl<>(result, pageable, count); - }; - - Class getEntityClass(EntityType metadataType) { - switch (metadataType) { - case BIOMATERIAL: - return Biomaterial.class; - case PROTOCOL: - return Protocol.class; - case PROJECT: - return Project.class; - case PROCESS: - return Process.class; - case FILE: - return File.class; - default: - throw new ResourceNotFoundException(); - } - } -} diff --git a/src/main/java/org/humancellatlas/ingest/query/Operator.java b/src/main/java/org/humancellatlas/ingest/query/Operator.java deleted file mode 100644 index 6dde472a8..000000000 --- a/src/main/java/org/humancellatlas/ingest/query/Operator.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.humancellatlas.ingest.query; - -public enum Operator { - GT, - GTE, - LT, - LTE, - IN, - NIN, - IS, - NE, - REGEX -} diff --git a/src/main/java/org/humancellatlas/ingest/query/QueryBuilder.java b/src/main/java/org/humancellatlas/ingest/query/QueryBuilder.java deleted file mode 100644 index 6e641e168..000000000 --- a/src/main/java/org/humancellatlas/ingest/query/QueryBuilder.java +++ /dev/null @@ -1,68 +0,0 @@ -package org.humancellatlas.ingest.query; - -import org.springframework.data.mongodb.InvalidMongoDbApiUsageException; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -@Component -public class QueryBuilder { - public Query build(List criteriaList, Boolean andCriteria){ - Query query = new Query(); - - List criterias = new ArrayList<>(); - - for (MetadataCriteria metadataCriteria : criteriaList) { - String field = metadataCriteria.getField(); - Criteria criteria = Criteria.where(field); - switch (metadataCriteria.getOperator()) { - case IS: - try { - criteria = criteria.is(metadataCriteria.getValue()); - } catch (InvalidMongoDbApiUsageException e) { - throw new IllegalArgumentException(e.getMessage(), e); - } - break; - case NE: - criteria = criteria.ne(metadataCriteria.getValue()); - break; - case GT: - criteria = criteria.gt(metadataCriteria.getValue()); - break; - case GTE: - criteria = criteria.gte(metadataCriteria.getValue()); - break; - case LT: - criteria = criteria.lt(metadataCriteria.getValue()); - break; - case LTE: - criteria = criteria.lte(metadataCriteria.getValue()); - break; - case IN: - criteria = criteria.in((Collection) metadataCriteria.getValue()); - break; - case NIN: - criteria = criteria.nin((Collection) metadataCriteria.getValue()); - break; - case REGEX: - criteria = criteria.regex((String) metadataCriteria.getValue(), "i"); - break; - default: - throw new IllegalArgumentException(String.format("MetadataCriteria %s is not supported.", metadataCriteria.getOperator())); - } - criterias.add(criteria); - } - - if (andCriteria) { - query.addCriteria(new Criteria().andOperator(criterias.toArray(new Criteria[criterias.size()]))); - } else { - query.addCriteria(new Criteria().orOperator(criterias.toArray(new Criteria[criterias.size()]))); - } - - return query; - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/schemas/Schema.java b/src/main/java/org/humancellatlas/ingest/schemas/Schema.java deleted file mode 100644 index f028422f6..000000000 --- a/src/main/java/org/humancellatlas/ingest/schemas/Schema.java +++ /dev/null @@ -1,102 +0,0 @@ -package org.humancellatlas.ingest.schemas; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Getter; -import org.humancellatlas.ingest.core.AbstractEntity; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static java.lang.String.format; - -@Getter -public class Schema extends AbstractEntity implements Comparable { - - private String highLevelEntity; - private String schemaVersion; - private String domainEntity; - private String subDomainEntity; - private String concreteEntity; - - private String compoundKeys; - - @JsonIgnore - private String schemaUri; - - public Schema(String highLevelEntity, String schemaVersion, String domainEntity, - String subDomainEntity, String concreteEntity, String schemaUri) { - this.highLevelEntity = highLevelEntity; - this.schemaVersion = schemaVersion; - this.domainEntity = domainEntity; - this.subDomainEntity = subDomainEntity; - this.concreteEntity = concreteEntity; - this.schemaUri = schemaUri; - this.compoundKeys = concatenateKeys(); - } - - private String concatenateKeys() { - return format("%s/%s/%s/%s", highLevelEntity, domainEntity, subDomainEntity, - concreteEntity); - } - - @Override - public int compareTo(Schema other) { - int difference = compoundKeys.compareTo(other.compoundKeys); - if (difference == 0) { - SemanticVersion otherSchemaVersion = SemanticVersion.parse(other.schemaVersion); - difference = SemanticVersion.parse(schemaVersion).compareTo(otherSchemaVersion); - } - return difference; - } - - private static class SemanticVersion implements Comparable { - - private static final Pattern VERSION_PATTERN = Pattern.compile("(?\\p{Digit}+)" + - "(.(?\\p{Digit}+))??(.(?\\p{Digit}+))??"); - - final int major; - final int minor; - int patch; - - SemanticVersion(int major, int minor, int patch) { - this.major = major; - this.minor = minor; - this.patch = patch; - } - - static SemanticVersion parse(String version) { - Matcher match = VERSION_PATTERN.matcher(version); - if (match.matches()) { - int major = parseVersionSegment(match, "major"); - int minor = parseVersionSegment(match, "minor"); - int patch = parseVersionSegment(match, "patch"); - return new SemanticVersion(major, minor, patch); - } else { - throw new RuntimeException("Invalid version format."); - } - } - - private static int parseVersionSegment(Matcher match, String versionSegment) { - String segmentValue = match.group(versionSegment); - int value = 0; - if (segmentValue != null) { - value = Integer.parseInt(segmentValue); - } - return value; - } - - @Override - public int compareTo(SemanticVersion other) { - int difference = major - other.major; - if (difference == 0) { - difference = minor - other.minor; - if (difference == 0) { - difference = patch - other.patch; - } - } - return difference; - } - - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/schemas/SchemaRepository.java b/src/main/java/org/humancellatlas/ingest/schemas/SchemaRepository.java deleted file mode 100644 index 8122d4bf8..000000000 --- a/src/main/java/org/humancellatlas/ingest/schemas/SchemaRepository.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.humancellatlas.ingest.schemas; - -import org.humancellatlas.ingest.core.Uuid; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.data.mongodb.repository.Query; -import org.springframework.data.repository.query.Param; -import org.springframework.data.rest.core.annotation.RestResource; -import org.springframework.web.bind.annotation.CrossOrigin; - -import java.util.List; -import java.util.stream.Stream; - -/** - * Created by rolando on 18/04/2018. - */ -@CrossOrigin -public interface SchemaRepository extends MongoRepository{ - @RestResource(exported = false) - S save(S schema); - - @RestResource(exported = false) - List save(Iterable schemas); - - @RestResource(exported = false) - void delete(Schema schema); - - @RestResource(exported = false) - List findByUuidEquals(Uuid uuid); - - @RestResource - Page findBySchemaVersionAfter(@Param("schema-version-range") String schemaVersionRange, Pageable pageable); - - @RestResource(rel = "querySchemas") - @Query("{$and :[" - + "?#{ [0] == null ? { $where : 'true'} : { 'highLevelEntity' : {'$regex' : [0]} } }," - + "?#{ [1] == null ? { $where : 'true'} : { 'concreteEntity' : {'$regex' : [1]} } }," - + "?#{ [2] == null ? { $where : 'true'} : { 'domainEntity' : {'$regex' : [2]} } }," - + "?#{ [3] == null ? { $where : 'true'} : { 'subDomainEntity' : {'$regex' : [3]} } }," - + "?#{ [4] == null ? { $where : 'true'} : { 'schemaVersion' : {'$regex' : [4]} } }" - + "]}") - Page querySchemas(@Param("high-level-entity") String highLevelEntity, - @Param("concrete-entity") String concreteEntity, - @Param("domain-entity") String domainEntity, - @Param("sub-domain-entity") String subDomainEntity, - @Param("schema-version") String schemaVersion, - Pageable pageable); - - @RestResource(exported = false) - Stream findAllByOrderBySchemaVersionDesc(); - - Page findAllByOrderBySchemaVersionDesc(Pageable pageable); - -} diff --git a/src/main/java/org/humancellatlas/ingest/schemas/SchemaService.java b/src/main/java/org/humancellatlas/ingest/schemas/SchemaService.java deleted file mode 100644 index a4109b875..000000000 --- a/src/main/java/org/humancellatlas/ingest/schemas/SchemaService.java +++ /dev/null @@ -1,157 +0,0 @@ -package org.humancellatlas.ingest.schemas; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.schemas.schemascraper.SchemaScraper; -import org.humancellatlas.ingest.schemas.schemascraper.impl.SchemaScrapeException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.env.Environment; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; - -import java.net.URI; -import java.util.*; -import java.util.stream.Collectors; - -@Service -@Getter -@NoArgsConstructor -@AllArgsConstructor -public class SchemaService { - - @Autowired - private SchemaRepository schemaRepository; - - @Autowired - private SchemaScraper schemaScraper; - - @Autowired - private Environment environment; - - private static final int EVERY_24_HOURS = 1000 * 60 * 60 * 24; - - public List filterLatestSchemas(String highLevelEntity) { - return getLatestSchemas().stream() - .filter(schema -> schema.getHighLevelEntity().matches(highLevelEntity)) - .collect(Collectors.toList()); - } - - public List getLatestSchemas() { - List allSchemas = schemaRepository.findAll(); - allSchemas.sort(Collections.reverseOrder()); - - Set latestSchemas = new LinkedHashSet<>(); - allSchemas.stream() - .map(LatestSchema::new) - .forEach(latestSchemas::add); - - return latestSchemas.stream() - .map(LatestSchema::getSchema) - .collect(Collectors.toList()); - } - - public Schema getLatestSchemaByEntityType(String highLevelEntity, String entityType) { - List allLatestSchema = filterLatestSchemas(highLevelEntity).stream() - .filter(schema -> schema.getConcreteEntity().matches(entityType)) - .collect(Collectors.toList()); - - return allLatestSchema.size() > 0 ? allLatestSchema.get(0) : null; - } - - @Scheduled(fixedDelay = EVERY_24_HOURS) - public void updateSchemasCollection() { - String schemaBaseUri = getSchemaBaseUri(); - - if (schemaBaseUri == null) - throw new SchemaScrapeException("SCHEMA_BASE_URI environmental variable should not be null."); - - if (schemaBaseUri.endsWith("/")) { - schemaBaseUri = schemaBaseUri.substring(0, schemaBaseUri.length() - 1); - } - - // TODO Find a way how to neatly exclude the files - schemaScraper.getAllSchemaURIs(URI.create(schemaBaseUri)).stream() - .filter(schemaUri -> !schemaUri.toString().contains("index.html") && !schemaUri.toString().contains("property_migrations")) - .forEach(this::doUpdate); - } - - public String getSchemaBaseUri() { - return environment.getProperty("SCHEMA_BASE_URI"); - } - - private void doUpdate(URI schemaUri) { - Schema schemaDocument = schemaDescriptionFromSchemaUri(schemaUri); - - UUID schemaUuid = UUID.nameUUIDFromBytes(schemaUri.toString().getBytes()); - schemaDocument.setUuid(new Uuid(schemaUuid.toString())); - - deleteMatchingSchemas(schemaUuid); - schemaRepository.save(schemaDocument); - } - - private void deleteMatchingSchemas(UUID schemaUuid) { - Collection matchingSchemas = schemaRepository - .findByUuidEquals(new Uuid(schemaUuid.toString())); - schemaRepository.deleteAll(matchingSchemas); - } - - public Collection schemaDescriptionFromSchemaUris(Collection schemaUris) { - return schemaUris.stream() - .map(this::schemaDescriptionFromSchemaUri) - .collect(Collectors.toList()); - } - - private Schema schemaDescriptionFromSchemaUri(URI schemaUri) { - String[] splitString = schemaUri.toString().split("/"); - String schemaFullUri = environment.getProperty("SCHEMA_BASE_URI") + schemaUri; - - if(splitString.length == 3) { // then this is a bundle schema - return new Schema(splitString[0], splitString[1], "", "", splitString[2], schemaFullUri); - } else if(splitString.length == 4) { - return new Schema(splitString[0], splitString[2], splitString[1], "", splitString[3], schemaFullUri); - } else if(splitString.length == 5) { - return new Schema(splitString[0], splitString[3], splitString[1], splitString[2], splitString[4], schemaFullUri); - } else { - throw new SchemaScrapeException("Couldn't construct a Schema document from URI: " + schemaFullUri); - } - } - - /** - * - * A wrapper for Schema documents used to define a looser equals()/hashCode() - * to determine equivalence of Schemas based only on a Schema's high level entity, - * type, etc. - * - */ - private class LatestSchema { - @Getter - private final Schema schema; - - LatestSchema(Schema schema) { - this.schema = schema; - } - - @Override - public boolean equals(Object to) { - if (to == this) return true; - if (!(to instanceof LatestSchema)) { - return false; - } - - LatestSchema schema = (LatestSchema) to; - return schema.hashCode() == this.hashCode(); - } - - @Override - public int hashCode(){ - int result = 17; - result = 31 * result + this.schema.getConcreteEntity().hashCode(); - result = 31 * result + this.schema.getHighLevelEntity().hashCode(); - result = 31 * result + this.schema.getDomainEntity().hashCode(); - result = 31 * result + this.schema.getSubDomainEntity().hashCode(); - return result; - } - } -} diff --git a/src/main/java/org/humancellatlas/ingest/schemas/schemascraper/SchemaScraper.java b/src/main/java/org/humancellatlas/ingest/schemas/schemascraper/SchemaScraper.java deleted file mode 100644 index e40f2655e..000000000 --- a/src/main/java/org/humancellatlas/ingest/schemas/schemascraper/SchemaScraper.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.humancellatlas.ingest.schemas.schemascraper; - - -import java.net.URI; -import java.util.Collection; - -/** - * Created by rolando on 19/04/2018. - * - * Collects schemas from schema.humancellatlas.org (or some other schema bucket) - * - */ -public interface SchemaScraper { - Collection getAllSchemaURIs(URI schemaBucketLocation); -} diff --git a/src/main/java/org/humancellatlas/ingest/schemas/schemascraper/impl/ListBucketResult.java b/src/main/java/org/humancellatlas/ingest/schemas/schemascraper/impl/ListBucketResult.java deleted file mode 100644 index 6a7d524f2..000000000 --- a/src/main/java/org/humancellatlas/ingest/schemas/schemascraper/impl/ListBucketResult.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.humancellatlas.ingest.schemas.schemascraper.impl; - -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; - -import java.util.ArrayList; -import java.util.List; - -/** - * Created by rolando on 19/04/2018. - */ - -public class ListBucketResult { - public ListBucketResult() {} - - @JacksonXmlProperty(localName = "Name") - public String name; - - @JacksonXmlProperty(localName = "Contents") - public List contents = new ArrayList<>(); - - static class Content { - Content() {} - @JacksonXmlProperty(localName = "Key") - public String key; - - public void setKey(String key) { - this.key = key; - } - - public String getKey() { - return this.key; - } - } -} diff --git a/src/main/java/org/humancellatlas/ingest/schemas/schemascraper/impl/S3BucketSchemaScraper.java b/src/main/java/org/humancellatlas/ingest/schemas/schemascraper/impl/S3BucketSchemaScraper.java deleted file mode 100644 index e278ad615..000000000 --- a/src/main/java/org/humancellatlas/ingest/schemas/schemascraper/impl/S3BucketSchemaScraper.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.humancellatlas.ingest.schemas.schemascraper.impl; - -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.dataformat.xml.JacksonXmlModule; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import lombok.NonNull; -import org.humancellatlas.ingest.schemas.schemascraper.SchemaScraper; -import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; -import org.springframework.stereotype.Service; -import org.springframework.web.client.RestTemplate; - -import java.io.IOException; -import java.net.URI; -import java.util.Collection; -import java.util.stream.Collectors; - -/** - * Created by rolando on 19/04/2018. - * - * Scrapes schemas from an s3 bucket's default XML file-list page - * - */ -@Service -public class S3BucketSchemaScraper implements SchemaScraper { - private final @NonNull RestTemplate restTemplate; - private final @NonNull XmlMapper xmlMapper; - - public S3BucketSchemaScraper(){ - this.restTemplate = new RestTemplate(new HttpComponentsClientHttpRequestFactory()); - - JacksonXmlModule xmlModule = new JacksonXmlModule(); - xmlModule.setDefaultUseWrapper(false); - XmlMapper xmlMapper = new XmlMapper(xmlModule); - xmlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - this.xmlMapper = xmlMapper; - - } - - @Override - public Collection getAllSchemaURIs(URI schemaBucketLocation) { - String bucketListingXmlString = this.restTemplate.getForObject(schemaBucketLocation, String.class); - try { - ListBucketResult listBucketResult = xmlMapper.readValue(bucketListingXmlString, ListBucketResult.class); - return listBucketResult.contents.stream() - .map(content -> URI.create(content.getKey())) - .collect(Collectors.toList()); - } catch (IOException e) { - throw new SchemaScrapeException("Failed to parse schema bucket xml at URL: " + schemaBucketLocation, e); - } - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/schemas/schemascraper/impl/SchemaScrapeException.java b/src/main/java/org/humancellatlas/ingest/schemas/schemascraper/impl/SchemaScrapeException.java deleted file mode 100644 index ae2d4393f..000000000 --- a/src/main/java/org/humancellatlas/ingest/schemas/schemascraper/impl/SchemaScrapeException.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.humancellatlas.ingest.schemas.schemascraper.impl; - -/** - * Created by rolando on 19/04/2018. - */ -public class SchemaScrapeException extends RuntimeException { - public SchemaScrapeException(String message) { - super(message); - } - - public SchemaScrapeException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/schemas/web/SchemaController.java b/src/main/java/org/humancellatlas/ingest/schemas/web/SchemaController.java deleted file mode 100644 index 8e544d859..000000000 --- a/src/main/java/org/humancellatlas/ingest/schemas/web/SchemaController.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.humancellatlas.ingest.schemas.web; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.schemas.Schema; -import org.humancellatlas.ingest.schemas.SchemaService; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageImpl; -import org.springframework.data.domain.Pageable; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.data.web.PagedResourcesAssembler; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RequestParam; - -import java.util.List; - -/** - * Created by rolando on 19/04/2018. - */ -@RepositoryRestController -@ExposesResourceFor(Schema.class) -@RequiredArgsConstructor -public class SchemaController { - private final @NonNull SchemaService schemaService; - private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; - - @RequestMapping(path = "/schemas/update", method = RequestMethod.POST) - ResponseEntity triggerSchemasUpdate() { - schemaService.updateSchemasCollection(); - return new ResponseEntity<>(HttpStatus.ACCEPTED); - } - - @RequestMapping(path = "/schemas/search/latestSchemas", method = RequestMethod.GET) - ResponseEntity latestSchemas(Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - List latestSchemas = schemaService.getLatestSchemas(); - Page latestSchemasPage = generatePageFromSchemaList(pageable, latestSchemas); - return ResponseEntity.ok(pagedResourcesAssembler.toResource(latestSchemasPage, resourceAssembler)); - } - - @RequestMapping(path = "/schemas/search/filterLatestSchemas", method = RequestMethod.GET) - ResponseEntity filterLatestSchemas(@RequestParam String highLevelEntity, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - List latestSchemas = schemaService.filterLatestSchemas(highLevelEntity); - Page latestSchemasPage = generatePageFromSchemaList(pageable, latestSchemas); - return ResponseEntity.ok(pagedResourcesAssembler.toResource(latestSchemasPage, resourceAssembler)); - } - - private Page generatePageFromSchemaList(Pageable pageable, List schemaList) { - List latestSchemasSubList = schemaList.subList((int) (pageable.getOffset()), - (int) (pageable.getOffset() + Math.min(pageable.getOffset() + pageable.getPageSize(), - schemaList.size() - pageable.getOffset()))); - - return new PageImpl<>(latestSchemasSubList, pageable, schemaList.size()); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/schemas/web/SchemaResourceProcessor.java b/src/main/java/org/humancellatlas/ingest/schemas/web/SchemaResourceProcessor.java deleted file mode 100644 index 3f825de2f..000000000 --- a/src/main/java/org/humancellatlas/ingest/schemas/web/SchemaResourceProcessor.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.humancellatlas.ingest.schemas.web; - -import org.humancellatlas.ingest.schemas.Schema; -import org.springframework.hateoas.Link; -import org.springframework.hateoas.Resource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.stereotype.Component; - -/** - * Created by rolando on 19/04/2018. - */ -@Component -public class SchemaResourceProcessor implements ResourceProcessor> { - - public Resource process(Resource resource) { - resource.add(new Link(resource.getContent().getSchemaUri(), "json-schema")); - return resource; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/schemas/web/SchemaSearchProcessor.java b/src/main/java/org/humancellatlas/ingest/schemas/web/SchemaSearchProcessor.java deleted file mode 100644 index 9b45ec167..000000000 --- a/src/main/java/org/humancellatlas/ingest/schemas/web/SchemaSearchProcessor.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.humancellatlas.ingest.schemas.web; - - -import org.humancellatlas.ingest.schemas.Schema; -import org.springframework.data.rest.webmvc.RepositorySearchesResource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.stereotype.Component; - -import static org.springframework.hateoas.mvc.ControllerLinkBuilder.linkTo; -import static org.springframework.hateoas.mvc.ControllerLinkBuilder.methodOn; - - -/** - * Created by rolando on 23/04/2018. - */ -@Component -public class SchemaSearchProcessor implements ResourceProcessor { - - @Override - public RepositorySearchesResource process(RepositorySearchesResource searchesResource) { - if(searchesResource.getDomainType().equals(Schema.class)) { - searchesResource.add(linkTo(methodOn(SchemaController.class).latestSchemas(null, null)).withRel("latestSchemas")); - searchesResource.add(linkTo(methodOn(SchemaController.class).filterLatestSchemas(null, null, null)).withRel("filterLatestSchemas")); - } - - return searchesResource; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/security/Account.java b/src/main/java/org/humancellatlas/ingest/security/Account.java deleted file mode 100644 index 45caa4b82..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/Account.java +++ /dev/null @@ -1,85 +0,0 @@ -package org.humancellatlas.ingest.security; - -import lombok.Getter; -import lombok.Setter; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.Document; - -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; - -@Document -public class Account { - - public static final Account GUEST = new GuestAccount(); - public static final Account SERVICE = new ServiceAccount(); - /** - * A Null Object subclass of Account that represents an unregistered Guest. - */ - private static class GuestAccount extends Account { - - private static final String EMPTY = ""; - - private GuestAccount() { - super(EMPTY, EMPTY); - setName(EMPTY); - addRole(Role.GUEST); - } - - } - - /** - * A Null Object subclass of Account that represents a service. - */ - private static class ServiceAccount extends Account { - - private static final String EMPTY = ""; - - private ServiceAccount() { - super(EMPTY, EMPTY); - setName(EMPTY); - addRole(Role.SERVICE); - } - - } - - @Id - @Getter - private String id; - - @Indexed(unique=true) - @Getter - private String providerReference; - - @Getter - @Setter - private String name; - - private Set roles = new HashSet<>(); - - //needed for reflection used by frameworks - private Account() {} - - public Account(String providerReference) { - this.providerReference = providerReference; - } - - public Account(String id, String providerReference) { - this.id = id; - this.providerReference = providerReference; - } - - /** - * @return an unmodifiable Set of Roles. - */ - public Set getRoles() { - return Collections.unmodifiableSet(roles); - } - - public void addRole(Role role) { - roles.add(role); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/AccountService.java b/src/main/java/org/humancellatlas/ingest/security/AccountService.java deleted file mode 100644 index 1668fed76..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/AccountService.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.humancellatlas.ingest.security; - -public interface AccountService { - - Account register(Account account); - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/CheckAllowed.java b/src/main/java/org/humancellatlas/ingest/security/CheckAllowed.java deleted file mode 100644 index cb0a93e7d..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/CheckAllowed.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.humancellatlas.ingest.security; - -import org.humancellatlas.ingest.security.exception.NotAllowedException; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * Annotation for allowing a method to proceed based on the evaluation of a given SpEL input. - * See: https://docs.spring.io/spring-framework/docs/3.1.0.M1/spring-framework-reference/html/expressions.html - * for SpEL docs - * - * Allows a value (SpEL input) and an optional custom exception - * - * E.g.: - * @CheckAllowed("#foo.bar") - * public void myMethod(Object foo) { ... } - * - * or: - * @CheckAllowed(value = "#foo.bar", exception = MyCustomException.class) - * public void myMethod(Object foo) { ... } - * - * See SecurityAspect for the aspect and advice that use this. - */ -@Target({ElementType.METHOD, ElementType.TYPE}) -@Retention(RetentionPolicy.RUNTIME) -public @interface CheckAllowed { - String value(); - Class exception() default NotAllowedException.class; -} diff --git a/src/main/java/org/humancellatlas/ingest/security/CorsConfig.java b/src/main/java/org/humancellatlas/ingest/security/CorsConfig.java deleted file mode 100644 index 9d908e5d1..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/CorsConfig.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.humancellatlas.ingest.security; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.servlet.config.annotation.CorsRegistry; -import org.springframework.web.servlet.config.annotation.EnableWebMvc; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; - -@Configuration -@EnableWebMvc -public class CorsConfig implements WebMvcConfigurer { - - @Override - public void addCorsMappings(CorsRegistry registry) { - registry.addMapping("/**").allowedMethods("GET", "PATCH", "POST", "PUT", "DELETE").allowedOrigins("*") - .allowedHeaders("*"); - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/security/DefaultAccountService.java b/src/main/java/org/humancellatlas/ingest/security/DefaultAccountService.java deleted file mode 100644 index d848a6a8b..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/DefaultAccountService.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.humancellatlas.ingest.security; - -import org.humancellatlas.ingest.security.exception.DuplicateAccount; -import org.springframework.stereotype.Component; - -@Component -public class DefaultAccountService implements AccountService { - - private final AccountRepository accountRepository; - - public DefaultAccountService(AccountRepository accountRepository) { - this.accountRepository = accountRepository; - } - - @Override - public Account register(Account account) { - account.addRole(Role.CONTRIBUTOR); - Account persistentAccount = accountRepository.findByProviderReference(account.getProviderReference()); - if (persistentAccount != null) { - throw new DuplicateAccount(); - } - return this.accountRepository.save(account); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/ElixirConfig.java b/src/main/java/org/humancellatlas/ingest/security/ElixirConfig.java deleted file mode 100644 index be19ec98a..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/ElixirConfig.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.humancellatlas.ingest.security; - -import org.humancellatlas.ingest.security.authn.provider.elixir.ElixirAaiAuthenticationProvider; -import org.humancellatlas.ingest.security.authn.provider.elixir.ElixirJwkVault; -import org.humancellatlas.ingest.security.common.jwk.JwkVault; -import org.humancellatlas.ingest.security.common.jwk.JwtVerifierResolver; -import org.humancellatlas.ingest.security.common.jwk.UrlJwkProviderResolver; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.security.authentication.AuthenticationProvider; - -@Configuration -public class ElixirConfig { - - public static final String ELIXIR = "elixir"; - - @Value("${AUTH_ISSUER}") - private String issuer; - - @Bean - @Qualifier(ELIXIR) - public JwtVerifierResolver elixirJwtVerifierResolver() { - var urlJwkProviderResolver = new UrlJwkProviderResolver(issuer + "/jwk"); - ElixirJwkVault jwkVault = new ElixirJwkVault(urlJwkProviderResolver); - return new JwtVerifierResolver(jwkVault, null, issuer); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/GcpConfig.java b/src/main/java/org/humancellatlas/ingest/security/GcpConfig.java deleted file mode 100644 index e9ec075c3..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/GcpConfig.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.humancellatlas.ingest.security; - -import org.humancellatlas.ingest.security.authn.provider.gcp.GcpDomainWhiteList; -import org.humancellatlas.ingest.security.authn.provider.gcp.GcpJwkVault; -import org.humancellatlas.ingest.security.authn.provider.gcp.GoogleServiceJwtAuthenticationProvider; -import org.humancellatlas.ingest.security.common.jwk.JwtVerifierResolver; -import org.humancellatlas.ingest.security.common.jwk.UrlJwkProviderResolver; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.security.authentication.AuthenticationProvider; - -import java.util.List; - -@Configuration -public class GcpConfig { - - public static final String GCP = "gcp"; - - @Value("${GCP_JWK_PROVIDER_BASE_URL}") - private String googleJwkProviderBaseUrl; - - @Value(value = "${SVC_AUTH_AUDIENCE}") - private String serviceAudience; - - @Value(value= "#{('${GCP_PROJECT_WHITELIST}').split(',')}") - private List projectWhitelist; - - @Bean(name=GCP) - public AuthenticationProvider gcpAuthenticationProvider() { - var urlJwkProviderResolver = new UrlJwkProviderResolver(googleJwkProviderBaseUrl); - var googleJwkVault = new GcpJwkVault(urlJwkProviderResolver); - var googleJwtVerifierResolver = new JwtVerifierResolver(googleJwkVault, serviceAudience, null); - return new GoogleServiceJwtAuthenticationProvider(new GcpDomainWhiteList(projectWhitelist), - googleJwtVerifierResolver); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/Role.java b/src/main/java/org/humancellatlas/ingest/security/Role.java deleted file mode 100644 index 915dacf84..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/Role.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.humancellatlas.ingest.security; - -import org.springframework.security.core.GrantedAuthority; - -public enum Role implements GrantedAuthority { - - WRANGLER, CONTRIBUTOR, GUEST, SERVICE; - - @Override - public String getAuthority() { - return name(); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/SecurityAspect.java b/src/main/java/org/humancellatlas/ingest/security/SecurityAspect.java deleted file mode 100644 index c640f6c68..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/SecurityAspect.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.humancellatlas.ingest.security; - -import org.aspectj.lang.JoinPoint; -import org.aspectj.lang.annotation.Aspect; -import org.aspectj.lang.annotation.Before; -import org.aspectj.lang.reflect.MethodSignature; -import org.humancellatlas.ingest.security.exception.NotAllowedException; -import org.springframework.expression.ExpressionParser; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.expression.spel.support.StandardEvaluationContext; -import org.springframework.stereotype.Component; - -import java.lang.reflect.Method; - -@Aspect -@Component -public class SecurityAspect { - /** - * Advice that runs before any method with the CheckAllowed annotation. - * Parses the given SpEL in the annotation and throws error if the result returns False. - * - * @param joinPoint - * @throws Throwable - */ - @Before("@annotation(org.humancellatlas.ingest.security.CheckAllowed) && execution(* *(..))") - public void checkAllowed(JoinPoint joinPoint) throws Throwable { - MethodSignature signature = (MethodSignature) joinPoint.getSignature(); - Method method = signature.getMethod(); - CheckAllowed annotation = method.getAnnotation(CheckAllowed.class); - Boolean isAllowed = parseExpression(signature.getParameterNames(), joinPoint.getArgs(), annotation.value()); - Class exceptionClass = annotation.exception(); - if (!isAllowed) { - throw exceptionClass.getDeclaredConstructor().newInstance(); - } - } - - private Boolean parseExpression(String[] params, Object[] args, String expression) { - ExpressionParser parser = new SpelExpressionParser(); - StandardEvaluationContext context = new StandardEvaluationContext(); - - for (int i = 0; i < params.length; i++) { - context.setVariable(params[i], args[i]); - } - - return parser.parseExpression(expression).getValue(context, Boolean.class); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/security/SecurityConfig.java b/src/main/java/org/humancellatlas/ingest/security/SecurityConfig.java deleted file mode 100644 index 244517b13..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/SecurityConfig.java +++ /dev/null @@ -1,117 +0,0 @@ -package org.humancellatlas.ingest.security; - -import com.auth0.spring.security.api.BearerSecurityContextRepository; -import com.auth0.spring.security.api.JwtAuthenticationEntryPoint; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Configuration; -import org.springframework.http.HttpMethod; -import org.springframework.security.authentication.AuthenticationProvider; -import org.springframework.security.config.annotation.web.builders.HttpSecurity; -import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; -import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; -import org.springframework.security.config.http.SessionCreationPolicy; -import org.springframework.security.web.util.matcher.AntPathRequestMatcher; - -import javax.servlet.http.HttpServletRequest; -import java.util.*; -import java.util.stream.Stream; - -import static java.util.stream.Collectors.toList; -import static org.humancellatlas.ingest.security.ElixirConfig.ELIXIR; -import static org.humancellatlas.ingest.security.GcpConfig.GCP; -import static org.humancellatlas.ingest.security.Role.*; -import static org.springframework.http.HttpMethod.*; - -@EnableWebSecurity -@Configuration -public class SecurityConfig extends WebSecurityConfigurerAdapter { - private static final String FORWARDED_HOST = "x-forwarded-host"; - - private static final List SECURED_ANT_PATHS = setupSecuredAntPaths(); - - private static final List SECURED_WRANGLER_ANT_PATHS = setupWranglerAntPaths(); - - // The following endpoints are only secured when accessed from the outside the cluster - - private static List setupSecuredAntPaths() { - List antPathMatchers = new ArrayList<>(); - antPathMatchers.addAll(defineAntPathMatchers(POST, "/**")); - antPathMatchers.addAll(defineAntPathMatchers(PATCH, "/projects/*")); - return Collections.unmodifiableList(antPathMatchers); - } - - private static List setupWranglerAntPaths() { - List antPathMatchers = new ArrayList<>(); - antPathMatchers.addAll(defineAntPathMatchers(GET, "/bundleManifests")); - antPathMatchers.addAll(defineAntPathMatchers(GET, "/submissionManifests")); - - antPathMatchers.addAll(defineAntPathMatchers(GET, "/submissionEnvelopes")); - antPathMatchers.addAll(defineAntPathMatchers(GET, "/biomaterials")); - antPathMatchers.addAll(defineAntPathMatchers(GET, "/files")); - antPathMatchers.addAll(defineAntPathMatchers(GET, "/processes")); - antPathMatchers.addAll(defineAntPathMatchers(GET, "/protocols")); - - antPathMatchers.addAll(defineAntPathMatchers(GET, "/projects")); - antPathMatchers.addAll(defineAntPathMatchers(PUT, "/**")); - antPathMatchers.addAll(defineAntPathMatchers(PATCH, "/**")); - antPathMatchers.addAll(defineAntPathMatchers(DELETE, "/**")); - return Collections.unmodifiableList(antPathMatchers); - } - - private static List defineAntPathMatchers(HttpMethod method, - String... patterns) { - return Stream.of(patterns) - .map(pattern -> new AntPathRequestMatcher(pattern, method.name())) - .collect(toList()); - } - - private final AuthenticationProvider gcpAuthenticationProvider; - private final AuthenticationProvider elixirAuthenticationProvider; - - public SecurityConfig(@Qualifier(GCP) AuthenticationProvider gcp, - @Qualifier(ELIXIR) AuthenticationProvider elixir) { - this.gcpAuthenticationProvider = gcp; - this.elixirAuthenticationProvider = elixir; - } - - @Override - protected void configure(HttpSecurity http) throws Exception { - http.authenticationProvider(elixirAuthenticationProvider) - .authenticationProvider(gcpAuthenticationProvider) - .securityContext().securityContextRepository(new BearerSecurityContextRepository()) - .and() - .exceptionHandling().authenticationEntryPoint(new JwtAuthenticationEntryPoint()) - .and() - .httpBasic().disable() - .csrf().disable() - .sessionManagement().sessionCreationPolicy(SessionCreationPolicy.STATELESS).and() - .cors().and() - .authorizeRequests() - .antMatchers(POST, "/submissionEnvelopes").authenticated() - .antMatchers(POST, "/projects").authenticated() - .antMatchers(POST, "/projects/suggestion").permitAll() - .antMatchers(POST, "/projects/catalogue").permitAll() - .antMatchers(GET, "/user/**").authenticated() - .antMatchers(GET, "/auth/account").authenticated() - .antMatchers(POST, "/auth/registration").hasAuthority(GUEST.name()) - .requestMatchers(SecurityConfig::isSecuredEndpointFromOutside).authenticated() - .requestMatchers(SecurityConfig::isSecuredWranglerEndpointFromOutside) - .hasAnyAuthority(WRANGLER.name(), SERVICE.name()) - .antMatchers(GET, "/**").authenticated(); - } - - private static Boolean isSecuredEndpointFromOutside(HttpServletRequest request) { - return SECURED_ANT_PATHS.stream().anyMatch(matcher -> matcher.matches(request)) && - SecurityConfig.isRequestOutsideProxy(request); - } - - private static Boolean isSecuredWranglerEndpointFromOutside(HttpServletRequest request) { - return SECURED_WRANGLER_ANT_PATHS.stream().anyMatch(matcher -> matcher.matches(request)) && - SecurityConfig.isRequestOutsideProxy(request); - } - - private static Boolean isRequestOutsideProxy(HttpServletRequest request) { - return Optional.ofNullable(request.getHeader(FORWARDED_HOST)).isPresent(); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/UserAuditing.java b/src/main/java/org/humancellatlas/ingest/security/UserAuditing.java deleted file mode 100644 index e2ee8a545..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/UserAuditing.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.humancellatlas.ingest.security; - -import org.springframework.data.domain.AuditorAware; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.context.SecurityContextHolder; -import org.springframework.stereotype.Component; - -import java.util.Optional; - -/* - * User auditing to get the current user to put into the database: Based on https://docs.spring.io/spring-data/jpa/docs/current/reference/html/#auditing - */ - -@Component -public class UserAuditing implements AuditorAware { - - @Override - public Optional getCurrentAuditor() { - - Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); - if (authentication == null || !authentication.isAuthenticated()) { - return Optional.empty(); - } - - Object principal = authentication.getPrincipal(); - if (Account.class.isAssignableFrom(principal.getClass())) { - Account account = (Account) authentication.getPrincipal(); - return Optional.of(account.getId()); - } else { - return Optional.ofNullable(principal.toString()); - } - } - -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/security/authn/oidc/OpenIdAuthentication.java b/src/main/java/org/humancellatlas/ingest/security/authn/oidc/OpenIdAuthentication.java deleted file mode 100644 index 11c05d93e..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/authn/oidc/OpenIdAuthentication.java +++ /dev/null @@ -1,76 +0,0 @@ -package org.humancellatlas.ingest.security.authn.oidc; - -import org.humancellatlas.ingest.security.Account; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.GrantedAuthority; - -import java.util.Collection; - -public class OpenIdAuthentication implements Authentication { - - private Account account; - private UserInfo userInfo; - - private boolean authenticated = false; - - public OpenIdAuthentication(final Account principal) { - account = Account.GUEST; - if (principal != null) { - account = principal; - } - } - - public OpenIdAuthentication(Account principal, UserInfo credentials) { - this(principal); - authenticateWith(credentials); - } - - public OpenIdAuthentication(UserInfo credentials) { - this(null, credentials); - } - - @Override - public Object getPrincipal() { - return account; - } - - @Override - public Object getCredentials() { - return userInfo; - } - - @Override - public Collection getAuthorities() { - return account.getRoles(); - } - - @Override - public String getName() { - return account.getProviderReference(); - } - - @Override - public Object getDetails() { - return userInfo; - } - - @Override - public boolean isAuthenticated() { - return authenticated; - } - - @Override - public void setAuthenticated(boolean isAuthenticated) throws IllegalArgumentException { - throw new IllegalArgumentException("Operation not supported. Use authenticateWith to set status."); - } - - public void authenticateWith(UserInfo credentials) { - this.userInfo = credentials; - if (credentials == null) { - authenticated = false; - return; - } - authenticated = account == Account.GUEST || account == Account.SERVICE || credentials.getSubjectId().equalsIgnoreCase(account.getProviderReference()); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/authn/oidc/UserInfo.java b/src/main/java/org/humancellatlas/ingest/security/authn/oidc/UserInfo.java deleted file mode 100644 index 6d5765058..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/authn/oidc/UserInfo.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.humancellatlas.ingest.security.authn.oidc; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.*; -import org.humancellatlas.ingest.security.Account; - -@NoArgsConstructor -@AllArgsConstructor -@Getter -@Setter(AccessLevel.PROTECTED) -public class UserInfo { - - @JsonProperty("sub") - private String subjectId; - - private String name; - - @JsonProperty("preferred_username") - private String preferredUsername; - - @JsonProperty("given_name") - private String givenName; - - @JsonProperty("family_name") - private String familyName; - - private String email; - - public UserInfo(String subjectId, String name) { - this.subjectId = subjectId; - this.name = name; - } - - public Account toAccount() { - Account account = new Account(subjectId); - account.setName(name); - return account; - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/authn/provider/auth0/UserJwt.java b/src/main/java/org/humancellatlas/ingest/security/authn/provider/auth0/UserJwt.java deleted file mode 100644 index f59eb688b..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/authn/provider/auth0/UserJwt.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.humancellatlas.ingest.security.authn.provider.auth0; - -import com.auth0.jwt.JWT; -import com.auth0.jwt.interfaces.DecodedJWT; -import com.auth0.spring.security.api.authentication.JwtAuthentication; - -public class UserJwt { - private DecodedJWT token; - - public UserJwt(JwtAuthentication jwtAuthentication) { - this.token = JWT.decode(jwtAuthentication.getToken()); - } - - public String getGroup() { - String claimName = "https://auth.data.humancellatlas.org/group"; - return token.getClaim(claimName).asString(); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/security/authn/provider/auth0/UserJwtAuthenticationProvider.java b/src/main/java/org/humancellatlas/ingest/security/authn/provider/auth0/UserJwtAuthenticationProvider.java deleted file mode 100644 index 01170e8c5..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/authn/provider/auth0/UserJwtAuthenticationProvider.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.humancellatlas.ingest.security.authn.provider.auth0; - -import com.auth0.spring.security.api.JwtAuthenticationProvider; -import com.auth0.spring.security.api.authentication.JwtAuthentication; -import org.humancellatlas.ingest.security.exception.InvalidUserGroup; -import org.springframework.security.authentication.AuthenticationProvider; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.AuthenticationException; - -import javax.annotation.Nullable; - -import static java.util.Optional.ofNullable; - -public class UserJwtAuthenticationProvider implements AuthenticationProvider { - private final JwtAuthenticationProvider delegate; - - public UserJwtAuthenticationProvider(JwtAuthenticationProvider delegate) { - this.delegate = delegate; - } - - @Override - public Authentication authenticate(Authentication authentication) throws AuthenticationException { - Authentication jwtAuthentication = delegate.authenticate(authentication); - ofNullable(jwtAuthentication).ifPresent(auth -> { - JwtAuthentication jwt = (JwtAuthentication) authentication; - UserJwt user = new UserJwt(jwt); - verifyUser(user); - }); - return jwtAuthentication; - } - - private void verifyUser(UserJwt user) { - String group = user.getGroup(); - if (group == null || !group.toLowerCase().equals("hca")) { - throw new InvalidUserGroup(group); - } - } - - @Override - public boolean supports(@Nullable Class authentication) { - return delegate.supports(authentication); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/authn/provider/elixir/ElixirAaiAuthenticationProvider.java b/src/main/java/org/humancellatlas/ingest/security/authn/provider/elixir/ElixirAaiAuthenticationProvider.java deleted file mode 100644 index 44b0aa2c0..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/authn/provider/elixir/ElixirAaiAuthenticationProvider.java +++ /dev/null @@ -1,93 +0,0 @@ -package org.humancellatlas.ingest.security.authn.provider.elixir; - -import com.auth0.jwt.JWT; -import com.auth0.jwt.exceptions.JWTVerificationException; -import com.auth0.jwt.exceptions.TokenExpiredException; -import com.auth0.jwt.interfaces.JWTVerifier; -import com.auth0.spring.security.api.authentication.JwtAuthentication; -import org.humancellatlas.ingest.security.Account; -import org.humancellatlas.ingest.security.AccountRepository; -import org.humancellatlas.ingest.security.authn.oidc.OpenIdAuthentication; -import org.humancellatlas.ingest.security.authn.oidc.UserInfo; -import org.humancellatlas.ingest.security.common.jwk.DelegatingJwtAuthentication; -import org.humancellatlas.ingest.security.common.jwk.JwtVerifierResolver; -import org.humancellatlas.ingest.security.exception.JwtVerificationFailed; -import org.humancellatlas.ingest.security.exception.UnlistedJwtIssuer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.security.authentication.AuthenticationProvider; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.AuthenticationException; -import org.springframework.stereotype.Component; -import org.springframework.web.reactive.function.client.WebClient; - -import static org.apache.http.HttpHeaders.AUTHORIZATION; -import static org.humancellatlas.ingest.security.ElixirConfig.ELIXIR; - -@Component -@Qualifier(ELIXIR) -public class ElixirAaiAuthenticationProvider implements AuthenticationProvider { - private static final Logger LOGGER = LoggerFactory.getLogger(ElixirAaiAuthenticationProvider.class); - - private final JwtVerifierResolver jwtVerifierResolver; - - private final AccountRepository accountRepository; - - private final WebClient webClient; - - public ElixirAaiAuthenticationProvider(@Qualifier(ELIXIR) JwtVerifierResolver jwtVerifierResolver, - AccountRepository accountRepository, WebClient.Builder webCliBuilder) { - this.jwtVerifierResolver = jwtVerifierResolver; - this.accountRepository = accountRepository; - webClient = webCliBuilder.build(); - } - - @Override - public Authentication authenticate(Authentication authentication) throws AuthenticationException { - if (!supports(authentication.getClass())) { - return null; - } - try { - JwtAuthentication jwt = (JwtAuthentication) authentication; - String token = jwt.getToken(); - String issuer = JWT.decode(token).getIssuer(); - verifyIssuer(issuer); - - JWTVerifier jwtVerifier = jwtVerifierResolver.resolve(jwt.getToken()); - DelegatingJwtAuthentication verifiedAuth = DelegatingJwtAuthentication.delegate(jwt, jwtVerifier); - - token = verifiedAuth.getToken(); - UserInfo userInfo = retrieveUserInfo(token); - - Account account = accountRepository.findByProviderReference(userInfo.getSubjectId()); - OpenIdAuthentication openIdAuth = new OpenIdAuthentication(account); - openIdAuth.authenticateWith(userInfo); - return openIdAuth; - } catch (TokenExpiredException e) { - throw new JwtVerificationFailed(e); - } catch (JWTVerificationException e) { - LOGGER.error("JWT verification failed: {}", e.getMessage()); - throw new JwtVerificationFailed(e); - } - } - - private UserInfo retrieveUserInfo(String token) { - return webClient.get() - .uri(String.format("%s/userinfo", jwtVerifierResolver.getIssuer())) - .header(AUTHORIZATION, String.format("Bearer %s", token)) - .retrieve() - .bodyToMono(UserInfo.class).block(); - } - - private void verifyIssuer(String issuer) { - if (!issuer.contains("elixir")) { - throw new UnlistedJwtIssuer(String.format("Not an Elxir AAI issued token: %s", issuer), issuer); - } - } - - @Override - public boolean supports(Class authentication) { - return JwtAuthentication.class.isAssignableFrom(authentication); - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/security/authn/provider/elixir/ElixirJwkVault.java b/src/main/java/org/humancellatlas/ingest/security/authn/provider/elixir/ElixirJwkVault.java deleted file mode 100644 index fc0e6e025..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/authn/provider/elixir/ElixirJwkVault.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.humancellatlas.ingest.security.authn.provider.elixir; - -import com.auth0.jwk.JwkException; -import com.auth0.jwk.UrlJwkProvider; -import com.auth0.jwt.interfaces.DecodedJWT; -import org.humancellatlas.ingest.security.common.jwk.JwkVault; -import org.humancellatlas.ingest.security.common.jwk.UrlJwkProviderResolver; - -import java.security.PublicKey; - -public class ElixirJwkVault implements JwkVault { - - private final UrlJwkProviderResolver urlJwkProviderResolver; - - public ElixirJwkVault(UrlJwkProviderResolver urlJwkProviderResolver) { - this.urlJwkProviderResolver = urlJwkProviderResolver; - } - - @Override - public PublicKey getPublicKey(DecodedJWT jwt) { - UrlJwkProvider jwkProvider = urlJwkProviderResolver.resolve(); - try { - var jwk = jwkProvider.get(jwt.getKeyId()); - return jwk.getPublicKey(); - } catch (JwkException e) { - throw new RuntimeException(e); - } - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/authn/provider/gcp/GcpDomainWhiteList.java b/src/main/java/org/humancellatlas/ingest/security/authn/provider/gcp/GcpDomainWhiteList.java deleted file mode 100644 index 0cd105031..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/authn/provider/gcp/GcpDomainWhiteList.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.humancellatlas.ingest.security.authn.provider.gcp; - -import java.util.Collections; -import java.util.List; - -import static java.lang.String.format; -import static java.util.Arrays.asList; - - -public class GcpDomainWhiteList { - - private final List domains; - - public GcpDomainWhiteList(List domains) { - this.domains = Collections.unmodifiableList(domains); - } - - public GcpDomainWhiteList(String... domains) { - this(asList(domains)); - } - - public boolean lists(String email) { - return domains.stream() - .map(domain -> format("@%s", domain)) - .anyMatch(email::endsWith); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/authn/provider/gcp/GcpJwkVault.java b/src/main/java/org/humancellatlas/ingest/security/authn/provider/gcp/GcpJwkVault.java deleted file mode 100755 index ddb0c453e..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/authn/provider/gcp/GcpJwkVault.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.humancellatlas.ingest.security.authn.provider.gcp; - -import com.auth0.jwk.JwkException; -import com.auth0.jwk.UrlJwkProvider; -import com.auth0.jwt.interfaces.DecodedJWT; -import org.humancellatlas.ingest.security.common.jwk.JwkVault; -import org.humancellatlas.ingest.security.common.jwk.UrlJwkProviderResolver; - -import java.security.PublicKey; - -public class GcpJwkVault implements JwkVault { - - private final UrlJwkProviderResolver urlJwkProviderResolver; - - public GcpJwkVault(UrlJwkProviderResolver urlJwkProviderResolver) { - this.urlJwkProviderResolver = urlJwkProviderResolver; - } - - @Override - public PublicKey getPublicKey(DecodedJWT jwt) { - var issuer = jwt.getIssuer(); - UrlJwkProvider jwkProvider = urlJwkProviderResolver.resolve(issuer); - try { - var jwk = jwkProvider.get(jwt.getKeyId()); - return jwk.getPublicKey(); - } catch (JwkException e) { - throw new RuntimeException(e); - } - } - -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/security/authn/provider/gcp/GoogleServiceJwtAuthenticationProvider.java b/src/main/java/org/humancellatlas/ingest/security/authn/provider/gcp/GoogleServiceJwtAuthenticationProvider.java deleted file mode 100644 index bf3ee519c..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/authn/provider/gcp/GoogleServiceJwtAuthenticationProvider.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.humancellatlas.ingest.security.authn.provider.gcp; - -import com.auth0.jwt.JWT; -import com.auth0.jwt.exceptions.JWTVerificationException; -import com.auth0.jwt.interfaces.DecodedJWT; -import com.auth0.jwt.interfaces.JWTVerifier; -import com.auth0.spring.security.api.authentication.JwtAuthentication; -import org.humancellatlas.ingest.security.Account; -import org.humancellatlas.ingest.security.authn.oidc.OpenIdAuthentication; -import org.humancellatlas.ingest.security.common.jwk.DelegatingJwtAuthentication; -import org.humancellatlas.ingest.security.common.jwk.JwtVerifierResolver; -import org.humancellatlas.ingest.security.exception.JwtVerificationFailed; -import org.humancellatlas.ingest.security.exception.UnlistedJwtIssuer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.security.authentication.AuthenticationProvider; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.AuthenticationException; - -public class GoogleServiceJwtAuthenticationProvider implements AuthenticationProvider { - - private static Logger logger = LoggerFactory.getLogger(GoogleServiceJwtAuthenticationProvider.class); - - private final JwtVerifierResolver jwtVerifierResolver; - - private final GcpDomainWhiteList projectWhitelist; - - public GoogleServiceJwtAuthenticationProvider(GcpDomainWhiteList projectWhitelist, - JwtVerifierResolver jwtVerifierResolver) { - this.jwtVerifierResolver = jwtVerifierResolver; - this.projectWhitelist = projectWhitelist; - } - - @Override - public boolean supports(Class authentication) { - return JwtAuthentication.class.isAssignableFrom(authentication); - } - - @Override - public Authentication authenticate(Authentication authentication) throws AuthenticationException { - if (!supports(authentication.getClass())) { - return null; - } - try { - JwtAuthentication jwt = (JwtAuthentication) authentication; - verifyIssuer(jwt); - - JWTVerifier jwtVerifier = jwtVerifierResolver.resolve(jwt.getToken()); - Authentication jwtAuth = DelegatingJwtAuthentication.delegate(jwt, jwtVerifier); - logger.info("Authenticated with jwt with scopes {}", jwtAuth.getAuthorities()); - - Account account = Account.SERVICE; - OpenIdAuthentication openIdAuth = new OpenIdAuthentication(account); - return openIdAuth; - } catch (JWTVerificationException e) { - logger.error("JWT verification failed: {}", e.getMessage()); - throw new JwtVerificationFailed(e); - } - } - - private void verifyIssuer(JwtAuthentication jwt) { - DecodedJWT token = JWT.decode(jwt.getToken()); - String issuer = token.getIssuer(); - - if (!projectWhitelist.lists(issuer)) { - throw UnlistedJwtIssuer.notWhitelisted(issuer); - } - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/common/jwk/Auth0JwtAuthentication.java b/src/main/java/org/humancellatlas/ingest/security/common/jwk/Auth0JwtAuthentication.java deleted file mode 100644 index 7b7aefad2..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/common/jwk/Auth0JwtAuthentication.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.humancellatlas.ingest.security.common.jwk; - -import org.springframework.security.core.Authentication; - -/** - * A custom Authentication interface based on Auth0's JwtAuthentication without the frustrating verify method - * that takes concrete JWTVerifier instead of the JWTVerifier interface (yeah, confusing). - */ -public interface Auth0JwtAuthentication extends Authentication { - - String getToken(); - - String getKeyId(); - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/common/jwk/DelegatingJwtAuthentication.java b/src/main/java/org/humancellatlas/ingest/security/common/jwk/DelegatingJwtAuthentication.java deleted file mode 100644 index 8be7c8c03..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/common/jwk/DelegatingJwtAuthentication.java +++ /dev/null @@ -1,75 +0,0 @@ -package org.humancellatlas.ingest.security.common.jwk; - -import com.auth0.jwt.interfaces.DecodedJWT; -import com.auth0.jwt.interfaces.JWTVerifier; -import com.auth0.spring.security.api.authentication.JwtAuthentication; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.GrantedAuthority; - -import java.util.Collection; - -public class DelegatingJwtAuthentication implements Auth0JwtAuthentication { - - private Authentication authentication; - - private DecodedJWT token; - - public static DelegatingJwtAuthentication delegate(JwtAuthentication source, JWTVerifier verifier) { - var authentication = source.verify(null); - DecodedJWT token = verifier.verify(source.getToken()); - return new DelegatingJwtAuthentication(authentication, token); - } - - private DelegatingJwtAuthentication(Authentication authentication, DecodedJWT token) { - this.authentication = authentication; - this.token = token; - } - - @Override - public String getToken() { - return token.getToken(); - } - - @Override - public String getKeyId() { - return token.getKeyId(); - } - - @Override - public Collection getAuthorities() { - return authentication.getAuthorities(); - } - - @Override - public Object getCredentials() { - return authentication.getCredentials(); - } - - @Override - public Object getDetails() { - return authentication.getDetails(); - } - - @Override - public Object getPrincipal() { - return authentication.getPrincipal(); - } - - @Override - public boolean isAuthenticated() { - //The construction of this object would only succeed if the token has first - //been successfully verified. - return true; - } - - @Override - public void setAuthenticated(boolean isAuthenticated) throws IllegalArgumentException { - throw new IllegalArgumentException("Authenticate through delegation to a new instance."); - } - - @Override - public String getName() { - return authentication.getName(); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/common/jwk/DelegatingJwtVerifier.java b/src/main/java/org/humancellatlas/ingest/security/common/jwk/DelegatingJwtVerifier.java deleted file mode 100644 index 8bdcb0ab9..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/common/jwk/DelegatingJwtVerifier.java +++ /dev/null @@ -1,88 +0,0 @@ -package org.humancellatlas.ingest.security.common.jwk; - -import com.auth0.jwt.JWT; -import com.auth0.jwt.algorithms.Algorithm; -import com.auth0.jwt.exceptions.JWTVerificationException; -import com.auth0.jwt.interfaces.DecodedJWT; -import com.auth0.jwt.interfaces.JWTVerifier; -import com.auth0.jwt.interfaces.Verification; - -/** - * A {@link JWTVerifier} that delegates to Auth0's {@link com.auth0.jwt.JWTVerifier} (which is another implementation - * of the interface). Yes, the interface and the default implementing class are named the same, which illustrates how - * confusing Auth0's library for processing JWTs can be. Hence, we have all these wrapper classes to hopefully help - * us deal with that. - */ -public class DelegatingJwtVerifier implements JWTVerifier { - - public static class Builder { - - private Verification verification; - - private String audience; - private String issuer; - - private Builder(Verification verification) { - this.verification = verification; - } - - public static Builder require(Algorithm algorithm) { - return new Builder(JWT.require(algorithm)); - } - - public Builder withAudience(String audience) { - this.audience = audience; - verification.withAudience(audience); - return this; - } - - public Builder withIssuer(String issuer) { - this.issuer = issuer; - verification.withIssuer(issuer); - return this; - } - - public JWTVerifier build() { - DelegatingJwtVerifier verifier = new DelegatingJwtVerifier(verification.build()); - verifier.audience = audience; - verifier.issuer = issuer; - return verifier; - } - - } - - private final JWTVerifier delegate; - - private String audience; - private String issuer; - - /** - * Effectively an alias for {@link Builder#require(Algorithm)}. - */ - public static Builder require(Algorithm algorithm) { - return Builder.require(algorithm); - } - - private DelegatingJwtVerifier(JWTVerifier delegate) { - this.delegate = delegate; - } - - public String getAudience() { - return audience; - } - - public String getIssuer() { - return issuer; - } - - @Override - public DecodedJWT verify(String token) throws JWTVerificationException { - return delegate.verify(token); - } - - @Override - public DecodedJWT verify(DecodedJWT jwt) throws JWTVerificationException { - return delegate.verify(jwt); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/common/jwk/JwtVerifierResolver.java b/src/main/java/org/humancellatlas/ingest/security/common/jwk/JwtVerifierResolver.java deleted file mode 100644 index 293a63e44..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/common/jwk/JwtVerifierResolver.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.humancellatlas.ingest.security.common.jwk; - -import com.auth0.jwt.JWT; -import com.auth0.jwt.algorithms.Algorithm; -import com.auth0.jwt.interfaces.DecodedJWT; -import com.auth0.jwt.interfaces.JWTVerifier; - -import java.security.interfaces.RSAPublicKey; -import java.util.Optional; - -/** - * Helper class whose main purpose is to create a {@link JWTVerifier} instance given a JWT string. - * - * This is part of the wrapper subsystem to help compartmentalise the area of the application that relies on Auth0's - * library for processing JWTs. - */ -public class JwtVerifierResolver { - - private final JwkVault jwkVault; - - private final Optional audience; - private final Optional issuer; - - public JwtVerifierResolver(JwkVault jwkVault, String audience, String issuer) { - this.jwkVault = jwkVault; - this.audience = Optional.ofNullable(audience); - this.issuer = Optional.ofNullable(issuer); - } - - public String getIssuer() { - return issuer.orElse(null); - } - - public JWTVerifier resolve(String jwt) { - DecodedJWT token = JWT.decode(jwt); - RSAPublicKey publicKey = (RSAPublicKey) jwkVault.getPublicKey(token); - DelegatingJwtVerifier.Builder builder = DelegatingJwtVerifier.require(Algorithm.RSA256(publicKey, null)); - audience.ifPresent(builder::withAudience); - builder.withIssuer(issuer.orElse(token.getIssuer())); - return builder.build(); - } - -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/security/common/jwk/RemoteJwkProvider.java b/src/main/java/org/humancellatlas/ingest/security/common/jwk/RemoteJwkProvider.java deleted file mode 100644 index 3ef012184..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/common/jwk/RemoteJwkProvider.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.humancellatlas.ingest.security.common.jwk; - -import com.auth0.jwk.UrlJwkProvider; - -import java.net.URL; - -/** - * A UrlJwkProvider that allows inspection of its internal URL. - */ -public class RemoteJwkProvider extends UrlJwkProvider { - - private final URL url; - - public RemoteJwkProvider(URL url) { - super(url); - this.url = url; - } - - public URL getUrl() { - return url; - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/common/jwk/UrlJwkProviderResolver.java b/src/main/java/org/humancellatlas/ingest/security/common/jwk/UrlJwkProviderResolver.java deleted file mode 100644 index e2913c148..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/common/jwk/UrlJwkProviderResolver.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.humancellatlas.ingest.security.common.jwk; - -import com.auth0.jwk.UrlJwkProvider; - -import java.net.MalformedURLException; -import java.net.URL; - -public class UrlJwkProviderResolver { - - private final URL baseUrl; - - public UrlJwkProviderResolver(String baseUrl) { - try { - this.baseUrl = new URL(baseUrl); - } catch (MalformedURLException e) { - throw new RuntimeException(e); - } - } - - //TODO cache providers based on relative path - public UrlJwkProvider resolve(String relativePath) { - try { - var providerUrl = new URL(baseUrl, relativePath); - return new RemoteJwkProvider(providerUrl); - } catch (MalformedURLException e) { - throw new RuntimeException(e); - } - } - - public UrlJwkProvider resolve() { - return new RemoteJwkProvider(this.baseUrl); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/exception/DuplicateAccount.java b/src/main/java/org/humancellatlas/ingest/security/exception/DuplicateAccount.java deleted file mode 100644 index 22a8e8ee6..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/exception/DuplicateAccount.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.humancellatlas.ingest.security.exception; - -public class DuplicateAccount extends RuntimeException { - - public DuplicateAccount() { - super("Operation failed due to Account duplication."); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/exception/InvalidUserGroup.java b/src/main/java/org/humancellatlas/ingest/security/exception/InvalidUserGroup.java deleted file mode 100644 index 8ee0720da..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/exception/InvalidUserGroup.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.humancellatlas.ingest.security.exception; - -import org.springframework.security.core.AuthenticationException; - -public class InvalidUserGroup extends AuthenticationException { - - public InvalidUserGroup(String group) { - super(String.format("Invalid user group, %s", group)); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/exception/NotAllowedException.java b/src/main/java/org/humancellatlas/ingest/security/exception/NotAllowedException.java deleted file mode 100644 index 5a89ec730..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/exception/NotAllowedException.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.humancellatlas.ingest.security.exception; - -public class NotAllowedException extends RuntimeException { - public NotAllowedException() { - super("Operation not allowed."); - } - - public NotAllowedException(String customMessage) { - super(customMessage); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/security/exception/UnlistedJwtIssuer.java b/src/main/java/org/humancellatlas/ingest/security/exception/UnlistedJwtIssuer.java deleted file mode 100644 index 2bed78392..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/exception/UnlistedJwtIssuer.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.humancellatlas.ingest.security.exception; - -import org.springframework.security.core.AuthenticationException; - -import static java.lang.String.format; - -public class UnlistedJwtIssuer extends AuthenticationException { - - private final String issuer; - - public UnlistedJwtIssuer(String issuer, String message) { - super(message); - this.issuer = issuer; - } - - public static UnlistedJwtIssuer notWhitelisted(String issuer) { - return new UnlistedJwtIssuer(format("Issuer [%s] is not specified in the whitelist.", issuer), issuer); - } - - public String getIssuer() { - return issuer; - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/security/web/AuthenticationController.java b/src/main/java/org/humancellatlas/ingest/security/web/AuthenticationController.java deleted file mode 100644 index 68228d1ca..000000000 --- a/src/main/java/org/humancellatlas/ingest/security/web/AuthenticationController.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.humancellatlas.ingest.security.web; - -import org.humancellatlas.ingest.security.Account; -import org.humancellatlas.ingest.security.AccountService; -import org.humancellatlas.ingest.security.Role; -import org.humancellatlas.ingest.security.authn.oidc.OpenIdAuthentication; -import org.humancellatlas.ingest.security.authn.oidc.UserInfo; -import org.humancellatlas.ingest.security.exception.DuplicateAccount; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.security.core.Authentication; -import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; - -import static org.springframework.http.MediaType.APPLICATION_JSON_UTF8_VALUE; - -@Controller -@RequestMapping("/auth") -public class AuthenticationController { - - private final AccountService accountService; - - public AuthenticationController(AccountService accountService) { - this.accountService = accountService; - } - - @PostMapping(path="/registration", produces=APPLICATION_JSON_UTF8_VALUE) - public ResponseEntity register(Authentication authentication) { - var openIdAuthentication = (OpenIdAuthentication) authentication; - var userInfo = (UserInfo) openIdAuthentication.getCredentials(); - try { - Account account = userInfo.toAccount(); - Account persistentAccount = accountService.register(account); - return ResponseEntity.ok().body(persistentAccount); - } catch (DuplicateAccount duplicateAccount) { - return ResponseEntity.status(HttpStatus.CONFLICT).build(); - } - } - - @GetMapping(path="/account", produces=APPLICATION_JSON_UTF8_VALUE) - public ResponseEntity getAccount(Authentication authentication) { - if (authentication.getAuthorities().contains(Role.GUEST)) { - return ResponseEntity.notFound().build(); - } - Account account = (Account) authentication.getPrincipal(); - return ResponseEntity.ok().body(account); - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/stagingjob/StagingJob.java b/src/main/java/org/humancellatlas/ingest/stagingjob/StagingJob.java deleted file mode 100644 index 7d30bb3bf..000000000 --- a/src/main/java/org/humancellatlas/ingest/stagingjob/StagingJob.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.humancellatlas.ingest.stagingjob; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.*; -import org.springframework.data.annotation.CreatedDate; -import org.springframework.data.annotation.Id; -import org.springframework.data.annotation.PersistenceConstructor; -import org.springframework.data.mongodb.core.index.CompoundIndex; -import org.springframework.data.mongodb.core.index.CompoundIndexes; -import org.springframework.data.mongodb.core.index.Indexed; -import org.springframework.data.mongodb.core.mapping.Document; -import org.springframework.hateoas.Identifiable; - -import java.time.Instant; -import java.util.UUID; - -@Getter -@CompoundIndexes({ - @CompoundIndex( - name = "stagingAreaUuidAndFileName", - def = "{'stagingAreaUuid' : 1, 'stagingAreaFileName' : 1}", - unique = true - ) -}) -@Document -@EqualsAndHashCode -@RequiredArgsConstructor -public class StagingJob implements Identifiable { - - @Id - private String id; - - @CreatedDate - private Instant createdDate; - - @Indexed - private final UUID stagingAreaUuid; - - private final String stagingAreaFileName; - - private String metadataUuid; - - @Setter - private String stagingAreaFileUri; - - @JsonCreator - @PersistenceConstructor - public StagingJob(@JsonProperty(value = "stagingAreaUuid") UUID stagingAreaUuid, - @JsonProperty(value = "metadataUuid") String metadataUuid, - @JsonProperty(value = "stagingAreaFileName") String stagingAreaFileName) { - this.stagingAreaUuid = stagingAreaUuid; - this.metadataUuid = metadataUuid; - this.stagingAreaFileName = stagingAreaFileName; - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/stagingjob/StagingJobRepository.java b/src/main/java/org/humancellatlas/ingest/stagingjob/StagingJobRepository.java deleted file mode 100644 index debf2933c..000000000 --- a/src/main/java/org/humancellatlas/ingest/stagingjob/StagingJobRepository.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.humancellatlas.ingest.stagingjob; - -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.data.repository.query.Param; -import org.springframework.data.rest.core.annotation.RestResource; -import org.springframework.web.bind.annotation.CrossOrigin; - -import java.util.UUID; - -@CrossOrigin -public interface StagingJobRepository extends MongoRepository { - @RestResource(exported = false) - T save(T stagingJob); - - @RestResource - void delete(StagingJob stagingJob); - - @RestResource(exported = false) - void deleteAllByStagingAreaUuid(UUID stagingAreaUuid); - - @RestResource(rel = "findByStagingAreaAndFileName") - T findByStagingAreaUuidAndStagingAreaFileName(@Param("stagingAreaUuid") UUID stagingAreaUuid, - @Param("stagingAreaFileName") String stagingAreaFileName); -} diff --git a/src/main/java/org/humancellatlas/ingest/stagingjob/StagingJobService.java b/src/main/java/org/humancellatlas/ingest/stagingjob/StagingJobService.java deleted file mode 100644 index b171c3c3e..000000000 --- a/src/main/java/org/humancellatlas/ingest/stagingjob/StagingJobService.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.humancellatlas.ingest.stagingjob; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.springframework.dao.DuplicateKeyException; -import org.springframework.stereotype.Service; - -import java.util.UUID; - -@Service -@RequiredArgsConstructor -public class StagingJobService { - - @NonNull - private final StagingJobRepository stagingJobRepository; - - public StagingJob register(StagingJob stagingJob) { - try { - return stagingJobRepository.save(stagingJob); - } catch (DuplicateKeyException e) { - throw new JobAlreadyRegisteredException(stagingJob.getStagingAreaUuid(), - stagingJob.getStagingAreaFileName()); - } - } - - @Deprecated - public StagingJob registerNewJob(UUID stagingAreaUuid, String stagingAreaFileName) { - try { - StagingJob stagingJob = new StagingJob(stagingAreaUuid, stagingAreaFileName); - return stagingJobRepository.save(stagingJob); - } catch (DuplicateKeyException e) { - throw new JobAlreadyRegisteredException(stagingAreaUuid, stagingAreaFileName); - } - } - - public StagingJob completeJob(StagingJob stagingJob, String stagingAreaUri) { - stagingJob.setStagingAreaFileUri(stagingAreaUri); - return stagingJobRepository.save(stagingJob); - } - - public void deleteJobsForStagingArea(UUID stagingAreaUuid) { - stagingJobRepository.deleteAllByStagingAreaUuid(stagingAreaUuid); - } - - public static class JobAlreadyRegisteredException extends IllegalStateException { - - public JobAlreadyRegisteredException(UUID stagingAreaUuid, String fileName) { - super(String.format("Staging job request already exists for file %s at upload area %s", - fileName, stagingAreaUuid)); - } - - } -} diff --git a/src/main/java/org/humancellatlas/ingest/stagingjob/web/StagingJobCompleteRequest.java b/src/main/java/org/humancellatlas/ingest/stagingjob/web/StagingJobCompleteRequest.java deleted file mode 100644 index 0d9ea0411..000000000 --- a/src/main/java/org/humancellatlas/ingest/stagingjob/web/StagingJobCompleteRequest.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.humancellatlas.ingest.stagingjob.web; - -import lombok.Data; - -@Data -public class StagingJobCompleteRequest { - private String stagingAreaFileUri; -} diff --git a/src/main/java/org/humancellatlas/ingest/stagingjob/web/StagingJobController.java b/src/main/java/org/humancellatlas/ingest/stagingjob/web/StagingJobController.java deleted file mode 100644 index a6ece67d9..000000000 --- a/src/main/java/org/humancellatlas/ingest/stagingjob/web/StagingJobController.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.humancellatlas.ingest.stagingjob.web; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.stagingjob.StagingJob; -import org.humancellatlas.ingest.stagingjob.StagingJobService; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; - -import java.util.UUID; - -@RepositoryRestController -@ExposesResourceFor(StagingJob.class) -@RequiredArgsConstructor -@RequestMapping("/stagingJobs") -public class StagingJobController { - - private final @NonNull StagingJobService stagingJobService; - - @PostMapping - public ResponseEntity createStagingJob(@RequestBody StagingJob stagingJob, - PersistentEntityResourceAssembler resourceAssembler) { - StagingJob persistentJob = stagingJobService.register(stagingJob); - return ResponseEntity.ok(resourceAssembler.toFullResource(persistentJob)); - } - - @PatchMapping(path = "/{stagingJob}" + Links.COMPLETE_STAGING_JOB_URL) - ResponseEntity completeStagingJob(@PathVariable("stagingJob") StagingJob stagingJob, - @RequestBody StagingJobCompleteRequest stagingJobCompleteRequest, - final PersistentEntityResourceAssembler resourceAssembler) { - StagingJob completedStagingJob = stagingJobService.completeJob( - stagingJob, - stagingJobCompleteRequest.getStagingAreaFileUri() - ); - - return ResponseEntity.ok(resourceAssembler.toFullResource(completedStagingJob)); - } - - @DeleteMapping - ResponseEntity deleteStagingJobs(@RequestParam("stagingAreaUuid") UUID stagingAreaUuid) { - stagingJobService.deleteJobsForStagingArea(stagingAreaUuid); - return new ResponseEntity(HttpStatus.NO_CONTENT); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/stagingjob/web/StagingJobCreateRequest.java b/src/main/java/org/humancellatlas/ingest/stagingjob/web/StagingJobCreateRequest.java deleted file mode 100644 index 1ce62cbbc..000000000 --- a/src/main/java/org/humancellatlas/ingest/stagingjob/web/StagingJobCreateRequest.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.humancellatlas.ingest.stagingjob.web; - -import lombok.Data; - -import java.util.UUID; - -@Data -public class StagingJobCreateRequest { - private UUID stagingAreaUuid; - private String stagingAreaFileName; -} diff --git a/src/main/java/org/humancellatlas/ingest/stagingjob/web/StagingJobResourceProcessor.java b/src/main/java/org/humancellatlas/ingest/stagingjob/web/StagingJobResourceProcessor.java deleted file mode 100644 index 005c5f2b7..000000000 --- a/src/main/java/org/humancellatlas/ingest/stagingjob/web/StagingJobResourceProcessor.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.humancellatlas.ingest.stagingjob.web; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.stagingjob.StagingJob; -import org.springframework.hateoas.EntityLinks; -import org.springframework.hateoas.Link; -import org.springframework.hateoas.Resource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.stereotype.Component; - -@RequiredArgsConstructor -@Component -public class StagingJobResourceProcessor implements ResourceProcessor> { - private final @NonNull EntityLinks entityLinks; - - private Link getCompleteStagingJobLink(StagingJob stagingJob) { - return entityLinks.linkForSingleResource(stagingJob) - .slash(Links.COMPLETE_STAGING_JOB_URL) - .withRel(Links.COMPLETE_STAGING_JOB_REL); - } - - @Override - public Resource process(Resource stagingJobResource) { - StagingJob stagingJob = stagingJobResource.getContent(); - - stagingJobResource.add(getCompleteStagingJobLink(stagingJob)); - - return stagingJobResource; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/stagingjob/web/StagingJobResourcesProcessor.java b/src/main/java/org/humancellatlas/ingest/stagingjob/web/StagingJobResourcesProcessor.java deleted file mode 100644 index 5ad5fb7b7..000000000 --- a/src/main/java/org/humancellatlas/ingest/stagingjob/web/StagingJobResourcesProcessor.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.humancellatlas.ingest.stagingjob.web; - -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.stagingjob.StagingJob; -import org.springframework.hateoas.Link; -import org.springframework.hateoas.Resource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.hateoas.Resources; -import org.springframework.stereotype.Component; - -import static org.springframework.hateoas.mvc.ControllerLinkBuilder.*; - -@Component -@RequiredArgsConstructor -public class StagingJobResourcesProcessor implements ResourceProcessor>> { - - private Link getDeleteByStagingAreaLink() { - return linkTo(methodOn(StagingJobController.class).deleteStagingJobs(null)).withRel("delete-staging-jobs"); - } - - @Override - public Resources> process(Resources> resources) { - resources.add(getDeleteByStagingAreaLink()); - return resources; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/state/MetadataDocumentEventHandler.java b/src/main/java/org/humancellatlas/ingest/state/MetadataDocumentEventHandler.java deleted file mode 100644 index 8499d0962..000000000 --- a/src/main/java/org/humancellatlas/ingest/state/MetadataDocumentEventHandler.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.humancellatlas.ingest.state; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.EntityType; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.springframework.data.rest.core.annotation.HandleAfterCreate; -import org.springframework.data.rest.core.annotation.RepositoryEventHandler; -import org.springframework.stereotype.Component; - - -@RepositoryEventHandler -@Component -@RequiredArgsConstructor -public class MetadataDocumentEventHandler { - private final @NonNull MessageRouter messageRouter; - - @HandleAfterCreate - public void metadataDocumentAfterCreate(MetadataDocument document) { - this.handleMetadataDocumentCreate(document); - } - - public void handleMetadataDocumentCreate(MetadataDocument document) { - messageRouter.routeValidationMessageFor(document); - messageRouter.routeStateTrackingUpdateMessageFor(document); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/state/MetadataStateChangeListener.java b/src/main/java/org/humancellatlas/ingest/state/MetadataStateChangeListener.java deleted file mode 100644 index 222e32534..000000000 --- a/src/main/java/org/humancellatlas/ingest/state/MetadataStateChangeListener.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.humancellatlas.ingest.state; - - -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.data.mongodb.core.mapping.event.AbstractMongoEventListener; -import org.springframework.data.mongodb.core.mapping.event.AfterSaveEvent; -import org.springframework.data.mongodb.core.mapping.event.BeforeConvertEvent; -import org.springframework.stereotype.Component; - -import java.util.Optional; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 12/09/17 - */ -@Component -@RequiredArgsConstructor -@Getter -public class MetadataStateChangeListener extends AbstractMongoEventListener { - private final MessageRouter messageRouter; - private final Logger log = LoggerFactory.getLogger(getClass()); - - protected Logger getLog() { - return log; - } - - @Override - public void onAfterSave(AfterSaveEvent event) { - MetadataDocument document = event.getSource(); - messageRouter.routeValidationMessageFor(document); - } - - - @Override - public void onBeforeConvert(BeforeConvertEvent event) { - MetadataDocument document = event.getSource(); - -// TODO Ideally, this should be being set when the submission is submitted. -// The exporter could set this. Putting this back here for now for convenience. - if (!Optional.ofNullable(document.getDcpVersion()).isPresent()) { - document.setDcpVersion(document.getSubmissionDate()); - } - - if (!Optional.ofNullable(document.getUuid()).isPresent()) { - document.setUuid(Uuid.newUuid()); - } - } -} diff --git a/src/main/java/org/humancellatlas/ingest/state/SubmissionState.java b/src/main/java/org/humancellatlas/ingest/state/SubmissionState.java deleted file mode 100644 index e239ab55f..000000000 --- a/src/main/java/org/humancellatlas/ingest/state/SubmissionState.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.humancellatlas.ingest.state; - -/** - * @author Simon Jupp - * @date 04/09/2017 - * Samples, Phenotypes and Ontologies Team, EMBL-EBI - */ -public enum SubmissionState { - PENDING, - DRAFT, - METADATA_VALIDATING, - METADATA_VALID, - METADATA_INVALID, - GRAPH_VALIDATION_REQUESTED, - GRAPH_VALIDATING, - GRAPH_VALID, - GRAPH_INVALID, - SUBMITTED, - PROCESSING, - ARCHIVING, - ARCHIVED, - EXPORTING, - EXPORTED, - CLEANUP, - COMPLETE -} diff --git a/src/main/java/org/humancellatlas/ingest/state/SubmissionStateChangeListener.java b/src/main/java/org/humancellatlas/ingest/state/SubmissionStateChangeListener.java deleted file mode 100644 index 6f84e2192..000000000 --- a/src/main/java/org/humancellatlas/ingest/state/SubmissionStateChangeListener.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.humancellatlas.ingest.state; - -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.mongodb.core.mapping.event.AbstractMongoEventListener; -import org.springframework.data.mongodb.core.mapping.event.AfterDeleteEvent; -import org.springframework.data.mongodb.core.mapping.event.AfterSaveEvent; -import org.springframework.stereotype.Component; - - -@Component -@RequiredArgsConstructor -@Getter -public class SubmissionStateChangeListener extends AbstractMongoEventListener { - @Autowired - @NonNull - private final MessageRouter messageRouter; - - private final Logger log = LoggerFactory.getLogger(getClass()); - - protected Logger getLog() { - return log; - } - - @Override - public void onAfterSave(AfterSaveEvent event) { - SubmissionEnvelope submissionEnvelope = event.getSource(); - - if(submissionEnvelope.getSubmissionState().equals(SubmissionState.CLEANUP)){ - log.info(String.format("Requesting cleanup for envelope with ID %s", submissionEnvelope.getId())); - this.messageRouter.routeRequestUploadAreaCleanup(submissionEnvelope); - } - } -} diff --git a/src/main/java/org/humancellatlas/ingest/state/SubmitAction.java b/src/main/java/org/humancellatlas/ingest/state/SubmitAction.java deleted file mode 100644 index f339fe82a..000000000 --- a/src/main/java/org/humancellatlas/ingest/state/SubmitAction.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.humancellatlas.ingest.state; - - -public enum SubmitAction { - ARCHIVE, - EXPORT, - CLEANUP, - EXPORT_METADATA -} diff --git a/src/main/java/org/humancellatlas/ingest/state/ValidationState.java b/src/main/java/org/humancellatlas/ingest/state/ValidationState.java deleted file mode 100644 index 9424750aa..000000000 --- a/src/main/java/org/humancellatlas/ingest/state/ValidationState.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.humancellatlas.ingest.state; - -import com.fasterxml.jackson.annotation.JsonCreator; - -/** - * Created by rolando on 07/09/2017. - */ -public enum ValidationState { - DRAFT, - VALIDATING, - VALID, - INVALID, - PROCESSING, - COMPLETE; - - @JsonCreator - public static ValidationState fromString(String key) { - return key == null - ? null - : ValidationState.valueOf(key.toUpperCase()); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/state/ValidationStateChangeListener.java b/src/main/java/org/humancellatlas/ingest/state/ValidationStateChangeListener.java deleted file mode 100644 index 1a5358fe8..000000000 --- a/src/main/java/org/humancellatlas/ingest/state/ValidationStateChangeListener.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.humancellatlas.ingest.state; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.EntityType; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.core.ValidationEvent; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectEventHandler; -import org.springframework.context.ApplicationListener; -import org.springframework.stereotype.Component; - -@Component -@RequiredArgsConstructor -public class ValidationStateChangeListener implements ApplicationListener { - private final @NonNull MessageRouter messageRouter; - private final @NonNull ProjectEventHandler projectEventHandler; - - @Override - public void onApplicationEvent(ValidationEvent event) { - MetadataDocument document = (MetadataDocument) event.getSource(); - messageRouter.routeStateTrackingUpdateMessageFor(document); - - if(document.getType().equals(EntityType.PROJECT)) { - projectEventHandler.validatedProject((Project) document); - } - } -} diff --git a/src/main/java/org/humancellatlas/ingest/state/ValidationStateEventPublisher.java b/src/main/java/org/humancellatlas/ingest/state/ValidationStateEventPublisher.java deleted file mode 100644 index 791e77bd9..000000000 --- a/src/main/java/org/humancellatlas/ingest/state/ValidationStateEventPublisher.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.humancellatlas.ingest.state; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.core.ValidationEvent; -import org.springframework.context.ApplicationEventPublisher; -import org.springframework.stereotype.Component; - -@Component -@RequiredArgsConstructor -public class ValidationStateEventPublisher { - private final @NonNull ApplicationEventPublisher applicationEventPublisher; - - public void publishValidationStateChangeEventFor(MetadataDocument metadataDocument) { - ValidationEvent validationEvent = new ValidationEvent(metadataDocument, - metadataDocument.getValidationState().toString()); - applicationEventPublisher.publishEvent(validationEvent); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/submission/SpreadsheetGenerationJob.java b/src/main/java/org/humancellatlas/ingest/submission/SpreadsheetGenerationJob.java deleted file mode 100644 index 099e61a38..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/SpreadsheetGenerationJob.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.humancellatlas.ingest.submission; - -import lombok.Data; - -import java.time.Instant; - -@Data -public class SpreadsheetGenerationJob { - private Instant finishedDate; - private Instant createdDate; -} diff --git a/src/main/java/org/humancellatlas/ingest/submission/StagingDetails.java b/src/main/java/org/humancellatlas/ingest/submission/StagingDetails.java deleted file mode 100644 index b233dcbc8..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/StagingDetails.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.humancellatlas.ingest.submission; - -import lombok.Data; -import org.humancellatlas.ingest.core.Uuid; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 15/09/17 - */ -@Data -class StagingDetails { - private Uuid stagingAreaUuid; - private StagingUrn stagingAreaLocation; -} diff --git a/src/main/java/org/humancellatlas/ingest/submission/StagingUrn.java b/src/main/java/org/humancellatlas/ingest/submission/StagingUrn.java deleted file mode 100644 index dbc1720c2..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/StagingUrn.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.humancellatlas.ingest.submission; - -import com.fasterxml.jackson.annotation.JsonCreator; -import lombok.Data; - -import java.net.URI; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 15/09/17 - */ -@Data -class StagingUrn { - private URI value; - - @JsonCreator - public StagingUrn(String name) { - this.value = URI.create(name); - - // test this uri is a URN - if (!value.isOpaque()) { - throw new IllegalArgumentException(String.format("Staging URN is malformed: %s", value.toString())); - } - } - - StagingUrn() { - - } -} diff --git a/src/main/java/org/humancellatlas/ingest/submission/SubmissionEnvelope.java b/src/main/java/org/humancellatlas/ingest/submission/SubmissionEnvelope.java deleted file mode 100644 index 9f050e04e..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/SubmissionEnvelope.java +++ /dev/null @@ -1,162 +0,0 @@ -package org.humancellatlas.ingest.submission; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; -import org.humancellatlas.ingest.core.AbstractEntity; -import org.humancellatlas.ingest.core.EntityType; -import org.humancellatlas.ingest.state.SubmissionState; -import org.humancellatlas.ingest.state.SubmitAction; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.data.mongodb.core.index.CompoundIndex; -import org.springframework.data.mongodb.core.mapping.Document; - -import java.util.*; - -@Getter -@Document -/* -Used as a workaround to inheritance issue. -Not proper to annotate uuid in parent class as we don't want uuid index for all subtypes. -*/ -@CompoundIndex(def = "{ 'uuid': 1 }", unique = true) -@EqualsAndHashCode(callSuper = true) -public class SubmissionEnvelope extends AbstractEntity { - private static final Logger log = LoggerFactory.getLogger(SubmissionEnvelope.class); - private @Setter - StagingDetails stagingDetails; - private SubmissionState submissionState; - private @Setter - Boolean triggersAnalysis; - private @Setter - Boolean isUpdate; - private @Setter - Set submitActions; - private @Setter - SpreadsheetGenerationJob lastSpreadsheetGenerationJob; - - public SubmissionEnvelope() { - super(EntityType.SUBMISSION); - this.submissionState = SubmissionState.PENDING; - this.triggersAnalysis = true; - this.isUpdate = false; - this.submitActions = new HashSet<>(); - } - - private static Logger getLog() { - return log; - } - - public static List allowedSubmissionStateTransitions(SubmissionState fromState) { - List allowedStates = new ArrayList<>(); - switch (fromState) { - case PENDING: - allowedStates.add(SubmissionState.DRAFT); - break; - case DRAFT: - allowedStates.add(SubmissionState.METADATA_VALIDATING); - break; - case METADATA_VALIDATING: - allowedStates.add(SubmissionState.DRAFT); - allowedStates.add(SubmissionState.METADATA_VALID); - allowedStates.add(SubmissionState.METADATA_INVALID); - break; - case METADATA_VALID: - allowedStates.add(SubmissionState.DRAFT); - allowedStates.add(SubmissionState.GRAPH_VALIDATION_REQUESTED); - break; - case METADATA_INVALID: - allowedStates.add(SubmissionState.DRAFT); - allowedStates.add(SubmissionState.METADATA_VALIDATING); - allowedStates.add(SubmissionState.GRAPH_VALIDATION_REQUESTED); - break; - case GRAPH_VALIDATION_REQUESTED: - allowedStates.add(SubmissionState.GRAPH_VALIDATING); - allowedStates.add(SubmissionState.DRAFT); - break; - case GRAPH_VALIDATING: - allowedStates.add(SubmissionState.GRAPH_VALID); - allowedStates.add(SubmissionState.GRAPH_INVALID); - allowedStates.add(SubmissionState.DRAFT); - break; - case GRAPH_INVALID: - allowedStates.add(SubmissionState.GRAPH_VALIDATION_REQUESTED); - allowedStates.add(SubmissionState.DRAFT); - break; - case GRAPH_VALID: - allowedStates.add(SubmissionState.SUBMITTED); - allowedStates.add(SubmissionState.DRAFT); - break; - case SUBMITTED: - allowedStates.add(SubmissionState.PROCESSING); - allowedStates.add(SubmissionState.EXPORTING); - break; - case PROCESSING: - allowedStates.add(SubmissionState.ARCHIVING); - break; - case ARCHIVING: - allowedStates.add(SubmissionState.ARCHIVED); - break; - case ARCHIVED: - allowedStates.add(SubmissionState.EXPORTING); - break; - case EXPORTED: - allowedStates.add(SubmissionState.CLEANUP); - break; - case CLEANUP: - allowedStates.add(SubmissionState.COMPLETE); - break; - default: - break; - } - return allowedStates; - } - - public List allowedSubmissionStateTransitions() { - return allowedSubmissionStateTransitions(getSubmissionState()); - } - - public void enactStateTransition(SubmissionState targetState) { - if (this.submissionState != targetState) { - this.submissionState = targetState; - } - } - - public boolean isOpen() { - List states = Arrays.asList(SubmissionState.values()); - return states.indexOf(this.getSubmissionState()) < states.indexOf(SubmissionState.SUBMITTED); - } - - private List getNonEditableStates() { - return Arrays.asList( - SubmissionState.PENDING, - SubmissionState.METADATA_VALIDATING, - SubmissionState.GRAPH_VALIDATION_REQUESTED, - SubmissionState.GRAPH_VALIDATING, - SubmissionState.EXPORTING, - SubmissionState.PROCESSING, - SubmissionState.CLEANUP, - SubmissionState.ARCHIVED, - SubmissionState.SUBMITTED - ); - } - - public boolean isEditable() { - return !this.getNonEditableStates().contains(this.submissionState); - } - - public boolean isSystemEditable() { - // The importer and validator have to add and edit metadata and perform linking while the submission - // may be in METADATA_VALIDATING - // So, this is used for the special case of allowing ingest components to function - // If anything, this highlights problems with the system design - we have many dependencies across domain - // boundaries - return this.getNonEditableStates().stream() - .filter(state -> state != SubmissionState.CLEANUP) - .filter(state -> state != SubmissionState.PENDING) - .filter(state -> state != SubmissionState.METADATA_VALIDATING) - .noneMatch(state -> state == this.submissionState); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeCreateHandler.java b/src/main/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeCreateHandler.java deleted file mode 100644 index 099261f7c..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeCreateHandler.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.humancellatlas.ingest.submission; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.amqp.rabbit.core.RabbitMessagingTemplate; -import org.springframework.data.rest.core.annotation.HandleAfterCreate; -import org.springframework.data.rest.core.annotation.HandleBeforeCreate; -import org.springframework.data.rest.core.annotation.RepositoryEventHandler; -import org.springframework.data.rest.core.config.RepositoryRestConfiguration; -import org.springframework.data.rest.core.mapping.ResourceMappings; -import org.springframework.stereotype.Component; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 15/09/17 - */ -@Component -@RepositoryEventHandler -@RequiredArgsConstructor -public class SubmissionEnvelopeCreateHandler { - private final @NonNull MessageRouter messageRouter; - private final @NonNull RabbitMessagingTemplate rabbitMessagingTemplate; - - private final @NonNull ResourceMappings mappings; - private final @NonNull RepositoryRestConfiguration config; - private final @NonNull Logger log = LoggerFactory.getLogger(getClass()); - - @HandleBeforeCreate - public boolean submissionEnvelopeBeforeCreate(SubmissionEnvelope submissionEnvelope) { - this.setUuid(submissionEnvelope); - return true; - } - - public SubmissionEnvelope setUuid(SubmissionEnvelope submissionEnvelope) { - submissionEnvelope.setUuid(Uuid.newUuid()); - return submissionEnvelope; - } - - @HandleAfterCreate - public boolean handleSubmissionEnvelopeCreationEvent(SubmissionEnvelope submissionEnvelope) { - return this.handleSubmissionEnvelopeCreation(submissionEnvelope); - } - - public boolean handleSubmissionEnvelopeCreation(SubmissionEnvelope submissionEnvelope) { - this.messageRouter.routeStateTrackingNewSubmissionEnvelope(submissionEnvelope); - this.messageRouter.routeRequestUploadAreaCredentials(submissionEnvelope); - log.info(String.format("Submission envelope with ID %s was created.", submissionEnvelope.getId())); - return true; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeMessageBuilder.java b/src/main/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeMessageBuilder.java deleted file mode 100644 index fdd444bcb..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeMessageBuilder.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.humancellatlas.ingest.submission; - -import org.humancellatlas.ingest.core.web.LinkGenerator; -import org.humancellatlas.ingest.messaging.model.SubmissionEnvelopeMessage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class SubmissionEnvelopeMessageBuilder { - public static SubmissionEnvelopeMessageBuilder using(LinkGenerator linkGenerator) { - return new SubmissionEnvelopeMessageBuilder(linkGenerator); - } - private LinkGenerator linkGenerator; - - private Class documentType; - private String submissionEnvelopeId; - private String submissionEnvelopeUuid; - - private final Logger log = LoggerFactory.getLogger(getClass()); - - protected Logger getLog() { - return log; - } - - private SubmissionEnvelopeMessageBuilder(LinkGenerator linkGenerator) { - this.linkGenerator = linkGenerator; - } - - public SubmissionEnvelopeMessageBuilder messageFor(SubmissionEnvelope submissionEnvelope) { - withDocumentType(submissionEnvelope.getClass()) - .withId(submissionEnvelope.getId()) - .withUuid(submissionEnvelope.getUuid().getUuid().toString()); - - return this; - } - - private SubmissionEnvelopeMessageBuilder withDocumentType(Class documentClass) { - this.documentType = documentClass; - - return this; - } - - private SubmissionEnvelopeMessageBuilder withId(String metadataDocId) { - this.submissionEnvelopeId = metadataDocId; - - return this; - } - - private SubmissionEnvelopeMessageBuilder withUuid(String uuid) { - this.submissionEnvelopeUuid = uuid; - - return this; - } - - public SubmissionEnvelopeMessage build() { - - String callbackLink = linkGenerator.createCallback(documentType, submissionEnvelopeId); - return new SubmissionEnvelopeMessage( - documentType.getSimpleName().toLowerCase(), - submissionEnvelopeId, - submissionEnvelopeUuid, - callbackLink); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeRepository.java b/src/main/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeRepository.java deleted file mode 100644 index cb8f7f6b5..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeRepository.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.humancellatlas.ingest.submission; - -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.state.SubmissionState; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.data.repository.query.Param; -import org.springframework.data.rest.core.annotation.RestResource; -import org.springframework.web.bind.annotation.CrossOrigin; - -import java.util.UUID; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 31/08/17 - */ -@CrossOrigin -public interface SubmissionEnvelopeRepository extends MongoRepository { - - @RestResource(exported = false) - SubmissionEnvelope findByUuid(Uuid uuid); - - @RestResource(rel = "findByUuid") - SubmissionEnvelope findByUuidUuid(@Param("uuid") UUID uuid); - - @RestResource(path = "findByUser", rel = "findByUser") - Page findByUser(@Param(value = "user") String user, Pageable pageable); - - Page findBySubmissionState(@Param("submissionState") SubmissionState submissionState, Pageable pageable); - - long countBySubmissionStateAndUser(SubmissionState submissionState, String user); - -} diff --git a/src/main/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeService.java b/src/main/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeService.java deleted file mode 100644 index 15a48ae70..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeService.java +++ /dev/null @@ -1,358 +0,0 @@ -package org.humancellatlas.ingest.submission; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.bundle.BundleManifestRepository; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.core.exception.StateTransitionNotAllowed; -import org.humancellatlas.ingest.core.service.MetadataCrudService; -import org.humancellatlas.ingest.errors.SubmissionErrorRepository; -import org.humancellatlas.ingest.exporter.Exporter; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.patch.PatchRepository; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.project.ProjectService; -import org.humancellatlas.ingest.project.WranglingState; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.humancellatlas.ingest.state.SubmissionState; -import org.humancellatlas.ingest.state.SubmitAction; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submissionmanifest.SubmissionManifestRepository; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.dao.OptimisticLockingFailureException; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Sort; -import org.springframework.retry.support.RetryTemplate; -import org.springframework.stereotype.Service; - -import java.text.DecimalFormat; -import java.time.Instant; -import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -@Service -@RequiredArgsConstructor -public class SubmissionEnvelopeService { - - @NonNull - private final MessageRouter messageRouter; - - @NonNull - private final Exporter exporter; - - @NonNull - private final ExecutorService executorService = Executors.newFixedThreadPool(5); - - @NonNull - private final SubmissionEnvelopeRepository submissionEnvelopeRepository; - - @NonNull - private final SubmissionEnvelopeCreateHandler submissionEnvelopeCreateHandler; - - @NonNull - private final SubmissionManifestRepository submissionManifestRepository; - - @NonNull - private final Logger log = LoggerFactory.getLogger(getClass()); - - @NonNull - private BundleManifestRepository bundleManifestRepository; - - @NonNull - private ProjectRepository projectRepository; - - @NonNull - private ProjectService projectService; - - @NonNull - private ProcessRepository processRepository; - - @NonNull - private ProtocolRepository protocolRepository; - - @NonNull - private FileRepository fileRepository; - - @NonNull - private BiomaterialRepository biomaterialRepository; - - @NonNull - private PatchRepository patchRepository; - - private final @NonNull MetadataCrudService metadataCrudService; - - @NonNull - private SubmissionErrorRepository submissionErrorRepository; - - public void handleSubmitRequest(SubmissionEnvelope envelope, List submitActions) { - getProject(envelope).ifPresentOrElse( - project -> { - if (!project.getValidationState().equals(ValidationState.VALID)) { - throw new StateTransitionNotAllowed( - String.format("Envelope with id %s cannot be submitted when the project is invalid.", envelope.getId()) - ); - } - }, - () -> { - throw new StateTransitionNotAllowed( - String.format("Envelope with id %s cannot be submitted without a project.", envelope.getId()) - ); - } - ); - - if (envelope.getSubmissionState() != SubmissionState.GRAPH_VALID) { - throw new StateTransitionNotAllowed( - String.format("Envelope with id %s cannot be submitted without a graph valid state", envelope.getId()) - ); - } - - if (isSubmitAction(submitActions)) { - envelope.setSubmitActions(new HashSet<>(submitActions)); - submissionEnvelopeRepository.save(envelope); - } else { - throw new IllegalArgumentException( - String.format( - "Envelope with id %s and state %s is submitted without the required submit actions", - envelope.getId(), - envelope.getSubmissionState() - ) - ); - } - handleEnvelopeStateUpdateRequest(envelope, SubmissionState.SUBMITTED); - } - - public void handleEnvelopeStateUpdateRequest(SubmissionEnvelope envelope, - SubmissionState state) { - if (envelope.getSubmissionState() == state) { - log.info(String.format( - "No Need to transition submissionEnvelope: %s already in state: %s", - envelope.getId(), - envelope.getSubmissionState() - )); - } else if (!envelope.allowedSubmissionStateTransitions().contains(state)) { - throw new StateTransitionNotAllowed(String.format( - "Envelope with id %s cannot be transitioned from state %s to state %s", - envelope.getId(), envelope.getSubmissionState(), state)); - } else { - messageRouter.routeStateTrackingUpdateMessageForEnvelopeEvent(envelope, state); - - if (state == SubmissionState.GRAPH_VALIDATION_REQUESTED) { - removeGraphValidationErrors(envelope); - } - } - } - - public void handleCommitSubmit(SubmissionEnvelope envelope) { - Set submitActions = envelope.getSubmitActions(); - if (submitActions.isEmpty()) { - log.info(String.format( - "No Submit Actions for submission: %s in state: %s", - envelope.getId(), - envelope.getSubmissionState() - )); - } else if (submitActions.contains(SubmitAction.ARCHIVE)) { - handleEnvelopeStateUpdateRequest(envelope, SubmissionState.PROCESSING); - archiveSubmission(envelope); - } else { - handleCommitArchived(envelope); - } - } - - public void handleCommitArchived(SubmissionEnvelope envelope) { - Set submitActions = envelope.getSubmitActions(); - if (submitActions.contains(SubmitAction.EXPORT)) { - handleEnvelopeStateUpdateRequest(envelope, SubmissionState.EXPORTING); - exportData(envelope); - } else if (submitActions.contains(SubmitAction.EXPORT_METADATA)) { - handleEnvelopeStateUpdateRequest(envelope, SubmissionState.EXPORTING); - generateSpreadsheet(envelope); - } else { - handleCommitExported(envelope); - } - } - - public void handleCommitExported(SubmissionEnvelope envelope) { - getProject(envelope).ifPresent(project -> projectService.updateWranglingState(project, WranglingState.SUBMITTED)); - if (envelope.getSubmitActions().contains(SubmitAction.CLEANUP)) { - cleanupSubmission(envelope); - } - } - - private void archiveSubmission(SubmissionEnvelope envelope) { - submit(exporter::exportManifests, envelope, "Archive Submission"); - } - - public void generateSpreadsheet(SubmissionEnvelope envelope) { - submit(exporter::generateSpreadsheet, envelope, "Generate Spreadsheet"); - } - - public void exportData(SubmissionEnvelope envelope) { - submit(exporter::exportData, envelope, "Export Data"); - } - - public void cleanupSubmission(SubmissionEnvelope envelope) { - try { - handleEnvelopeStateUpdateRequest(envelope, SubmissionState.CLEANUP); - } catch (Exception e) { - log.error(String.format("Uncaught Exception sending message to cleanup upload area for submission %s", envelope.getId()), e); - } - } - - public SubmissionEnvelope createUpdateSubmissionEnvelope() { - SubmissionEnvelope updateSubmissionEnvelope = new SubmissionEnvelope(); - submissionEnvelopeCreateHandler.setUuid(updateSubmissionEnvelope); - updateSubmissionEnvelope.setIsUpdate(true); - return createSubmissionEnvelope(updateSubmissionEnvelope); - } - - public SubmissionEnvelope createSubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { - SubmissionEnvelope insertedSubmissionEnvelope = submissionEnvelopeRepository.insert(submissionEnvelope); - submissionEnvelopeCreateHandler.handleSubmissionEnvelopeCreation(submissionEnvelope); - return insertedSubmissionEnvelope; - } - - public void deleteSubmission(SubmissionEnvelope submissionEnvelope, boolean forceDelete) { - if (!(submissionEnvelope.isOpen() || forceDelete)) - throw new UnsupportedOperationException("Cannot delete submission if it is already submitted!"); - - RetryTemplate retry = RetryTemplate.builder() - .maxAttempts(5) - .fixedBackoff(75) - .retryOn(OptimisticLockingFailureException.class) - .build(); - retry.execute(context -> { - cleanupLinksToSubmissionMetadata(submissionEnvelope); - return null; - }); - - biomaterialRepository.deleteBySubmissionEnvelope(submissionEnvelope); - processRepository.deleteBySubmissionEnvelope(submissionEnvelope); - protocolRepository.deleteBySubmissionEnvelope(submissionEnvelope); - fileRepository.deleteBySubmissionEnvelope(submissionEnvelope); - bundleManifestRepository.deleteByEnvelopeUuid(submissionEnvelope.getUuid().getUuid().toString()); - patchRepository.deleteBySubmissionEnvelope(submissionEnvelope); - submissionManifestRepository.deleteBySubmissionEnvelope(submissionEnvelope); - submissionErrorRepository.deleteBySubmissionEnvelope(submissionEnvelope); - submissionEnvelopeRepository.delete(submissionEnvelope); - - this.messageRouter.routeRequestUploadAreaCleanup(submissionEnvelope); - } - - - /** - * Ensures that any links to metadata in the submission are removed. - * - * @param submissionEnvelope - */ - private void cleanupLinksToSubmissionMetadata(SubmissionEnvelope submissionEnvelope) { - long startTime = System.currentTimeMillis(); - - processRepository.findBySubmissionEnvelope(submissionEnvelope) - .forEach(metadataCrudService::removeLinksToDocument); - - protocolRepository.findBySubmissionEnvelope(submissionEnvelope) - .forEach(metadataCrudService::removeLinksToDocument); - - bundleManifestRepository.findByEnvelopeUuid(submissionEnvelope.getUuid().getUuid().toString()) - .forEach(bundleManifest -> processRepository.findByInputBundleManifestsContains(bundleManifest) - .forEach(process -> { - process.getInputBundleManifests().remove(bundleManifest); - processRepository.save(process); - })); - - fileRepository.findBySubmissionEnvelope(submissionEnvelope) - .forEach(metadataCrudService::removeLinksToDocument); - - // project cleanup - projectRepository.findBySubmissionEnvelope(submissionEnvelope) - .forEach(project -> { - project.setSubmissionEnvelope(null); // TODO: address this; we should implement project containers that aren't deleted as part of deleteSubmission() - projectRepository.save(project); - }); - - projectRepository.findBySubmissionEnvelopesContains(submissionEnvelope) - .forEach(project -> { - project.getSubmissionEnvelopes().remove(submissionEnvelope); - projectRepository.save(project); - }); - - long endTime = System.currentTimeMillis(); - float duration = ((float) (endTime - startTime)) / 1000; - String durationStr = new DecimalFormat("#,###.##").format(duration); - log.info("cleanup link time: {} s", durationStr); - } - - private boolean isSubmitAction(List submitActions) { - return submitActions.contains(SubmitAction.ARCHIVE) - || submitActions.contains(SubmitAction.EXPORT) - || submitActions.contains(SubmitAction.EXPORT_METADATA); - } - - private void removeGraphValidationErrors(SubmissionEnvelope submissionEnvelope) { - biomaterialRepository.saveAll( - biomaterialRepository.findBySubmissionEnvelope(submissionEnvelope) - .peek(biomaterial -> biomaterial.setGraphValidationErrors(new ArrayList<>())) - .collect(Collectors.toList()) - ); - - processRepository.saveAll( - processRepository.findBySubmissionEnvelope(submissionEnvelope) - .peek(process -> process.setGraphValidationErrors(new ArrayList<>())) - .collect(Collectors.toList()) - ); - - protocolRepository.saveAll( - protocolRepository.findBySubmissionEnvelope(submissionEnvelope) - .peek(protocol -> protocol.setGraphValidationErrors(new ArrayList<>())) - .collect(Collectors.toList()) - ); - - fileRepository.saveAll( - fileRepository.findBySubmissionEnvelope(submissionEnvelope) - .peek(file -> file.setGraphValidationErrors(new ArrayList<>())) - .collect(Collectors.toList()) - ); - } - - public Optional getSubmissionContentLastUpdated(SubmissionEnvelope submissionEnvelope) { - PageRequest request = PageRequest.of(0, 1, new Sort(Sort.Direction.DESC, "updateDate")); - List projects = projectRepository.findBySubmissionEnvelopesContaining(submissionEnvelope, request).getContent(); - List biomaterials = biomaterialRepository.findBySubmissionEnvelope(submissionEnvelope, request).getContent(); - List protocols = protocolRepository.findBySubmissionEnvelope(submissionEnvelope, request).getContent(); - List processes = processRepository.findBySubmissionEnvelope(submissionEnvelope, request).getContent(); - List files = fileRepository.findBySubmissionEnvelope(submissionEnvelope, request).getContent(); - - return Stream.of(projects, biomaterials, protocols, processes, files) - .flatMap(List::stream) - .map(MetadataDocument::getUpdateDate) - .max(Instant::compareTo); - } - - public Optional getProject(SubmissionEnvelope submissionEnvelope) { - return projectRepository.findBySubmissionEnvelopesContains(submissionEnvelope).findFirst(); - } - - private void submit(Consumer submissionAction, SubmissionEnvelope submission, String actionName) { - executorService.submit(() -> { - try { - submissionAction.accept(submission); - } catch (Exception e) { - log.error(String.format("Uncaught Exception sending message %s for Submission %s", - actionName, submission.getId()), e); - } - }); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/submission/SubmissionStateMachineService.java b/src/main/java/org/humancellatlas/ingest/submission/SubmissionStateMachineService.java deleted file mode 100644 index d1ca3531c..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/SubmissionStateMachineService.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.humancellatlas.ingest.submission; - -import lombok.AllArgsConstructor; -import lombok.NonNull; -import org.humancellatlas.ingest.config.ConfigurationService; -import org.humancellatlas.ingest.state.ValidationState; -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.stereotype.Service; -import org.springframework.web.client.RestOperations; -import org.springframework.web.client.RestTemplate; -import org.springframework.web.util.UriComponentsBuilder; - -import java.net.URI; -import java.util.Map; -import java.util.UUID; - -/** - * Created by rolando on 05/09/2018. - */ -@Service -@AllArgsConstructor -public class SubmissionStateMachineService { - private final @NonNull RestOperations restOperations = new RestTemplate( ); - private final @NonNull ConfigurationService configurationService; - - private static HttpEntity DEFAULT_HTTP_ENTITY = defaultHttpEntity(); - - public Map documentStatesForEnvelope(SubmissionEnvelope submissionEnvelope) { - UUID envelopeUuid = submissionEnvelope.getUuid().getUuid(); - ParameterizedTypeReference> documentStatesType = new ParameterizedTypeReference>() {}; - URI documentStatesUri = UriComponentsBuilder.newInstance() - .scheme(configurationService.getStateTrackerScheme()) - .host(configurationService.getStateTrackerHost()) - .port(configurationService.getStateTrackerPort()) - .pathSegment(configurationService.getDocumentStatesPath(), envelopeUuid.toString()) - .build() - .toUri(); - - return restOperations.exchange(documentStatesUri, - HttpMethod.GET, - DEFAULT_HTTP_ENTITY, - documentStatesType) - .getBody(); - } - - private static HttpEntity defaultHttpEntity() { - return new HttpEntity<>(null, uriListHeaders()); - } - - private static HttpHeaders uriListHeaders() { - HttpHeaders headers = new HttpHeaders(); - headers.add("Content-type", "application/json"); - return headers; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/submission/exception/NotAllowedDuringSubmissionStateException.java b/src/main/java/org/humancellatlas/ingest/submission/exception/NotAllowedDuringSubmissionStateException.java deleted file mode 100644 index 3e447d035..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/exception/NotAllowedDuringSubmissionStateException.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.humancellatlas.ingest.submission.exception; - -import org.humancellatlas.ingest.security.exception.NotAllowedException; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; - -public class NotAllowedDuringSubmissionStateException extends NotAllowedException { - public NotAllowedDuringSubmissionStateException() { - super("Operation not allowed during the current submission state for the envelope."); - } - - public NotAllowedDuringSubmissionStateException(SubmissionEnvelope submissionEnvelope) { - super(String.format("Operation not allowed during the current submission state %s for the envelope %s", - submissionEnvelope.getSubmissionState(), submissionEnvelope.getUuid()) - ); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/submission/web/MongoAggregationUtils.java b/src/main/java/org/humancellatlas/ingest/submission/web/MongoAggregationUtils.java deleted file mode 100644 index 872761fbc..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/web/MongoAggregationUtils.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.humancellatlas.ingest.submission.web; - -import org.bson.Document; -import org.springframework.data.mongodb.core.aggregation.Aggregation; -import org.springframework.data.mongodb.core.aggregation.AggregationOperation; - -import java.util.List; -import java.util.stream.Collectors; - -public class MongoAggregationUtils { - public static Aggregation aggregationPipelineFromStrings(List jsonStages) { - return Aggregation.newAggregation( - jsonStages.stream() - .map(Document::parse) - .map((Document d) -> (AggregationOperation) context -> context.getMappedObject(d)) - .collect(Collectors.toList()) - ); - } -} \ No newline at end of file diff --git a/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionController.java b/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionController.java deleted file mode 100644 index 875098b44..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionController.java +++ /dev/null @@ -1,381 +0,0 @@ -package org.humancellatlas.ingest.submission.web; - -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.bundle.BundleManifest; -import org.humancellatlas.ingest.bundle.BundleManifestRepository; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.exporter.Exporter; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.process.ProcessService; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.humancellatlas.ingest.protocol.ProtocolService; -import org.humancellatlas.ingest.security.CheckAllowed; -import org.humancellatlas.ingest.state.SubmissionState; -import org.humancellatlas.ingest.state.SubmitAction; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeService; -import org.humancellatlas.ingest.submission.SubmissionStateMachineService; -import org.humancellatlas.ingest.submission.exception.NotAllowedDuringSubmissionStateException; -import org.humancellatlas.ingest.submissionmanifest.SubmissionManifest; -import org.humancellatlas.ingest.submissionmanifest.SubmissionManifestRepository; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.data.web.PagedResourcesAssembler; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.http.HttpEntity; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; - -import java.time.Instant; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -/** - * Spring controller that will handle submission events on a {@link SubmissionEnvelope} - * - * @author Tony Burdett - * @date 31/08/17 - */ -@RepositoryRestController -@ExposesResourceFor(SubmissionEnvelope.class) -@RequiredArgsConstructor -@Getter -public class SubmissionController { - - private final @NonNull Exporter exporter; - - private final @NonNull SubmissionEnvelopeService submissionEnvelopeService; - private final @NonNull SubmissionStateMachineService submissionStateMachineService; - private final @NonNull ProcessService processService; - private final @NonNull ProtocolService protocolService; - - private final @NonNull SubmissionEnvelopeRepository submissionEnvelopeRepository; - private final @NonNull FileRepository fileRepository; - private final @NonNull ProjectRepository projectRepository; - private final @NonNull ProtocolRepository protocolRepository; - private final @NonNull BiomaterialRepository biomaterialRepository; - private final @NonNull ProcessRepository processRepository; - private final @NonNull BundleManifestRepository bundleManifestRepository; - private final @NonNull SubmissionManifestRepository submissionManifestRepository; - - private final MessageRouter messageRouter; - - private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; - private final @NonNull Logger log = LoggerFactory.getLogger(getClass()); - - - @PostMapping("/submissionEnvelopes" + Links.UPDATE_SUBMISSION_URL) - ResponseEntity createUpdateSubmission( - final PersistentEntityResourceAssembler resourceAssembler) { - SubmissionEnvelope updateSubmission = getSubmissionEnvelopeService().createUpdateSubmissionEnvelope(); - return ResponseEntity.ok(resourceAssembler.toFullResource(updateSubmission)); - } - - @GetMapping({ - "/submissionEnvelopes/{sub_id}" + Links.PROJECTS_URL, - "/submissionEnvelopes/{sub_id}" + Links.SUBMISSION_RELATED_PROJECTS_URL - }) - ResponseEntity getProjects(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - Page projects = getProjectRepository().findBySubmissionEnvelopesContaining(submissionEnvelope, pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(projects, resourceAssembler)); - } - - @GetMapping("/submissionEnvelopes/{sub_id}/biomaterials") - ResponseEntity getBiomaterials(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - Page biomaterials = getBiomaterialRepository().findBySubmissionEnvelope(submissionEnvelope, pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(biomaterials, resourceAssembler)); - } - - - @GetMapping("/submissionEnvelopes/{sub_id}/processes") - ResponseEntity getProcesses(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - Page processes = getProcessRepository().findBySubmissionEnvelope(submissionEnvelope, pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(processes - , resourceAssembler)); - } - - @GetMapping("/submissionEnvelopes/{sub_id}/protocols") - ResponseEntity getProtocols(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - Page protocols = protocolService.retrieve(submissionEnvelope, pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(protocols, resourceAssembler)); - } - - @GetMapping("/submissionEnvelopes/{sub_id}/files") - ResponseEntity getFiles(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - Page files = getFileRepository().findBySubmissionEnvelope(submissionEnvelope, pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(files, resourceAssembler)); - } - - @GetMapping("/submissionEnvelopes/{sub_id}/bundleManifests") - ResponseEntity getBundleManifests(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - Pageable pageable, - final PersistentEntityResourceAssembler resourceAssembler) { - Page bundleManifests = getBundleManifestRepository().findByEnvelopeUuid(submissionEnvelope.getUuid().getUuid().toString(), pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(bundleManifests, resourceAssembler)); - } - - @GetMapping("/submissionEnvelopes/{sub_id}/submissionManifest") - ResponseEntity getSubmissionManifests(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - final PersistentEntityResourceAssembler resourceAssembler) { - Optional submissionManifest = Optional.ofNullable(getSubmissionManifestRepository().findBySubmissionEnvelopeId(submissionEnvelope.getId())); - if (submissionManifest.isPresent()) { - return ResponseEntity.ok(resourceAssembler.toFullResource(submissionManifest.get())); - } else { - return ResponseEntity.notFound().build(); - } - } - - @GetMapping("/submissionEnvelopes/{sub_id}/biomaterials/{state}") - ResponseEntity getSamplesWithValidationState(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, @PathVariable("state") String state, - Pageable pageable, final PersistentEntityResourceAssembler resourceAssembler) { - Page biomaterials = getBiomaterialRepository().findBySubmissionEnvelopeAndValidationState(submissionEnvelope, ValidationState.valueOf(state.toUpperCase()), pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(biomaterials, resourceAssembler)); - } - - @GetMapping("/submissionEnvelopes/{sub_id}/processes/{state}") - ResponseEntity getProcessesWithValidationState(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, @PathVariable("state") String state, - Pageable pageable, final PersistentEntityResourceAssembler resourceAssembler) { - Page processes = getProcessRepository().findBySubmissionEnvelopeAndValidationState(submissionEnvelope, ValidationState.valueOf(state.toUpperCase()), pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(processes, resourceAssembler)); - } - - @GetMapping("/submissionEnvelopes/{sub_id}/protocols/{state}") - ResponseEntity getProtocolsWithValidationState(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, @PathVariable("state") String state, - Pageable pageable, final PersistentEntityResourceAssembler resourceAssembler) { - Page protocols = getProtocolRepository().findBySubmissionEnvelopeAndValidationState(submissionEnvelope, ValidationState.valueOf(state.toUpperCase()), pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(protocols, resourceAssembler)); - } - - @GetMapping("/submissionEnvelopes/{sub_id}/files/{state}") - ResponseEntity getFilesWithValidationState(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, @PathVariable("state") String state, - Pageable pageable, final PersistentEntityResourceAssembler resourceAssembler) { - Page files = getFileRepository().findBySubmissionEnvelopeAndValidationState(submissionEnvelope, ValidationState.valueOf(state.toUpperCase()), pageable); - return ResponseEntity.ok(getPagedResourcesAssembler().toResource(files, resourceAssembler)); - } - - @CheckAllowed(value = "#submissionEnvelope.isEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @PutMapping("/submissionEnvelopes/{id}" + Links.SUBMIT_URL) - HttpEntity submitEnvelopeRequest(@PathVariable("id") SubmissionEnvelope submissionEnvelope, - @RequestBody(required = false) List submitActionParam, - final PersistentEntityResourceAssembler resourceAssembler) { - List submitActions = Optional.ofNullable( - submitActionParam.stream().map(submitAction -> { - return SubmitAction.valueOf(submitAction.toUpperCase()); - }).collect(Collectors.toList()) - ).orElse(List.of(SubmitAction.ARCHIVE, SubmitAction.EXPORT, SubmitAction.CLEANUP)); - - submissionEnvelopeService.handleSubmitRequest(submissionEnvelope, submitActions); - return ResponseEntity.accepted().body(resourceAssembler.toFullResource(submissionEnvelope)); - } - - @CheckAllowed(value = "#submissionEnvelope.isEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @PutMapping("/submissionEnvelopes/{id}" + Links.ARCHIVED_URL) - HttpEntity completeArchivingEnvelopeRequest(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - submissionEnvelopeService.handleEnvelopeStateUpdateRequest(submissionEnvelope, SubmissionState.ARCHIVED); - return ResponseEntity.accepted().body(resourceAssembler.toFullResource(submissionEnvelope)); - } - - @CheckAllowed(value = "#submissionEnvelope.isEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @PutMapping("/submissionEnvelopes/{id}" + Links.EXPORT_URL) - HttpEntity exportEnvelopeRequest(@PathVariable("id") SubmissionEnvelope submissionEnvelope, - final PersistentEntityResourceAssembler resourceAssembler) { - submissionEnvelopeService.exportData(submissionEnvelope); - return ResponseEntity.accepted().body(resourceAssembler.toFullResource(submissionEnvelope)); - } - - @CheckAllowed(value = "#submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @PutMapping("/submissionEnvelopes/{id}" + Links.CLEANUP_URL) - HttpEntity cleanupEnvelopeRequest(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - submissionEnvelopeService.handleEnvelopeStateUpdateRequest(submissionEnvelope, SubmissionState.CLEANUP); - return ResponseEntity.accepted().body(resourceAssembler.toFullResource(submissionEnvelope)); - } - - @CheckAllowed(value = "#submissionEnvelope.isSystemEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @PutMapping("/submissionEnvelopes/{id}" + Links.COMPLETE_URL) - HttpEntity completeEnvelopeRequest(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - submissionEnvelopeService.handleEnvelopeStateUpdateRequest(submissionEnvelope, SubmissionState.COMPLETE); - return ResponseEntity.accepted().body(resourceAssembler.toFullResource(submissionEnvelope)); - } - - private HttpEntity enactStateTransition(SubmissionState state, SubmissionEnvelope envelope, final PersistentEntityResourceAssembler resourceAssembler) { - envelope.enactStateTransition(state); - getSubmissionEnvelopeRepository().save(envelope); - return ResponseEntity.accepted().body(resourceAssembler.toFullResource(envelope)); - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_DRAFT_URL) - HttpEntity enactDraftEnvelope(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - return this.enactStateTransition(SubmissionState.DRAFT, submissionEnvelope, resourceAssembler); - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_METADATA_VALIDATING_URL) - HttpEntity enactValidatingEnvelope(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - return this.enactStateTransition(SubmissionState.METADATA_VALIDATING, submissionEnvelope, resourceAssembler); - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_METADATA_INVALID_URL) - HttpEntity enactInvalidEnvelope(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - return this.enactStateTransition(SubmissionState.METADATA_INVALID, submissionEnvelope, resourceAssembler); - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_METADATA_VALID_URL) - HttpEntity enactValidEnvelope(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - return this.enactStateTransition(SubmissionState.METADATA_VALID, submissionEnvelope, resourceAssembler); - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_SUBMIT_URL) - HttpEntity enactSubmitEnvelope(@PathVariable("id") SubmissionEnvelope submissionEnvelope, - final PersistentEntityResourceAssembler resourceAssembler) { - HttpEntity response = this.enactStateTransition(SubmissionState.SUBMITTED, submissionEnvelope, resourceAssembler); - log.info(String.format("Submission envelope with ID %s was submitted.", submissionEnvelope.getId())); - submissionEnvelopeService.handleCommitSubmit(submissionEnvelope); - return response; - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_PROCESSING_URL) - HttpEntity enactProcessEnvelope(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - return this.enactStateTransition(SubmissionState.PROCESSING, submissionEnvelope, resourceAssembler); - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_ARCHIVING_URL) - HttpEntity enactArchivingEnvelope(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - return this.enactStateTransition(SubmissionState.ARCHIVING, submissionEnvelope, resourceAssembler); - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_ARCHIVED_URL) - HttpEntity enactArchivedEnvelope(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - HttpEntity response = this.enactStateTransition(SubmissionState.ARCHIVED, submissionEnvelope, resourceAssembler); - log.info(String.format("Submission envelope with ID %s was archived.", submissionEnvelope.getId())); - submissionEnvelopeService.handleCommitArchived(submissionEnvelope); - return response; - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_EXPORTING_URL) - HttpEntity enactExportingEnvelope(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - return this.enactStateTransition(SubmissionState.EXPORTING, submissionEnvelope, resourceAssembler); - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_EXPORTED_URL) - HttpEntity enactExportedEnvelope(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - HttpEntity response = this.enactStateTransition(SubmissionState.EXPORTED, submissionEnvelope, resourceAssembler); - log.info(String.format("Submission envelope with ID %s was exported.", submissionEnvelope.getId())); - submissionEnvelopeService.handleCommitExported(submissionEnvelope); - return response; - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_CLEANUP_URL) - HttpEntity enactCleanupEnvelope(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - return this.enactStateTransition(SubmissionState.CLEANUP, submissionEnvelope, resourceAssembler); - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_COMPLETE_URL) - HttpEntity enactCompleteEnvelope(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - return this.enactStateTransition(SubmissionState.COMPLETE, submissionEnvelope, resourceAssembler); - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_GRAPH_VALIDATION_REQUESTED_URL) - HttpEntity enactGraphValidationRequested(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - HttpEntity response = this.enactStateTransition(SubmissionState.GRAPH_VALIDATION_REQUESTED, submissionEnvelope, resourceAssembler); - messageRouter.routeGraphValidationMessageFor(submissionEnvelope); - return response; - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_GRAPH_VALIDATING_URL) - HttpEntity enactGraphValidating(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - return this.enactStateTransition(SubmissionState.GRAPH_VALIDATING, submissionEnvelope, resourceAssembler); - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_GRAPH_VALID_URL) - HttpEntity enactGraphValid(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - return this.enactStateTransition(SubmissionState.GRAPH_VALID, submissionEnvelope, resourceAssembler); - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_GRAPH_INVALID_URL) - HttpEntity enactGraphInvalid(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - return this.enactStateTransition(SubmissionState.GRAPH_INVALID, submissionEnvelope, resourceAssembler); - } - - private HttpEntity performStateUpdateRequest(SubmissionState state, SubmissionEnvelope envelope, final PersistentEntityResourceAssembler resourceAssembler) { - submissionEnvelopeService.handleEnvelopeStateUpdateRequest(envelope, state); - return ResponseEntity.accepted().body(resourceAssembler.toFullResource(envelope)); - } - - @CheckAllowed(value = "#submissionEnvelope.isEditable()", exception = NotAllowedDuringSubmissionStateException.class) - @PutMapping("/submissionEnvelopes/{id}" + Links.GRAPH_VALIDATION_REQUESTED_URL) - HttpEntity requestGraphValidation(@PathVariable("id") SubmissionEnvelope submissionEnvelope, - final PersistentEntityResourceAssembler resourceAssembler) { - // Used by the user (UI) to start the validation process - return this.performStateUpdateRequest(SubmissionState.GRAPH_VALIDATION_REQUESTED, submissionEnvelope, resourceAssembler); - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.GRAPH_VALIDATING_URL) - HttpEntity requestGraphValidating(@PathVariable("id") SubmissionEnvelope submissionEnvelope, - final PersistentEntityResourceAssembler resourceAssembler) { - // Used by ingest-graph-validator to notify that the graph is validating - return this.performStateUpdateRequest(SubmissionState.GRAPH_VALIDATING, submissionEnvelope, resourceAssembler); - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.GRAPH_VALID_URL) - HttpEntity requestGraphValid(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - // used by ingest-graph-validator to notify that graph is valid - return this.performStateUpdateRequest(SubmissionState.GRAPH_VALID, submissionEnvelope, resourceAssembler); - } - - @PutMapping("/submissionEnvelopes/{id}" + Links.GRAPH_INVALID_URL) - HttpEntity requestGraphInvalid(@PathVariable("id") SubmissionEnvelope submissionEnvelope, final PersistentEntityResourceAssembler resourceAssembler) { - // used by ingest-graph-validator to notify that graph is invalid - return this.performStateUpdateRequest(SubmissionState.GRAPH_INVALID, submissionEnvelope, resourceAssembler); - } - - @GetMapping("/submissionEnvelopes/{id}" + Links.SUBMISSION_DOCUMENTS_SM_URL) - ResponseEntity getDocumentStateMachineReport(@PathVariable("id") SubmissionEnvelope submissionEnvelope) { - return ResponseEntity.ok(getSubmissionStateMachineService().documentStatesForEnvelope(submissionEnvelope)); - } - - @GetMapping("/submissionEnvelopes/{id}/sync") - HttpEntity forceStateCheck(@PathVariable("id") SubmissionEnvelope submissionEnvelope) { - // TODO: if really needed, modify this method to ask the state tracker component for an update - return ResponseEntity.noContent().build(); - } - - @DeleteMapping("/submissionEnvelopes/{id}") - HttpEntity forceDeleteSubmission(@PathVariable("id") SubmissionEnvelope submissionEnvelope, - @RequestParam(name = "force", required = false, defaultValue = "false") boolean forceDelete) { - getSubmissionEnvelopeService().deleteSubmission(submissionEnvelope, forceDelete); - return ResponseEntity.accepted().build(); - } - - @GetMapping("/submissionEnvelopes/{id}" + Links.SUBMISSION_CONTENT_LAST_UPDATED_URL) - ResponseEntity getContentLastUpdated(@PathVariable("id") SubmissionEnvelope submissionEnvelope) { - Optional lastUpdateDate = submissionEnvelopeService.getSubmissionContentLastUpdated(submissionEnvelope); - return ResponseEntity.ok(lastUpdateDate.isPresent() ? lastUpdateDate.get().toString() : null); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionEnvelopeCollectionResourceProcessor.java b/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionEnvelopeCollectionResourceProcessor.java deleted file mode 100644 index 0280a508b..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionEnvelopeCollectionResourceProcessor.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.humancellatlas.ingest.submission.web; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.rest.webmvc.RepositoryLinksResource; -import org.springframework.hateoas.*; -import org.springframework.stereotype.Component; - -@Component -@RequiredArgsConstructor -public class SubmissionEnvelopeCollectionResourceProcessor implements ResourceProcessor { - private final @NonNull EntityLinks entityLinks; - - private Link getUpdateSubmissionsLink() { - return entityLinks.linkFor(SubmissionEnvelope.class) - .slash(Links.UPDATE_SUBMISSION_URL) - .withRel(Links.UPDATE_SUBMISSION_REL); - } - - - @Override - public RepositoryLinksResource process(RepositoryLinksResource resource) { - resource.add(getUpdateSubmissionsLink()); - return resource; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionEnvelopeResourceProcessor.java b/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionEnvelopeResourceProcessor.java deleted file mode 100644 index a4cdb11ec..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionEnvelopeResourceProcessor.java +++ /dev/null @@ -1,321 +0,0 @@ -package org.humancellatlas.ingest.submission.web; - -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.state.SubmissionState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submissionmanifest.SubmissionManifest; -import org.humancellatlas.ingest.submissionmanifest.SubmissionManifestRepository; -import org.springframework.hateoas.EntityLinks; -import org.springframework.hateoas.Link; -import org.springframework.hateoas.Resource; -import org.springframework.hateoas.ResourceProcessor; -import org.springframework.stereotype.Component; - -import java.util.Arrays; -import java.util.Optional; - -/** - * Javadocs go here! - * - * @author Tony Burdett - * @date 31/08/17 - */ -@Component -@RequiredArgsConstructor -public class SubmissionEnvelopeResourceProcessor implements ResourceProcessor> { - private final @NonNull EntityLinks entityLinks; - private final @NonNull SubmissionManifestRepository submissionManifestRepository; - - private Link getBiomaterialsLink(SubmissionEnvelope submissionEnvelope) { - return entityLinks.linkForSingleResource(submissionEnvelope) - .slash(Links.BIOMATERIALS_URL) - .withRel(Links.BIOMATERIALS_REL); - } - - private Link getProcessesLink(SubmissionEnvelope submissionEnvelope) { - return entityLinks.linkForSingleResource(submissionEnvelope) - .slash(Links.PROCESSES_URL) - .withRel(Links.PROCESSES_REL); - } - - private Link getFilesLink(SubmissionEnvelope submissionEnvelope) { - return entityLinks.linkForSingleResource(submissionEnvelope) - .slash(Links.FILES_URL) - .withRel(Links.FILES_REL); - } - - private Link getProjectsLink(SubmissionEnvelope submissionEnvelope) { - return entityLinks.linkForSingleResource(submissionEnvelope) - .slash(Links.PROJECTS_URL) - .withRel(Links.PROJECTS_REL); - } - - private Link getProtocolsLink(SubmissionEnvelope submissionEnvelope) { - return entityLinks.linkForSingleResource(submissionEnvelope) - .slash(Links.PROTOCOLS_URL) - .withRel(Links.PROTOCOLS_REL); - } - - private Link getBundleManifestsLink(SubmissionEnvelope submissionEnvelope) { - return entityLinks.linkForSingleResource(submissionEnvelope) - .slash(Links.BUNDLE_MANIFESTS_URL) - .withRel(Links.BUNDLE_MANIFESTS_REL); - } - - private Link getSubmissionManifestsLink(SubmissionEnvelope submissionEnvelope) { - return entityLinks.linkForSingleResource(submissionEnvelope) - .slash(Links.SUBMISSION_MANIFEST_URL) - .withRel(Links.SUBMISSION_MANIFEST_REL); - } - - private Link getExportJobsLink(SubmissionEnvelope submissionEnvelope) { - return entityLinks.linkForSingleResource(submissionEnvelope) - .slash(Links.EXPORT_JOBS_URL) - .withRel(Links.EXPORT_JOBS_REL); - } - - private Link getSubmissionErrorsLink(SubmissionEnvelope submissionEnvelope) { - return entityLinks.linkForSingleResource(submissionEnvelope) - .slash(Links.SUBMISSION_ERRORS_URL) - .withRel(Links.SUBMISSION_ERRORS_REL); - } - - private Link getSubmissionSummary(SubmissionEnvelope submissionEnvelope) { - return entityLinks.linkForSingleResource(submissionEnvelope) - .slash(Links.SUBMISSION_SUMMARY_URL) - .withRel(Links.SUBMISSION_SUMMARY_REL); - } - - private Link getSubmissionLinkingMap(SubmissionEnvelope submissionEnvelope) { - return entityLinks.linkForSingleResource(submissionEnvelope) - .slash(Links.SUBMISSION_LINKING_MAP_URL) - .withRel(Links.SUBMISSION_LINKING_MAP_REL); - } - - private Link getSubmissionContentLastUpdatedLink(SubmissionEnvelope submissionEnvelope) { - return entityLinks.linkForSingleResource(submissionEnvelope) - .slash(Links.SUBMISSION_CONTENT_LAST_UPDATED_URL) - .withRel(Links.SUBMISSION_CONTENT_LAST_UPDATED_REL); - } - - private Link getSubmissionRelatedProjectLink(SubmissionEnvelope submissionEnvelope) { - return entityLinks.linkForSingleResource(submissionEnvelope) - .slash(Links.SUBMISSION_RELATED_PROJECTS_URL) - .withRel(Links.SUBMISSION_RELATED_PROJECTS_REL); - } - - private Link getSubmissionDocumentStateLink(SubmissionEnvelope submissionEnvelope) { - return entityLinks.linkForSingleResource(submissionEnvelope) - .slash(Links.SUBMISSION_DOCUMENTS_SM_URL) - .withRel(Links.SUBMISSION_DOCUMENTS_SM_REL); - } - - private Optional getStateTransitionLink(SubmissionEnvelope submissionEnvelope, SubmissionState targetState) { - Optional transitionResourceName = getSubresourceNameForRequestSubmissionState(submissionEnvelope, targetState); - if (transitionResourceName.isPresent()) { - Optional rel = getRelNameForRequestSubmissionState(targetState); - if (rel.isPresent()) { - return Optional.of(entityLinks.linkForSingleResource(submissionEnvelope) - .slash(transitionResourceName.get()) - .withRel(rel.get())); - } else { - throw new RuntimeException(String.format("Unexpected link/rel mismatch exception (link = '%s', rel = " + - "'%s')", transitionResourceName.toString(), rel.toString())); - } - } else { - return Optional.empty(); - } - } - - private Optional getCommitStateTransitionLink(SubmissionEnvelope submissionEnvelope, SubmissionState targetState) { - Optional transitionResourceName = getSubresourceNameForCommitSubmissionState(targetState); - if (transitionResourceName.isPresent()) { - Optional rel = getRelNameForCommitSubmissionState(targetState); - if (rel.isPresent()) { - return Optional.of(entityLinks.linkForSingleResource(submissionEnvelope) - .slash(transitionResourceName.get()) - .withRel(rel.get())); - } else { - throw new RuntimeException(String.format("Unexpected link/rel mismatch exception (link = '%s', rel = " + - "'%s')", transitionResourceName.toString(), rel.toString())); - } - } else { - return Optional.empty(); - } - } - - private Optional getRelNameForRequestSubmissionState(SubmissionState submissionState) { - switch (submissionState) { - case GRAPH_VALIDATION_REQUESTED: - return Optional.of(Links.GRAPH_VALIDATION_REQUESTED_REL); - case GRAPH_VALIDATING: - return Optional.of(Links.GRAPH_VALIDATING_REL); - case GRAPH_VALID: - return Optional.of(Links.GRAPH_VALID_REL); - case GRAPH_INVALID: - return Optional.of(Links.GRAPH_INVALID_REL); - case SUBMITTED: - return Optional.of(Links.SUBMIT_REL); - case ARCHIVED: - return Optional.of(Links.ARCHIVED_REL); - case EXPORTING: - return Optional.of(Links.EXPORT_REL); - case CLEANUP: - return Optional.of(Links.CLEANUP_REL); - case COMPLETE: - return Optional.of(Links.COMPLETE_REL); - default: - // default returns no links (not expecting external user interaction) - return Optional.empty(); - } - } - - private Optional getSubmitLink(SubmissionEnvelope submissionEnvelope){ - SubmissionManifest submissionManifest = this.submissionManifestRepository.findBySubmissionEnvelopeId(submissionEnvelope.getId()); - - if(submissionManifest == null) - return Optional.of(Links.SUBMIT_URL); - else if(submissionManifest.getExpectedLinks() !=null && submissionManifest.getExpectedLinks().equals(submissionManifest.getActualLinks())){ - return Optional.of(Links.SUBMIT_URL); - } else { - return Optional.empty(); - } - } - - private Optional getSubresourceNameForRequestSubmissionState(SubmissionEnvelope submissionEnvelope, SubmissionState submissionState) { - switch (submissionState) { - case SUBMITTED: - return this.getSubmitLink(submissionEnvelope); - case ARCHIVED: - return Optional.of(Links.ARCHIVED_URL); - case EXPORTING: - return Optional.of(Links.EXPORT_URL); - case CLEANUP: - return Optional.of(Links.CLEANUP_URL); - case COMPLETE: - return Optional.of(Links.COMPLETE_URL); - default: - // default returns no subresource name (not expecting external user interaction) - return Optional.empty(); - } - } - - private Optional getRelNameForCommitSubmissionState(SubmissionState submissionState) { - switch (submissionState) { - case DRAFT: - return Optional.of(Links.COMMIT_DRAFT_REL); - case METADATA_VALIDATING: - return Optional.of(Links.COMMIT_METADATA_VALIDATING_REL); - case METADATA_INVALID: - return Optional.of(Links.COMMIT_METADATA_INVALID_REL); - case METADATA_VALID: - return Optional.of(Links.COMMIT_METADATA_VALID_REL); - case GRAPH_VALIDATION_REQUESTED: - return Optional.of(Links.COMMIT_GRAPH_VALIDATION_REQUESTED_REL); - case GRAPH_VALIDATING: - return Optional.of(Links.COMMIT_GRAPH_VALIDATING_REL); - case GRAPH_VALID: - return Optional.of(Links.COMMIT_GRAPH_VALID_REL); - case GRAPH_INVALID: - return Optional.of(Links.COMMIT_GRAPH_INVALID_REL); - case SUBMITTED: - return Optional.of(Links.COMMIT_SUBMIT_REL); - case PROCESSING: - return Optional.of(Links.COMMIT_PROCESSING_REL); - case ARCHIVING: - return Optional.of(Links.COMMIT_ARCHIVING_REL); - case ARCHIVED: - return Optional.of(Links.COMMIT_ARCHIVED_REL); - case EXPORTING: - return Optional.of(Links.COMMIT_EXPORTING_REL); - case EXPORTED: - return Optional.of(Links.COMMIT_EXPORTED_REL); - case CLEANUP: - return Optional.of(Links.COMMIT_CLEANUP_REL); - case COMPLETE: - return Optional.of(Links.COMMIT_COMPLETE_REL); - default: - // default returns no links (not expecting external user interaction) - return Optional.empty(); - } - } - - private Optional getSubresourceNameForCommitSubmissionState(SubmissionState submissionState) { - switch (submissionState) { - case DRAFT: - return Optional.of(Links.COMMIT_DRAFT_URL); - case METADATA_VALIDATING: - return Optional.of(Links.COMMIT_METADATA_VALIDATING_URL); - case METADATA_INVALID: - return Optional.of(Links.COMMIT_METADATA_INVALID_URL); - case METADATA_VALID: - return Optional.of(Links.COMMIT_METADATA_VALID_URL); - case GRAPH_VALIDATION_REQUESTED: - return Optional.of(Links.COMMIT_GRAPH_VALIDATION_REQUESTED_URL); - case GRAPH_VALIDATING: - return Optional.of(Links.COMMIT_GRAPH_VALIDATING_URL); - case GRAPH_VALID: - return Optional.of(Links.COMMIT_GRAPH_VALID_URL); - case GRAPH_INVALID: - return Optional.of(Links.COMMIT_GRAPH_INVALID_URL); - case SUBMITTED: - return Optional.of(Links.COMMIT_SUBMIT_URL); - case PROCESSING: - return Optional.of(Links.COMMIT_PROCESSING_URL); - case ARCHIVING: - return Optional.of(Links.COMMIT_ARCHIVING_URL); - case ARCHIVED: - return Optional.of(Links.COMMIT_ARCHIVED_URL); - case EXPORTING: - return Optional.of(Links.COMMIT_EXPORTING_URL); - case EXPORTED: - return Optional.of(Links.COMMIT_EXPORTED_URL); - case CLEANUP: - return Optional.of(Links.COMMIT_CLEANUP_URL); - case COMPLETE: - return Optional.of(Links.COMMIT_COMPLETE_URL); - default: - // default returns no subresource name (not expecting external user interaction) - return Optional.empty(); - } - } - - @Override - public Resource process(Resource resource) { - SubmissionEnvelope submissionEnvelope = resource.getContent(); - - // add subresource links for each type of metadata document in a submission envelope - resource.add(getBiomaterialsLink(submissionEnvelope)); - resource.add(getProcessesLink(submissionEnvelope)); - resource.add(getFilesLink(submissionEnvelope)); - resource.add(getProjectsLink(submissionEnvelope)); - resource.add(getProtocolsLink(submissionEnvelope)); - resource.add(getBundleManifestsLink(submissionEnvelope)); - resource.add(getSubmissionManifestsLink(submissionEnvelope)); - resource.add(getExportJobsLink(submissionEnvelope)); - resource.add(getSubmissionErrorsLink(submissionEnvelope)); - resource.add(getSubmissionDocumentStateLink(submissionEnvelope)); - resource.add(getSubmissionSummary(submissionEnvelope)); - resource.add(getSubmissionLinkingMap(submissionEnvelope)); - resource.add(getSubmissionContentLastUpdatedLink(submissionEnvelope)); - resource.add(getSubmissionRelatedProjectLink(submissionEnvelope)); - - // add subresource links for allowed state transition requests - submissionEnvelope.allowedSubmissionStateTransitions().stream() - .map(submissionState -> getStateTransitionLink(submissionEnvelope, submissionState)) - .filter(Optional::isPresent) - .map(Optional::get) - .forEach(resource::add); - - // add subresource links for state tracker to commit state transitions - Arrays.stream(SubmissionState.values()) - .map(submissionState -> getCommitStateTransitionLink(submissionEnvelope, submissionState)) - .filter(Optional::isPresent) - .map(Optional::get) - .forEach(resource::add); - - return resource; - } -} diff --git a/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionLinkMapController.java b/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionLinkMapController.java deleted file mode 100644 index acb0b5904..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionLinkMapController.java +++ /dev/null @@ -1,141 +0,0 @@ -package org.humancellatlas.ingest.submission.web; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.NonNull; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.core.web.Links; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; - -import java.util.*; - -@RestController -public class SubmissionLinkMapController { - - @Autowired - BiomaterialRepository biomaterialRepository; - @Autowired - FileRepository fileRepository; - @Autowired - ProcessRepository processRepository; - @Autowired - ProtocolRepository protocolRepository; - - @Autowired - SubmissionLinkMapRepository submissionLinkMapRepository; - - @NonNull - private final Logger log = LoggerFactory.getLogger(getClass()); - - @RequestMapping(path = "/submissionEnvelopes/{sub_id}" + Links.SUBMISSION_LINKING_MAP_URL, - method = RequestMethod.GET) - @ResponseBody - public SubmissionLinkingMap getSubmissionLinkMap(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope) { - return new SubmissionLinkingMap(submissionEnvelope); - } - - @Getter - public class SubmissionLinkingMap { - final Map processes = new Hashtable<>(); - final Map protocols = new Hashtable<>(); - final Map biomaterials = new Hashtable<>(); - final Map files = new Hashtable<>(); - - public SubmissionLinkingMap(SubmissionEnvelope submissionEnvelope) { - log.info("before processes"); - getProcessLinksUsingAggregation(submissionEnvelope); - log.info("found {} processes", processes.size()); - log.info("before biomaterials"); - findBiomaterialsLinkedProcessesForSubmission(submissionEnvelope); - log.info("found {} biomaterials", biomaterials.size()); - log.info("before files"); - findFilesLinkedProcessesForSubmission(submissionEnvelope); - log.info("found {} files", files.size()); - } - - private void findBiomaterialsLinkedProcessesForSubmission(SubmissionEnvelope submissionEnvelope) { - submissionLinkMapRepository.findLinkedProcessesByEntityTypeAndSubmission(submissionEnvelope, "biomaterial") - .forEach(bioMaterialsAndProcesses -> this.biomaterials.compute(bioMaterialsAndProcesses.entityId, - (_processId, plm) -> { - BiomaterialLinkingMap biomaterialLinkingMap = Optional.ofNullable(plm) - .orElse(new BiomaterialLinkingMap()); - biomaterialLinkingMap.inputToProcesses.addAll(bioMaterialsAndProcesses.inputToProcesses); - biomaterialLinkingMap.derivedByProcesses.addAll(bioMaterialsAndProcesses.derivedByProcesses); - return biomaterialLinkingMap; - })); - } - - private void findFilesLinkedProcessesForSubmission(SubmissionEnvelope submissionEnvelope) { - submissionLinkMapRepository.findLinkedProcessesByEntityTypeAndSubmission(submissionEnvelope, "file") - .forEach(filesAndProcesses -> this.files.compute(filesAndProcesses.entityId, - (_processId, plm) -> { - FileLinkingMap fileLinkingMap = Optional.ofNullable(plm) - .orElse(new FileLinkingMap()); - fileLinkingMap.inputToProcesses.addAll(filesAndProcesses.inputToProcesses); - fileLinkingMap.derivedByProcesses.addAll(filesAndProcesses.derivedByProcesses); - return fileLinkingMap; - })); - } - - private void getProcessLinksUsingAggregation(SubmissionEnvelope submissionEnvelope) { - submissionLinkMapRepository.findProcessInputBiomaterials(submissionEnvelope) - .forEach(processAndInputBiomaterials -> this.processes.compute(processAndInputBiomaterials.processId, - (_processId, plm) -> { - ProcessLinkingMap processLinkingMap = Optional.ofNullable(plm) - .orElse(new ProcessLinkingMap()); - processLinkingMap.inputBiomaterials.addAll(processAndInputBiomaterials.inputBiomaterials); - return processLinkingMap; - })); - submissionLinkMapRepository.findProcessInputFiles(submissionEnvelope) - .forEach((ProcessAndInputFiles processAndInputFiles) -> this.processes.compute(processAndInputFiles.processId, - (_processId, plm) -> { - ProcessLinkingMap processLinkingMap = Optional.ofNullable(plm) - .orElse(new ProcessLinkingMap()); - processLinkingMap.inputFiles.addAll(processAndInputFiles.inputFiles); - return processLinkingMap; - })); - log.info("processes: before protocols"); - submissionLinkMapRepository.findProcessProtocols(submissionEnvelope) - .forEach(process -> this.processes.compute(process.entityId, - (_processId, plm) -> { - ProcessLinkingMap processLinkingMap = Optional.ofNullable(plm) - .orElse(new ProcessLinkingMap()); - processLinkingMap.protocols.addAll(process.protocols); - return processLinkingMap; - })); - log.info("processes: finished protocols"); - - } - - } - - @Getter - @AllArgsConstructor - public static class ProcessLinkingMap { - final Collection protocols = new HashSet<>(); - final Collection inputBiomaterials = new HashSet<>(); - final Collection inputFiles = new HashSet<>(); - } - - @Getter - @NoArgsConstructor - public static class BiomaterialLinkingMap { - final Collection derivedByProcesses = new HashSet<>(); - final Collection inputToProcesses = new HashSet<>(); - } - - @Getter - @NoArgsConstructor - public static class FileLinkingMap { - final Collection derivedByProcesses = new HashSet<>(); - final Collection inputToProcesses = new HashSet<>(); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionLinkMapRepository.java b/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionLinkMapRepository.java deleted file mode 100644 index 0ede115c8..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionLinkMapRepository.java +++ /dev/null @@ -1,200 +0,0 @@ -package org.humancellatlas.ingest.submission.web; - -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.Sort; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.aggregation.Aggregation; -import org.springframework.data.mongodb.core.aggregation.AggregationResults; -import org.springframework.data.mongodb.core.aggregation.ConvertOperators.ToString; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.stereotype.Repository; - -import java.util.List; - -import static org.springframework.data.mongodb.core.aggregation.Aggregation.*; -import static org.springframework.data.mongodb.core.aggregation.ArrayOperators.ArrayElemAt.arrayOf; -import static org.springframework.data.mongodb.core.aggregation.Fields.UNDERSCORE_ID; -import static org.springframework.data.mongodb.core.aggregation.ObjectOperators.ObjectToArray.valueOfToArray; - - -class ProcessAndInputBiomaterials { - String processId; - List inputBiomaterials; -} - -class ProcessAndInputFiles { - String processId; - List inputFiles; -} - -class EntityWithInputsAndDerivedBy { - String entityId; - List inputToProcesses; - List derivedByProcesses; - -} -class EntityWithProtocols { - String entityId; - List protocols; -} -@Repository -public class SubmissionLinkMapRepository { - - @Autowired - MongoTemplate mongoTemplate; - - List findProcessInputBiomaterials(SubmissionEnvelope submissionEnvelope) { - String entity_type = "biomaterial"; - Aggregation agg = buildAggregationQueryForProcessInputs(submissionEnvelope, entity_type, "inputBiomaterials"); - AggregationResults processAndInputBiomaterials = mongoTemplate.aggregate(agg, - entity_type, ProcessAndInputBiomaterials.class - ); - return processAndInputBiomaterials.getMappedResults(); - } - - private static Aggregation buildAggregationQueryForProcessInputs(SubmissionEnvelope submissionEnvelope, String entity_type, String inputBiomaterials) { - return newAggregation( - project("inputToProcesses", UNDERSCORE_ID) - .and(arrayOf(valueOfToArray("submissionEnvelope")) - .elementAt(1)) - .as("submission_id"), - project("inputToProcesses", UNDERSCORE_ID) - .and(ToString.toString("$submission_id.v")) - .as("submission_id"), - match(Criteria.where("submission_id") - .is(submissionEnvelope.getId())), - unwind("inputToProcesses"), - project(UNDERSCORE_ID) - .and(arrayOf(valueOfToArray("inputToProcesses")) - .elementAt(1)) - .as("process_id"), - project("process_id") - .and(UNDERSCORE_ID).as(entity_type + "_id"), - group("$process_id.v") - .addToSet(ToString.toString("$" + entity_type + "_id")) - .as(inputBiomaterials), - project(inputBiomaterials) - .and(ToString.toString("$_id")) - .as("processId"), - sort(Sort.Direction.DESC, "processId") - ); - } - - List findProcessInputFiles(SubmissionEnvelope submissionEnvelope) { - String entity_type = "file"; - Aggregation agg = buildAggregationQueryForProcessInputs(submissionEnvelope, entity_type, "inputFiles"); - AggregationResults processAndInputFiles = mongoTemplate.aggregate(agg, - entity_type, ProcessAndInputFiles.class - ); - return processAndInputFiles.getMappedResults(); - } - - List findLinkedProcessesByEntityTypeAndSubmission(SubmissionEnvelope submissionEnvelope, String entity_type) { - List jsonStages = List.of( - " {\n" + - " $project: {\n" + - " submission_id: { $arrayElemAt: [{ $objectToArray: \"$submissionEnvelope\" }, 1] },\n" + - " inputToProcesses: 1,\n" + - " \"derivedByProcesses\": 1,\n" + - " }\n" + - " }", - " {\n" + - " $project: {\n" + - " submission_id: { $toString: '$submission_id.v' },\n" + - " inputToProcesses: 1,\n" + - " \"derivedByProcesses\": 1,\n" + - " }\n" + - " }", - String.format("{ \"$match\": { \"submission_id\": \"%s\", } }", submissionEnvelope.getId()), - "{\n" + - " $project: {\n" + - " \"inputToProcesses\": {\n" + - " $map: {\n" + - " input: \"$inputToProcesses\",\n" + - " as: \"process_id\",\n" + - " in: { $arrayElemAt: [{ $objectToArray: \"$$process_id\" }, 1] }\n" + - " }\n" + - " },\n" + - " \"derivedByProcesses\": {\n" + - " $map: {\n" + - " input: \"$derivedByProcesses\",\n" + - " as: \"process_id\",\n" + - " in: { $arrayElemAt: [{ $objectToArray: \"$$process_id\" }, 1] }\n" + - " }\n" + - " },\n" + - " }\n" + - " }", - "{\n" + - " $project: {\n" + - " _id: 0" + - " entityId: {$toString:\"$_id\"}," + - " \"inputToProcesses\": {\n" + - " $map: {\n" + - " input: \"$inputToProcesses\",\n" + - " as: \"process_id\",\n" + - " in: {$toString: \"$$process_id.v\"}\n" + - " }\n" + - " },\n" + - " \"derivedByProcesses\": {\n" + - " $map: {\n" + - " input: \"$derivedByProcesses\",\n" + - " as: \"process_id\",\n" + - " in: {$toString: \"$$process_id.v\"}\n" + - " }\n" + - " },\n" + - " }\n" + - " }" - ); - Aggregation aggregation = MongoAggregationUtils.aggregationPipelineFromStrings(jsonStages); - return mongoTemplate - .aggregate(aggregation, entity_type, EntityWithInputsAndDerivedBy.class) - .getMappedResults(); - } - - public List findProcessProtocols(SubmissionEnvelope submissionEnvelope) { - List jsonStages = List.of( - "{\n" + - " $project: {\n" + - " submission_id: { $arrayElemAt: [{ $objectToArray: \"$submissionEnvelope\" }, 1] },\n" + - " protocols: 1,\n" + - " }\n" + - " },\n", - " {\n" + - " $project: {\n" + - " submission_id: { $toString: '$submission_id.v' },\n" + - " protocols: 1,\n" + - " }\n" + - " },\n", - String.format(" { \"$match\": { \"submission_id\": \"%s\", } },\n",submissionEnvelope.getId()), - " {\n" + - " $project: {\n" + - " \"protocols\": {\n" + - " $map: {\n" + - " input: \"$protocols\",\n" + - " as: \"protocol_id\",\n" + - " in: { $arrayElemAt: [{ $objectToArray: \"$$protocol_id\" }, 1] }\n" + - " }\n" + - " },\n" + - " }\n" + - " },\n", - " {\n" + - " $project: {\n" + - " _id:0,\n" + - " entityId: {$toString:\"$_id\"},\n" + - " \"protocols\": {\n" + - " $map: {\n" + - " input: \"$protocols\",\n" + - " as: \"protocol_id\",\n" + - " in: {$toString: \"$$protocol_id.v\"}\n" + - " }\n" + - " },\n" + - " }\n" + - " }\n" - ); - Aggregation aggregation = MongoAggregationUtils.aggregationPipelineFromStrings(jsonStages); - return mongoTemplate - .aggregate(aggregation, "process", EntityWithProtocols.class) - .getMappedResults(); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionSummaryController.java b/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionSummaryController.java deleted file mode 100644 index 9fdcadaaf..000000000 --- a/src/main/java/org/humancellatlas/ingest/submission/web/SubmissionSummaryController.java +++ /dev/null @@ -1,102 +0,0 @@ -package org.humancellatlas.ingest.submission.web; - -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.file.ValidationErrorType; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.ResponseBody; -import org.springframework.web.bind.annotation.RestController; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -@RestController -public class SubmissionSummaryController { - - @Autowired - BiomaterialRepository biomaterialRepository; - @Autowired - FileRepository fileRepository; - @Autowired - ProcessRepository processRepository; - @Autowired - ProtocolRepository protocolRepository; - - - @RequestMapping(path = "/submissionEnvelopes/{sub_id}/summary", method = RequestMethod.GET) - @ResponseBody - public SubmissionSummary submissionSummary(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope) { - SubmissionSummary summary = new SubmissionSummary(); - summary.setUuid(submissionEnvelope.getUuid()); - summary.setTotalBiomaterials(biomaterialRepository.countBySubmissionEnvelope(submissionEnvelope)); - summary.setTotalFiles(fileRepository.countBySubmissionEnvelope(submissionEnvelope)); - summary.setTotalProcesses(processRepository.countBySubmissionEnvelope(submissionEnvelope)); - summary.setTotalProtocols(protocolRepository.countBySubmissionEnvelope(submissionEnvelope)); - - long invalidBiomaterials = biomaterialRepository.countBySubmissionEnvelopeAndValidationState(submissionEnvelope, ValidationState.INVALID); - // Setting a special graphInvalid[type] property until dcp-546 is done - // This allows us to filter by graph invalid entities until we consolidate the graphValidationState into - // validationState - long graphInvalidBiomaterials = biomaterialRepository.countBySubmissionEnvelopeAndCountWithGraphValidationErrors(submissionEnvelope.getId()); - - long invalidFiles = fileRepository.countBySubmissionEnvelopeAndValidationState(submissionEnvelope, ValidationState.INVALID); - long graphInvalidFiles = fileRepository.countBySubmissionEnvelopeAndCountWithGraphValidationErrors(submissionEnvelope.getId()); - long fileMetadataErrors = fileRepository.countBySubmissionEnvelopeIdAndErrorType(submissionEnvelope.getId(), ValidationErrorType.METADATA_ERROR.name()); - long missingFiles = fileRepository.countBySubmissionEnvelopeIdAndErrorType(submissionEnvelope.getId(), ValidationErrorType.FILE_NOT_UPLOADED.name()); - long fileErrors = fileRepository.countBySubmissionEnvelopeIdAndErrorType(submissionEnvelope.getId(), ValidationErrorType.FILE_ERROR.name()); - - long invalidProcesses = processRepository.countBySubmissionEnvelopeAndValidationState(submissionEnvelope, ValidationState.INVALID); - long graphInvalidProcesses = processRepository.countBySubmissionEnvelopeAndCountWithGraphValidationErrors(submissionEnvelope.getId()); - long invalidProtocols = protocolRepository.countBySubmissionEnvelopeAndValidationState(submissionEnvelope, ValidationState.INVALID); - long graphInvalidProtocols = protocolRepository.countBySubmissionEnvelopeAndCountWithGraphValidationErrors(submissionEnvelope.getId()); - - - long totalInvalid = invalidBiomaterials + graphInvalidBiomaterials - + (fileMetadataErrors + missingFiles + fileErrors + graphInvalidFiles) - + invalidProcesses + graphInvalidProcesses - + invalidProtocols + graphInvalidProtocols; - - summary.setInvalidBiomaterials(invalidBiomaterials); - summary.setGraphInvalidBiomaterials(graphInvalidBiomaterials); - - summary.setInvalidFiles(invalidFiles); - summary.setGraphInvalidFiles(graphInvalidFiles); - summary.setFileMetadataErrors(fileMetadataErrors); - summary.setMissingFiles(missingFiles); - summary.setFileErrors(fileErrors); - - summary.setInvalidProcesses(invalidProcesses); - summary.setGraphInvalidProcesses(graphInvalidProcesses); - - summary.setInvalidProtocols(invalidProtocols); - summary.setGraphInvalidProtocols(graphInvalidProtocols); - - summary.setTotalInvalid(totalInvalid); - - return summary; - } - - @Getter - @Setter - @NoArgsConstructor - public class SubmissionSummary { - - private Uuid uuid; - private Long totalBiomaterials, invalidBiomaterials, graphInvalidBiomaterials; - private Long totalFiles, invalidFiles, graphInvalidFiles, fileMetadataErrors, missingFiles, fileErrors; - private Long totalProcesses, invalidProcesses, graphInvalidProcesses; - private Long totalProtocols, invalidProtocols, graphInvalidProtocols; - private Long totalInvalid; - - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/submissionmanifest/SubmissionManifest.java b/src/main/java/org/humancellatlas/ingest/submissionmanifest/SubmissionManifest.java deleted file mode 100644 index b123feb07..000000000 --- a/src/main/java/org/humancellatlas/ingest/submissionmanifest/SubmissionManifest.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.humancellatlas.ingest.submissionmanifest; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NonNull; -import lombok.Setter; -import org.humancellatlas.ingest.core.AbstractEntity; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.mongodb.core.mapping.DBRef; - -/** - * Created by rolando on 30/05/2018. - */ -@AllArgsConstructor -@Getter -public class SubmissionManifest extends AbstractEntity { - private final Integer expectedBiomaterials; - private final Integer expectedProcesses; - private final Integer expectedFiles; - private final Integer expectedProtocols; - private final Integer expectedProjects; - - private @Setter Integer actualLinks = 0; - private final Integer expectedLinks; - - private final Integer totalCount; - - - @Setter private @DBRef(lazy = true) SubmissionEnvelope submissionEnvelope; -} diff --git a/src/main/java/org/humancellatlas/ingest/submissionmanifest/SubmissionManifestRepository.java b/src/main/java/org/humancellatlas/ingest/submissionmanifest/SubmissionManifestRepository.java deleted file mode 100644 index 65a2ec697..000000000 --- a/src/main/java/org/humancellatlas/ingest/submissionmanifest/SubmissionManifestRepository.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.humancellatlas.ingest.submissionmanifest; - -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.web.bind.annotation.CrossOrigin; - -/** - * Created by rolando on 30/05/2018. - */ -@CrossOrigin -public interface SubmissionManifestRepository extends MongoRepository { - S findBySubmissionEnvelopeId(String envelopeId); - - Long deleteBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); -} diff --git a/src/main/java/org/humancellatlas/ingest/submissionmanifest/web/SubmissionManifestController.java b/src/main/java/org/humancellatlas/ingest/submissionmanifest/web/SubmissionManifestController.java deleted file mode 100644 index dc2378cb9..000000000 --- a/src/main/java/org/humancellatlas/ingest/submissionmanifest/web/SubmissionManifestController.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.humancellatlas.ingest.submissionmanifest.web; - -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submissionmanifest.SubmissionManifest; -import org.humancellatlas.ingest.submissionmanifest.SubmissionManifestRepository; -import org.springframework.data.rest.webmvc.PersistentEntityResource; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.rest.webmvc.RepositoryRestController; -import org.springframework.hateoas.ExposesResourceFor; -import org.springframework.hateoas.Resource; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; - -/** - * Created by rolando on 30/05/2018. - */ -@RepositoryRestController -@ExposesResourceFor(SubmissionManifest.class) -@RequiredArgsConstructor -@Getter -public class SubmissionManifestController { - private final @NonNull SubmissionManifestRepository submissionManifestRepository; - - @RequestMapping(path = "submissionEnvelopes/{sub_id}/submissionManifest", method = RequestMethod.POST) - ResponseEntity> addManifestToEnvelope(@PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, - @RequestBody SubmissionManifest submissionManifest, - PersistentEntityResourceAssembler assembler) { - submissionManifest.setSubmissionEnvelope(submissionEnvelope); - SubmissionManifest manifest = submissionManifestRepository.save(submissionManifest); - PersistentEntityResource resource = assembler.toFullResource(manifest); - return ResponseEntity.accepted().body(resource); - } -} diff --git a/src/main/java/org/humancellatlas/ingest/user/UserController.java b/src/main/java/org/humancellatlas/ingest/user/UserController.java deleted file mode 100644 index 99c1ba396..000000000 --- a/src/main/java/org/humancellatlas/ingest/user/UserController.java +++ /dev/null @@ -1,148 +0,0 @@ -package org.humancellatlas.ingest.user; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.security.Account; -import org.humancellatlas.ingest.security.AccountRepository; -import org.humancellatlas.ingest.security.Role; -import org.humancellatlas.ingest.state.SubmissionState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.humancellatlas.ingest.submission.web.SubmissionEnvelopeResourceProcessor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.rest.core.support.SelfLinkProvider; -import org.springframework.data.rest.webmvc.RepositoryLinksResource; -import org.springframework.data.web.PagedResourcesAssembler; -import org.springframework.hateoas.*; -import org.springframework.hateoas.mvc.ControllerLinkBuilder; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.context.SecurityContextHolder; -import org.springframework.web.bind.annotation.CrossOrigin; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.ResponseBody; -import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestParam; - - -import java.util.Optional; - -import static org.springframework.http.MediaType.APPLICATION_JSON_UTF8_VALUE; - -@RestController -@CrossOrigin -@RequestMapping("/user") -public class UserController implements ResourceProcessor { - - @Autowired - SubmissionEnvelopeRepository submissionEnvelopeRepository; - - @Autowired - ProjectRepository projectRepository; - - @Autowired - AccountRepository accountRepository; - - @Autowired - private PagedResourcesAssembler submissionEnvelopePagedResourcesAssembler; - - @Autowired - private PagedResourcesAssembler projectPagedResourcesAssembler; - - @Autowired - private SubmissionEnvelopeResourceProcessor submissionEnvelopeResourceProcessor; - - @Autowired - private SelfLinkProvider linkProvider; - - @Autowired - private EntityLinks entityLinks; - - @RequestMapping(value = "/summary") - @ResponseBody - public Summary summary() { - String user = getCurrentAccount().getId(); - long pendingSubmissions = submissionEnvelopeRepository.countBySubmissionStateAndUser(SubmissionState.PENDING, user); - long draftSubmissions = submissionEnvelopeRepository.countBySubmissionStateAndUser(SubmissionState.DRAFT, user); - long completedSubmissions = submissionEnvelopeRepository.countBySubmissionStateAndUser(SubmissionState.COMPLETE, user); - long projects = projectRepository.countByUser(user); - return new Summary(pendingSubmissions, draftSubmissions, completedSubmissions, projects); - } - - @RequestMapping(value = "/submissionEnvelopes") - public PagedResources> getUserSubmissionEnvelopes(Pageable pageable) { - Page submissionEnvelopes = submissionEnvelopeRepository.findByUser(getCurrentAccount().getId(), pageable); - PagedResources> pagedResources = submissionEnvelopePagedResourcesAssembler.toResource(submissionEnvelopes); - for (Resource resource : pagedResources) { - resource.add(entityLinks.linkForSingleResource(resource.getContent()).withRel(Link.REL_SELF)); - submissionEnvelopeResourceProcessor.process(resource); - } - return pagedResources; - } - - @RequestMapping(value = "/projects") - public PagedResources> getUserProjects(Pageable pageable) { - Page projects = Page.empty(); - - if (getCurrentAccount().getRoles().contains(Role.WRANGLER)) { - projects = projectRepository.findByUserOrPrimaryWrangler(getCurrentAccount().getId(), getCurrentAccount().getId(), pageable); - } else { - projects = projectRepository.findByUser(getCurrentAccount().getId(), pageable); - } - - PagedResources> pagedResources = projectPagedResourcesAssembler.toResource(projects); - for (Resource resource : pagedResources) { - resource.add(entityLinks.linkForSingleResource(resource.getContent()).withRel(Link.REL_SELF)); - } - return pagedResources; - } - - private Account getCurrentAccount() { - Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); - Account account = (Account) authentication.getPrincipal(); - return account; - } - - @GetMapping(path = "/list", produces=APPLICATION_JSON_UTF8_VALUE) - ResponseEntity listUsers(Authentication authentication, - @RequestParam("role") Optional role) { - if (!authentication.getAuthorities().contains(Role.WRANGLER)) - return new ResponseEntity<>("Unauthorized", HttpStatus.UNAUTHORIZED); - - return role.map(value -> ResponseEntity.ok(accountRepository.findAccountByRoles(value))) - .orElseGet(() -> ResponseEntity.ok(accountRepository.findAll())); - } - - - @Override - public RepositoryLinksResource process(RepositoryLinksResource resource) { - resource.add(ControllerLinkBuilder.linkTo(UserController.class).withRel("user")); - return resource; - } - - @Getter - @Setter - @NoArgsConstructor - public class Summary { - - private Long draftSubmissions = 0L; - private Long pendingSubmissions = 0L; - private Long completedSubmissions = 0L; - private Long projects = 0L; - - public Summary(long pendingSubmissions, long draftSubmissions, long completedSubmissions, long projects) { - this.pendingSubmissions = pendingSubmissions; - this.draftSubmissions = draftSubmissions; - this.completedSubmissions = completedSubmissions; - this.projects = projects; - } - } - -} diff --git a/src/main/java/org/humancellatlas/ingest/IngestCoreApplication.java b/src/main/java/uk/ac/ebi/subs/ingest/IngestCoreApplication.java similarity index 61% rename from src/main/java/org/humancellatlas/ingest/IngestCoreApplication.java rename to src/main/java/uk/ac/ebi/subs/ingest/IngestCoreApplication.java index 8aa6b30b6..8119fa8c5 100644 --- a/src/main/java/org/humancellatlas/ingest/IngestCoreApplication.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/IngestCoreApplication.java @@ -1,4 +1,4 @@ -package org.humancellatlas.ingest; +package uk.ac.ebi.subs.ingest; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -6,16 +6,12 @@ import org.springframework.context.annotation.PropertySources; import org.springframework.retry.annotation.EnableRetry; - @SpringBootApplication @EnableRetry -@PropertySources({ - @PropertySource("classpath:application.properties") -}) +@PropertySources({@PropertySource("classpath:application.properties")}) public class IngestCoreApplication { - public static void main(String[] args) { - SpringApplication.run(IngestCoreApplication.class, args); - } - + public static void main(String[] args) { + SpringApplication.run(IngestCoreApplication.class, args); + } } diff --git a/src/main/java/uk/ac/ebi/subs/ingest/archiving/Error.java b/src/main/java/uk/ac/ebi/subs/ingest/archiving/Error.java new file mode 100644 index 000000000..fedd57217 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/archiving/Error.java @@ -0,0 +1,10 @@ +package uk.ac.ebi.subs.ingest.archiving; + +import lombok.Data; + +@Data +public class Error { + private String errorCode; + private String message; + private Object details; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveEntity.java b/src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveEntity.java new file mode 100644 index 000000000..470ae2335 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveEntity.java @@ -0,0 +1,52 @@ +package uk.ac.ebi.subs.ingest.archiving.entity; + +import java.net.URI; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.springframework.data.annotation.CreatedDate; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.DBRef; +import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.hateoas.Identifiable; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import lombok.Getter; +import lombok.Setter; +import uk.ac.ebi.subs.ingest.archiving.Error; +import uk.ac.ebi.subs.ingest.archiving.submission.ArchiveSubmission; + +@Getter +@Setter +@Document +public class ArchiveEntity implements Identifiable { + @DBRef(lazy = true) + ArchiveSubmission archiveSubmission; + + @Id @JsonIgnore private String id; + + @CreatedDate private Instant created; + + private ArchiveEntityType type; + + private String alias; + + @Indexed(unique = true) + private String dspUuid; + + private URI dspUrl; + + private String accession; + + private Object conversion; + + private Set metadataUuids; + + private Set accessionedMetadataUuids; + + private List errors = new ArrayList<>(); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveEntityRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveEntityRepository.java new file mode 100644 index 000000000..7d687871c --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveEntityRepository.java @@ -0,0 +1,27 @@ +package uk.ac.ebi.subs.ingest.archiving.entity; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.rest.core.annotation.RestResource; +import org.springframework.web.bind.annotation.CrossOrigin; + +import uk.ac.ebi.subs.ingest.archiving.submission.ArchiveSubmission; + +@CrossOrigin +public interface ArchiveEntityRepository extends MongoRepository { + Page findByArchiveSubmission( + ArchiveSubmission archiveSubmission, Pageable pageable); + + Page findByAlias(String alias, Pageable pageable); + + ArchiveEntity findByArchiveSubmissionAndAlias(ArchiveSubmission archiveSubmission, String alias); + + ArchiveEntity findByDspUuid(String dspUuid); + + Page findByArchiveSubmissionAndType( + ArchiveSubmission archiveSubmission, ArchiveEntityType archiveEntityType, Pageable pageable); + + @RestResource(exported = false) + Long deleteByArchiveSubmission(ArchiveSubmission archiveSubmission); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveEntityType.java b/src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveEntityType.java new file mode 100644 index 000000000..f50df70b1 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveEntityType.java @@ -0,0 +1,18 @@ +package uk.ac.ebi.subs.ingest.archiving.entity; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@JsonSerialize(using = ArchiveEntityTypeSerializer.class) +public enum ArchiveEntityType { + SAMPLE("sample"), + PROJECT("project"), + STUDY("study"), + SEQUENCING_EXPERIMENT("sequencingExperiment"), + SEQUENCING_RUN("sequencingRun"); + + protected String type; + + ArchiveEntityType(String type) { + this.type = type; + } +} diff --git a/src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveEntityTypeSerializer.java b/src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveEntityTypeSerializer.java similarity index 53% rename from src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveEntityTypeSerializer.java rename to src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveEntityTypeSerializer.java index 949a98250..2c896b3a5 100644 --- a/src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveEntityTypeSerializer.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveEntityTypeSerializer.java @@ -1,15 +1,17 @@ -package org.humancellatlas.ingest.archiving.entity; +package uk.ac.ebi.subs.ingest.archiving.entity; + +import java.io.IOException; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; -import java.io.IOException; - public class ArchiveEntityTypeSerializer extends JsonSerializer { - @Override - public void serialize(ArchiveEntityType value, JsonGenerator generator, SerializerProvider serializers) throws IOException { - generator.writeString(value.type); - } + @Override + public void serialize( + ArchiveEntityType value, JsonGenerator generator, SerializerProvider serializers) + throws IOException { + generator.writeString(value.type); + } } diff --git a/src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveJob.java b/src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveJob.java new file mode 100644 index 000000000..44fc50d47 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveJob.java @@ -0,0 +1,37 @@ +package uk.ac.ebi.subs.ingest.archiving.entity; + +import java.time.Instant; +import java.util.Map; + +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import lombok.Data; + +@Data +@Document +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ArchiveJob { + + protected @Id String id; + private String submissionUuid; + private Instant createdDate; + private Instant responseDate; + private ArchiveJobStatus overallStatus; + private Map resultsFromArchives; + + public enum ArchiveJobStatus { + PENDING("Pending"), + RUNNING("Running"), + FAILED("Failed"), + COMPLETED("Completed"); + + final String status; + + ArchiveJobStatus(String status) { + this.status = status; + } + } +} diff --git a/src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveJobRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveJobRepository.java similarity index 64% rename from src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveJobRepository.java rename to src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveJobRepository.java index efbbc585f..2f9334442 100644 --- a/src/main/java/org/humancellatlas/ingest/archiving/entity/ArchiveJobRepository.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/archiving/entity/ArchiveJobRepository.java @@ -1,6 +1,5 @@ -package org.humancellatlas.ingest.archiving.entity; +package uk.ac.ebi.subs.ingest.archiving.entity; -import org.humancellatlas.ingest.core.Uuid; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.mongodb.repository.MongoRepository; @@ -9,5 +8,5 @@ @CrossOrigin public interface ArchiveJobRepository extends MongoRepository { - Page findBySubmissionUuid(String submissionUuid, Pageable pageable); + Page findBySubmissionUuid(String submissionUuid, Pageable pageable); } diff --git a/src/main/java/org/humancellatlas/ingest/archiving/submission/ArchiveSubmission.java b/src/main/java/uk/ac/ebi/subs/ingest/archiving/submission/ArchiveSubmission.java similarity index 56% rename from src/main/java/org/humancellatlas/ingest/archiving/submission/ArchiveSubmission.java rename to src/main/java/uk/ac/ebi/subs/ingest/archiving/submission/ArchiveSubmission.java index a1d314522..edf84f62b 100644 --- a/src/main/java/org/humancellatlas/ingest/archiving/submission/ArchiveSubmission.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/archiving/submission/ArchiveSubmission.java @@ -1,44 +1,38 @@ -package org.humancellatlas.ingest.archiving.submission; +package uk.ac.ebi.subs.ingest.archiving.submission; + +import java.net.URI; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import lombok.Setter; -import org.humancellatlas.ingest.archiving.Error; import org.springframework.data.annotation.CreatedDate; import org.springframework.data.annotation.Id; import org.springframework.data.mongodb.core.mapping.Document; import org.springframework.hateoas.Identifiable; -import java.net.URI; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; +import com.fasterxml.jackson.annotation.JsonIgnore; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import uk.ac.ebi.subs.ingest.archiving.Error; @Getter @Document @RequiredArgsConstructor public class ArchiveSubmission implements Identifiable { - @Id - @JsonIgnore - private String id; + @Id @JsonIgnore private String id; - @CreatedDate - private Instant created; + @CreatedDate private Instant created; - @Setter - private String dspUuid; + @Setter private String dspUuid; - @Setter - private URI dspUrl; + @Setter private URI dspUrl; - @Setter - private String submissionUuid; + @Setter private String submissionUuid; - @Setter - private Object fileUploadPlan; + @Setter private Object fileUploadPlan; - private @Setter - List errors = new ArrayList<>(); + private @Setter List errors = new ArrayList<>(); } diff --git a/src/main/java/org/humancellatlas/ingest/archiving/submission/ArchiveSubmissionRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/archiving/submission/ArchiveSubmissionRepository.java similarity index 62% rename from src/main/java/org/humancellatlas/ingest/archiving/submission/ArchiveSubmissionRepository.java rename to src/main/java/uk/ac/ebi/subs/ingest/archiving/submission/ArchiveSubmissionRepository.java index d4ec877b9..641704963 100644 --- a/src/main/java/org/humancellatlas/ingest/archiving/submission/ArchiveSubmissionRepository.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/archiving/submission/ArchiveSubmissionRepository.java @@ -1,4 +1,4 @@ -package org.humancellatlas.ingest.archiving.submission; +package uk.ac.ebi.subs.ingest.archiving.submission; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; @@ -7,8 +7,7 @@ @CrossOrigin public interface ArchiveSubmissionRepository extends MongoRepository { - ArchiveSubmission findByDspUuid(String dspUuid); - - Page findBySubmissionUuid(String submissionUuid, Pageable pageable); + ArchiveSubmission findByDspUuid(String dspUuid); + Page findBySubmissionUuid(String submissionUuid, Pageable pageable); } diff --git a/src/main/java/uk/ac/ebi/subs/ingest/archiving/submission/web/ArchiveJobController.java b/src/main/java/uk/ac/ebi/subs/ingest/archiving/submission/web/ArchiveJobController.java new file mode 100644 index 000000000..f4826c6d6 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/archiving/submission/web/ArchiveJobController.java @@ -0,0 +1,64 @@ +package uk.ac.ebi.subs.ingest.archiving.submission.web; + +import java.net.URI; +import java.time.Instant; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.rest.webmvc.BasePathAwareController; +import org.springframework.data.rest.webmvc.PersistentEntityResource; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; + +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.archiving.entity.ArchiveJob; +import uk.ac.ebi.subs.ingest.archiving.entity.ArchiveJobRepository; + +@RepositoryRestController +@ExposesResourceFor(ArchiveJob.class) +@BasePathAwareController +@RequiredArgsConstructor +public class ArchiveJobController { + + private static final Logger LOGGER = LoggerFactory.getLogger(ArchiveJobController.class); + + private final ArchiveJobRepository archiveJobRepository; + + @PostMapping("/archiveJobs") + ResponseEntity createArchiveJob( + @RequestBody ArchiveJob archiveJob, PersistentEntityResourceAssembler resourceAssembler) { + initResource(archiveJob); + + final ArchiveJob persistedArchiveJob = archiveJobRepository.save(archiveJob); + final PersistentEntityResource entityResource = + resourceAssembler.toFullResource(persistedArchiveJob); + return ResponseEntity.created(URI.create(entityResource.getId().getHref())) + .contentType(MediaType.APPLICATION_JSON) + .body(entityResource); + } + + private void initResource(ArchiveJob archiveJob) { + archiveJob.setCreatedDate(Instant.now()); + archiveJob.setOverallStatus(ArchiveJob.ArchiveJobStatus.PENDING); + } + + @GetMapping("/archiveJobs/{id}") + ResponseEntity getArchiveJob( + @PathVariable String id, PersistentEntityResourceAssembler resourceAssembler) { + Optional archiveJob = archiveJobRepository.findById(id); + + if (archiveJob.isEmpty()) { + return ResponseEntity.notFound().build(); + } + + return ResponseEntity.ok().body(resourceAssembler.toFullResource(archiveJob.get())); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/archiving/submission/web/ArchiveSubmissionController.java b/src/main/java/uk/ac/ebi/subs/ingest/archiving/submission/web/ArchiveSubmissionController.java new file mode 100644 index 000000000..8f2f40fa2 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/archiving/submission/web/ArchiveSubmissionController.java @@ -0,0 +1,63 @@ +package uk.ac.ebi.subs.ingest.archiving.submission.web; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.rest.webmvc.PersistentEntityResource; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.data.web.PagedResourcesAssembler; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.hateoas.Resource; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.archiving.entity.ArchiveEntity; +import uk.ac.ebi.subs.ingest.archiving.entity.ArchiveEntityRepository; +import uk.ac.ebi.subs.ingest.archiving.submission.ArchiveSubmission; +import uk.ac.ebi.subs.ingest.archiving.submission.ArchiveSubmissionRepository; + +@RepositoryRestController +@RequiredArgsConstructor +@ExposesResourceFor(ArchiveSubmission.class) +@Getter +public class ArchiveSubmissionController { + private final @NonNull ArchiveSubmissionRepository archiveSubmissionRepository; + private final @NonNull ArchiveEntityRepository archiveEntityRepository; + + private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; + + @RequestMapping(path = "archiveSubmissions/{sub_id}/entities", method = RequestMethod.POST) + ResponseEntity> addEntity( + @PathVariable("sub_id") ArchiveSubmission archiveSubmission, + @RequestBody ArchiveEntity entity, + PersistentEntityResourceAssembler assembler) { + entity.setArchiveSubmission(archiveSubmission); + entity = archiveEntityRepository.save(entity); + PersistentEntityResource resource = assembler.toFullResource(entity); + return ResponseEntity.accepted().body(resource); + } + + @RequestMapping(path = "archiveSubmissions/{sub_id}/entities", method = RequestMethod.GET) + ResponseEntity addEntity( + @PathVariable("sub_id") ArchiveSubmission archiveSubmission, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page archiveEntities = + archiveEntityRepository.findByArchiveSubmission(archiveSubmission, pageable); + return ResponseEntity.ok( + pagedResourcesAssembler.toResource(archiveEntities, resourceAssembler)); + } + + @RequestMapping(path = "archiveSubmissions/{sub_id}", method = RequestMethod.DELETE) + ResponseEntity deleteSubmission(@PathVariable("sub_id") ArchiveSubmission archiveSubmission) { + archiveEntityRepository.deleteByArchiveSubmission(archiveSubmission); + archiveSubmissionRepository.delete(archiveSubmission); + return ResponseEntity.accepted().build(); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/archiving/submission/web/ArchiveSubmissionResourceProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/archiving/submission/web/ArchiveSubmissionResourceProcessor.java new file mode 100644 index 000000000..4b1369fef --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/archiving/submission/web/ArchiveSubmissionResourceProcessor.java @@ -0,0 +1,32 @@ +package uk.ac.ebi.subs.ingest.archiving.submission.web; + +import org.springframework.hateoas.EntityLinks; +import org.springframework.hateoas.Link; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.stereotype.Component; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.archiving.submission.ArchiveSubmission; + +@Component +@RequiredArgsConstructor +public class ArchiveSubmissionResourceProcessor + implements ResourceProcessor> { + private final @NonNull EntityLinks entityLinks; + + private Link getEntitiesLink(ArchiveSubmission archiveSubmission) { + return entityLinks + .linkForSingleResource(archiveSubmission) + .slash("/entities") + .withRel("entities"); + } + + @Override + public Resource process(Resource resource) { + ArchiveSubmission archiveSubmission = resource.getContent(); + resource.add(getEntitiesLink(archiveSubmission)); + return resource; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/audit/AuditEntry.java b/src/main/java/uk/ac/ebi/subs/ingest/audit/AuditEntry.java new file mode 100644 index 000000000..7d8a1dbc3 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/audit/AuditEntry.java @@ -0,0 +1,36 @@ +package uk.ac.ebi.subs.ingest.audit; + +import java.time.Instant; + +import org.springframework.data.annotation.CreatedBy; +import org.springframework.data.annotation.CreatedDate; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.DBRef; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import lombok.Getter; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.core.AbstractEntity; + +@Getter +public class AuditEntry { + protected @Id @JsonIgnore String id; + @NonNull private final AuditType auditType; + private final Object before; + private final Object after; + private @CreatedDate Instant date; + // todo: @CreatedBy isn't working, need to figure out why + private @CreatedBy String user; + + @DBRef(lazy = true) + @JsonIgnore + final @NonNull private AbstractEntity entity; + + public AuditEntry(AuditType auditType, Object before, Object after, AbstractEntity entity) { + this.auditType = auditType; + this.before = before; + this.after = after; + this.entity = entity; + } +} diff --git a/src/main/java/org/humancellatlas/ingest/audit/AuditEntryRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/audit/AuditEntryRepository.java similarity index 60% rename from src/main/java/org/humancellatlas/ingest/audit/AuditEntryRepository.java rename to src/main/java/uk/ac/ebi/subs/ingest/audit/AuditEntryRepository.java index bdcf5b0fb..ddeb76e18 100644 --- a/src/main/java/org/humancellatlas/ingest/audit/AuditEntryRepository.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/audit/AuditEntryRepository.java @@ -1,17 +1,16 @@ -package org.humancellatlas.ingest.audit; +package uk.ac.ebi.subs.ingest.audit; + +import java.util.List; -import org.humancellatlas.ingest.core.AbstractEntity; import org.springframework.data.mongodb.repository.MongoRepository; import org.springframework.data.rest.core.annotation.RestResource; import org.springframework.stereotype.Repository; -import java.util.List; - +import uk.ac.ebi.subs.ingest.core.AbstractEntity; @Repository -@RestResource(exported=false) +@RestResource(exported = false) public interface AuditEntryRepository extends MongoRepository { - List findByEntityEqualsOrderByDateDesc(AbstractEntity entity); + List findByEntityEqualsOrderByDateDesc(AbstractEntity entity); } - diff --git a/src/main/java/uk/ac/ebi/subs/ingest/audit/AuditEntryService.java b/src/main/java/uk/ac/ebi/subs/ingest/audit/AuditEntryService.java new file mode 100644 index 000000000..cc921a8d3 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/audit/AuditEntryService.java @@ -0,0 +1,23 @@ +package uk.ac.ebi.subs.ingest.audit; + +import java.util.List; + +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.AbstractEntity; + +@Service +@RequiredArgsConstructor +public class AuditEntryService { + + private final AuditEntryRepository auditEntryRepository; + + public void addAuditEntry(AuditEntry auditEntry) { + auditEntryRepository.save(auditEntry); + } + + public List getAuditEntriesForAbstractEntity(AbstractEntity entity) { + return auditEntryRepository.findByEntityEqualsOrderByDateDesc(entity); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/audit/AuditType.java b/src/main/java/uk/ac/ebi/subs/ingest/audit/AuditType.java new file mode 100644 index 000000000..496421f3b --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/audit/AuditType.java @@ -0,0 +1,18 @@ +package uk.ac.ebi.subs.ingest.audit; + +import com.fasterxml.jackson.annotation.JsonValue; + +public enum AuditType { + STATUS_UPDATED("Status updated"); + + protected String value; + + AuditType(String value) { + this.value = value; + } + + @JsonValue + public String getValue() { + return this.value; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/biomaterial/Biomaterial.java b/src/main/java/uk/ac/ebi/subs/ingest/biomaterial/Biomaterial.java new file mode 100644 index 000000000..47a69945f --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/biomaterial/Biomaterial.java @@ -0,0 +1,159 @@ +package uk.ac.ebi.subs.ingest.biomaterial; + +import static com.fasterxml.jackson.annotation.JsonProperty.Access.READ_ONLY; + +import java.util.HashSet; +import java.util.Set; + +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.DBRef; +import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.data.rest.core.annotation.RestResource; +import org.springframework.web.bind.annotation.CrossOrigin; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.project.Project; + +/** Created by rolando on 16/02/2018. */ +@CrossOrigin +@Getter +@Document +@NoArgsConstructor +public class Biomaterial extends MetadataDocument { + @Indexed + @Setter + @DBRef(lazy = true) + private Project project; + + @RestResource + @DBRef(lazy = true) + private Set projects = new HashSet<>(); + + @Indexed + @RestResource + @DBRef(lazy = true) + private Set inputToProcesses = new HashSet<>(); + + @Indexed + @RestResource + @DBRef(lazy = true) + private Set derivedByProcesses = new HashSet<>(); + + @Indexed + @RestResource + @DBRef(lazy = true) + private Set parentBiomaterials = new HashSet<>(); + + @Indexed + @RestResource + @DBRef(lazy = true) + private Set childBiomaterials = new HashSet<>(); + + @JsonCreator + public Biomaterial(@JsonProperty("content") Object content) { + super(EntityType.BIOMATERIAL, content); + } + + /** + * Adds to the collection of processes that this biomaterial serves as an input to + * + * @param process the process to add + * @return a reference to this biomaterial + */ + public Biomaterial addAsInputToProcess(final Process process) { + this.inputToProcesses.add(process); + return this; + } + + /** + * Adds to the collection of processes that this biomaterial was derived by + * + * @param process the process to add + * @return a reference to this biomaterial + */ + public Biomaterial addAsDerivedByProcess(final Process process) { + this.derivedByProcesses.add(process); + return this; + } + + @JsonProperty(access = READ_ONLY) + public boolean isLinked() { + return !inputToProcesses.isEmpty() || !derivedByProcesses.isEmpty(); + } + + /** + * Removes a process from the collection of processes that this biomaterial serves as an input to + * + * @param process the process to remove + * @return a reference to this biomaterial + */ + public Biomaterial removeAsInputToProcess(final Process process) { + this.inputToProcesses.remove(process); + return this; + } + + /** + * Removes a process from the collection of processes that this biomaterial was derived by + * + * @param process the process to remove + * @return a reference to this biomaterial + */ + public Biomaterial removeAsDerivedByProcess(final Process process) { + this.derivedByProcesses.remove(process); + return this; + } + + /** + * Adds a child biomaterial and sets this biomaterial as the parent of the child + * + * @param childBiomaterial the child biomaterial to add + * @return a reference to this biomaterial + */ + public Biomaterial addChildBiomaterial(final Biomaterial childBiomaterial) { + childBiomaterial.addParentBiomaterial(this); + this.childBiomaterials.add(childBiomaterial); + return this; + } + + /** + * Removes a child biomaterial and clears this biomaterial as the parent of the child + * + * @param childBiomaterial the child biomaterial to remove + * @return a reference to this biomaterial + */ + public Biomaterial removeChildBiomaterial(final Biomaterial childBiomaterial) { + childBiomaterial.removeParentBiomaterial(this); + this.childBiomaterials.remove(childBiomaterial); + return this; + } + + /** + * Adds a parent biomaterial + * + * @param parentBiomaterial the parent biomaterial to add + * @return a reference to this biomaterial + */ + public Biomaterial addParentBiomaterial(final Biomaterial parentBiomaterial) { + this.parentBiomaterials.add(parentBiomaterial); + return this; + } + + /** + * Removes a parent biomaterial + * + * @param parentBiomaterial the parent biomaterial to remove + * @return a reference to this biomaterial + */ + public Biomaterial removeParentBiomaterial(final Biomaterial parentBiomaterial) { + this.parentBiomaterials.remove(parentBiomaterial); + return this; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/biomaterial/BiomaterialRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/biomaterial/BiomaterialRepository.java new file mode 100644 index 000000000..09b923940 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/biomaterial/BiomaterialRepository.java @@ -0,0 +1,100 @@ +package uk.ac.ebi.subs.ingest.biomaterial; + +import java.util.Collection; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.data.rest.core.annotation.RestResource; +import org.springframework.web.bind.annotation.CrossOrigin; + +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.security.RowLevelFilterSecurity; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@CrossOrigin +@RowLevelFilterSecurity( + expression = + "(#filterObject.project != null)" + + "? " + + " (" + + " #authentication.authorities.![authority].contains(" + + " 'ROLE_access_' +#filterObject.project.uuid?.toString()) " + + " or " + + " #authentication.authorities.![authority].contains('ROLE_SERVICE') " + + " or " + + " #filterObject.project.content['dataAccess']['type'] " + + " eq T(uk.ac.ebi.subs.ingest.project.DataAccessTypes).OPEN.label" + + " )" + + ":true", + ignoreClasses = {Project.class}) +public interface BiomaterialRepository extends MongoRepository { + + @RestResource(rel = "findAllByUuid", path = "findAllByUuid") + Page findByUuid(@Param("uuid") Uuid uuid, Pageable pageable); + + @RestResource(rel = "findByUuid", path = "findByUuid") + Optional findByUuidUuidAndIsUpdateFalse(@Param("uuid") UUID uuid); + + Page findBySubmissionEnvelope( + SubmissionEnvelope submissionEnvelope, Pageable pageable); + + Page findByProject(Project project, Pageable pageable); + + @RestResource(exported = false) + Stream findByProject(Project project); + + @RestResource(exported = false) + Stream findByProjectsContaining(Project project); + + @RestResource(exported = false) + Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + @RestResource(exported = false) + Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + @RestResource(exported = false) + Long deleteBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + @RestResource(rel = "findBySubmissionAndValidationState") + public Page findBySubmissionEnvelopeAndValidationState( + @Param("envelopeUri") SubmissionEnvelope submissionEnvelope, + @Param("state") ValidationState state, + Pageable pageable); + + @Query( + value = + "{'submissionEnvelope.id': ?0, graphValidationErrors: { $exists: true, $not: {$size: 0} } }") + @RestResource(rel = "findBySubmissionIdWithGraphValidationErrors") + public Page findBySubmissionIdWithGraphValidationErrors( + @Param("envelopeId") String envelopeId, Pageable pageable); + + @RestResource(exported = false) + Stream findByInputToProcessesContains(Process process); + + Page findByInputToProcessesContaining(Process process, Pageable pageable); + + @RestResource(exported = false) + Stream findByDerivedByProcessesContains(Process process); + + Page findByDerivedByProcessesContaining(Process process, Pageable pageable); + + long countBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + long countBySubmissionEnvelopeAndValidationState( + SubmissionEnvelope submissionEnvelope, ValidationState validationState); + + @Query( + value = + "{'submissionEnvelope.id': ?0, graphValidationErrors: { $exists: true, $not: {$size: 0} } }", + count = true) + long countBySubmissionEnvelopeAndCountWithGraphValidationErrors(String submissionEnvelopeId); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/biomaterial/BiomaterialService.java b/src/main/java/uk/ac/ebi/subs/ingest/biomaterial/BiomaterialService.java new file mode 100644 index 000000000..15b4fbd0d --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/biomaterial/BiomaterialService.java @@ -0,0 +1,109 @@ +package uk.ac.ebi.subs.ingest.biomaterial; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@Service +@RequiredArgsConstructor +@Getter +public class BiomaterialService { + private final @NonNull SubmissionEnvelopeRepository submissionEnvelopeRepository; + private final @NonNull BiomaterialRepository biomaterialRepository; + private final @NonNull ProcessRepository processRepository; + private final @NonNull ProjectRepository projectRepository; + private final @NonNull MetadataUpdateService metadataUpdateService; + private final @NonNull MetadataCrudService metadataCrudService; + private final Logger log = LoggerFactory.getLogger(getClass()); + + protected Logger getLog() { + return log; + } + + public Biomaterial addBiomaterialToSubmissionEnvelope( + final SubmissionEnvelope submissionEnvelope, final Biomaterial biomaterial) { + if (!biomaterial.getIsUpdate()) { + projectRepository + .findBySubmissionEnvelopesContains(submissionEnvelope) + .findFirst() + .ifPresent( + project -> { + biomaterial.setProject(project); + biomaterial.getProjects().add(project); + }); + return metadataCrudService.addToSubmissionEnvelopeAndSave(biomaterial, submissionEnvelope); + } else { + return metadataUpdateService.acceptUpdate(biomaterial, submissionEnvelope); + } + } + + public Biomaterial addChildBiomaterial( + final String parentId, final Biomaterial childBiomaterial) { + final Biomaterial parentBiomaterial = + biomaterialRepository + .findById(parentId) + .orElseThrow(() -> new RuntimeException("Parent biomaterial not found")); + + parentBiomaterial.addChildBiomaterial(childBiomaterial); + biomaterialRepository.save(childBiomaterial); + + return biomaterialRepository.save(parentBiomaterial); + } + + public Biomaterial removeChildBiomaterial(final String parentId, final String childId) { + final Biomaterial parentBiomaterial = + biomaterialRepository + .findById(parentId) + .orElseThrow(() -> new RuntimeException("Parent biomaterial not found")); + + final Biomaterial childBiomaterial = + biomaterialRepository + .findById(childId) + .orElseThrow(() -> new RuntimeException("Child biomaterial not found")); + + parentBiomaterial.removeChildBiomaterial(childBiomaterial); + biomaterialRepository.save(childBiomaterial); + + return biomaterialRepository.save(parentBiomaterial); + } + + public Biomaterial addParentBiomaterial( + final String childId, final Biomaterial parentBiomaterial) { + final Biomaterial childBiomaterial = + biomaterialRepository + .findById(childId) + .orElseThrow(() -> new RuntimeException("Child biomaterial not found")); + + childBiomaterial.addParentBiomaterial(parentBiomaterial); + biomaterialRepository.save(parentBiomaterial); + + return biomaterialRepository.save(childBiomaterial); + } + + public Biomaterial removeParentBiomaterial(final String childId, final String parentId) { + final Biomaterial childBiomaterial = + biomaterialRepository + .findById(childId) + .orElseThrow(() -> new RuntimeException("Child biomaterial not found")); + + final Biomaterial parentBiomaterial = + biomaterialRepository + .findById(parentId) + .orElseThrow(() -> new RuntimeException("Parent biomaterial not found")); + + childBiomaterial.removeParentBiomaterial(parentBiomaterial); + biomaterialRepository.save(parentBiomaterial); + + return biomaterialRepository.save(childBiomaterial); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/biomaterial/web/BiomaterialController.java b/src/main/java/uk/ac/ebi/subs/ingest/biomaterial/web/BiomaterialController.java new file mode 100644 index 000000000..b62d2788c --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/biomaterial/web/BiomaterialController.java @@ -0,0 +1,245 @@ +package uk.ac.ebi.subs.ingest.biomaterial.web; + +import static org.springframework.data.rest.webmvc.RestMediaTypes.TEXT_URI_LIST_VALUE; +import static org.springframework.web.bind.annotation.RequestMethod.POST; +import static org.springframework.web.bind.annotation.RequestMethod.PUT; + +import java.lang.reflect.InvocationTargetException; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.rest.webmvc.PersistentEntityResource; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.data.web.PagedResourcesAssembler; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.Resources; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialService; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataLinkingService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.core.service.UriToEntityConversionService; +import uk.ac.ebi.subs.ingest.core.service.ValidationStateChangeService; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.security.CheckAllowed; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.exception.NotAllowedDuringSubmissionStateException; + +@RepositoryRestController +@RequiredArgsConstructor +@ExposesResourceFor(Biomaterial.class) +@Getter +public class BiomaterialController { + private final @NonNull ProcessRepository processRepository; + private final @NonNull BiomaterialService biomaterialService; + private final @NonNull BiomaterialRepository biomaterialRepository; + private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; + private final @NonNull MetadataCrudService metadataCrudService; + private final @NonNull MetadataUpdateService metadataUpdateService; + private @Autowired ValidationStateChangeService validationStateChangeService; + private @Autowired UriToEntityConversionService uriToEntityConversionService; + private @Autowired MetadataLinkingService metadataLinkingService; + + @CheckAllowed( + value = "#submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PostMapping(path = "submissionEnvelopes/{sub_id}/biomaterials") + ResponseEntity> addBiomaterialToEnvelope( + @PathVariable("sub_id") final SubmissionEnvelope submissionEnvelope, + @RequestBody final Biomaterial biomaterial, + @RequestParam("updatingUuid") final Optional updatingUuid, + final PersistentEntityResourceAssembler assembler) { + updatingUuid.ifPresent( + uuid -> { + biomaterial.setUuid(new Uuid(uuid.toString())); + biomaterial.setIsUpdate(true); + }); + final Biomaterial entity = + biomaterialService.addBiomaterialToSubmissionEnvelope(submissionEnvelope, biomaterial); + final PersistentEntityResource resource = assembler.toFullResource(entity); + + return ResponseEntity.accepted().body(resource); + } + + @CheckAllowed( + value = "#submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @RequestMapping( + path = "submissionEnvelopes/{sub_id}/biomaterials/{id}", + method = RequestMethod.PUT) + ResponseEntity> linkBiomaterialToEnvelope( + @PathVariable("sub_id") final SubmissionEnvelope submissionEnvelope, + @PathVariable("id") final Biomaterial biomaterial, + final PersistentEntityResourceAssembler assembler) { + final Biomaterial entity = + biomaterialService.addBiomaterialToSubmissionEnvelope(submissionEnvelope, biomaterial); + final PersistentEntityResource resource = assembler.toFullResource(entity); + + return ResponseEntity.accepted().body(resource); + } + + @CheckAllowed( + value = "#biomaterial.submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PatchMapping(path = "/biomaterials/{id}") + HttpEntity patchBiomaterial( + @PathVariable("id") final Biomaterial biomaterial, + @RequestBody final ObjectNode patch, + final PersistentEntityResourceAssembler assembler) { + final List allowedFields = + List.of("content", "validationErrors", "graphValidationErrors"); + final ObjectNode validPatch = patch.retain(allowedFields); + final Biomaterial updatedBiomaterial = metadataUpdateService.update(biomaterial, validPatch); + final PersistentEntityResource resource = assembler.toFullResource(updatedBiomaterial); + + return ResponseEntity.accepted().body(resource); + } + + @CheckAllowed( + value = "#biomaterial.submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @RequestMapping( + path = "/biomaterials/{id}/inputToProcesses", + method = {PUT, POST}, + consumes = {TEXT_URI_LIST_VALUE}) + HttpEntity linkBiomaterialAsInputToProcesses( + @PathVariable("id") final Biomaterial biomaterial, + @RequestBody final Resources incoming, + final HttpMethod requestMethod) + throws URISyntaxException, + InvocationTargetException, + NoSuchMethodException, + IllegalAccessException { + final List processes = + uriToEntityConversionService.convertLinks(incoming.getLinks(), Process.class); + metadataLinkingService.updateLinks( + biomaterial, processes, "inputToProcesses", requestMethod.equals(HttpMethod.PUT)); + + return ResponseEntity.ok().build(); + } + + @CheckAllowed( + value = "#biomaterial.submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @RequestMapping( + path = "/biomaterials/{id}/derivedByProcesses", + method = {PUT, POST}, + consumes = {TEXT_URI_LIST_VALUE}) + HttpEntity linkBiomaterialAsDerivedByProcesses( + @PathVariable("id") final Biomaterial biomaterial, + @RequestBody final Resources incoming, + final HttpMethod requestMethod) + throws URISyntaxException, + InvocationTargetException, + NoSuchMethodException, + IllegalAccessException { + final List processes = + uriToEntityConversionService.convertLinks(incoming.getLinks(), Process.class); + metadataLinkingService.updateLinks( + biomaterial, processes, "derivedByProcesses", requestMethod.equals(HttpMethod.PUT)); + + return ResponseEntity.ok().build(); + } + + @CheckAllowed( + value = "#biomaterial.submissionEnvelope.isEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @DeleteMapping(path = "/biomaterials/{id}/inputToProcesses/{processId}") + HttpEntity unlinkBiomaterialAsInputToProcesses( + @PathVariable("id") final Biomaterial biomaterial, + @PathVariable("processId") final Process process) + throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + metadataLinkingService.removeLink(biomaterial, process, "inputToProcesses"); + + return ResponseEntity.noContent().build(); + } + + @CheckAllowed( + value = "#biomaterial.submissionEnvelope.isEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @DeleteMapping(path = "/biomaterials/{id}/derivedByProcesses/{processId}") + HttpEntity unlinkBiomaterialAsDerivedProcesses( + @PathVariable("id") final Biomaterial biomaterial, + @PathVariable("processId") final Process process) + throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + metadataLinkingService.removeLink(biomaterial, process, "derivedByProcesses"); + + return ResponseEntity.noContent().build(); + } + + @DeleteMapping(path = "/biomaterials/{id}") + @CheckAllowed( + value = "#biomaterial.submissionEnvelope.isEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + ResponseEntity deleteBiomaterial(@PathVariable("id") final Biomaterial biomaterial) + throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + metadataLinkingService.removeLinks(biomaterial, "inputToProcesses"); + metadataLinkingService.removeLinks(biomaterial, "derivedByProcesses"); + metadataCrudService.deleteDocument(biomaterial); + + return ResponseEntity.noContent().build(); + } + + @PostMapping("/biomaterials/{parentId}/childBiomaterials") + public ResponseEntity> addChildBiomaterial( + @PathVariable final String parentId, + @RequestBody final Biomaterial childBiomaterial, + final PersistentEntityResourceAssembler assembler) { + biomaterialService.addChildBiomaterial(parentId, childBiomaterial); + final PersistentEntityResource resource = assembler.toFullResource(childBiomaterial); + + return ResponseEntity.accepted().body(resource); + } + + @DeleteMapping("/biomaterials/{parentId}/childBiomaterials/{childId}") + public ResponseEntity> removeChildBiomaterial( + @PathVariable final String parentId, + @PathVariable final String childId, + final PersistentEntityResourceAssembler assembler) { + final Biomaterial updatedParent = biomaterialService.removeChildBiomaterial(parentId, childId); + final PersistentEntityResource resource = assembler.toFullResource(updatedParent); + + return ResponseEntity.accepted().body(resource); + } + + @PostMapping("/biomaterials/{childId}/parentBiomaterials") + public ResponseEntity> addParentBiomaterial( + @PathVariable final String childId, + @RequestBody final Biomaterial parentBiomaterial, + final PersistentEntityResourceAssembler assembler) { + final Biomaterial updatedChild = + biomaterialService.addParentBiomaterial(childId, parentBiomaterial); + final PersistentEntityResource resource = assembler.toFullResource(updatedChild); + + return ResponseEntity.accepted().body(resource); + } + + @DeleteMapping("/biomaterials/{childId}/parentBiomaterials/{parentId}") + public ResponseEntity> removeParentBiomaterial( + @PathVariable final String childId, + @PathVariable final String parentId, + final PersistentEntityResourceAssembler assembler) { + Biomaterial updatedChild = biomaterialService.removeParentBiomaterial(childId, parentId); + final PersistentEntityResource resource = assembler.toFullResource(updatedChild); + + return ResponseEntity.accepted().body(resource); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifest.java b/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifest.java new file mode 100644 index 000000000..784f3f876 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifest.java @@ -0,0 +1,37 @@ +package uk.ac.ebi.subs.ingest.bundle; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.hateoas.Identifiable; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +/** Created by rolando on 05/09/2017. */ +@AllArgsConstructor +@Getter +@Document +@EqualsAndHashCode +public class BundleManifest implements Identifiable { + private @Id @JsonIgnore String id; + + @Indexed private final String bundleUuid; + @Indexed private final String bundleVersion; + + private final String envelopeUuid; + + private final List dataFiles; + private final Map> fileBiomaterialMap; + private final Map> fileProcessMap; + private final Map> fileProjectMap; + private final Map> fileProtocolMap; + private final Map> fileFilesMap; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifestRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifestRepository.java new file mode 100644 index 000000000..3a5ae0d57 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifestRepository.java @@ -0,0 +1,29 @@ +package uk.ac.ebi.subs.ingest.bundle; + +import java.util.Optional; +import java.util.stream.Stream; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.repository.query.Param; +import org.springframework.data.rest.core.annotation.RestResource; +import org.springframework.web.bind.annotation.CrossOrigin; + +/** Created by rolando on 05/09/2017. */ +@CrossOrigin +public interface BundleManifestRepository + extends MongoRepository, BundleManifestRepositoryCustom { + Page findByBundleUuid(@Param("uuid") String uuid, Pageable pageable); + + Optional findTopByBundleUuidOrderByBundleVersionDesc(String uuid); + + Page findByEnvelopeUuid(String uuid, Pageable pageable); + + Page findAll(Pageable pageable); + + Long deleteByEnvelopeUuid(String uuid); + + @RestResource(exported = false) + Stream findByEnvelopeUuid(String envelopeUuid); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifestRepositoryCustom.java b/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifestRepositoryCustom.java new file mode 100644 index 000000000..38aa86c99 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifestRepositoryCustom.java @@ -0,0 +1,11 @@ +package uk.ac.ebi.subs.ingest.bundle; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; + +import uk.ac.ebi.subs.ingest.project.Project; + +public interface BundleManifestRepositoryCustom { + Page findBundleManifestsByProjectAndBundleType( + Project project, BundleType bundleType, Pageable pageable); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifestRepositoryImpl.java b/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifestRepositoryImpl.java new file mode 100644 index 000000000..5eef4481c --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifestRepositoryImpl.java @@ -0,0 +1,52 @@ +package uk.ac.ebi.subs.ingest.bundle; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; + +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +public class BundleManifestRepositoryImpl implements BundleManifestRepositoryCustom { + + private final MongoTemplate mongoTemplate; + + @Autowired + public BundleManifestRepositoryImpl(final MongoTemplate mongoTemplate) { + this.mongoTemplate = mongoTemplate; + } + + @Override + public Page findBundleManifestsByProjectAndBundleType( + Project project, BundleType bundleType, Pageable pageable) { + SubmissionEnvelope submissionEnvelope = project.getSubmissionEnvelope(); + String submissionUuid = submissionEnvelope.getUuid().getUuid().toString(); + String projectUuid = project.getUuid().getUuid().toString(); + + Query query = new Query(); + query.addCriteria(Criteria.where("fileProjectMap." + projectUuid).exists(true)); + + if (bundleType != null) { + if (bundleType.equals(BundleType.PRIMARY)) { + query.addCriteria(Criteria.where("envelopeUuid").is(submissionUuid)); + } else if (bundleType.equals(BundleType.ANALYSIS)) { + // TODO This might not be the best criteria to query analysis bundles. Might need to remodel + // bundle manifest. + query.addCriteria(Criteria.where("envelopeUuid").ne(submissionUuid)); + } + } + + query.with(pageable); + + List result = mongoTemplate.find(query, BundleManifest.class); + long count = mongoTemplate.count(query, BundleManifest.class); + Page bundleManifestPage = new PageImpl<>(result, pageable, count); + return bundleManifestPage; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifestSearchProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifestSearchProcessor.java new file mode 100644 index 000000000..0e35d7392 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifestSearchProcessor.java @@ -0,0 +1,27 @@ +package uk.ac.ebi.subs.ingest.bundle; + +import static org.springframework.hateoas.mvc.ControllerLinkBuilder.methodOn; + +import org.springframework.data.rest.webmvc.RepositorySearchesResource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.hateoas.mvc.ControllerLinkBuilder; +import org.springframework.stereotype.Component; + +import uk.ac.ebi.subs.ingest.bundle.web.BundleManifestController; + +@Component +public class BundleManifestSearchProcessor + implements ResourceProcessor { + @Override + public RepositorySearchesResource process(RepositorySearchesResource resource) { + if (resource.getDomainType().equals(BundleManifest.class)) { + resource.add( + ControllerLinkBuilder.linkTo( + methodOn(BundleManifestController.class) + .findBundleManifestsByProjectUuidAndBundleType(null, null, null, null)) + .withRel("findBundleManifestsByProjectUuidAndBundleType")); + } + + return resource; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifestService.java b/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifestService.java new file mode 100644 index 000000000..3ccd717c9 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleManifestService.java @@ -0,0 +1,122 @@ +package uk.ac.ebi.subs.ingest.bundle; + +import java.text.DecimalFormat; +import java.util.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; + +@Service +@RequiredArgsConstructor +public class BundleManifestService { + private final @NonNull BundleManifestRepository bundleManifestRepository; + private final Logger log = LoggerFactory.getLogger(getClass()); + + public Map> bundleManifestsForDocuments( + Collection documents) { + + Map> hits = new HashMap<>(); + + long fileStartTime = System.currentTimeMillis(); + Iterator iterator = allManifestsIterator(); + + while (iterator.hasNext()) { + BundleManifest bundleManifest = iterator.next(); + documents.forEach( + document -> { + String documentUuid = document.getUuid().getUuid().toString(); + String bundleUuid = bundleManifest.getBundleUuid(); + EntityType documentType = document.getType(); + entityMapFromManifest(documentType, bundleManifest) + .ifPresent( + entityMap -> { + if (entityMap.containsKey(documentUuid)) { + if (hits.containsKey(bundleUuid)) { + hits.get(bundleUuid).add(document); + } else { + hits.put(bundleUuid, new HashSet<>(Collections.singletonList(document))); + } + } + }); + }); + } + + long fileEndTime = System.currentTimeMillis(); + float fileQueryTime = ((float) (fileEndTime - fileStartTime)) / 1000; + String fileQt = new DecimalFormat("#,###.##").format(fileQueryTime); + log.info("Finding bundles to update took {}s", fileQt); + log.info("documentsToUpdate: {}, bundlesToUpdate:{}", documents.size(), hits.keySet().size()); + return hits; + } + + private Iterator allManifestsIterator() { + Iterator manifestsIterator = + new Iterator() { + Pageable pageable = new PageRequest(0, 5000); + Page pagedBundleManifests = null; + Queue bundleManifests; + + private void fetch(Pageable pageable) { + pagedBundleManifests = bundleManifestRepository.findAll(pageable); + bundleManifests = new LinkedList<>(pagedBundleManifests.getContent()); + } + + @Override + public boolean hasNext() { + return (pagedBundleManifests == null + || bundleManifests.size() > 0 + || pagedBundleManifests.hasNext()); + } + + @Override + public BundleManifest next() { + BundleManifest bundleManifest = null; + + if (pagedBundleManifests == null) { + fetch(pageable); + } + + if (bundleManifests.size() == 0 && pagedBundleManifests.hasNext()) { + fetch(pagedBundleManifests.nextPageable()); + } + + if (bundleManifests.size() > 0) { + bundleManifest = bundleManifests.remove(); + } + + return bundleManifest; + } + }; + + return manifestsIterator; + } + + private Optional>> entityMapFromManifest( + EntityType entityType, BundleManifest bundleManifest) { + if (entityType.equals(EntityType.BIOMATERIAL)) { + return Optional.ofNullable(bundleManifest.getFileBiomaterialMap()); + } else if (entityType.equals(EntityType.FILE)) { + return Optional.ofNullable(bundleManifest.getFileFilesMap()); + } else if (entityType.equals(EntityType.PROTOCOL)) { + return Optional.ofNullable(bundleManifest.getFileProtocolMap()); + } else if (entityType.equals(EntityType.PROCESS)) { + return Optional.ofNullable(bundleManifest.getFileProcessMap()); + } else if (entityType.equals(EntityType.PROJECT)) { + return Optional.ofNullable(bundleManifest.getFileProjectMap()); + } else { + throw new RuntimeException( + String.format( + "Bundle manifest %s contains no entity map for entity type %s", + bundleManifest.getId(), entityType.toString())); + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleType.java b/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleType.java new file mode 100644 index 000000000..9e089a432 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/bundle/BundleType.java @@ -0,0 +1,6 @@ +package uk.ac.ebi.subs.ingest.bundle; + +public enum BundleType { + PRIMARY, + ANALYSIS +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/bundle/web/BundleManifestController.java b/src/main/java/uk/ac/ebi/subs/ingest/bundle/web/BundleManifestController.java new file mode 100644 index 000000000..9bbc7e1bd --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/bundle/web/BundleManifestController.java @@ -0,0 +1,48 @@ +package uk.ac.ebi.subs.ingest.bundle.web; + +import java.util.Optional; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.data.web.PagedResourcesAssembler; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.bundle.BundleManifest; +import uk.ac.ebi.subs.ingest.bundle.BundleType; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.project.ProjectService; + +@RepositoryRestController +@RequiredArgsConstructor +@ExposesResourceFor(BundleManifest.class) +@Getter +public class BundleManifestController { + private final @NonNull ProjectService projectService; + + private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; + + @RequestMapping( + path = "/projects/search/findBundleManifestsByProjectUuidAndBundleType", + method = RequestMethod.GET) + public ResponseEntity findBundleManifestsByProjectUuidAndBundleType( + @RequestParam("projectUuid") Uuid projectUuid, + @RequestParam("bundleType") Optional bundleType, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + + Page bundleManifests = + this.projectService.findBundleManifestsByProjectUuidAndBundleType( + projectUuid, bundleType.orElse(null), pageable); + return ResponseEntity.ok( + pagedResourcesAssembler.toResource(bundleManifests, resourceAssembler)); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/config/ConfigurationService.java b/src/main/java/uk/ac/ebi/subs/ingest/config/ConfigurationService.java new file mode 100644 index 000000000..95f9ee37b --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/config/ConfigurationService.java @@ -0,0 +1,55 @@ +package uk.ac.ebi.subs.ingest.config; + +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import lombok.Getter; + +/** Created by rolando on 05/09/2018. */ +@Component +public class ConfigurationService implements InitializingBean { + @Value("${STATE_TRACKER_SCHEME:http}") + private String stateTrackerSchemeString; + + @Value("${STATE_TRACKER_HOST:localhost}") + private String stateTrackerHostString; + + @Value("${STATE_TRACKER_PORT:8999}") + private String stateTrackerPortString; + + @Value("${STATE_TRACKER_DOCUMENT_STATES_PATH:machine-reports}") + private String documentStatesPathString; + + @Value("${STATE_TRACKER_DOCUMENT_STATES_UPDATE_PATH:state-updates/metadata-documents}") + private String documentStatesUpdatePathString; + + @Value("${STATE_TRACKER_DOCUMENT_PARAM:metadataDocumentId}") + private String documentIdParamNameString; + + @Value("${STATE_TRACKER_DOCUMENT_PARAM:envelopeId}") + private String envelopeIdParamNameString; + + @Getter private String stateTrackerScheme; + @Getter private String stateTrackerHost; + @Getter private int stateTrackerPort; + @Getter private String documentStatesPath; + @Getter private String documentStatesUpdatePath; + @Getter private String documentIdParamName; + @Getter private String envelopeIdParamName; + + private void init() { + this.stateTrackerScheme = this.stateTrackerSchemeString; + this.stateTrackerHost = this.stateTrackerHostString; + this.stateTrackerPort = Integer.parseInt(this.stateTrackerPortString); + this.documentStatesPath = this.documentStatesPathString; + this.documentStatesUpdatePath = this.documentStatesUpdatePathString; + this.documentIdParamName = this.documentIdParamNameString; + this.envelopeIdParamName = this.envelopeIdParamNameString; + } + + @Override + public void afterPropertiesSet() throws Exception { + init(); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/config/ConvertersConfiguration.java b/src/main/java/uk/ac/ebi/subs/ingest/config/ConvertersConfiguration.java new file mode 100644 index 000000000..43a606fa4 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/config/ConvertersConfiguration.java @@ -0,0 +1,19 @@ +package uk.ac.ebi.subs.ingest.config; + +import java.util.Arrays; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; + +import uk.ac.ebi.subs.ingest.project.DataAccessTypesReadConverter; +import uk.ac.ebi.subs.ingest.project.DataAccessTypesWriteConverter; + +@Configuration +public class ConvertersConfiguration { + @Bean + public MongoCustomConversions mongoCustomConversions() { + return new MongoCustomConversions( + Arrays.asList(new DataAccessTypesReadConverter(), new DataAccessTypesWriteConverter())); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/config/MigrationConfiguration.java b/src/main/java/uk/ac/ebi/subs/ingest/config/MigrationConfiguration.java new file mode 100644 index 000000000..cd2ef871d --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/config/MigrationConfiguration.java @@ -0,0 +1,16 @@ +package uk.ac.ebi.subs.ingest.config; + +import org.springframework.context.annotation.Configuration; + +@Configuration +public class MigrationConfiguration { + /*@Value("${spring.data.mongodb.uri}") + private String mongoURI; + + @Bean + public Mongobee Configure() { + Mongobee runner = new Mongobee(mongoURI); + runner.setChangeLogsScanPackage("org.humancellatlas.ingest.migrations"); + return runner; + }*/ +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/config/MongoConfiguration.java b/src/main/java/uk/ac/ebi/subs/ingest/config/MongoConfiguration.java new file mode 100644 index 000000000..bd6c6697b --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/config/MongoConfiguration.java @@ -0,0 +1,24 @@ +package uk.ac.ebi.subs.ingest.config; + +import java.util.Arrays; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.convert.CustomConversions; +import org.springframework.data.mongodb.config.EnableMongoAuditing; +import org.springframework.data.mongodb.core.convert.MongoCustomConversions; + +import uk.ac.ebi.subs.ingest.project.DataAccessTypesReadConverter; +import uk.ac.ebi.subs.ingest.project.DataAccessTypesWriteConverter; + +@Configuration +@EnableMongoAuditing(auditorAwareRef = "userAuditing") +public class MongoConfiguration { + + @Bean + public CustomConversions customConversions() { + + return new MongoCustomConversions( + Arrays.asList(new DataAccessTypesReadConverter(), new DataAccessTypesWriteConverter())); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/AbstractEntity.java b/src/main/java/uk/ac/ebi/subs/ingest/core/AbstractEntity.java new file mode 100644 index 000000000..439a0b75c --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/AbstractEntity.java @@ -0,0 +1,64 @@ +package uk.ac.ebi.subs.ingest.core; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import org.springframework.data.annotation.*; +import org.springframework.hateoas.Identifiable; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 30/08/17 + */ +@Getter +@ToString +@JsonIgnoreProperties( + value = {"type"}, + allowGetters = true) +@EqualsAndHashCode +public abstract class AbstractEntity implements Identifiable { + protected @Id @JsonIgnore String id; + + // This alias is used to ensure the 'id' field is included in JSON responses + // for compatibility with morphic API clients. + @JsonProperty("id") + private String jsonIdAlias; + + private @Version Long version; + + private @CreatedDate Instant submissionDate; + + private @LastModifiedDate Instant updateDate; + + private @CreatedBy String user; + + private @LastModifiedBy String lastModifiedUser; + + private EntityType type; + + private @Setter Uuid uuid; + + private @Setter List events = new ArrayList<>(); + + protected AbstractEntity(EntityType type) { + this.type = type; + } + + protected AbstractEntity() {} + + public String getJsonIdAlias() { + return id; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/Accession.java b/src/main/java/uk/ac/ebi/subs/ingest/core/Accession.java new file mode 100644 index 000000000..74ef7d264 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/Accession.java @@ -0,0 +1,30 @@ +package uk.ac.ebi.subs.ingest.core; + +import lombok.Data; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 30/08/17 + */ +@Data +public class Accession { + private String number; + + protected Accession() { + this.number = null; + } + + public Accession(String number) { + if (!isValid(number)) { + throw new IllegalArgumentException( + String.format("Accession number '%s' is not a valid format ", number)); + } + this.number = number; + } + + public static boolean isValid(String number) { + return !number.isEmpty(); // todo might want to regex this, maybe this service doesn't care + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/Checksums.java b/src/main/java/uk/ac/ebi/subs/ingest/core/Checksums.java new file mode 100644 index 000000000..3f0c7d45e --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/Checksums.java @@ -0,0 +1,30 @@ +package uk.ac.ebi.subs.ingest.core; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.Data; + +/** Created by rolando on 06/09/2017. */ +@Data +public class Checksums { + private String sha1; + private String sha256; + private String crc32c; + private String md5; + + @JsonProperty("s3_etag") + private String s3Etag; + + public Checksums(String sha1, String sha256, String crc32c, String s3Etag) { + this.sha1 = sha1; + this.sha256 = sha256; + this.crc32c = crc32c; + this.s3Etag = s3Etag; + } + + public Checksums(String md5) { + this.md5 = md5; + } + + protected Checksums() {} +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/DescriptiveSchema.java b/src/main/java/uk/ac/ebi/subs/ingest/core/DescriptiveSchema.java new file mode 100644 index 000000000..badc721f1 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/DescriptiveSchema.java @@ -0,0 +1,15 @@ +package uk.ac.ebi.subs.ingest.core; + +public interface DescriptiveSchema { + String getDescribedBy(); + + void setDescribedBy(String describedBy); + + String getSchemaVersion(); + + void setSchemaVersion(String schemaVersion); + + String getSchemaType(); + + void setSchemaType(String schemaType); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/EntityType.java b/src/main/java/uk/ac/ebi/subs/ingest/core/EntityType.java new file mode 100644 index 000000000..59a868f43 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/EntityType.java @@ -0,0 +1,18 @@ +package uk.ac.ebi.subs.ingest.core; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 30/08/17 + */ +public enum EntityType { + SUBMISSION, + PROJECT, + BIOMATERIAL, + PROCESS, + PROTOCOL, + FILE, + STUDY, + DATASET +} diff --git a/src/main/java/org/humancellatlas/ingest/core/Event.java b/src/main/java/uk/ac/ebi/subs/ingest/core/Event.java similarity index 69% rename from src/main/java/org/humancellatlas/ingest/core/Event.java rename to src/main/java/uk/ac/ebi/subs/ingest/core/Event.java index 819596e89..8566c0702 100644 --- a/src/main/java/org/humancellatlas/ingest/core/Event.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/Event.java @@ -1,4 +1,4 @@ -package org.humancellatlas.ingest.core; +package uk.ac.ebi.subs.ingest.core; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -12,5 +12,5 @@ @RequiredArgsConstructor @Getter public abstract class Event { - private final SubmissionDate submissionDate; + private final SubmissionDate submissionDate; } diff --git a/src/main/java/org/humancellatlas/ingest/core/JsonPatch.java b/src/main/java/uk/ac/ebi/subs/ingest/core/JsonPatch.java similarity index 62% rename from src/main/java/org/humancellatlas/ingest/core/JsonPatch.java rename to src/main/java/uk/ac/ebi/subs/ingest/core/JsonPatch.java index fea7b0d96..3bff5dbb6 100644 --- a/src/main/java/org/humancellatlas/ingest/core/JsonPatch.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/JsonPatch.java @@ -1,14 +1,14 @@ -package org.humancellatlas.ingest.core; - +package uk.ac.ebi.subs.ingest.core; import com.fasterxml.jackson.databind.JsonNode; + import lombok.AllArgsConstructor; import lombok.Data; @Data @AllArgsConstructor public class JsonPatch { - private JsonNode patch; + private JsonNode patch; - public JsonPatch(){} -} \ No newline at end of file + public JsonPatch() {} +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/MetadataDocument.java b/src/main/java/uk/ac/ebi/subs/ingest/core/MetadataDocument.java new file mode 100644 index 000000000..5654aac20 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/MetadataDocument.java @@ -0,0 +1,140 @@ +package uk.ac.ebi.subs.ingest.core; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.DBRef; +import org.springframework.data.mongodb.core.mapping.Field; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 31/08/17 + */ +@Getter +@EqualsAndHashCode(callSuper = true, exclude = "contentLastModified") +public abstract class MetadataDocument extends AbstractEntity { + + @Setter private Instant firstDcpVersion; + + private Instant dcpVersion; + + private Instant contentLastModified; + + private Object content; + + // This property holds the reference to the submissionEnvelope this metadata document was part of. + // A metadata document is part of one submissionEnvelope. + // The other end of this relationship can be defined as a set of metadataDocuments in a + // submissionEnvelope. + @Indexed + @Setter + @DBRef(lazy = true) + private SubmissionEnvelope submissionEnvelope; + + @Field("accessions") + private List accessions; + + // private @Setter Accession accession; + + private @Setter ValidationState validationState; + + private @Setter List validationErrors; + + private @Setter List graphValidationErrors; + + private @Setter @Field Boolean isUpdate = false; + + private static final Logger log = LoggerFactory.getLogger(SubmissionEnvelope.class); + + protected static Logger getLog() { + return log; + } + + protected MetadataDocument() {} + + protected MetadataDocument(EntityType type, Object content) { + super(type); + this.content = content; + this.contentLastModified = Instant.now(); + this.validationState = ValidationState.DRAFT; + } + + public static List allowedStateTransitions(ValidationState fromState) { + List allowedStates = new ArrayList<>(); + switch (fromState) { + case DRAFT: + allowedStates.add(ValidationState.VALIDATING); + break; + case VALIDATING: + allowedStates.add(ValidationState.VALID); + allowedStates.add(ValidationState.INVALID); + break; + case VALID: + allowedStates.add(ValidationState.PROCESSING); + allowedStates.add(ValidationState.DRAFT); + break; + case INVALID: + allowedStates.add(ValidationState.DRAFT); + break; + case PROCESSING: + allowedStates.add(ValidationState.COMPLETE); + allowedStates.add(ValidationState.DRAFT); + break; + default: + getLog() + .warn( + String.format( + "There are no legal state transitions for '%s' state", fromState.name())); + break; + } + return allowedStates; + } + + public List allowedStateTransitions() { + return allowedStateTransitions(getValidationState()); + } + + public MetadataDocument enactStateTransition(ValidationState targetState) { + this.validationState = targetState; + return this; + } + + public void setContent(Object content) { + if (this.content == null || !this.content.equals(content)) { + this.content = content; + this.contentLastModified = Instant.now(); + this.setDcpVersion(this.contentLastModified); + } + } + + public MetadataDocument setDcpVersion(Instant dcpVersion) { + // DCP version should never be set to null + if (dcpVersion != null) { + if (this.dcpVersion == null) { + this.firstDcpVersion = dcpVersion; + } + this.dcpVersion = dcpVersion; + } + return this; + } + + public List getAccessions() { + return accessions; + } + + public void setAccessions(List accessions) { + this.accessions = accessions; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/MetadataDocumentMessageBuilder.java b/src/main/java/uk/ac/ebi/subs/ingest/core/MetadataDocumentMessageBuilder.java new file mode 100644 index 000000000..e00d4c690 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/MetadataDocumentMessageBuilder.java @@ -0,0 +1,103 @@ +package uk.ac.ebi.subs.ingest.core; + +import java.time.Instant; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.hateoas.Identifiable; + +import uk.ac.ebi.subs.ingest.core.web.LinkGenerator; +import uk.ac.ebi.subs.ingest.messaging.model.MetadataDocumentMessage; +import uk.ac.ebi.subs.ingest.state.ValidationState; + +public class MetadataDocumentMessageBuilder { + + private LinkGenerator linkGenerator; + + private Class documentType; + private String metadataDocId; + private String metadataDocUuid; + private String envelopeId; + private String envelopeUuid; + private Instant metadataDocVersion; + private ValidationState validationState; + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private MetadataDocumentMessageBuilder(LinkGenerator linkGenerator) { + this.linkGenerator = linkGenerator; + } + + public static MetadataDocumentMessageBuilder using(LinkGenerator linkGenerator) { + return new MetadataDocumentMessageBuilder(linkGenerator); + } + + protected Logger getLog() { + return log; + } + + public MetadataDocumentMessageBuilder messageFor(MetadataDocument metadataDocument) { + MetadataDocumentMessageBuilder builder = + withDocumentType(metadataDocument.getClass()).withId(metadataDocument.getId()); + Uuid metadataDocumentUuid = metadataDocument.getUuid(); + if (metadataDocumentUuid != null && metadataDocumentUuid.getUuid() != null) { + builder = builder.withUuid(metadataDocument.getUuid().getUuid().toString()); + } + builder = builder.withVersion(metadataDocument.getDcpVersion()); + + return builder; + } + + private MetadataDocumentMessageBuilder withDocumentType( + Class documentClass) { + this.documentType = documentClass; + return this; + } + + private MetadataDocumentMessageBuilder withId(String metadataDocId) { + this.metadataDocId = metadataDocId; + + return this; + } + + private MetadataDocumentMessageBuilder withUuid(String metadataDocUuid) { + this.metadataDocUuid = metadataDocUuid; + + return this; + } + + private MetadataDocumentMessageBuilder withVersion(Instant metadataDocVersion) { + this.metadataDocVersion = metadataDocVersion; + + return this; + } + + public MetadataDocumentMessageBuilder withValidationState(ValidationState validationState) { + this.validationState = validationState; + + return this; + } + + public MetadataDocumentMessageBuilder withEnvelopeId(String envelopeId) { + this.envelopeId = envelopeId; + + return this; + } + + public MetadataDocumentMessageBuilder withEnvelopeUuid(String envelopeUuid) { + this.envelopeUuid = envelopeUuid; + + return this; + } + + public MetadataDocumentMessage build() { + String callbackLink = linkGenerator.createCallback(documentType, metadataDocId); + return new MetadataDocumentMessage( + documentType.getSimpleName().toLowerCase(), + metadataDocId, + metadataDocUuid, + validationState, + callbackLink, + envelopeId); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/SubmissionDate.java b/src/main/java/uk/ac/ebi/subs/ingest/core/SubmissionDate.java new file mode 100644 index 000000000..c8c73ea62 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/SubmissionDate.java @@ -0,0 +1,32 @@ +package uk.ac.ebi.subs.ingest.core; + +import java.util.Date; + +import lombok.Data; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 30/08/17 + */ +@Data +public class SubmissionDate { + private Date date; + + protected SubmissionDate() { + this.date = null; + } + + public SubmissionDate(Date date) { + if (!isValid(date)) { + throw new IllegalArgumentException( + String.format("Submission date '%s' is in the future!", date)); + } + this.date = date; + } + + public static boolean isValid(Date date) { + return !date.after(new Date()); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/UpdateDate.java b/src/main/java/uk/ac/ebi/subs/ingest/core/UpdateDate.java new file mode 100644 index 000000000..95de759bf --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/UpdateDate.java @@ -0,0 +1,31 @@ +package uk.ac.ebi.subs.ingest.core; + +import java.util.Date; + +import lombok.Data; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 30/08/17 + */ +@Data +public class UpdateDate { + private Date date; + + protected UpdateDate() { + this.date = null; + } + + public UpdateDate(Date date) { + if (!isValid(date)) { + throw new IllegalArgumentException(String.format("Update date '%s' is in the future!", date)); + } + this.date = date; + } + + public static boolean isValid(Date date) { + return !date.after(new Date()); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/Uuid.java b/src/main/java/uk/ac/ebi/subs/ingest/core/Uuid.java new file mode 100644 index 000000000..ce7b602e5 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/Uuid.java @@ -0,0 +1,39 @@ +package uk.ac.ebi.subs.ingest.core; + +import java.util.UUID; + +import com.fasterxml.jackson.annotation.JsonCreator; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 31/08/17 + */ +@Data +@EqualsAndHashCode +public class Uuid { + private UUID uuid; + + @JsonCreator + public Uuid(String name) { + // throws IllegalArgumentException if not valid + this.uuid = UUID.fromString(name); + } + + public Uuid() {} + + public static Uuid newUuid() { + Uuid uuid = new Uuid(); + uuid.setUuid(UUID.randomUUID()); + return uuid; + } + + @Override + public String toString() { + return uuid.toString(); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/ValidationChecksum.java b/src/main/java/uk/ac/ebi/subs/ingest/core/ValidationChecksum.java new file mode 100644 index 000000000..b94f81c44 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/ValidationChecksum.java @@ -0,0 +1,9 @@ +package uk.ac.ebi.subs.ingest.core; + +import lombok.Data; + +/** Created by rolando on 11/09/2017. */ +@Data +public class ValidationChecksum { + private String md5 = ""; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/ValidationEvent.java b/src/main/java/uk/ac/ebi/subs/ingest/core/ValidationEvent.java new file mode 100644 index 000000000..919c6552a --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/ValidationEvent.java @@ -0,0 +1,25 @@ +package uk.ac.ebi.subs.ingest.core; + +import org.springframework.context.ApplicationEvent; + +import lombok.Getter; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 12/09/17 + */ +@Getter +public class ValidationEvent extends ApplicationEvent { + private String message; + + public ValidationEvent(Object source, String message) { + super(source); + this.message = message; + } + + public String getMessage() { + return message; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/exception/CoreEntityNotFoundException.java b/src/main/java/uk/ac/ebi/subs/ingest/core/exception/CoreEntityNotFoundException.java new file mode 100644 index 000000000..70bef33e4 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/exception/CoreEntityNotFoundException.java @@ -0,0 +1,10 @@ +package uk.ac.ebi.subs.ingest.core.exception; + +/** Created by rolando on 07/09/2017. */ +public class CoreEntityNotFoundException extends Exception { + public CoreEntityNotFoundException() {} + + public CoreEntityNotFoundException(String message) { + super(message); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/exception/MultipleOpenSubmissionsException.java b/src/main/java/uk/ac/ebi/subs/ingest/core/exception/MultipleOpenSubmissionsException.java new file mode 100644 index 000000000..23b26eda0 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/exception/MultipleOpenSubmissionsException.java @@ -0,0 +1,25 @@ +package uk.ac.ebi.subs.ingest.core.exception; + +public class MultipleOpenSubmissionsException extends RuntimeException { + + public MultipleOpenSubmissionsException() { + super(); + } + + public MultipleOpenSubmissionsException(String message) { + super(message); + } + + public MultipleOpenSubmissionsException(String message, Throwable cause) { + super(message, cause); + } + + public MultipleOpenSubmissionsException(Throwable cause) { + super(cause); + } + + protected MultipleOpenSubmissionsException( + String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/exception/RedundantUpdateException.java b/src/main/java/uk/ac/ebi/subs/ingest/core/exception/RedundantUpdateException.java new file mode 100644 index 000000000..74e4876e0 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/exception/RedundantUpdateException.java @@ -0,0 +1,9 @@ +package uk.ac.ebi.subs.ingest.core.exception; + +public class RedundantUpdateException extends RuntimeException { + public RedundantUpdateException() {} + + public RedundantUpdateException(String message) { + super(message); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/exception/StateTransitionNotAllowed.java b/src/main/java/uk/ac/ebi/subs/ingest/core/exception/StateTransitionNotAllowed.java new file mode 100644 index 000000000..8967dd5b7 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/exception/StateTransitionNotAllowed.java @@ -0,0 +1,10 @@ +package uk.ac.ebi.subs.ingest.core.exception; + +/** Created by rolando on 10/03/2018. */ +public class StateTransitionNotAllowed extends RuntimeException { + public StateTransitionNotAllowed() {} + + public StateTransitionNotAllowed(String message) { + super(message); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/service/MetadataCrudService.java b/src/main/java/uk/ac/ebi/subs/ingest/core/service/MetadataCrudService.java new file mode 100644 index 000000000..1317ad293 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/service/MetadataCrudService.java @@ -0,0 +1,85 @@ +package uk.ac.ebi.subs.ingest.core.service; + +import java.util.Collection; + +import org.springframework.stereotype.Service; + +import lombok.AllArgsConstructor; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.core.service.strategy.MetadataCrudStrategy; +import uk.ac.ebi.subs.ingest.core.service.strategy.impl.*; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Service +@AllArgsConstructor +public class MetadataCrudService { + private final @NonNull BiomaterialCrudStrategy biomaterialCrudStrategy; + private final @NonNull ProcessCrudStrategy processCrudStrategy; + private final @NonNull ProtocolCrudStrategy protocolCrudStrategy; + private final @NonNull ProjectCrudStrategy projectCrudStrategy; + private final @NonNull FileCrudStrategy fileCrudStrategy; + private final @NonNull StudyCrudStrategy studyCrudStrategy; + private final @NonNull DatasetCrudStrategy datasetCrudStrategy; + + private MetadataCrudStrategy crudStrategyForMetadataType(EntityType metadataType) { + switch (metadataType) { + case BIOMATERIAL: + return biomaterialCrudStrategy; + case PROCESS: + return processCrudStrategy; + case PROTOCOL: + return protocolCrudStrategy; + case PROJECT: + return projectCrudStrategy; + case FILE: + return fileCrudStrategy; + case STUDY: + return studyCrudStrategy; + case DATASET: + return datasetCrudStrategy; + default: + throw new RuntimeException(String.format("No such metadata type: %s", metadataType)); + } + } + + public T save(T metadataDocument) { + MetadataCrudStrategy crudStrategy = crudStrategyForMetadataType(metadataDocument.getType()); + return (T) crudStrategy.saveMetadataDocument(metadataDocument); + } + + public T setValidationState( + EntityType entityType, String entityId, ValidationState validationState) { + MetadataCrudStrategy crudStrategy = crudStrategyForMetadataType(entityType); + T document = (T) crudStrategy.findMetadataDocument(entityId); + document.setValidationState(validationState); + return (T) crudStrategy.saveMetadataDocument(document); + } + + public T addToSubmissionEnvelopeAndSave( + T metadataDocument, SubmissionEnvelope submissionEnvelope) { + metadataDocument.setSubmissionEnvelope(submissionEnvelope); + return (T) + (crudStrategyForMetadataType(metadataDocument.getType()) + .saveMetadataDocument(metadataDocument)); + } + + public T findOriginalByUuid(String uuid, EntityType entityType) { + return (T) crudStrategyForMetadataType(entityType).findOriginalByUuid(uuid); + } + + public Collection findAllBySubmission( + SubmissionEnvelope submissionEnvelope, EntityType entityType) { + return crudStrategyForMetadataType(entityType).findAllBySubmissionEnvelope(submissionEnvelope); + } + + public void removeLinksToDocument(T metadataDocument) { + crudStrategyForMetadataType(metadataDocument.getType()).removeLinksToDocument(metadataDocument); + } + + public void deleteDocument(T metadataDocument) { + crudStrategyForMetadataType(metadataDocument.getType()).deleteDocument(metadataDocument); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/service/MetadataDifferService.java b/src/main/java/uk/ac/ebi/subs/ingest/core/service/MetadataDifferService.java new file mode 100644 index 000000000..b28491eba --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/service/MetadataDifferService.java @@ -0,0 +1,39 @@ +package uk.ac.ebi.subs.ingest.core.service; + +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.zjsonpatch.JsonDiff; + +import lombok.AllArgsConstructor; +import uk.ac.ebi.subs.ingest.core.JsonPatch; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; + +@Service +@AllArgsConstructor +public class MetadataDifferService { + + public boolean anyDifference(MetadataDocument source, MetadataDocument target) { + ObjectMapper objectMapper = new ObjectMapper(); + + JsonNode sourceContent = objectMapper.valueToTree(source.getContent()); + JsonNode targetContent = objectMapper.valueToTree(target.getContent()); + + return !sourceContent.equals(targetContent); + } + + public JsonPatch generatePatch( + T originalDocument, T updateDocument) { + ObjectMapper objectMapper = new ObjectMapper(); + + JsonNode sourceContent = objectMapper.valueToTree(originalDocument.getContent()); + JsonNode targetContent = objectMapper.valueToTree(updateDocument.getContent()); + + return this.generatePatch(sourceContent, targetContent); + } + + public JsonPatch generatePatch(JsonNode source, JsonNode target) { + return new JsonPatch(JsonDiff.asJson(source, target)); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/service/MetadataLinkingService.java b/src/main/java/uk/ac/ebi/subs/ingest/core/service/MetadataLinkingService.java new file mode 100644 index 000000000..ff875ddd7 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/service/MetadataLinkingService.java @@ -0,0 +1,137 @@ +package uk.ac.ebi.subs.ingest.core.service; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.*; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.OptimisticLockingFailureException; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.stereotype.Service; + +import reactor.util.StringUtils; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Service +public class MetadataLinkingService { + + private static final long RETRY_BACKOFF_MS = 100; + private static final int RETRY_MAX_ATTEMPTS = 5; + + private ValidationStateChangeService validationStateChangeService; + + private MongoTemplate mongoTemplate; + + @Autowired + public MetadataLinkingService( + ValidationStateChangeService validationStateChangeService, MongoTemplate mongoTemplate) { + this.validationStateChangeService = validationStateChangeService; + this.mongoTemplate = mongoTemplate; + } + + public T updateLinks( + T targetEntity, List entitiesToLink, String linkProperty, Boolean replace) + throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + if (replace) { + replaceLinks(targetEntity, entitiesToLink, linkProperty); + } else { + addLinks(targetEntity, entitiesToLink, linkProperty); + } + return targetEntity; + } + + public T addLinks( + T targetEntity, List entitiesToLink, String linkProperty) + throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + Method method = getGetterMethod(targetEntity, linkProperty); + Set linkedEntities = (Set) invoke(targetEntity, method); + entitiesToLink.forEach( + doc -> { + linkedEntities.add(doc); + }); + mongoTemplate.save(targetEntity); + + setValidationStateToDraftIfGraphValid(targetEntity); + + return targetEntity; + } + + public T replaceLinks( + T targetEntity, List entitiesToLink, String linkProperty) + throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + Method method = getGetterMethod(targetEntity, linkProperty); + Set linkedEntities = (Set) invoke(targetEntity, method); + linkedEntities.clear(); + linkedEntities.addAll(entitiesToLink); + mongoTemplate.save(targetEntity); + + setValidationStateToDraftIfGraphValid(targetEntity); + + return targetEntity; + } + + public T removeLinks( + T targetEntity, String linkProperty) + throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + Method method = getGetterMethod(targetEntity, linkProperty); + Set linkedEntities = (Set) invoke(targetEntity, method); + linkedEntities.clear(); + mongoTemplate.save(targetEntity); + + setValidationStateToDraftIfGraphValid(targetEntity); + + return targetEntity; + } + + public T removeLink( + T targetEntity, S entityToUnlink, String linkProperty) + throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + Method method = getGetterMethod(targetEntity, linkProperty); + Set linkedEntities = (Set) invoke(targetEntity, method); + linkedEntities.remove(entityToUnlink); + mongoTemplate.save(targetEntity); + + setValidationStateToDraftIfGraphValid(targetEntity); + + return targetEntity; + } + + private Method getGetterMethod( + T metadataDocument, String linkProperty) throws NoSuchMethodException { + return metadataDocument.getClass().getMethod("get" + StringUtils.capitalize(linkProperty)); + } + + private Object invoke(T metadataDocument, Method method) + throws IllegalAccessException, InvocationTargetException { + return method.invoke(metadataDocument); + } + + private void setValidationStateToDraftIfGraphValid(MetadataDocument... entities) { + Arrays.stream(entities) + .forEach( + entity -> { + SubmissionEnvelope submission = entity.getSubmissionEnvelope(); + if (submission != null + && submission.getSubmissionState().equals(SubmissionState.GRAPH_VALID)) { + setToDraft(entity); + } + }); + } + + private void setToDraft(MetadataDocument entity) { + RetryTemplate retry = + RetryTemplate.builder() + .maxAttempts(RETRY_MAX_ATTEMPTS) + .fixedBackoff(RETRY_BACKOFF_MS) + .retryOn(OptimisticLockingFailureException.class) + .build(); + retry.execute( + context -> + validationStateChangeService.changeValidationState( + entity.getType(), entity.getId(), ValidationState.DRAFT)); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/service/MetadataUpdateService.java b/src/main/java/uk/ac/ebi/subs/ingest/core/service/MetadataUpdateService.java new file mode 100644 index 000000000..47d9d5eea --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/service/MetadataUpdateService.java @@ -0,0 +1,66 @@ +package uk.ac.ebi.subs.ingest.core.service; + +import java.util.Optional; + +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import lombok.AllArgsConstructor; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.core.exception.RedundantUpdateException; +import uk.ac.ebi.subs.ingest.patch.JsonPatcher; +import uk.ac.ebi.subs.ingest.patch.PatchService; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@AllArgsConstructor +@Service +public class MetadataUpdateService { + private final @NonNull MetadataDifferService metadataDifferService; + private final @NonNull MetadataCrudService metadataCrudService; + private final @NonNull PatchService patchService; + + private final @NonNull ValidationStateChangeService validationStateChangeService; + private final @NonNull JsonPatcher jsonPatcher; + + public T update(T metadataDocument, ObjectNode patch) { + ObjectMapper mapper = new ObjectMapper(); + + Boolean contentChanged = + Optional.ofNullable(patch.get("content")) + .map(content -> !content.equals(mapper.valueToTree(metadataDocument.getContent()))) + .orElse(false); + + T patchedMetadata = jsonPatcher.merge(patch, metadataDocument); + T doc = metadataCrudService.save(patchedMetadata); + + if (contentChanged) { + validationStateChangeService.changeValidationState( + doc.getType(), doc.getId(), ValidationState.DRAFT); + } + + return doc; + } + + public T acceptUpdate( + T updateDocument, SubmissionEnvelope submissionEnvelope) { + T originalDocument = + metadataCrudService.findOriginalByUuid( + updateDocument.getUuid().getUuid().toString(), updateDocument.getType()); + + if (metadataDifferService.anyDifference(originalDocument, updateDocument)) { + T savedUpdateDocument = + metadataCrudService.addToSubmissionEnvelopeAndSave(updateDocument, submissionEnvelope); + patchService.storePatch(originalDocument, savedUpdateDocument, submissionEnvelope); + return savedUpdateDocument; + } else { + throw new RedundantUpdateException( + String.format( + "Attempted to update %s document at %s with contents of %s but there is no diff", + updateDocument.getType(), originalDocument.getId(), updateDocument.getId())); + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/service/UriToEntityConversionService.java b/src/main/java/uk/ac/ebi/subs/ingest/core/service/UriToEntityConversionService.java new file mode 100644 index 000000000..42044848a --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/service/UriToEntityConversionService.java @@ -0,0 +1,44 @@ +package uk.ac.ebi.subs.ingest.core.service; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.data.mapping.context.PersistentEntities; +import org.springframework.data.repository.support.Repositories; +import org.springframework.data.repository.support.RepositoryInvokerFactory; +import org.springframework.data.rest.core.UriToEntityConverter; +import org.springframework.hateoas.Link; +import org.springframework.stereotype.Service; + +@Service +public class UriToEntityConversionService { + + private UriToEntityConverter converter; + + @Autowired + public UriToEntityConversionService( + PersistentEntities entities, + RepositoryInvokerFactory invokerFactory, + Repositories repositories) { + converter = new UriToEntityConverter(entities, invokerFactory, repositories); + } + + public T convert(Object source, TypeDescriptor sourceType, TypeDescriptor targetType) { + return (T) converter.convert(source, sourceType, targetType); + } + + public T convertLink(Link link, TypeDescriptor targetType) throws URISyntaxException { + URI uri = new URI(link.getHref()); + return (T) convert(uri, TypeDescriptor.valueOf(URI.class), targetType); + } + + public List convertLinks(List links, Class clazz) throws URISyntaxException { + List list = new ArrayList<>(); + for (Link link : links) list.add((T) convertLink(link, TypeDescriptor.valueOf(clazz))); + return list; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/service/ValidationStateChangeService.java b/src/main/java/uk/ac/ebi/subs/ingest/core/service/ValidationStateChangeService.java new file mode 100644 index 000000000..187fc83f4 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/service/ValidationStateChangeService.java @@ -0,0 +1,26 @@ +package uk.ac.ebi.subs.ingest.core.service; + +import org.springframework.stereotype.Service; + +import lombok.AllArgsConstructor; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.state.ValidationStateEventPublisher; + +@Service +@AllArgsConstructor +public class ValidationStateChangeService { + private final @NonNull MetadataCrudService metadataCrudService; + + private final @NonNull ValidationStateEventPublisher validationStateEventPublisher; + + public T changeValidationState( + EntityType metadataType, String metadataId, ValidationState validationState) { + T metadataDocument = + metadataCrudService.setValidationState(metadataType, metadataId, validationState); + validationStateEventPublisher.publishValidationStateChangeEventFor(metadataDocument); + return metadataDocument; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/MetadataCrudStrategy.java b/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/MetadataCrudStrategy.java new file mode 100644 index 000000000..479748edb --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/MetadataCrudStrategy.java @@ -0,0 +1,23 @@ +package uk.ac.ebi.subs.ingest.core.service.strategy; + +import java.util.Collection; +import java.util.stream.Stream; + +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +public interface MetadataCrudStrategy { + T saveMetadataDocument(T document); + + T findMetadataDocument(String id); + + T findOriginalByUuid(String uuid); + + Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + void removeLinksToDocument(T document); + + void deleteDocument(T document); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/BiomaterialCrudStrategy.java b/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/BiomaterialCrudStrategy.java new file mode 100644 index 000000000..dc64a60b1 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/BiomaterialCrudStrategy.java @@ -0,0 +1,69 @@ +package uk.ac.ebi.subs.ingest.core.service.strategy.impl; + +import java.util.Collection; +import java.util.UUID; +import java.util.stream.Stream; + +import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.stereotype.Component; + +import lombok.AllArgsConstructor; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.core.service.strategy.MetadataCrudStrategy; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@AllArgsConstructor +@Component +public class BiomaterialCrudStrategy implements MetadataCrudStrategy { + private final @NonNull BiomaterialRepository biomaterialRepository; + private final @NonNull MessageRouter messageRouter; + + @Override + public Biomaterial saveMetadataDocument(Biomaterial document) { + return biomaterialRepository.save(document); + } + + @Override + public Biomaterial findMetadataDocument(String id) { + return biomaterialRepository + .findById(id) + .orElseThrow( + () -> { + throw new ResourceNotFoundException(); + }); + } + + @Override + public Biomaterial findOriginalByUuid(String uuid) { + return biomaterialRepository + .findByUuidUuidAndIsUpdateFalse(UUID.fromString(uuid)) + .orElseThrow( + () -> { + throw new ResourceNotFoundException(); + }); + } + + @Override + public Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { + return biomaterialRepository.findBySubmissionEnvelope(submissionEnvelope); + } + + @Override + public Collection findAllBySubmissionEnvelope( + SubmissionEnvelope submissionEnvelope) { + return biomaterialRepository.findAllBySubmissionEnvelope(submissionEnvelope); + } + + @Override + public void removeLinksToDocument(Biomaterial document) { + throw new UnsupportedOperationException("Not supported"); + } + + @Override + public void deleteDocument(Biomaterial document) { + biomaterialRepository.delete(document); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/DatasetCrudStrategy.java b/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/DatasetCrudStrategy.java new file mode 100644 index 000000000..2fe370eda --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/DatasetCrudStrategy.java @@ -0,0 +1,59 @@ +package uk.ac.ebi.subs.ingest.core.service.strategy.impl; + +import java.util.Collection; +import java.util.stream.Stream; + +import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.stereotype.Component; + +import lombok.AllArgsConstructor; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.core.service.strategy.MetadataCrudStrategy; +import uk.ac.ebi.subs.ingest.dataset.Dataset; +import uk.ac.ebi.subs.ingest.dataset.DatasetRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Component +@AllArgsConstructor +public class DatasetCrudStrategy implements MetadataCrudStrategy { + private final @NonNull DatasetRepository datasetRepository; + + @Override + public Dataset saveMetadataDocument(final Dataset document) { + return datasetRepository.save(document); + } + + @Override + public Dataset findMetadataDocument(final String id) { + return datasetRepository + .findById(id) + .orElseThrow( + () -> { + throw new ResourceNotFoundException(); + }); + } + + @Override + public Dataset findOriginalByUuid(final String uuid) { + return null; + } + + @Override + public Stream findBySubmissionEnvelope(final SubmissionEnvelope submissionEnvelope) { + return null; + } + + @Override + public Collection findAllBySubmissionEnvelope( + final SubmissionEnvelope submissionEnvelope) { + return null; + } + + @Override + public void removeLinksToDocument(final Dataset document) {} + + @Override + public void deleteDocument(final Dataset document) { + datasetRepository.delete(document); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/FileCrudStrategy.java b/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/FileCrudStrategy.java new file mode 100644 index 000000000..b16946f0a --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/FileCrudStrategy.java @@ -0,0 +1,77 @@ +package uk.ac.ebi.subs.ingest.core.service.strategy.impl; + +import java.util.Collection; +import java.util.UUID; +import java.util.stream.Stream; + +import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.stereotype.Component; + +import lombok.AllArgsConstructor; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.core.service.strategy.MetadataCrudStrategy; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Component +@AllArgsConstructor +public class FileCrudStrategy implements MetadataCrudStrategy { + private final @NonNull FileRepository fileRepository; + private final @NonNull ProjectRepository projectRepository; + private final @NonNull MessageRouter messageRouter; + + @Override + public File saveMetadataDocument(File document) { + return fileRepository.save(document); + } + + @Override + public File findMetadataDocument(String id) { + return fileRepository + .findById(id) + .orElseThrow( + () -> { + throw new ResourceNotFoundException(); + }); + } + + @Override + public File findOriginalByUuid(String uuid) { + return fileRepository + .findByUuidUuidAndIsUpdateFalse(UUID.fromString(uuid)) + .orElseThrow( + () -> { + throw new ResourceNotFoundException(); + }); + } + + @Override + public Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { + return fileRepository.findBySubmissionEnvelope(submissionEnvelope); + } + + @Override + public Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { + return fileRepository.findAllBySubmissionEnvelope(submissionEnvelope); + } + + @Override + public void removeLinksToDocument(File document) { + projectRepository + .findBySupplementaryFilesContains(document) + .forEach( + project -> { + project.getSupplementaryFiles().remove(document); + projectRepository.save(project); + }); + } + + @Override + public void deleteDocument(File document) { + removeLinksToDocument(document); + fileRepository.delete(document); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/ProcessCrudStrategy.java b/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/ProcessCrudStrategy.java new file mode 100644 index 000000000..ebd6847ba --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/ProcessCrudStrategy.java @@ -0,0 +1,100 @@ +package uk.ac.ebi.subs.ingest.core.service.strategy.impl; + +import java.util.Collection; +import java.util.UUID; +import java.util.stream.Stream; + +import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.stereotype.Component; + +import lombok.AllArgsConstructor; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.core.service.strategy.MetadataCrudStrategy; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Component +@AllArgsConstructor +public class ProcessCrudStrategy implements MetadataCrudStrategy { + private final @NonNull ProcessRepository processRepository; + private final @NonNull FileRepository fileRepository; + private final @NonNull BiomaterialRepository biomaterialRepository; + private final @NonNull MessageRouter messageRouter; + + @Override + public Process saveMetadataDocument(Process document) { + return processRepository.save(document); + } + + @Override + public Process findMetadataDocument(String id) { + return processRepository + .findById(id) + .orElseThrow( + () -> { + throw new ResourceNotFoundException(); + }); + } + + @Override + public Process findOriginalByUuid(String uuid) { + return processRepository + .findByUuidUuidAndIsUpdateFalse(UUID.fromString(uuid)) + .orElseThrow( + () -> { + throw new ResourceNotFoundException(); + }); + } + + @Override + public Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { + return processRepository.findBySubmissionEnvelope(submissionEnvelope); + } + + @Override + public Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { + return processRepository.findAllBySubmissionEnvelope(submissionEnvelope); + } + + @Override + public void removeLinksToDocument(Process document) { + fileRepository + .findByInputToProcessesContains(document) + .forEach( + file -> { + file.getInputToProcesses().remove(document); + fileRepository.save(file); + }); + fileRepository + .findByDerivedByProcessesContains(document) + .forEach( + file -> { + file.getDerivedByProcesses().remove(document); + fileRepository.save(file); + }); + biomaterialRepository + .findByInputToProcessesContains(document) + .forEach( + biomaterial -> { + biomaterial.getInputToProcesses().remove(document); + biomaterialRepository.save(biomaterial); + }); + biomaterialRepository + .findByDerivedByProcessesContains(document) + .forEach( + biomaterial -> { + biomaterial.getDerivedByProcesses().remove(document); + biomaterialRepository.save(biomaterial); + }); + } + + @Override + public void deleteDocument(Process document) { + removeLinksToDocument(document); + processRepository.delete(document); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/ProjectCrudStrategy.java b/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/ProjectCrudStrategy.java new file mode 100644 index 000000000..e889859fa --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/ProjectCrudStrategy.java @@ -0,0 +1,118 @@ +package uk.ac.ebi.subs.ingest.core.service.strategy.impl; + +import java.util.Collection; +import java.util.UUID; +import java.util.stream.Stream; + +import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.stereotype.Component; + +import lombok.AllArgsConstructor; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.core.service.strategy.MetadataCrudStrategy; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Component +@AllArgsConstructor +public class ProjectCrudStrategy implements MetadataCrudStrategy { + private final @NonNull ProjectRepository projectRepository; + private final @NonNull ProtocolRepository protocolRepository; + private final @NonNull ProcessRepository processRepository; + private final @NonNull FileRepository fileRepository; + private final @NonNull BiomaterialRepository biomaterialRepository; + private final @NonNull MessageRouter messageRouter; + + @Override + public Project saveMetadataDocument(Project document) { + return projectRepository.save(document); + } + + @Override + public Project findMetadataDocument(String id) { + return projectRepository + .findById(id) + .orElseThrow( + () -> { + throw new ResourceNotFoundException(); + }); + } + + @Override + public Project findOriginalByUuid(String uuid) { + return projectRepository + .findByUuidUuidAndIsUpdateFalse(UUID.fromString(uuid)) + .orElseThrow( + () -> { + throw new ResourceNotFoundException(); + }); + } + + @Override + public Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { + return projectRepository.findBySubmissionEnvelope(submissionEnvelope); + } + + @Override + public Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { + return projectRepository.findAllBySubmissionEnvelope(submissionEnvelope); + } + + @Override + public void removeLinksToDocument(Project document) { + biomaterialRepository + .findByProject(document) + .forEach( + biomaterial -> { + biomaterial.setProject(null); + biomaterialRepository.save(biomaterial); + }); + biomaterialRepository + .findByProjectsContaining(document) + .forEach( + biomaterial -> { + biomaterial.getProjects().remove(document); + biomaterialRepository.save(biomaterial); + }); + fileRepository + .findByProject(document) + .forEach( + file -> { + file.setProject(null); + fileRepository.save(file); + }); + processRepository + .findByProject(document) + .forEach( + process -> { + process.setProject(null); + processRepository.save(process); + }); + processRepository + .findByProjectsContaining(document) + .forEach( + process -> { + process.getProjects().remove(document); + processRepository.save(process); + }); + protocolRepository + .findByProject(document) + .forEach( + protocol -> { + protocol.setProject(null); + protocolRepository.save(protocol); + }); + } + + @Override + public void deleteDocument(Project document) { + removeLinksToDocument(document); + projectRepository.delete(document); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/ProtocolCrudStrategy.java b/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/ProtocolCrudStrategy.java new file mode 100644 index 000000000..1e396556d --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/ProtocolCrudStrategy.java @@ -0,0 +1,77 @@ +package uk.ac.ebi.subs.ingest.core.service.strategy.impl; + +import java.util.Collection; +import java.util.UUID; +import java.util.stream.Stream; + +import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.stereotype.Component; + +import lombok.AllArgsConstructor; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.core.service.strategy.MetadataCrudStrategy; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Component +@AllArgsConstructor +public class ProtocolCrudStrategy implements MetadataCrudStrategy { + private final @NonNull ProtocolRepository protocolRepository; + private final @NonNull ProcessRepository processRepository; + private final @NonNull MessageRouter messageRouter; + + @Override + public Protocol saveMetadataDocument(Protocol document) { + return protocolRepository.save(document); + } + + @Override + public Protocol findMetadataDocument(String id) { + return protocolRepository + .findById(id) + .orElseThrow( + () -> { + throw new ResourceNotFoundException(); + }); + } + + @Override + public Protocol findOriginalByUuid(String uuid) { + return protocolRepository + .findByUuidUuidAndIsUpdateFalse(UUID.fromString(uuid)) + .orElseThrow( + () -> { + throw new ResourceNotFoundException(); + }); + } + + @Override + public Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { + return protocolRepository.findBySubmissionEnvelope(submissionEnvelope); + } + + @Override + public Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { + return protocolRepository.findAllBySubmissionEnvelope(submissionEnvelope); + } + + @Override + public void removeLinksToDocument(Protocol document) { + processRepository + .findByProtocolsContains(document) + .forEach( + process -> { + process.getProtocols().remove(document); + processRepository.save(process); + }); + } + + @Override + public void deleteDocument(Protocol document) { + removeLinksToDocument(document); + protocolRepository.delete(document); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/StudyCrudStrategy.java b/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/StudyCrudStrategy.java new file mode 100644 index 000000000..f7e03bb8e --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/service/strategy/impl/StudyCrudStrategy.java @@ -0,0 +1,58 @@ +package uk.ac.ebi.subs.ingest.core.service.strategy.impl; + +import java.util.Collection; +import java.util.UUID; +import java.util.stream.Stream; + +import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.stereotype.Component; + +import lombok.AllArgsConstructor; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.core.service.strategy.MetadataCrudStrategy; +import uk.ac.ebi.subs.ingest.study.Study; +import uk.ac.ebi.subs.ingest.study.StudyRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Component +@AllArgsConstructor +public class StudyCrudStrategy implements MetadataCrudStrategy { + private final @NonNull StudyRepository studyRepository; + + @Override + public Study saveMetadataDocument(Study document) { + return studyRepository.save(document); + } + + @Override + public Study findMetadataDocument(String id) { + return studyRepository.findById(id).orElseThrow(ResourceNotFoundException::new); + } + + @Override + public Study findOriginalByUuid(String uuid) { + return studyRepository + .findByUuidUuidAndIsUpdateFalse(UUID.fromString(uuid)) + .orElseThrow(ResourceNotFoundException::new); + } + + @Override + public Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { + return studyRepository.findBySubmissionEnvelope(submissionEnvelope); + } + + @Override + public Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope) { + return studyRepository.findAllBySubmissionEnvelope(submissionEnvelope); + } + + @Override + public void removeLinksToDocument(Study document) {} + + @Override + public void deleteDocument(Study document) { + // TODO: Check what links need to be removed when deleting a Study document + // removeLinksToDocument(document); + studyRepository.delete(document); + } +} diff --git a/src/main/java/org/humancellatlas/ingest/core/web/ExceptionInfo.java b/src/main/java/uk/ac/ebi/subs/ingest/core/web/ExceptionInfo.java similarity index 61% rename from src/main/java/org/humancellatlas/ingest/core/web/ExceptionInfo.java rename to src/main/java/uk/ac/ebi/subs/ingest/core/web/ExceptionInfo.java index 4da78c8c4..e6bcb4c8e 100644 --- a/src/main/java/org/humancellatlas/ingest/core/web/ExceptionInfo.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/web/ExceptionInfo.java @@ -1,4 +1,4 @@ -package org.humancellatlas.ingest.core.web; +package uk.ac.ebi.subs.ingest.core.web; import lombok.AllArgsConstructor; import lombok.Getter; @@ -13,6 +13,6 @@ @AllArgsConstructor @Getter public class ExceptionInfo { - private final @NonNull String url; - private final @NonNull String exceptionMessage; + private final @NonNull String url; + private final @NonNull String exceptionMessage; } diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/web/GlobalStateExceptionHandler.java b/src/main/java/uk/ac/ebi/subs/ingest/core/web/GlobalStateExceptionHandler.java new file mode 100644 index 000000000..0c9f22dd3 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/web/GlobalStateExceptionHandler.java @@ -0,0 +1,184 @@ +package uk.ac.ebi.subs.ingest.core.web; + +import javax.servlet.http.HttpServletRequest; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.dao.OptimisticLockingFailureException; +import org.springframework.data.repository.support.QueryMethodParameterConversionException; +import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.security.access.AccessDeniedException; +import org.springframework.security.authentication.AuthenticationCredentialsNotFoundException; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.server.ResponseStatusException; + +import uk.ac.ebi.subs.ingest.core.exception.MultipleOpenSubmissionsException; +import uk.ac.ebi.subs.ingest.core.exception.RedundantUpdateException; +import uk.ac.ebi.subs.ingest.core.exception.StateTransitionNotAllowed; +import uk.ac.ebi.subs.ingest.security.exception.NotAllowedException; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 27/09/17 + */ +@ControllerAdvice +public class GlobalStateExceptionHandler { + private final Logger log = LoggerFactory.getLogger(getClass()); + + protected Logger getLog() { + return log; + } + + @ResponseStatus(HttpStatus.CONFLICT) + @ExceptionHandler(IllegalStateException.class) + public @ResponseBody ExceptionInfo handleIllegalStateException( + HttpServletRequest request, Exception e) { + getLog() + .warn( + String.format( + "Attempted an illegal state transition at '%s';" + + "this will generate a CONFLICT RESPONSE", + request.getRequestURL().toString())); + getLog().debug("Handling IllegalStateException and returning CONFLICT response", e); + return new ExceptionInfo(request.getRequestURL().toString(), e.getLocalizedMessage()); + } + + @ResponseStatus(HttpStatus.CONFLICT) + @ExceptionHandler(OptimisticLockingFailureException.class) + public @ResponseBody ExceptionInfo handleOptimisticLock(HttpServletRequest request, Exception e) { + getLog() + .warn( + String.format( + "Attempt a failed save, likely due to multiple requests, at '%s'; " + + "this will generate a CONFLICT RESPONSE", + request.getRequestURL().toString())); + getLog().debug("Handling OptimisticLockingFailureException and returning CONFLICT response", e); + return new ExceptionInfo(request.getRequestURL().toString(), e.getLocalizedMessage()); + } + + @ResponseStatus(HttpStatus.BAD_REQUEST) + @ExceptionHandler({ + IllegalArgumentException.class, + HttpMessageNotReadableException.class, + RedundantUpdateException.class, + MultipleOpenSubmissionsException.class, + QueryMethodParameterConversionException.class + }) + public @ResponseBody ExceptionInfo handleIllegalArgument( + HttpServletRequest request, Exception e) { + getLog() + .warn( + String.format( + "Caught an illegal argument at '%s %s'. error: %s; " + + "this will generate a BAD_REQUEST RESPONSE", + request.getMethod(), request.getRequestURL().toString(), e.getLocalizedMessage())); + getLog().error("Handling IllegalArgumentException and returning BAD_REQUEST response", e); + return new ExceptionInfo(request.getRequestURL().toString(), e.getLocalizedMessage()); + } + + @ResponseStatus(HttpStatus.NOT_FOUND) + @ExceptionHandler(ResourceNotFoundException.class) + public @ResponseBody ExceptionInfo handleResourceNotFound( + HttpServletRequest request, Exception e) { + getLog() + .warn( + String.format( + "Caught a resource not found exception argument at '%s'; " + + "this will generate a NOT_FOUND RESPONSE. Error message: %s", + request.getRequestURL().toString(), e.getLocalizedMessage())); + getLog().warn("Handling ResourceNotFoundException and returning NOT_FOUND response"); + return new ExceptionInfo(request.getRequestURL().toString(), e.getLocalizedMessage()); + } + + @ResponseStatus(HttpStatus.BAD_REQUEST) + @ExceptionHandler(StateTransitionNotAllowed.class) + public @ResponseBody ExceptionInfo handleStateTransitionNotAllowed( + HttpServletRequest request, Exception e) { + getLog() + .warn( + String.format( + "Caught a state transition not allowed exception at '%s'; " + + "this will generate a BAD_REQUEST RESPONSE", + request.getRequestURL().toString())); + getLog().debug("Handling StateTransitionNotAllowed and returning BAD_REQUEST response", e); + return new ExceptionInfo(request.getRequestURL().toString(), e.getLocalizedMessage()); + } + + @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) + @ExceptionHandler(RuntimeException.class) + public @ResponseBody ExceptionInfo handleRuntimeException( + HttpServletRequest request, Exception e) { + getLog() + .error( + String.format( + "Runtime exception encountered on %s request to resource %s ", + request.getMethod(), request.getRequestURL().toString()), + e); + getLog().error("Handling RuntimeException and returning INTERNAL_SERVER_ERROR response"); + return new ExceptionInfo(request.getRequestURL().toString(), "Unexpected server error"); + } + + @ResponseStatus(HttpStatus.FORBIDDEN) + @ExceptionHandler(NotAllowedException.class) + public @ResponseBody ExceptionInfo handNotAllowedException( + HttpServletRequest request, Exception e) { + getLog() + .error( + String.format( + "Not allowed exception encountered on %s request to resource %s ", + request.getMethod(), request.getRequestURL().toString()), + e); + getLog() + .debug( + String.format( + "Handling NotAllowedException and returning FORBIDDEN response during %s request to %s", + request.getMethod(), request.getRequestURL().toString()), + e); + return new ExceptionInfo(request.getRequestURL().toString(), e.getLocalizedMessage()); + } + + @ResponseStatus(HttpStatus.METHOD_NOT_ALLOWED) + @ExceptionHandler(UnsupportedOperationException.class) + public @ResponseBody ExceptionInfo handleUnsupportedOperationException( + HttpServletRequest request, Exception e) { + getLog() + .error( + String.format( + "UnsupportedOperationException encountered on %s request to resource %s ", + request.getMethod(), request.getRequestURL().toString()), + e); + getLog() + .debug( + "Handling UnsupportedOperationException and returning METHOD_NOT_ALLOWED response", e); + return new ExceptionInfo(request.getRequestURL().toString(), e.getLocalizedMessage()); + } + + @ResponseStatus(HttpStatus.FORBIDDEN) + @ExceptionHandler(AccessDeniedException.class) + public @ResponseBody ExceptionInfo handleAccessDeniedException( + HttpServletRequest request, Exception e) { + getLog().info("access denied: {} {}", request.getMethod(), request.getRequestURL()); + return new ExceptionInfo(request.getRequestURL().toString(), e.getLocalizedMessage()); + } + + @ResponseStatus(HttpStatus.UNAUTHORIZED) + @ExceptionHandler(AuthenticationCredentialsNotFoundException.class) + public @ResponseBody ExceptionInfo handleAccessCredentialsNotFoundException( + HttpServletRequest request, Exception e) { + getLog().info("unauthorized {} {}", request.getMethod(), request.getRequestURL()); + return new ExceptionInfo(request.getRequestURL().toString(), e.getLocalizedMessage()); + } + + @ExceptionHandler(ResponseStatusException.class) + public ResponseEntity handleResponseStatusException(final ResponseStatusException ex) { + return new ResponseEntity<>(ex.getReason(), ex.getStatus()); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/web/LinkGenerator.java b/src/main/java/uk/ac/ebi/subs/ingest/core/web/LinkGenerator.java new file mode 100644 index 000000000..f8783e9f9 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/web/LinkGenerator.java @@ -0,0 +1,6 @@ +package uk.ac.ebi.subs.ingest.core.web; + +public interface LinkGenerator { + + String createCallback(Class documentType, String documentId); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/web/Links.java b/src/main/java/uk/ac/ebi/subs/ingest/core/web/Links.java new file mode 100644 index 000000000..19083af2b --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/web/Links.java @@ -0,0 +1,168 @@ +package uk.ac.ebi.subs.ingest.core.web; + +/** + * Enumerates the relations that are available in this application to provide some stability across + * different implementations. These should not change without serious motivation, as will likely + * cause backwards-compatibility breaking changes for clients. + * + * @author Tony Burdett + * @date 05/09/17 + */ +public class Links { + // Links to request state changes for submission envelopes + + public static final String UPDATE_SUBMISSION_URL = "/updateSubmissions"; + public static final String UPDATE_SUBMISSION_REL = "updateSubmissions"; + + public static final String SUBMIT_URL = "/submissionEvent"; + public static final String SUBMIT_REL = "submit"; + + public static final String DRAFT_REL = "draft"; + public static final String DRAFT_URL = "/draftEvent"; + public static final String METADATA_VALIDATING_REL = "validating"; + public static final String METADATA_VALIDATING_URL = "/validatingEvent"; + public static final String METADATA_VALID_REL = "valid"; + public static final String METADATA_VALID_URL = "/validEvent"; + public static final String GRAPH_VALIDATION_REQUESTED_REL = "graphValidationRequested"; + public static final String GRAPH_VALIDATION_REQUESTED_URL = "/graphValidationRequestedEvent"; + public static final String GRAPH_VALIDATING_REL = "graphValidating"; + public static final String GRAPH_VALIDATING_URL = "/graphValidatingEvent"; + public static final String GRAPH_VALID_REL = "graphValid"; + public static final String GRAPH_VALID_URL = "/graphValidEvent"; + public static final String GRAPH_INVALID_REL = "graphInvalid"; + public static final String GRAPH_INVALID_URL = "/graphInvalidEvent"; + public static final String INVALID_REL = "invalid"; + public static final String INVALID_URL = "/invalidEvent"; + public static final String PROCESSING_REL = "processing"; + public static final String PROCESSING_URL = "/processingEvent"; + public static final String ARCHIVING_REL = "archiving"; + public static final String ARCHIVING_URL = "/archivingEvent"; + public static final String ARCHIVED_REL = "archived"; + public static final String ARCHIVED_URL = "/archivedEvent"; + public static final String EXPORT_REL = "export"; + public static final String EXPORT_URL = "/exportEvent"; + public static final String EXPORTED_REL = "exported"; + public static final String EXPORTED_URL = "/exportedEvent"; + public static final String CLEANUP_REL = "cleanup"; + public static final String CLEANUP_URL = "/cleanupEvent"; + public static final String COMPLETE_REL = "complete"; + public static final String COMPLETE_URL = "/completionEvent"; + + // links to commit state changes + public static final String COMMIT_SUBMIT_URL = "/commitSubmissionEvent"; + public static final String COMMIT_SUBMIT_REL = "commitSubmit"; + + public static final String COMMIT_DRAFT_REL = "commitDraft"; + public static final String COMMIT_DRAFT_URL = "/commitDraftEvent"; + public static final String COMMIT_METADATA_VALIDATING_REL = "commitValidating"; + public static final String COMMIT_METADATA_VALIDATING_URL = "/commitValidatingEvent"; + public static final String COMMIT_METADATA_VALID_REL = "commitValid"; + public static final String COMMIT_METADATA_VALID_URL = "/commitValidEvent"; + public static final String COMMIT_METADATA_INVALID_REL = "commitInvalid"; + public static final String COMMIT_METADATA_INVALID_URL = "/commitInvalidEvent"; + public static final String COMMIT_GRAPH_VALIDATION_REQUESTED_REL = + "commitGraphValidationRequested"; + public static final String COMMIT_GRAPH_VALIDATION_REQUESTED_URL = + "/commitGraphValidationRequestedEvent"; + public static final String COMMIT_GRAPH_VALIDATING_REL = "commitGraphValidating"; + public static final String COMMIT_GRAPH_VALIDATING_URL = "/commitGraphValidatingEvent"; + public static final String COMMIT_GRAPH_VALID_REL = "commitGraphValid"; + public static final String COMMIT_GRAPH_VALID_URL = "/commitGraphValidEvent"; + public static final String COMMIT_GRAPH_INVALID_REL = "commitGraphInvalid"; + public static final String COMMIT_GRAPH_INVALID_URL = "/commitGraphInvalidEvent"; + public static final String COMMIT_PROCESSING_REL = "commitProcessing"; + public static final String COMMIT_PROCESSING_URL = "/commitProcessingEvent"; + public static final String COMMIT_ARCHIVING_REL = "commitArchiving"; + public static final String COMMIT_ARCHIVING_URL = "/commitArchivingEvent"; + public static final String COMMIT_ARCHIVED_REL = "commitArchived"; + public static final String COMMIT_ARCHIVED_URL = "/commitArchivedEvent"; + public static final String COMMIT_EXPORTING_REL = "commitExporting"; + public static final String COMMIT_EXPORTING_URL = "/commitExportingEvent"; + public static final String COMMIT_EXPORTED_REL = "commitExported"; + public static final String COMMIT_EXPORTED_URL = "/commitExportedEvent"; + public static final String COMMIT_CLEANUP_REL = "commitCleanup"; + public static final String COMMIT_CLEANUP_URL = "/commitCleanupEvent"; + public static final String COMMIT_COMPLETE_REL = "commitComplete"; + public static final String COMMIT_COMPLETE_URL = "/commitCompleteEvent"; + + // Links to entities for submission envelopes + public static final String BIOMATERIALS_URL = "/biomaterials"; + public static final String BIOMATERIALS_REL = "biomaterials"; + public static final String PROCESSES_URL = "/processes"; + public static final String PROCESSES_REL = "processes"; + public static final String FILES_URL = "/files"; + public static final String FILES_REL = "files"; + public static final String PROJECTS_URL = "/projects"; + public static final String PROJECTS_REL = "projects"; + + public static final String STUDIES_URL = "/studies"; + public static final String STUDIES_REL = "studies"; + public static final String DATASETS_URL = "/datasets"; + public static final String DATASETS_REL = "datasets"; + public static final String PROTOCOLS_URL = "/protocols"; + public static final String PROTOCOLS_REL = "protocols"; + public static final String BUNDLE_MANIFESTS_URL = "/bundleManifests"; + public static final String BUNDLE_MANIFESTS_REL = "bundleManifests"; + public static final String SUBMISSION_MANIFEST_URL = "/submissionManifest"; + public static final String SUBMISSION_MANIFEST_REL = "submissionManifest"; + public static final String SUBMISSION_ERRORS_URL = "/submissionErrors"; + public static final String SUBMISSION_ERRORS_REL = "submissionEnvelopeErrors"; + public static final String SUBMISSION_SUMMARY_URL = "/summary"; + public static final String SUBMISSION_SUMMARY_REL = "summary"; + public static final String SUBMISSION_CONTENT_LAST_UPDATED_URL = "/contentLastUpdated"; + public static final String SUBMISSION_CONTENT_LAST_UPDATED_REL = "contentLastUpdated"; + public static final String SUBMISSION_LINKING_MAP_URL = "/linkingMap"; + public static final String SUBMISSION_LINKING_MAP_REL = "linkingMap"; + public static final String SUBMISSION_RELATED_PROJECTS_URL = "/relatedProjects"; + public static final String SUBMISSION_RELATED_PROJECTS_REL = "relatedProjects"; + + public static final String SUBMISSION_RELATED_STUDIES_URL = "/relatedStudies"; + public static final String SUBMISSION_RELATED_STUDIES_REL = "relatedStudies"; + public static final String SUBMISSION_RELATED_DATASETS_URL = "/relatedDatasets"; + public static final String SUBMISSION_RELATED_DATASETS_REL = "relatedDatasets"; + + public static final String SUBMISSION_DOCUMENTS_SM_URL = "/documentSmReport"; + public static final String SUBMISSION_DOCUMENTS_SM_REL = "documentSmReport"; + + // Links to entities for projects + public static final String AUDIT_LOGS_URL = "/auditLogs"; + public static final String AUDIT_LOGS_REL = "auditLogs"; + + // Links for analyses + public static final String BUNDLE_REF_URL = "/bundleReferences"; + public static final String BUNDLE_REF_REL = "inputBundleReferences"; + public static final String BUNDLE_REF_OLD_EVIL_REL = "add-input-bundles"; + public static final String FILE_REF_URL = "/fileReference"; + public static final String FILE_REF_REL = "inputFileReferences"; + public static final String FILE_REF_OLD_EVIL_REL = "add-file-reference"; + + // Links from Processes + public static final String INPUT_BIOMATERIALS_URL = "/inputBiomaterials"; + public static final String INPUT_BIOMATERIALS_REL = "inputBiomaterials"; + public static final String INPUT_FILES_URL = "/inputFiles"; + public static final String INPUT_FILES_REL = "inputFiles"; + + public static final String DERIVED_BY_BIOMATERIALS_URL = "/derivedBiomaterials"; + public static final String DERIVED_BY_BIOMATERIALS_REL = "derivedBiomaterials"; + public static final String DERIVED_BY_FILES_URL = "/derivedFiles"; + public static final String DERIVED_BY_FILES_REL = "derivedFiles"; + + // Links from Files + public static final String FILE_VALIDATION_JOB_URL = "/validationJob"; + public static final String FILE_VALIDATION_JOB_REL = "validationJob"; + + // Links from StagingJobs + public static final String COMPLETE_STAGING_JOB_URL = "/complete"; + public static final String COMPLETE_STAGING_JOB_REL = "completeStagingJob"; + + // Links to ExportJobs + public static final String EXPORT_JOBS_URL = "/exportJobs"; + public static final String EXPORT_JOBS_REL = "exportJobs"; + + public static final String EXPORT_JOB_ENTITIES_URL = "/entities"; + public static final String EXPORT_JOB_ENTITIES_REL = "exportEntities"; + public static final String EXPORT_JOB_ENTITIES_BY_STATUS_REL = "exportEntitiesByStatus"; + + public static final String EXPORT_JOB_FIND_URL = "/find"; + public static final String EXPORT_JOB_FIND_REL = "find"; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/web/MetadataController.java b/src/main/java/uk/ac/ebi/subs/ingest/core/web/MetadataController.java new file mode 100644 index 000000000..77ba4c9f9 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/web/MetadataController.java @@ -0,0 +1,112 @@ +package uk.ac.ebi.subs.ingest.core.web; + +import java.util.List; +import java.util.Optional; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.data.web.PagedResourcesAssembler; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.hateoas.PagedResources; +import org.springframework.hateoas.Resource; +import org.springframework.http.HttpEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.core.service.ValidationStateChangeService; +import uk.ac.ebi.subs.ingest.query.MetadataCriteria; +import uk.ac.ebi.subs.ingest.query.MetadataQueryService; +import uk.ac.ebi.subs.ingest.state.ValidationState; + +@RequiredArgsConstructor +@RepositoryRestController +@ExposesResourceFor(MetadataDocument.class) +public class MetadataController { + private final @NonNull ValidationStateChangeService validationStateChangeService; + private final @NonNull MetadataQueryService metadataQueryService; + private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; + + @PutMapping("/{metadataType}/{id}" + Links.DRAFT_URL) + HttpEntity draftEvent( + @PathVariable("metadataType") String metadataType, + @PathVariable("id") String metadataId, + PersistentEntityResourceAssembler assembler) { + MetadataDocument metadataDocument = + validationStateChangeService.changeValidationState( + entityTypeForCollection(metadataType), metadataId, ValidationState.DRAFT); + return ResponseEntity.accepted().body(assembler.toFullResource(metadataDocument)); + } + + @PutMapping("/{metadataType}/{id}" + Links.METADATA_VALIDATING_URL) + HttpEntity validatingEvent( + @PathVariable("metadataType") String metadataType, + @PathVariable("id") String metadataId, + PersistentEntityResourceAssembler assembler) { + MetadataDocument metadataDocument = + validationStateChangeService.changeValidationState( + entityTypeForCollection(metadataType), metadataId, ValidationState.VALIDATING); + return ResponseEntity.accepted().body(assembler.toFullResource(metadataDocument)); + } + + @PutMapping("/{metadataType}/{id}" + Links.METADATA_VALID_URL) + HttpEntity validEvent( + @PathVariable("metadataType") String metadataType, + @PathVariable("id") String metadataId, + PersistentEntityResourceAssembler assembler) { + MetadataDocument metadataDocument = + validationStateChangeService.changeValidationState( + entityTypeForCollection(metadataType), metadataId, ValidationState.VALID); + return ResponseEntity.accepted().body(assembler.toFullResource(metadataDocument)); + } + + @PutMapping("/{metadataType}/{id}" + Links.INVALID_URL) + HttpEntity invalidEvent( + @PathVariable("metadataType") String metadataType, + @PathVariable("id") String metadataId, + PersistentEntityResourceAssembler assembler) { + MetadataDocument metadataDocument = + validationStateChangeService.changeValidationState( + entityTypeForCollection(metadataType), metadataId, ValidationState.INVALID); + return ResponseEntity.accepted().body(assembler.toFullResource(metadataDocument)); + } + + @PostMapping("/{metadataType}/query") + ResponseEntity>> query( + @PathVariable("metadataType") String metadataType, + @RequestBody List criteriaList, + @RequestParam("operator") Optional operator, + Pageable pageable, + final PersistentEntityResourceAssembler assembler) { + Boolean andCriteria = operator.map("and"::equalsIgnoreCase).orElse(false); + Page docs = + metadataQueryService.findByCriteria( + entityTypeForCollection(metadataType), criteriaList, andCriteria, pageable); + return ResponseEntity.ok(pagedResourcesAssembler.toResource(docs, assembler)); + } + + private EntityType entityTypeForCollection(String collection) { + switch (collection) { + case "biomaterials": + return EntityType.BIOMATERIAL; + case "protocols": + return EntityType.PROTOCOL; + case "projects": + return EntityType.PROJECT; + case "studies": + return EntityType.STUDY; + case "processes": + return EntityType.PROCESS; + case "files": + return EntityType.FILE; + default: + throw new ResourceNotFoundException(); + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/web/MetadataDocumentResourceProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/core/web/MetadataDocumentResourceProcessor.java new file mode 100644 index 000000000..59395c277 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/web/MetadataDocumentResourceProcessor.java @@ -0,0 +1,116 @@ +package uk.ac.ebi.subs.ingest.core.web; + +import java.util.Optional; + +import org.springframework.data.rest.webmvc.support.RepositoryEntityLinks; +import org.springframework.hateoas.EntityLinks; +import org.springframework.hateoas.Link; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.stereotype.Component; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.patch.Patch; +import uk.ac.ebi.subs.ingest.state.ValidationState; + +@Component +@RequiredArgsConstructor +public class MetadataDocumentResourceProcessor + implements ResourceProcessor> { + + private final @NonNull EntityLinks entityLinks; + + @NonNull private final RepositoryEntityLinks repositoryEntityLinks; + + private Optional getStateTransitionLink( + MetadataDocument metadataDocument, ValidationState targetState) { + Optional transitionResourceName = getSubresourceNameForValidationState(targetState); + if (transitionResourceName.isPresent()) { + Optional rel = getRelNameForValidationState(targetState); + if (rel.isPresent()) { + return Optional.of( + entityLinks + .linkForSingleResource(metadataDocument) + .slash(transitionResourceName.get()) + .withRel(rel.get())); + } else { + String messageTemplate = + "Unexpected link/rel mismatch exception " + "(link = '%s', rel = '%s')"; + throw new RuntimeException( + String.format(messageTemplate, transitionResourceName.toString(), rel.toString())); + } + } else { + return Optional.empty(); + } + } + + private Optional getRelNameForValidationState(ValidationState validationState) { + switch (validationState) { + case DRAFT: + return Optional.of(Links.DRAFT_REL); + case VALIDATING: + return Optional.of(Links.METADATA_VALIDATING_REL); + case VALID: + return Optional.of(Links.METADATA_VALID_REL); + case INVALID: + return Optional.of(Links.INVALID_REL); + case PROCESSING: + return Optional.of(Links.PROCESSING_REL); + case COMPLETE: + return Optional.of(Links.COMPLETE_REL); + default: + // default returns no links (not expecting external user interaction) + return Optional.empty(); + } + } + + private Optional getSubresourceNameForValidationState(ValidationState validationState) { + switch (validationState) { + case DRAFT: + return Optional.of(Links.DRAFT_URL); + case VALIDATING: + return Optional.of(Links.METADATA_VALIDATING_URL); + case VALID: + return Optional.of(Links.METADATA_VALID_URL); + case INVALID: + return Optional.of(Links.INVALID_URL); + case PROCESSING: + return Optional.of(Links.PROCESSING_URL); + case COMPLETE: + return Optional.of(Links.COMPLETE_URL); + default: + // default returns no links (not expecting external user interaction) + return Optional.empty(); + } + } + + @Override + public Resource process(Resource resource) { + MetadataDocument metadataDocument = resource.getContent(); + addStateLinks(resource, metadataDocument); + if (metadataDocument.getIsUpdate()) { + addPatchLink(resource, metadataDocument.getId()); + } + return resource; + } + + private void addStateLinks( + Resource resource, MetadataDocument metadataDocument) { + metadataDocument.allowedStateTransitions().stream() + .map(validationState -> getStateTransitionLink(metadataDocument, validationState)) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(resource::add); + } + + private void addPatchLink(Resource resource, String documentId) { + Link link = + repositoryEntityLinks + .linkToSearchResource(Patch.class, "WithUpdateDocument") + .withRel("patch") + .expand(documentId); + resource.add(link); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/core/web/SpringLinkGenerator.java b/src/main/java/uk/ac/ebi/subs/ingest/core/web/SpringLinkGenerator.java new file mode 100644 index 000000000..e034addb4 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/core/web/SpringLinkGenerator.java @@ -0,0 +1,18 @@ +package uk.ac.ebi.subs.ingest.core.web; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.rest.core.mapping.ResourceMappings; +import org.springframework.data.rest.core.mapping.ResourceMetadata; +import org.springframework.stereotype.Component; + +@Component +public class SpringLinkGenerator implements LinkGenerator { + + @Autowired private ResourceMappings resourceMappings; + + @Override + public String createCallback(Class documentType, String documentId) { + ResourceMetadata metadata = resourceMappings.getMetadataFor(documentType); + return String.format("/%s/%s", metadata.getRel(), documentId); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/dataset/Dataset.java b/src/main/java/uk/ac/ebi/subs/ingest/dataset/Dataset.java new file mode 100644 index 000000000..cc26403a6 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/dataset/Dataset.java @@ -0,0 +1,57 @@ +package uk.ac.ebi.subs.ingest.dataset; + +import java.util.*; + +import javax.validation.constraints.NotNull; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.Getter; +import lombok.Setter; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.protocol.Protocol; + +@Getter +@JsonIgnoreProperties({ + "firstDcpVersion", + "dcpVersion", + "validationState", + "validationErrors", + "graphValidationErrors", + "isUpdate" +}) +public class Dataset extends MetadataDocument { + private Set dataFiles = new HashSet<>(); + + private Set biomaterials = new HashSet<>(); + + private Set protocols = new HashSet<>(); + + private Set processes = new HashSet<>(); + + @Setter private String comment; + + @JsonCreator + public Dataset(@JsonProperty("content") final Object content) { + super(EntityType.DATASET, content); + } + + public void addBiomaterial(@NotNull final String biomaterialId) { + this.biomaterials.add(biomaterialId); + } + + public void addProtocol(@NotNull final Protocol protocol) { + this.protocols.add(protocol); + } + + public void addProcess(@NotNull final String processId) { + this.processes.add(processId); + } + + public void addFile(@NotNull final String dataFileId) { + this.dataFiles.add(dataFileId); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/dataset/DatasetEventHandler.java b/src/main/java/uk/ac/ebi/subs/ingest/dataset/DatasetEventHandler.java new file mode 100644 index 000000000..f313eae8f --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/dataset/DatasetEventHandler.java @@ -0,0 +1,25 @@ +package uk.ac.ebi.subs.ingest.dataset; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import lombok.RequiredArgsConstructor; + +@Component +@RequiredArgsConstructor +public class DatasetEventHandler { + private final Logger log = LoggerFactory.getLogger(getClass()); + + public void registeredDataset(Dataset dataset) { + log.info("A new dataset [" + dataset.getUuid() + "] has been registered."); + } + + public void updatedDataset(Dataset dataset) { + log.info("Updated dataset: {}", dataset.getUuid()); + } + + public void deletedDataset(String id) { + log.info("Deleted dataset with ID: {}", id); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/dataset/DatasetRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/dataset/DatasetRepository.java new file mode 100644 index 000000000..cc355ecf2 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/dataset/DatasetRepository.java @@ -0,0 +1,29 @@ +package uk.ac.ebi.subs.ingest.dataset; + +import java.util.Optional; +import java.util.UUID; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.repository.query.Param; +import org.springframework.data.rest.core.annotation.RestResource; +import org.springframework.web.bind.annotation.CrossOrigin; + +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +/** Javadocs go here! */ +@CrossOrigin +public interface DatasetRepository extends MongoRepository { + @RestResource(rel = "findAllByUuid", path = "findAllByUuid") + Page findByUuid(@Param("uuid") Uuid uuid, Pageable pageable); + + @RestResource(exported = false) + Optional findByUuid(Uuid uuid); + + @RestResource(rel = "findByUuid", path = "findByUuid") + Optional findByUuidUuidAndIsUpdateFalse(@Param("uuid") UUID uuid); + + Page findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope, Pageable pageable); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/dataset/DatasetService.java b/src/main/java/uk/ac/ebi/subs/ingest/dataset/DatasetService.java new file mode 100644 index 000000000..7e01046ee --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/dataset/DatasetService.java @@ -0,0 +1,235 @@ +package uk.ac.ebi.subs.ingest.dataset; + +import java.util.Optional; + +import javax.validation.constraints.NotNull; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.server.ResponseStatusException; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.dataset.util.UploadAreaUtil; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Service +@RequiredArgsConstructor +public class DatasetService { + @Autowired private final MongoTemplate mongoTemplate; + private final @NonNull DatasetRepository datasetRepository; + private final @NotNull BiomaterialRepository biomaterialRepository; + private final @NonNull ProcessRepository processRepository; + private final @NotNull ProtocolRepository protocolRepository; + private final @NotNull FileRepository fileRepository; + private final @NonNull MetadataCrudService metadataCrudService; + private final @NonNull MetadataUpdateService metadataUpdateService; + private final @NonNull DatasetEventHandler datasetEventHandler; + private final @NotNull UploadAreaUtil uploadAreaUtil; + private final Logger log = LoggerFactory.getLogger(getClass()); + + public Dataset register(final Dataset dataset) { + final Dataset persistentDataset = datasetRepository.save(dataset); + + uploadAreaUtil.createDataFilesUploadArea(dataset); + datasetEventHandler.registeredDataset(persistentDataset); + + return persistentDataset; + } + + public Dataset update(final Dataset dataset, final ObjectNode patch) { + final String datasetId = dataset.getId(); + final Optional existingDatasetOptional = datasetRepository.findById(datasetId); + + if (existingDatasetOptional.isEmpty()) { + log.warn("Dataset not found with ID: {}", datasetId); + + throw new ResponseStatusException( + HttpStatus.NOT_FOUND, "Dataset not found with ID: " + datasetId); + } + + final Dataset updatedDataset = + metadataUpdateService.update(existingDatasetOptional.get(), patch); + + datasetEventHandler.updatedDataset(updatedDataset); + + return updatedDataset; + } + + public void delete(final String datasetId, final boolean deleteLinkedEntities) { + final Optional deleteDatasetOptional = datasetRepository.findById(datasetId); + + if (deleteDatasetOptional.isEmpty()) { + log.warn("Dataset not found with ID: {}", datasetId); + + throw new ResponseStatusException( + HttpStatus.NOT_FOUND, "Dataset not found with ID: " + datasetId); + } + + final Dataset deleteDataset = deleteDatasetOptional.get(); + + if (deleteLinkedEntities) { + deleteDataset + .getBiomaterials() + .forEach( + id -> { + try { + final Optional biomaterial = biomaterialRepository.findById(id); + + biomaterial.ifPresent(metadataCrudService::deleteDocument); + } catch (final Exception e) { + log.info("Biomaterial not found with ID " + id); + } + }); + + deleteDataset + .getProcesses() + .forEach( + id -> { + try { + final Optional process = processRepository.findById(id); + + process.ifPresent(metadataCrudService::deleteDocument); + } catch (final Exception e) { + log.info("Process not found with ID " + id); + } + }); + + deleteDataset + .getDataFiles() + .forEach( + id -> { + try { + final Optional file = fileRepository.findById(id); + + file.ifPresent(metadataCrudService::deleteDocument); + } catch (final Exception e) { + log.info("File not found with ID " + id); + } + }); + + uploadAreaUtil.deleteDataFilesAndUploadArea(datasetId); + } + + metadataCrudService.deleteDocument(deleteDataset); + datasetEventHandler.deletedDataset(datasetId); + } + + public Dataset replace(final String datasetId, final Dataset updatedDataset) { + final Optional existingDatasetOptional = datasetRepository.findById(datasetId); + + if (existingDatasetOptional.isEmpty()) { + log.warn("Dataset not found with ID: {}", datasetId); + throw new ResponseStatusException( + HttpStatus.NOT_FOUND, "Dataset not found with ID: " + datasetId); + } + + datasetRepository.save(updatedDataset); + datasetEventHandler.updatedDataset(updatedDataset); + + return updatedDataset; + } + + public Dataset addDatasetToSubmissionEnvelope( + final SubmissionEnvelope submissionEnvelope, final Dataset dataset) { + if (!dataset.getIsUpdate()) { + final Dataset savedDataset = + metadataCrudService.addToSubmissionEnvelopeAndSave(dataset, submissionEnvelope); + + uploadAreaUtil.createDataFilesUploadArea(savedDataset); + + return savedDataset; + } else { + return metadataUpdateService.acceptUpdate(dataset, submissionEnvelope); + } + } + + public final Dataset addFileToDataset(final Dataset dataset, final String id) { + final String datasetId = dataset.getId(); + + datasetRepository + .findById(datasetId) + .orElseThrow(() -> new ResourceNotFoundException("Dataset: " + datasetId)); + fileRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("File: " + id)); + + dataset.addFile(id); + // file.addDataset(dataset); + + // fileRepository.save(file); + final Dataset updatedDataset = datasetRepository.save(dataset); + + return updatedDataset; + } + + public final Dataset addBiomaterialToDataset(final Dataset dataset, final String id) { + final String datasetId = dataset.getId(); + + datasetRepository + .findById(datasetId) + .orElseThrow(() -> new ResourceNotFoundException("Dataset: " + datasetId)); + biomaterialRepository + .findById(id) + .orElseThrow(() -> new ResourceNotFoundException("Biomaterial: " + id)); + + dataset.addBiomaterial(id); + // biomaterial.addDataset(dataset); + + // biomaterialRepository.save(biomaterial); + + final Dataset updatedDataset = datasetRepository.save(dataset); + + return updatedDataset; + } + + public final Dataset linkProtocolToDataset(final Dataset dataset, final Protocol protocol) { + final String datasetId = dataset.getId(); + final String protocolId = protocol.getId(); + + datasetRepository + .findById(datasetId) + .orElseThrow(() -> new ResourceNotFoundException("Dataset: " + datasetId)); + protocolRepository + .findById(protocolId) + .orElseThrow(() -> new ResourceNotFoundException("Protocol: " + protocolId)); + + dataset.addProtocol(protocol); + + return datasetRepository.save(dataset); + } + + public final Dataset addProcessToDataset(final Dataset dataset, final String id) { + final String datasetId = dataset.getId(); + + datasetRepository + .findById(datasetId) + .orElseThrow(() -> new ResourceNotFoundException("Dataset: " + datasetId)); + processRepository + .findById(id) + .orElseThrow(() -> new ResourceNotFoundException("Process: " + id)); + + dataset.addProcess(id); + // process.addDataset(dataset); + + // processRepository.save(process); + final Dataset updatedDataset = datasetRepository.save(dataset); + + return updatedDataset; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/dataset/util/UploadAreaUtil.java b/src/main/java/uk/ac/ebi/subs/ingest/dataset/util/UploadAreaUtil.java new file mode 100644 index 000000000..7735f16b7 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/dataset/util/UploadAreaUtil.java @@ -0,0 +1,86 @@ +package uk.ac.ebi.subs.ingest.dataset.util; + +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.server.ResponseStatusException; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.*; + +import lombok.extern.slf4j.Slf4j; +import uk.ac.ebi.subs.ingest.dataset.Dataset; + +@Service +@Slf4j +public class UploadAreaUtil { + public void createDataFilesUploadArea(final Dataset dataset) { + final String datasetId = dataset.getId(); + + try { + final AmazonS3 s3Client = + AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build(); + + if (!s3Client.doesBucketExistV2(datasetId)) { + s3Client.createBucket(new CreateBucketRequest(datasetId)); + dataset.setComment( + "Upload area created for this dataset with name " + + dataset.getId() + + " Please use the morphic-util tool to upload your data files to the dataset upload area"); + } else { + log.info("Bucket already exists for dataset with ID " + datasetId); + + dataset.setComment( + "Upload area available for this dataset with name " + + dataset.getId() + + " Please use the morphic-util tool to upload your data files to the dataset upload area"); + } + } catch (AmazonServiceException ase) { + throw new ResponseStatusException( + HttpStatus.INTERNAL_SERVER_ERROR, "AmazonServiceException: " + ase.getMessage()); + } catch (AmazonClientException ace) { + throw new ResponseStatusException( + HttpStatus.INTERNAL_SERVER_ERROR, "AmazonClientException: " + ace.getMessage()); + } + } + + public void deleteDataFilesAndUploadArea(final String datasetId) { + try { + final AmazonS3 s3Client = + AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build(); + + if (s3Client.doesBucketExistV2(datasetId)) { + // List and delete all objects in the bucket + final ListObjectsV2Request listObjectsV2Request = + new ListObjectsV2Request().withBucketName(datasetId); + ListObjectsV2Result result; + + do { + result = s3Client.listObjectsV2(listObjectsV2Request); + + for (final S3ObjectSummary objectSummary : result.getObjectSummaries()) { + s3Client.deleteObject(new DeleteObjectRequest(datasetId, objectSummary.getKey())); + } + + listObjectsV2Request.setContinuationToken(result.getNextContinuationToken()); + } while (result.isTruncated()); + + // Delete the bucket + s3Client.deleteBucket(datasetId); + + log.info("Bucket and all contents deleted for dataset with ID " + datasetId); + } else { + log.info("Bucket does not exist for dataset with ID " + datasetId); + } + } catch (AmazonServiceException ase) { + throw new ResponseStatusException( + HttpStatus.INTERNAL_SERVER_ERROR, "AmazonServiceException: " + ase.getMessage()); + } catch (AmazonClientException ace) { + throw new ResponseStatusException( + HttpStatus.INTERNAL_SERVER_ERROR, "AmazonClientException: " + ace.getMessage()); + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/dataset/web/DatasetController.java b/src/main/java/uk/ac/ebi/subs/ingest/dataset/web/DatasetController.java new file mode 100644 index 000000000..65698cc4f --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/dataset/web/DatasetController.java @@ -0,0 +1,171 @@ +package uk.ac.ebi.subs.ingest.dataset.web; + +import java.util.Optional; +import java.util.UUID; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.hateoas.Resource; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.dataset.Dataset; +import uk.ac.ebi.subs.ingest.dataset.DatasetService; +import uk.ac.ebi.subs.ingest.security.CheckAllowed; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.exception.NotAllowedDuringSubmissionStateException; + +/** Controller for managing Datasets. */ +@RepositoryRestController +@ExposesResourceFor(Dataset.class) +@RequiredArgsConstructor +@Getter +public class DatasetController { + private static final Logger LOGGER = LoggerFactory.getLogger(DatasetController.class); + private final @NonNull DatasetService datasetService; + + /** + * Update an existing dataset. + * + * @param dataset The dataset to update. + * @param patch The patch containing updates. + * @param assembler The resource assembler. + * @return The updated dataset as a resource. + */ + @PatchMapping("/datasets/{datasetId}") + public ResponseEntity> update( + @PathVariable("datasetId") final Dataset dataset, + @RequestBody final ObjectNode patch, + final PersistentEntityResourceAssembler assembler) { + return ResponseEntity.ok() + .body(assembler.toFullResource(datasetService.update(dataset, patch))); + } + + /** + * Delete a dataset. + * + * @param datasetId The ID of the dataset to delete. + * @return No content response. + */ + // TODO: check why not authenticated + @DeleteMapping("/datasets/{datasetId}") + public ResponseEntity delete( + @PathVariable final String datasetId, + @RequestParam(name = "deleteLinkedEntities", required = false, defaultValue = "false") + boolean deleteLinkedEntities) { + datasetService.delete(datasetId, deleteLinkedEntities); + return ResponseEntity.noContent().build(); + } + + /** + * Add a dataset to a submission envelope. + * + * @param submissionEnvelope The submission envelope. + * @param dataset The dataset to add. + * @param updatingUuid Optional UUID for updating. + * @param assembler The resource assembler. + * @return The added dataset as a resource. + */ + @CheckAllowed( + value = "#submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PostMapping(path = "/submissionEnvelopes/{sub_id}/datasets") + public ResponseEntity> addDatasetToEnvelopeAndLink( + @PathVariable("sub_id") final SubmissionEnvelope submissionEnvelope, + @RequestBody final Dataset dataset, + @RequestParam("updatingUuid") final Optional updatingUuid, + final PersistentEntityResourceAssembler assembler) { + updatingUuid.ifPresent( + uuid -> { + dataset.setUuid(new Uuid(uuid.toString())); + dataset.setIsUpdate(true); + }); + + final Dataset savedDataset = + datasetService.addDatasetToSubmissionEnvelope(submissionEnvelope, dataset); + + return ResponseEntity.accepted().body(assembler.toFullResource(savedDataset)); + } + + /** + * Link a submission envelope to a dataset. + * + * @param dataset The dataset to link. + * @param submissionEnvelope The submission envelope. + * @param assembler The resource assembler. + * @return The linked dataset as a resource. + */ + @CheckAllowed( + value = "#submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PutMapping(path = "/submissionEnvelopes/{sub_id}/datasets/{dataset_id}") + public ResponseEntity> linkSubmissionToDataset( + @PathVariable("sub_id") final SubmissionEnvelope submissionEnvelope, + @PathVariable("dataset_id") final Dataset dataset, + final PersistentEntityResourceAssembler assembler) { + final Dataset savedDataset = + datasetService.addDatasetToSubmissionEnvelope(submissionEnvelope, dataset); + + return ResponseEntity.accepted().body(assembler.toFullResource(savedDataset)); + } + + /** + * Link a biomaterial to a dataset. + * + * @param dataset The dataset. + * @param id The id of the biomaterial to link. + * @param assembler The resource assembler. + * @return The updated dataset as a resource. + */ + @PutMapping("/datasets/{dataset_id}/biomaterials/{biomaterial_id}") + public ResponseEntity> addBiomaterialToDataset( + @PathVariable("dataset_id") final Dataset dataset, + @PathVariable("biomaterial_id") final String id, + final PersistentEntityResourceAssembler assembler) { + return ResponseEntity.accepted() + .body(assembler.toFullResource(datasetService.addBiomaterialToDataset(dataset, id))); + } + + /** + * Link a file to a dataset. + * + * @param dataset The dataset. + * @param id The id of the file to link. + * @param assembler The resource assembler. + * @return The updated dataset as a resource. + */ + @PutMapping("/datasets/{dataset_id}/files/{file_id}") + public ResponseEntity> addFileToDataset( + @PathVariable("dataset_id") final Dataset dataset, + @PathVariable("file_id") final String id, + final PersistentEntityResourceAssembler assembler) { + return ResponseEntity.accepted() + .body(assembler.toFullResource(datasetService.addFileToDataset(dataset, id))); + } + + /** + * Link a process to a dataset. + * + * @param dataset The dataset. + * @param id The id of the process to link. + * @param assembler The resource assembler. + * @return The updated dataset as a resource. + */ + @PutMapping("/datasets/{dataset_id}/processes/{process_id}") + public ResponseEntity> addProcessToDataset( + @PathVariable("dataset_id") final Dataset dataset, + @PathVariable("process_id") final String id, + final PersistentEntityResourceAssembler assembler) { + return ResponseEntity.accepted() + .body(assembler.toFullResource(datasetService.addProcessToDataset(dataset, id))); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/errors/IngestError.java b/src/main/java/uk/ac/ebi/subs/ingest/errors/IngestError.java new file mode 100644 index 000000000..e33c995c4 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/errors/IngestError.java @@ -0,0 +1,26 @@ +package uk.ac.ebi.subs.ingest.errors; + +import java.net.URI; + +import org.zalando.problem.Problem; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@JsonIgnoreProperties({"parameters", "status"}) +public class IngestError implements Problem { + private URI type; + private String title; + private String detail; + private URI instance; + + IngestError(Problem problem) { + this.type = problem.getType(); + this.title = problem.getTitle(); + this.detail = problem.getDetail(); + } +} diff --git a/src/main/java/org/humancellatlas/ingest/errors/SubmissionError.java b/src/main/java/uk/ac/ebi/subs/ingest/errors/SubmissionError.java similarity index 56% rename from src/main/java/org/humancellatlas/ingest/errors/SubmissionError.java rename to src/main/java/uk/ac/ebi/subs/ingest/errors/SubmissionError.java index de6d28f15..472965e29 100644 --- a/src/main/java/org/humancellatlas/ingest/errors/SubmissionError.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/errors/SubmissionError.java @@ -1,18 +1,20 @@ -package org.humancellatlas.ingest.errors; +package uk.ac.ebi.subs.ingest.errors; + +import java.util.UUID; + +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.DBRef; +import org.springframework.hateoas.Identifiable; +import org.zalando.problem.Problem; import com.fasterxml.jackson.annotation.JsonIgnore; + import lombok.AccessLevel; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.Setter; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.DBRef; -import org.springframework.hateoas.Identifiable; -import org.zalando.problem.Problem; - -import java.util.UUID; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; @Data @NoArgsConstructor @@ -20,12 +22,15 @@ @EqualsAndHashCode(callSuper = true) public class SubmissionError extends IngestError implements Identifiable { - @JsonIgnore @DBRef(lazy = true) private SubmissionEnvelope submissionEnvelope; - @Id private String id; + @JsonIgnore + @DBRef(lazy = true) + private SubmissionEnvelope submissionEnvelope; + + @Id private String id; - SubmissionError(SubmissionEnvelope submissionEnvelope, Problem submissionProblem) { - super(submissionProblem); - this.id = UUID.randomUUID().toString(); - this.submissionEnvelope = submissionEnvelope; - } + SubmissionError(SubmissionEnvelope submissionEnvelope, Problem submissionProblem) { + super(submissionProblem); + this.id = UUID.randomUUID().toString(); + this.submissionEnvelope = submissionEnvelope; + } } diff --git a/src/main/java/org/humancellatlas/ingest/errors/SubmissionErrorRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/errors/SubmissionErrorRepository.java similarity index 50% rename from src/main/java/org/humancellatlas/ingest/errors/SubmissionErrorRepository.java rename to src/main/java/uk/ac/ebi/subs/ingest/errors/SubmissionErrorRepository.java index d3ae8997d..a0d184e86 100644 --- a/src/main/java/org/humancellatlas/ingest/errors/SubmissionErrorRepository.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/errors/SubmissionErrorRepository.java @@ -1,17 +1,18 @@ -package org.humancellatlas.ingest.errors; +package uk.ac.ebi.subs.ingest.errors; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.mongodb.repository.MongoRepository; import org.springframework.data.rest.core.annotation.RestResource; import org.springframework.web.bind.annotation.CrossOrigin; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + @CrossOrigin public interface SubmissionErrorRepository extends MongoRepository { - Page findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope, - Pageable pageable); + Page findBySubmissionEnvelope( + SubmissionEnvelope submissionEnvelope, Pageable pageable); - @RestResource(exported = false) - Long deleteBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + @RestResource(exported = false) + Long deleteBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); } diff --git a/src/main/java/uk/ac/ebi/subs/ingest/errors/SubmissionErrorService.java b/src/main/java/uk/ac/ebi/subs/ingest/errors/SubmissionErrorService.java new file mode 100644 index 000000000..ba52a9336 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/errors/SubmissionErrorService.java @@ -0,0 +1,30 @@ +package uk.ac.ebi.subs.ingest.errors; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; +import org.zalando.problem.Problem; + +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Service +public class SubmissionErrorService { + @Autowired private SubmissionErrorRepository submissionErrorRepository; + + public Page getErrorsFromEnvelope( + SubmissionEnvelope submissionEnvelope, Pageable pageable) { + return submissionErrorRepository.findBySubmissionEnvelope(submissionEnvelope, pageable); + } + + public SubmissionError addErrorToEnvelope( + SubmissionEnvelope submissionEnvelope, Problem submissionProblem) { + SubmissionError submissionError = new SubmissionError(submissionEnvelope, submissionProblem); + submissionErrorRepository.insert(submissionError); + return submissionError; + } + + public void deleteSubmissionEnvelopeErrors(SubmissionEnvelope submissionEnvelope) { + submissionErrorRepository.deleteBySubmissionEnvelope(submissionEnvelope); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/errors/web/SubmissionErrorController.java b/src/main/java/uk/ac/ebi/subs/ingest/errors/web/SubmissionErrorController.java new file mode 100644 index 000000000..a77b80447 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/errors/web/SubmissionErrorController.java @@ -0,0 +1,65 @@ +package uk.ac.ebi.subs.ingest.errors.web; + +import java.net.URI; + +import org.springframework.data.domain.Pageable; +import org.springframework.data.rest.webmvc.PersistentEntityResource; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.data.web.PagedResourcesAssembler; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.hateoas.PagedResources; +import org.springframework.hateoas.Resource; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.errors.IngestError; +import uk.ac.ebi.subs.ingest.errors.SubmissionError; +import uk.ac.ebi.subs.ingest.errors.SubmissionErrorService; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@RepositoryRestController +@ExposesResourceFor(SubmissionError.class) +@RequiredArgsConstructor +public class SubmissionErrorController { + private final @NonNull SubmissionErrorService submissionErrorService; + private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; + + @DeleteMapping(path = "submissionEnvelopes/{sub_id}" + Links.SUBMISSION_ERRORS_URL) + public ResponseEntity deleteSubmissionEnvelopeErrors( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope) { + submissionErrorService.deleteSubmissionEnvelopeErrors(submissionEnvelope); + return ResponseEntity.noContent().build(); + } + + @GetMapping(path = "submissionEnvelopes/{sub_id}" + Links.SUBMISSION_ERRORS_URL) + public ResponseEntity>> getSubmissionEnvelopeErrors( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + return ResponseEntity.ok( + pagedResourcesAssembler.toResource( + submissionErrorService.getErrorsFromEnvelope(submissionEnvelope, pageable), + resourceAssembler)); + } + + @PostMapping(path = "submissionEnvelopes/{sub_id}" + Links.SUBMISSION_ERRORS_URL) + public ResponseEntity addErrorToEnvelope( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + @RequestBody IngestError ingestError, + final PersistentEntityResourceAssembler resourceAssembler) { + SubmissionError submissionError = + submissionErrorService.addErrorToEnvelope(submissionEnvelope, ingestError); + PersistentEntityResource submissionErrorResource = + resourceAssembler.toFullResource(submissionError); + return ResponseEntity.created(URI.create(submissionErrorResource.getId().getHref())) + .body(submissionErrorResource); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/errors/web/SubmissionErrorResourceProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/errors/web/SubmissionErrorResourceProcessor.java new file mode 100644 index 000000000..8ea6cc7bd --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/errors/web/SubmissionErrorResourceProcessor.java @@ -0,0 +1,31 @@ +package uk.ac.ebi.subs.ingest.errors.web; + +import java.net.URI; + +import org.springframework.hateoas.EntityLinks; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.stereotype.Component; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.errors.SubmissionError; + +@Component +@RequiredArgsConstructor +public class SubmissionErrorResourceProcessor + implements ResourceProcessor> { + private final @NonNull EntityLinks entityLinks; + + @Override + public Resource process(Resource resource) { + resource.getContent().setInstance(URI.create(resource.getId().getHref())); + resource.add( + entityLinks + .linkForSingleResource(resource.getContent().getSubmissionEnvelope()) + .slash(Links.SUBMISSION_ERRORS_URL) + .withRel(Links.SUBMISSION_ERRORS_REL)); + return resource; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/export/ExportError.java b/src/main/java/uk/ac/ebi/subs/ingest/export/ExportError.java new file mode 100644 index 000000000..21446b104 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/ExportError.java @@ -0,0 +1,11 @@ +package uk.ac.ebi.subs.ingest.export; + +import lombok.Data; +import lombok.NonNull; + +@Data +public class ExportError { + private final String errorCode; + @NonNull private final String message; + private final Object details; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/export/ExportState.java b/src/main/java/uk/ac/ebi/subs/ingest/export/ExportState.java new file mode 100644 index 000000000..f86275bee --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/ExportState.java @@ -0,0 +1,8 @@ +package uk.ac.ebi.subs.ingest.export; + +public enum ExportState { + EXPORTING, + FAILED, + EXPORTED, + DEPRECATED +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/export/destination/ExportDestination.java b/src/main/java/uk/ac/ebi/subs/ingest/export/destination/ExportDestination.java new file mode 100644 index 000000000..8453ceb82 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/destination/ExportDestination.java @@ -0,0 +1,12 @@ +package uk.ac.ebi.subs.ingest.export.destination; + +import java.util.Map; + +import lombok.Data; + +@Data +public class ExportDestination { + private final ExportDestinationName name; + private final String version; + private final Map context; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/export/destination/ExportDestinationName.java b/src/main/java/uk/ac/ebi/subs/ingest/export/destination/ExportDestinationName.java new file mode 100644 index 000000000..7aa1eb86a --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/destination/ExportDestinationName.java @@ -0,0 +1,9 @@ +package uk.ac.ebi.subs.ingest.export.destination; + +public enum ExportDestinationName { + DCP, + DSP, + ENA, + BIO_SAMPLES, + BIO_STUDIES +} diff --git a/src/main/java/org/humancellatlas/ingest/export/entity/ExportEntity.java b/src/main/java/uk/ac/ebi/subs/ingest/export/entity/ExportEntity.java similarity index 53% rename from src/main/java/org/humancellatlas/ingest/export/entity/ExportEntity.java rename to src/main/java/uk/ac/ebi/subs/ingest/export/entity/ExportEntity.java index 08862f949..14874a268 100644 --- a/src/main/java/org/humancellatlas/ingest/export/entity/ExportEntity.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/entity/ExportEntity.java @@ -1,11 +1,10 @@ -package org.humancellatlas.ingest.export.entity; +package uk.ac.ebi.subs.ingest.export.entity; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Builder; -import lombok.Data; -import org.humancellatlas.ingest.export.ExportError; -import org.humancellatlas.ingest.export.ExportState; -import org.humancellatlas.ingest.export.job.ExportJob; import org.springframework.data.annotation.CreatedDate; import org.springframework.data.annotation.Id; import org.springframework.data.mongodb.core.index.Indexed; @@ -14,34 +13,31 @@ import org.springframework.data.rest.core.annotation.RestResource; import org.springframework.hateoas.Identifiable; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import com.fasterxml.jackson.annotation.JsonIgnore; + +import lombok.Builder; +import lombok.Data; +import uk.ac.ebi.subs.ingest.export.ExportError; +import uk.ac.ebi.subs.ingest.export.ExportState; +import uk.ac.ebi.subs.ingest.export.job.ExportJob; @Data @Builder @Document public class ExportEntity implements Identifiable { - @Id - @JsonIgnore - private String id; - - @Indexed - @DBRef(lazy = true) - @RestResource(exported = false) - @JsonIgnore - private ExportJob exportJob; + @Id @JsonIgnore private String id; - @Indexed - private ExportState status; + @Indexed + @DBRef(lazy = true) + @RestResource(exported = false) + @JsonIgnore + private ExportJob exportJob; - @CreatedDate - private Instant createdDate; + @Indexed private ExportState status; - private Map context; + @CreatedDate private Instant createdDate; - @Builder.Default - private List errors = new ArrayList<>(); + private Map context; + @Builder.Default private List errors = new ArrayList<>(); } diff --git a/src/main/java/org/humancellatlas/ingest/export/entity/ExportEntityRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/export/entity/ExportEntityRepository.java similarity index 54% rename from src/main/java/org/humancellatlas/ingest/export/entity/ExportEntityRepository.java rename to src/main/java/uk/ac/ebi/subs/ingest/export/entity/ExportEntityRepository.java index ec16ec153..4c06740f0 100644 --- a/src/main/java/org/humancellatlas/ingest/export/entity/ExportEntityRepository.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/entity/ExportEntityRepository.java @@ -1,18 +1,20 @@ -package org.humancellatlas.ingest.export.entity; +package uk.ac.ebi.subs.ingest.export.entity; -import org.humancellatlas.ingest.export.ExportState; -import org.humancellatlas.ingest.export.job.ExportJob; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.mongodb.repository.MongoRepository; import org.springframework.data.rest.core.annotation.RestResource; import org.springframework.web.bind.annotation.CrossOrigin; +import uk.ac.ebi.subs.ingest.export.ExportState; +import uk.ac.ebi.subs.ingest.export.job.ExportJob; + @CrossOrigin @RestResource(exported = false) public interface ExportEntityRepository extends MongoRepository { - Page findByExportJob(ExportJob exportJob, Pageable pageable); + Page findByExportJob(ExportJob exportJob, Pageable pageable); - Page findByExportJobAndStatus(ExportJob exportJob, ExportState exportState, Pageable pageable); + Page findByExportJobAndStatus( + ExportJob exportJob, ExportState exportState, Pageable pageable); } diff --git a/src/main/java/uk/ac/ebi/subs/ingest/export/entity/ExportEntityService.java b/src/main/java/uk/ac/ebi/subs/ingest/export/entity/ExportEntityService.java new file mode 100644 index 000000000..a2340729b --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/entity/ExportEntityService.java @@ -0,0 +1,25 @@ +package uk.ac.ebi.subs.ingest.export.entity; + +import org.springframework.stereotype.Component; + +import lombok.AllArgsConstructor; +import uk.ac.ebi.subs.ingest.export.entity.web.ExportEntityRequest; +import uk.ac.ebi.subs.ingest.export.job.ExportJob; + +@Component +@AllArgsConstructor +public class ExportEntityService { + private final ExportEntityRepository exportEntityRepository; + + public ExportEntity createExportEntity( + ExportJob exportJob, ExportEntityRequest exportEntityRequest) { + ExportEntity newExportEntity = + ExportEntity.builder() + .exportJob(exportJob) + .status(exportEntityRequest.getStatus()) + .context(exportEntityRequest.getContext()) + .errors(exportEntityRequest.getErrors()) + .build(); + return exportEntityRepository.insert(newExportEntity); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/export/entity/web/ExportEntityController.java b/src/main/java/uk/ac/ebi/subs/ingest/export/entity/web/ExportEntityController.java new file mode 100644 index 000000000..65e0fd881 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/entity/web/ExportEntityController.java @@ -0,0 +1,82 @@ +package uk.ac.ebi.subs.ingest.export.entity.web; + +import java.net.URI; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.rest.webmvc.PersistentEntityResource; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.data.web.PagedResourcesAssembler; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; + +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.export.ExportState; +import uk.ac.ebi.subs.ingest.export.entity.ExportEntity; +import uk.ac.ebi.subs.ingest.export.entity.ExportEntityRepository; +import uk.ac.ebi.subs.ingest.export.entity.ExportEntityService; +import uk.ac.ebi.subs.ingest.export.job.ExportJob; + +@RepositoryRestController +@RequiredArgsConstructor +@ExposesResourceFor(ExportEntity.class) +public class ExportEntityController { + private final ExportEntityService exportEntityService; + private final ExportEntityRepository exportEntityRepository; + private final PagedResourcesAssembler pagedAssembler; + + @GetMapping(path = Links.EXPORT_JOBS_URL + "/{id}" + Links.EXPORT_JOB_ENTITIES_URL) + public ResponseEntity getExportJobEntities( + @PathVariable("id") ExportJob exportJob, + @RequestParam(name = "status", required = false) ExportState exportState, + Pageable pageable, + PersistentEntityResourceAssembler assembler) { + if (exportJob == null) { + return ResponseEntity.notFound().build(); + } + Page entityPage; + if (exportState == null) { + entityPage = exportEntityRepository.findByExportJob(exportJob, pageable); + } else { + entityPage = + exportEntityRepository.findByExportJobAndStatus(exportJob, exportState, pageable); + } + return ResponseEntity.ok(pagedAssembler.toResource(entityPage, assembler)); + } + + @GetMapping( + path = Links.EXPORT_JOBS_URL + "/{job_id}" + Links.EXPORT_JOB_ENTITIES_URL + "/{entity_id}") + ResponseEntity getExportJobEntity( + @PathVariable("job_id") ExportJob exportJob, + @PathVariable("entity_id") ExportEntity exportEntity, + PersistentEntityResourceAssembler assembler) { + if (exportJob == null || exportEntity == null) { + return ResponseEntity.notFound().build(); + } + PersistentEntityResource newExportEntityResource = assembler.toFullResource(exportEntity); + return ResponseEntity.ok(newExportEntityResource); + } + + @PostMapping(path = Links.EXPORT_JOBS_URL + "/{id}" + Links.EXPORT_JOB_ENTITIES_URL) + ResponseEntity createExportEntity( + @PathVariable("id") ExportJob exportJob, + @RequestBody ExportEntityRequest exportEntityRequest, + PersistentEntityResourceAssembler resourceAssembler) { + if (exportJob == null) { + return ResponseEntity.notFound().build(); + } + ExportEntity newExportEntity = + exportEntityService.createExportEntity(exportJob, exportEntityRequest); + PersistentEntityResource newExportEntityResource = + resourceAssembler.toFullResource(newExportEntity); + return ResponseEntity.created(URI.create(newExportEntityResource.getId().getHref())) + .body(newExportEntityResource); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/export/entity/web/ExportEntityRequest.java b/src/main/java/uk/ac/ebi/subs/ingest/export/entity/web/ExportEntityRequest.java new file mode 100644 index 000000000..1725ee1c8 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/entity/web/ExportEntityRequest.java @@ -0,0 +1,18 @@ +package uk.ac.ebi.subs.ingest.export.entity.web; + +import java.util.List; +import java.util.Map; + +import lombok.Data; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.export.ExportError; +import uk.ac.ebi.subs.ingest.export.ExportState; + +@Data +public class ExportEntityRequest { + @NonNull ExportState status; + + @NonNull Map context; + + @NonNull List errors; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/export/entity/web/ExportEntityResourceProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/export/entity/web/ExportEntityResourceProcessor.java new file mode 100644 index 000000000..9a95da95e --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/entity/web/ExportEntityResourceProcessor.java @@ -0,0 +1,41 @@ +package uk.ac.ebi.subs.ingest.export.entity.web; + +import org.springframework.hateoas.EntityLinks; +import org.springframework.hateoas.Link; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.stereotype.Component; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.export.entity.ExportEntity; +import uk.ac.ebi.subs.ingest.export.job.ExportJob; + +@Component +@RequiredArgsConstructor +public class ExportEntityResourceProcessor implements ResourceProcessor> { + private final @NonNull EntityLinks entityLinks; + + private Link getSelfLink(ExportEntity exportEntity) { + return entityLinks + .linkForSingleResource(exportEntity.getExportJob()) + .slash(Links.EXPORT_JOB_ENTITIES_URL) + .slash(exportEntity.getId()) + .withSelfRel(); + } + + private Link getExportJobLink(ExportJob exportJob) { + return entityLinks.linkForSingleResource(exportJob).withRel("exportJob"); + } + + @Override + public Resource process(Resource resource) { + ExportEntity exportEntity = resource.getContent(); + resource.removeLinks(); + resource.add(getSelfLink(exportEntity)); + resource.add(getSelfLink(exportEntity).withRel("exportEntity")); + resource.add(getExportJobLink(exportEntity.getExportJob())); + return resource; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/export/job/ExportJob.java b/src/main/java/uk/ac/ebi/subs/ingest/export/job/ExportJob.java new file mode 100644 index 000000000..4588c6a78 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/job/ExportJob.java @@ -0,0 +1,81 @@ +package uk.ac.ebi.subs.ingest.export.job; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.springframework.data.annotation.CreatedDate; +import org.springframework.data.annotation.Id; +import org.springframework.data.annotation.LastModifiedDate; +import org.springframework.data.mongodb.core.index.CompoundIndex; +import org.springframework.data.mongodb.core.index.CompoundIndexes; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.DBRef; +import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.data.rest.core.annotation.RestResource; +import org.springframework.hateoas.Identifiable; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import lombok.Builder; +import lombok.Data; +import uk.ac.ebi.subs.ingest.core.web.LinkGenerator; +import uk.ac.ebi.subs.ingest.export.ExportError; +import uk.ac.ebi.subs.ingest.export.ExportState; +import uk.ac.ebi.subs.ingest.export.destination.ExportDestination; +import uk.ac.ebi.subs.ingest.messaging.model.ExportSubmissionMessage; +import uk.ac.ebi.subs.ingest.messaging.model.SpreadsheetGenerationMessage; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Data +@Builder +@Document +@CompoundIndexes({ + @CompoundIndex(name = "exportDestinationName", def = "{ 'destination.name': 1 }"), + @CompoundIndex(name = "exportDestinationVersion", def = "{ 'destination.version': 1 }") +}) +public class ExportJob implements Identifiable { + @Id @JsonIgnore private String id; + + @CreatedDate @Builder.Default private Instant createdDate = Instant.now(); + + @Indexed + @DBRef(lazy = true) + @RestResource(exported = false) + @JsonIgnore + private final SubmissionEnvelope submission; + + private final ExportDestination destination; + + @Indexed @Builder.Default private ExportState status = ExportState.EXPORTING; + + @LastModifiedDate private Instant updatedDate; + + private Map context; + + @Builder.Default private List errors = new ArrayList<>(); + + public ExportSubmissionMessage toExportSubmissionMessage( + LinkGenerator linkGenerator, Map context) { + String callbackLink = linkGenerator.createCallback(getClass(), getId()); + return new ExportSubmissionMessage( + getId(), + submission.getUuid().getUuid().toString(), + destination.getContext().get("projectUuid").toString(), + callbackLink, + context); + } + + public SpreadsheetGenerationMessage toGenerateSubmissionMessage( + LinkGenerator linkGenerator, Map context) { + // TODO: unify with toExportSubmissionMessage + String callbackLink = linkGenerator.createCallback(getClass(), getId()); + return new SpreadsheetGenerationMessage( + getId(), + submission.getUuid().getUuid().toString(), + destination.getContext().get("projectUuid").toString(), + callbackLink, + context); + } +} diff --git a/src/main/java/org/humancellatlas/ingest/export/job/ExportJobRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/export/job/ExportJobRepository.java similarity index 60% rename from src/main/java/org/humancellatlas/ingest/export/job/ExportJobRepository.java rename to src/main/java/uk/ac/ebi/subs/ingest/export/job/ExportJobRepository.java index dacdc19cd..a040e69aa 100644 --- a/src/main/java/org/humancellatlas/ingest/export/job/ExportJobRepository.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/job/ExportJobRepository.java @@ -1,12 +1,13 @@ -package org.humancellatlas.ingest.export.job; +package uk.ac.ebi.subs.ingest.export.job; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.mongodb.repository.MongoRepository; import org.springframework.web.bind.annotation.CrossOrigin; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + @CrossOrigin public interface ExportJobRepository extends MongoRepository { - Page findBySubmission(SubmissionEnvelope submissionEnvelope, Pageable pageable); + Page findBySubmission(SubmissionEnvelope submissionEnvelope, Pageable pageable); } diff --git a/src/main/java/uk/ac/ebi/subs/ingest/export/job/ExportJobService.java b/src/main/java/uk/ac/ebi/subs/ingest/export/job/ExportJobService.java new file mode 100644 index 000000000..b91c05d0f --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/job/ExportJobService.java @@ -0,0 +1,94 @@ +package uk.ac.ebi.subs.ingest.export.job; + +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.domain.Example; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Component; + +import lombok.AllArgsConstructor; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.export.ExportState; +import uk.ac.ebi.subs.ingest.export.destination.ExportDestination; +import uk.ac.ebi.subs.ingest.export.destination.ExportDestinationName; +import uk.ac.ebi.subs.ingest.export.job.web.ExportJobRequest; +import uk.ac.ebi.subs.ingest.exporter.Exporter; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@Component +@AllArgsConstructor +public class ExportJobService { + private final ExportJobRepository exportJobRepository; + private final SubmissionEnvelopeRepository submissionEnvelopeRepository; + private final Exporter exporter; + private final @NonNull ExecutorService executorService = Executors.newFixedThreadPool(5); + private final @NonNull Logger log = LoggerFactory.getLogger(getClass()); + + public ExportJob createExportJob( + SubmissionEnvelope submissionEnvelope, ExportJobRequest exportJobRequest) { + ExportJob newExportJob = + ExportJob.builder() + .submission(submissionEnvelope) + .destination(exportJobRequest.getDestination()) + .context(exportJobRequest.getContext()) + .build(); + return exportJobRepository.insert(newExportJob); + } + + public Page find( + UUID submissionUuid, + ExportState exportState, + ExportDestinationName destinationName, + String version, + Pageable pageable) { + SubmissionEnvelope submissionEnvelope = + submissionEnvelopeRepository.findByUuidUuid(submissionUuid); + ExportJob exportJobProbe = + ExportJob.builder() + .submission(submissionEnvelope) + .status(exportState) + .destination(new ExportDestination(destinationName, version, null)) + .build(); + return this.exportJobRepository.findAll(Example.of(exportJobProbe), pageable); + } + + public ExportJob updateContext(ExportJob exportJob, Map context) { + exportJob.getContext().putAll(context); + var savedJob = exportJobRepository.save(exportJob); + if (context.getOrDefault("dataFileTransfer", "").equals("COMPLETE")) { + submit(exporter::generateSpreadsheet, exportJob, "spreadsheetGeneration"); + } else if (context.getOrDefault("spreadsheetGeneration", "").equals("COMPLETE")) { + submit(exporter::exportMetadata, exportJob, "exportMetadata"); + } + return savedJob; + } + + private void submit(Consumer exportAction, ExportJob exportJob, String actionName) { + String submissionUuid = exportJob.getSubmission().getUuid().getUuid().toString(); + executorService.submit( + () -> { + try { + log.info( + "submitting export action {} for export job {} for submission {}", + actionName, + exportJob.getId(), + submissionUuid); + exportAction.accept(exportJob); + } catch (Exception e) { + log.error( + String.format( + "Uncaught Exception sending message %s for Export Job %s for submission %s", + actionName, exportJob.getId(), submissionUuid), + e); + } + }); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/export/job/web/ExportJobController.java b/src/main/java/uk/ac/ebi/subs/ingest/export/job/web/ExportJobController.java new file mode 100644 index 000000000..3c5941a99 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/job/web/ExportJobController.java @@ -0,0 +1,86 @@ +package uk.ac.ebi.subs.ingest.export.job.web; + +import java.net.URI; +import java.util.Map; +import java.util.UUID; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.rest.webmvc.PersistentEntityResource; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.data.web.PagedResourcesAssembler; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.hateoas.PagedResources; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.export.ExportState; +import uk.ac.ebi.subs.ingest.export.destination.ExportDestinationName; +import uk.ac.ebi.subs.ingest.export.job.ExportJob; +import uk.ac.ebi.subs.ingest.export.job.ExportJobRepository; +import uk.ac.ebi.subs.ingest.export.job.ExportJobService; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@RepositoryRestController +@RequiredArgsConstructor +@ExposesResourceFor(ExportJob.class) +public class ExportJobController { + private final ExportJobService exportJobService; + private final ExportJobRepository exportJobRepository; + private final PagedResourcesAssembler pagedResourcesAssembler; + + @GetMapping(path = "/submissionEnvelopes/{id}" + Links.EXPORT_JOBS_URL) + ResponseEntity getExportJobsForSubmission( + @PathVariable("id") SubmissionEnvelope submission, + Pageable pageable, + PersistentEntityResourceAssembler resourceAssembler) { + if (submission == null) { + return ResponseEntity.notFound().build(); + } + return ResponseEntity.ok( + pagedResourcesAssembler.toResource( + exportJobRepository.findBySubmission(submission, pageable), resourceAssembler)); + } + + @PostMapping(path = "/submissionEnvelopes/{id}" + Links.EXPORT_JOBS_URL) + ResponseEntity createExportJob( + @PathVariable("id") SubmissionEnvelope submission, + @RequestBody ExportJobRequest exportJobRequest, + PersistentEntityResourceAssembler resourceAssembler) { + if (submission == null) { + return ResponseEntity.notFound().build(); + } + ExportJob newExportJob = exportJobService.createExportJob(submission, exportJobRequest); + PersistentEntityResource newExportJobResource = resourceAssembler.toFullResource(newExportJob); + return ResponseEntity.created(URI.create(newExportJobResource.getId().getHref())) + .body(newExportJobResource); + } + + @GetMapping(path = Links.EXPORT_JOBS_URL + "/search" + Links.EXPORT_JOB_FIND_URL) + ResponseEntity> findExportJobs( + @RequestParam("submissionUuid") UUID submissionUuid, + @RequestParam("status") ExportState exportState, + @RequestParam("destination") ExportDestinationName exportDestinationName, + @RequestParam("version") String destinationVersion, + Pageable pageable, + PersistentEntityResourceAssembler resourceAssembler) { + String version = destinationVersion.isEmpty() ? null : destinationVersion; + Page searchResults = + exportJobService.find( + submissionUuid, exportState, exportDestinationName, version, pageable); + return ResponseEntity.ok(pagedResourcesAssembler.toResource(searchResults, resourceAssembler)); + } + + @PatchMapping(Links.EXPORT_JOBS_URL + "/{id}/context") + ResponseEntity patchExportJobContext( + @PathVariable("id") ExportJob exportJob, + @RequestBody Map context, + PersistentEntityResourceAssembler assembler) { + ExportJob updatedExportJob = exportJobService.updateContext(exportJob, context); + PersistentEntityResource resource = assembler.toFullResource(updatedExportJob); + return ResponseEntity.accepted().body(resource); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/export/job/web/ExportJobRequest.java b/src/main/java/uk/ac/ebi/subs/ingest/export/job/web/ExportJobRequest.java new file mode 100644 index 000000000..11aeb928e --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/job/web/ExportJobRequest.java @@ -0,0 +1,14 @@ +package uk.ac.ebi.subs.ingest.export.job.web; + +import java.util.Map; + +import lombok.Data; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.export.destination.ExportDestination; + +@Data +public class ExportJobRequest { + @NonNull ExportDestination destination; + + @NonNull Map context; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/export/job/web/ExportJobResourceProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/export/job/web/ExportJobResourceProcessor.java new file mode 100644 index 000000000..9ef49ae1a --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/job/web/ExportJobResourceProcessor.java @@ -0,0 +1,46 @@ +package uk.ac.ebi.subs.ingest.export.job.web; + +import org.springframework.hateoas.EntityLinks; +import org.springframework.hateoas.Link; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.stereotype.Component; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.export.job.ExportJob; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Component +@RequiredArgsConstructor +public class ExportJobResourceProcessor implements ResourceProcessor> { + private final @NonNull EntityLinks entityLinks; + + private Link getEntitiesLink(ExportJob exportJob) { + return entityLinks + .linkForSingleResource(exportJob) + .slash(Links.EXPORT_JOB_ENTITIES_URL) + .withRel(Links.EXPORT_JOB_ENTITIES_REL); + } + + private Link getEntitiesStatusLink(ExportJob exportJob) { + return entityLinks + .linkForSingleResource(exportJob) + .slash(Links.EXPORT_JOB_ENTITIES_URL + "?status={status}") + .withRel(Links.EXPORT_JOB_ENTITIES_BY_STATUS_REL); + } + + private Link getSubmissionLink(SubmissionEnvelope submission) { + return entityLinks.linkForSingleResource(submission).withRel("submission"); + } + + @Override + public Resource process(Resource resource) { + ExportJob exportJob = resource.getContent(); + resource.add(getEntitiesLink(exportJob)); + resource.add(getEntitiesStatusLink(exportJob)); + resource.add(getSubmissionLink(exportJob.getSubmission())); + return resource; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/export/job/web/ExportJobSearchProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/export/job/web/ExportJobSearchProcessor.java new file mode 100644 index 000000000..0fb0c1ea5 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/export/job/web/ExportJobSearchProcessor.java @@ -0,0 +1,28 @@ +package uk.ac.ebi.subs.ingest.export.job.web; + +import static org.springframework.hateoas.mvc.ControllerLinkBuilder.linkTo; +import static org.springframework.hateoas.mvc.ControllerLinkBuilder.methodOn; + +import org.springframework.data.rest.webmvc.RepositorySearchesResource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.stereotype.Component; + +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.export.job.ExportJob; + +@Component +public class ExportJobSearchProcessor implements ResourceProcessor { + + @Override + public RepositorySearchesResource process(RepositorySearchesResource searchesResource) { + if (searchesResource.getDomainType().equals(ExportJob.class)) { + searchesResource.add( + linkTo( + methodOn(ExportJobController.class) + .findExportJobs(null, null, null, null, null, null)) + .withRel(Links.EXPORT_JOB_FIND_REL)); + } + + return searchesResource; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/exporter/DefaultExporter.java b/src/main/java/uk/ac/ebi/subs/ingest/exporter/DefaultExporter.java new file mode 100644 index 000000000..542230c99 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/exporter/DefaultExporter.java @@ -0,0 +1,148 @@ +package uk.ac.ebi.subs.ingest.exporter; + +import static uk.ac.ebi.subs.ingest.export.destination.ExportDestinationName.DCP; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.commons.collections4.ListUtils; +import org.json.simple.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import uk.ac.ebi.subs.ingest.export.destination.ExportDestination; +import uk.ac.ebi.subs.ingest.export.job.ExportJob; +import uk.ac.ebi.subs.ingest.export.job.ExportJobRepository; +import uk.ac.ebi.subs.ingest.export.job.web.ExportJobRequest; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.process.ProcessService; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Component +public class DefaultExporter implements Exporter { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + @Autowired private ProcessService processService; + + @Autowired private ProcessRepository processRepository; + + @Autowired private ExportJobRepository exportJobRepository; + + @Autowired private ProjectRepository projectRepository; + + @Autowired private MessageRouter messageRouter; + + /** + * Divides a set of process IDs into lists of size partitionSize + * + * @param processIds + * @param partitionSize + * @return A collection of partitionSize sized lists of processes + */ + private static List> partitionProcessIds( + Collection processIds, int partitionSize) { + return ListUtils.partition(new ArrayList<>(processIds), partitionSize); + } + + @Override + public void exportManifests(SubmissionEnvelope envelope) { + Collection assayingProcessIds = processService.findAssays(envelope); + + log.info( + String.format( + "Found %s assays processes for envelope with ID %s", + assayingProcessIds.size(), envelope.getId())); + + int totalCount = assayingProcessIds.size(); + ExperimentProcess.IndexCounter counter = new ExperimentProcess.IndexCounter(totalCount); + + int partitionSize = 500; + partitionProcessIds(assayingProcessIds, partitionSize).stream() + .flatMap(processIdBatch -> processService.getProcesses(processIdBatch)) + .map(process -> ExperimentProcess.from(process, counter)) + .forEach(messageRouter::sendManifestForExport); + } + + @Override + public void exportData(SubmissionEnvelope envelope) { + Project project = + projectRepository.findBySubmissionEnvelopesContains(envelope).findFirst().orElseThrow(); + var destinationContext = new JSONObject(); + destinationContext.put("projectUuid", project.getUuid().getUuid().toString()); + + var exportJobContext = new JSONObject(); + exportJobContext.put("dataFileTransfer", false); + ExportJob exportJob = createDcpExportJob(envelope, destinationContext, exportJobContext); + + var messageContext = new JSONObject(); + messageRouter.sendSubmissionForDataExport(exportJob, messageContext); + } + + @Override + public void generateSpreadsheet(SubmissionEnvelope submissionEnvelope) { + Project project = + projectRepository + .findBySubmissionEnvelopesContains(submissionEnvelope) + .findFirst() + .orElseThrow(); + var destinationContext = new JSONObject(); + destinationContext.put("projectUuid", project.getUuid().getUuid().toString()); + + var exportJob = createDcpExportJob(submissionEnvelope, destinationContext, new JSONObject()); + generateSpreadsheet(exportJob); + } + + @Override + public void exportMetadata(ExportJob exportJob) { + var submission = exportJob.getSubmission(); + Collection assayingProcessIds = processService.findAssays(submission); + exportJob.getContext().put("totalAssayCount", assayingProcessIds.size()); + exportJobRepository.save(exportJob); + updateDcpVersionAndSendMessageForEachProcess(assayingProcessIds, exportJob); + } + + private ExportJob createDcpExportJob( + SubmissionEnvelope submissionEnvelope, + JSONObject destinationContext, + JSONObject exportJobContext) { + ExportDestination exportDestination = new ExportDestination(DCP, "v2", destinationContext); + ExportJobRequest exportJobRequest = new ExportJobRequest(exportDestination, exportJobContext); + ExportJob newExportJob = + ExportJob.builder() + .submission(submissionEnvelope) + .destination(exportJobRequest.getDestination()) + .context(exportJobRequest.getContext()) + .build(); + return exportJobRepository.insert(newExportJob); + } + + private void updateDcpVersionAndSendMessageForEachProcess( + Collection assayingProcessIds, ExportJob exportJob) { + int totalCount = assayingProcessIds.size(); + ExperimentProcess.IndexCounter counter = new ExperimentProcess.IndexCounter(totalCount); + + int partitionSize = 500; + partitionProcessIds(assayingProcessIds, partitionSize).stream() + .flatMap(processIdBatch -> processService.getProcesses(processIdBatch)) + .map(process -> (Process) process.setDcpVersion(exportJob.getCreatedDate())) + .map(process -> processRepository.save(process)) + .map(process -> ExperimentProcess.from(process, counter)) + .forEach(exportData -> messageRouter.sendExperimentForExport(exportData, exportJob, null)); + } + + @Override + public void generateSpreadsheet(ExportJob exportJob) { + exportJob.getContext().put("spreadsheetGeneration", false); + exportJobRepository.save(exportJob); + var messageContext = new JSONObject(); + messageRouter.sendGenerateSpreadsheet(exportJob, messageContext); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/exporter/ExperimentProcess.java b/src/main/java/uk/ac/ebi/subs/ingest/exporter/ExperimentProcess.java new file mode 100644 index 000000000..fd5273bff --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/exporter/ExperimentProcess.java @@ -0,0 +1,106 @@ +package uk.ac.ebi.subs.ingest.exporter; + +import java.time.Instant; +import java.util.Map; +import java.util.UUID; + +import uk.ac.ebi.subs.ingest.core.web.LinkGenerator; +import uk.ac.ebi.subs.ingest.export.job.ExportJob; +import uk.ac.ebi.subs.ingest.messaging.model.ExportEntityMessage; +import uk.ac.ebi.subs.ingest.messaging.model.ManifestMessage; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +public class ExperimentProcess { + + private final int index; + private final int totalCount; + + private final Process process; + + private final SubmissionEnvelope submissionEnvelope; + + private final Project project; + + public ExperimentProcess( + int index, int totalCount, Process process, SubmissionEnvelope submission, Project project) { + this.index = index; + this.totalCount = totalCount; + this.process = process; + this.submissionEnvelope = submission; + this.project = project; + } + + public static ExperimentProcess from(Process process, IndexCounter counter) { + return new ExperimentProcess( + counter.next(), + counter.totalCount, + process, + process.getSubmissionEnvelope(), + process.getProject()); + } + + public static class IndexCounter { + int base; + int totalCount; + + IndexCounter(int totalCount) { + this.base = 0; + this.totalCount = totalCount; + } + + int next() { + return base++; + } + } + + public Integer getIndex() { + return index; + } + + public Integer getTotalCount() { + return totalCount; + } + + public Process getProcess() { + return process; + } + + public SubmissionEnvelope getSubmissionEnvelope() { + return submissionEnvelope; + } + + public ExportEntityMessage toExportEntityMessage( + LinkGenerator linkGenerator, ExportJob exportJob, Map context) { + String callbackLink = linkGenerator.createCallback(process.getClass(), process.getId()); + return new ExportEntityMessage( + exportJob.getId(), + process.getId(), + process.getUuid().toString(), + callbackLink, + process.getClass().getSimpleName().toLowerCase(), + submissionEnvelope.getId(), + submissionEnvelope.getUuid().toString(), + project.getId(), + process.getProject().getUuid().toString(), + index, + totalCount, + context); + } + + public ManifestMessage toManifestMessage(LinkGenerator linkGenerator) { + String callbackLink = linkGenerator.createCallback(process.getClass(), process.getId()); + return new ManifestMessage( + UUID.randomUUID(), + Instant.now().toString(), + process.getId(), + process.getUuid().toString(), + callbackLink, + process.getClass().getSimpleName(), + submissionEnvelope.getId(), + submissionEnvelope.getUuid().toString(), + index, + totalCount); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/exporter/Exporter.java b/src/main/java/uk/ac/ebi/subs/ingest/exporter/Exporter.java new file mode 100644 index 000000000..4e092c863 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/exporter/Exporter.java @@ -0,0 +1,17 @@ +package uk.ac.ebi.subs.ingest.exporter; + +import uk.ac.ebi.subs.ingest.export.job.ExportJob; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +public interface Exporter { + + void exportManifests(SubmissionEnvelope submissionEnvelope); + + void exportMetadata(ExportJob exportJob); + + void generateSpreadsheet(SubmissionEnvelope envelope); + + void exportData(SubmissionEnvelope submissionEnvelope); + + void generateSpreadsheet(ExportJob exportJob); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/file/File.java b/src/main/java/uk/ac/ebi/subs/ingest/file/File.java new file mode 100644 index 000000000..c1560b4ce --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/file/File.java @@ -0,0 +1,132 @@ +package uk.ac.ebi.subs.ingest.file; + +import static com.fasterxml.jackson.annotation.JsonProperty.Access.READ_ONLY; + +import java.util.*; + +import javax.validation.constraints.NotNull; + +import org.springframework.data.mongodb.core.index.CompoundIndex; +import org.springframework.data.mongodb.core.index.CompoundIndexes; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.DBRef; +import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.data.rest.core.annotation.RestResource; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import uk.ac.ebi.subs.ingest.core.Checksums; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Getter +@Setter +@Document +@CompoundIndexes({ + @CompoundIndex(name = "validationId", def = "{ 'validationJob.validationId': 1 }") +}) +@EqualsAndHashCode( + callSuper = true, + exclude = {"project", "inputToProcesses", "derivedByProcesses"}) +public class File extends MetadataDocument { + @Indexed + private @Setter @DBRef(lazy = true) Project project; + + @Indexed @RestResource private Set inputToProcesses = new HashSet<>(); + + @Indexed @RestResource private Set derivedByProcesses = new HashSet<>(); + + @Indexed private String fileName; + private String cloudUrl; + + private Checksums checksums; + private Checksums lastExportedChecksums; + + private ValidationJob validationJob; + private FileArchiveResult fileArchiveResult; + private UUID validationId; + @NotNull private UUID dataFileUuid; + private Long size; + private String fileContentType; + + public File() { + super(EntityType.FILE, null); + initFile(); + } + + @JsonCreator + public File(@JsonProperty("content") Object content, @JsonProperty("fileName") String fileName) { + super(EntityType.FILE, content); + this.setFileName(fileName); + initFile(); + } + + private void initFile() { + setDataFileUuid(UUID.randomUUID()); + } + + /** + * Adds to the collection of processes that this file serves as an input to + * + * @param process the process to add + * @return a reference to this file + */ + public File addAsInputToProcess(Process process) { + this.inputToProcesses.add(process); + + return this; + } + + /** + * Adds to the collection of processes that this file was derived by + * + * @param process the process to add + * @return a reference to this file + */ + public File addAsDerivedByProcess(Process process) { + // XXX why we're implementing this check here but not above?? + String processId = process.getId(); + + boolean processInList = + derivedByProcesses.stream().map(Process::getId).anyMatch(id -> id.equals(processId)); + if (!processInList) { + this.derivedByProcesses.add(process); + } + return this; + } + + public void addToAnalysis(Process analysis) { + // TODO check if this File and the Analysis belong to the same Submission? + SubmissionEnvelope submissionEnvelope = analysis.getSubmissionEnvelope(); + super.setSubmissionEnvelope(submissionEnvelope); + addAsDerivedByProcess(analysis); + } + + @JsonProperty(access = READ_ONLY) + public boolean isLinked() { + return !inputToProcesses.isEmpty() || !derivedByProcesses.isEmpty(); + } + + /** + * Removes a process to the collection of processes that this file was derived by + * + * @param process the process to add + * @return a reference to this file + */ + public File removeAsDerivedByProcess(Process process) { + this.derivedByProcesses.remove(process); + return this; + } + + public File removeAsInputToProcess(Process process) { + this.inputToProcesses.remove(process); + return this; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/file/FileAlreadyExistsException.java b/src/main/java/uk/ac/ebi/subs/ingest/file/FileAlreadyExistsException.java new file mode 100644 index 000000000..03427adb4 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/file/FileAlreadyExistsException.java @@ -0,0 +1,20 @@ +package uk.ac.ebi.subs.ingest.file; + +import lombok.Getter; +import lombok.Setter; + +/** Created by rolando on 04/06/2018. */ +public class FileAlreadyExistsException extends RuntimeException { + @Getter @Setter private String fileName; + + public FileAlreadyExistsException() {} + + public FileAlreadyExistsException(String message) { + super(message); + } + + public FileAlreadyExistsException(String message, String fileName) { + super(message); + this.fileName = fileName; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/file/FileArchiveResult.java b/src/main/java/uk/ac/ebi/subs/ingest/file/FileArchiveResult.java new file mode 100644 index 000000000..d7eebecfb --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/file/FileArchiveResult.java @@ -0,0 +1,16 @@ +package uk.ac.ebi.subs.ingest.file; + +import java.time.Instant; + +import lombok.Data; + +@Data +public class FileArchiveResult { + private Instant lastArchived; + private Boolean compressed; + private String md5; + private String enaUploadPath; + private String error; + + protected FileArchiveResult() {} +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/file/FileRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/file/FileRepository.java new file mode 100644 index 000000000..e8668345d --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/file/FileRepository.java @@ -0,0 +1,125 @@ +package uk.ac.ebi.subs.ingest.file; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.data.rest.core.annotation.RestResource; +import org.springframework.web.bind.annotation.CrossOrigin; + +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.security.RowLevelFilterSecurity; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +/** Created by rolando on 06/09/2017. */ +@CrossOrigin +@RowLevelFilterSecurity( + expression = + "(#filterObject.project != null)" + + "? " + + " (" + + " #authentication.authorities.![authority].contains(" + + " 'ROLE_access_' +#filterObject.project.uuid?.toString()) " + + " or " + + " #authentication.authorities.![authority].contains('ROLE_SERVICE') " + + " or " + + " #filterObject.project.content['dataAccess']['type'] " + + " eq T(uk.ac.ebi.subs.ingest.project.DataAccessTypes).OPEN.label" + + " )" + + ":true", + ignoreClasses = {Project.class}) +public interface FileRepository extends MongoRepository { + + @RestResource(rel = "findAllByUuid", path = "findAllByUuid") + Page findByUuid(@Param("uuid") Uuid uuid, Pageable pageable); + + @RestResource(rel = "findByUuid", path = "findByUuid") + Optional findByUuidUuidAndIsUpdateFalse(@Param("uuid") UUID uuid); + + Page findByProject(Project project, Pageable pageable); + + @RestResource(exported = false) + Stream findByProject(Project project); + + @RestResource(rel = "findBySubmissionEnvelope") + Page findBySubmissionEnvelope( + @Param("envelopeUri") SubmissionEnvelope submissionEnvelope, Pageable pageable); + + @RestResource(exported = false) + Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + List findBySubmissionEnvelopeAndFileName( + SubmissionEnvelope submissionEnvelope, String fileName); + + @RestResource(rel = "findBySubmissionAndValidationState") + public Page findBySubmissionEnvelopeAndValidationState( + @Param("envelopeUri") SubmissionEnvelope submissionEnvelope, + @Param("state") ValidationState state, + Pageable pageable); + + @Query( + value = + "{'submissionEnvelope.id': ?0, graphValidationErrors: { $exists: true, $not: {$size: 0} } }") + @RestResource(rel = "findBySubmissionIdWithGraphValidationErrors") + public Page findBySubmissionIdWithGraphValidationErrors( + @Param("envelopeId") String envelopeId, Pageable pageable); + + @RestResource(exported = false) + Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + @RestResource(exported = false) + Long deleteBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + @RestResource(rel = "findByValidationId") + File findByValidationJobValidationId(@Param("validationId") UUID id); + + @RestResource(exported = false) + Stream findByInputToProcessesContains(Process process); + + Page findByInputToProcessesContaining(Process process, Pageable pageable); + + @RestResource(exported = false) + Stream findByDerivedByProcessesContains(Process process); + + Page findByDerivedByProcessesContaining(Process process, Pageable pageable); + + long countBySubmissionEnvelopeAndValidationState( + SubmissionEnvelope submissionEnvelope, ValidationState validationState); + + long countBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + @Query( + value = "{'submissionEnvelope.id': ?0, validationErrors: {$elemMatch: {errorType: ?1} }}", + count = true) + long countBySubmissionEnvelopeIdAndErrorType( + @Param("id") String submissionEnvelopeId, @Param("errorType") String errorType); + + @Query(value = "{'submissionEnvelope.id': ?0, validationErrors: {$elemMatch: {errorType: ?1} }}") + Page findBySubmissionEnvelopeIdAndErrorType( + @Param("id") String submissionEnvelopeId, + @Param("errorType") String errorType, + Pageable pageable); + + @Query( + value = + "{'submissionEnvelope.id': ?0, validationErrors: {$not: {$elemMatch: {errorType: ?1} }}}", + count = true) + long countBySubmissionEnvelopeIdAndNotErrorType( + @Param("id") String submissionEnvelopeId, @Param("errorType") String errorType); + + @Query( + value = + "{'submissionEnvelope.id': ?0, graphValidationErrors: { $exists: true, $not: {$size: 0} } }", + count = true) + long countBySubmissionEnvelopeAndCountWithGraphValidationErrors(String submissionEnvelopeId); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/file/FileService.java b/src/main/java/uk/ac/ebi/subs/ingest/file/FileService.java new file mode 100644 index 000000000..82ad6bea8 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/file/FileService.java @@ -0,0 +1,160 @@ +package uk.ac.ebi.subs.ingest.file; + +import java.util.List; +import java.util.Optional; + +import javax.validation.Valid; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.dao.OptimisticLockingFailureException; +import org.springframework.retry.annotation.Backoff; +import org.springframework.retry.annotation.Retryable; +import org.springframework.stereotype.Service; +import org.springframework.validation.annotation.Validated; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.core.Checksums; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.exception.CoreEntityNotFoundException; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.file.web.FileMessage; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.state.MetadataDocumentEventHandler; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 06/09/17 + */ +@Service +@RequiredArgsConstructor +@Getter +@Validated +public class FileService { + private final @NonNull SubmissionEnvelopeRepository submissionEnvelopeRepository; + private final @NonNull FileRepository fileRepository; + private final @NonNull BiomaterialRepository biomaterialRepository; + private final @NonNull ProcessRepository processRepository; + private final @NonNull ProjectRepository projectRepository; + private final @NonNull MetadataDocumentEventHandler metadataDocumentEventHandler; + private final @NonNull MetadataCrudService metadataCrudService; + private final @NonNull MetadataUpdateService metadataUpdateService; + + private final Logger log = LoggerFactory.getLogger(getClass()); + + public File addFileToSubmissionEnvelope(SubmissionEnvelope submissionEnvelope, @Valid File file) { + if (!fileRepository + .findBySubmissionEnvelopeAndFileName(submissionEnvelope, file.getFileName()) + .isEmpty()) { + throw new FileAlreadyExistsException( + String.format( + "File with name %s already exists in envelope %s", + file.getFileName(), submissionEnvelope.getId())); + } else { + projectRepository + .findBySubmissionEnvelopesContains(submissionEnvelope) + .findFirst() + .ifPresent(file::setProject); + File createdFile = + metadataCrudService.addToSubmissionEnvelopeAndSave(file, submissionEnvelope); + metadataDocumentEventHandler.handleMetadataDocumentCreate(createdFile); + return createdFile; + } + } + + public File addFileValidationJob(File file, ValidationJob validationJob) { + if (file.getChecksums().getSha1().equals(validationJob.getChecksums().getSha1())) { + file.setValidationJob(validationJob); + return fileRepository.save(file); + } else { + throw new IllegalStateException( + String.format( + "Failed to create validation job for file with ID %s : checksums mismatch", + file.getId())); + } + } + + public void createFileFromFileMessage(FileMessage fileMessage) + throws CoreEntityNotFoundException { + String envelopeUuid = fileMessage.getStagingAreaId(); + SubmissionEnvelope envelope = findEnvelope(envelopeUuid); + try { + addFileToSubmissionEnvelope(envelope, new File(null, fileMessage.getFileName())); + } catch (FileAlreadyExistsException e) { + log.info( + String.format( + "File listener attempted to create a File resource with name %s but it already existed for envelope %s", + fileMessage.getFileName(), envelope.getId())); + } + } + + @Retryable( + value = OptimisticLockingFailureException.class, + maxAttempts = 5, + backoff = @Backoff(delay = 500, maxDelay = 60000, multiplier = 2)) + public File updateFileFromFileMessage(FileMessage fileMessage) + throws CoreEntityNotFoundException { + String envelopeUuid = fileMessage.getStagingAreaId(); + SubmissionEnvelope envelope = findEnvelope(envelopeUuid); + return findAndUpdateFile(fileMessage, envelope); + } + + private File findAndUpdateFile(FileMessage fileMessage, SubmissionEnvelope envelope) { + String fileName = fileMessage.getFileName(); + File file = findFile(fileName, envelope); + + String newFileUrl = fileMessage.getCloudUrl(); + Checksums checksums = fileMessage.getChecksums(); + Long size = fileMessage.getSize(); + String contentType = fileMessage.getContentType(); + + log.info( + String.format( + "Updating file with cloudUrl %s and submission UUID %s", + newFileUrl, envelope.getUuid())); + + file.setCloudUrl(newFileUrl); + file.setChecksums(checksums); + file.setSize(size); + file.setFileContentType(contentType); + file.enactStateTransition(ValidationState.DRAFT); + File updatedFile = fileRepository.save(file); + + log.info( + String.format( + "File validation state is %s for file with cloudUrl %s and submission UUID %s ", + updatedFile.getValidationState(), file.getCloudUrl(), envelope.getUuid())); + + return updatedFile; + } + + private SubmissionEnvelope findEnvelope(String envelopeUuid) throws CoreEntityNotFoundException { + return Optional.ofNullable(submissionEnvelopeRepository.findByUuid(new Uuid(envelopeUuid))) + .orElseThrow( + () -> + new CoreEntityNotFoundException( + String.format("Couldn't find envelope with with uuid %s", envelopeUuid))); + } + + private File findFile(String fileName, SubmissionEnvelope envelope) { + List filesInEnvelope = + fileRepository.findBySubmissionEnvelopeAndFileName(envelope, fileName); + + if (filesInEnvelope.size() != 1) { + throw new RuntimeException( + String.format( + "Expected 1 file with name %s, but found %s", fileName, filesInEnvelope.size())); + } + return filesInEnvelope.get(0); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/file/ValidationErrorType.java b/src/main/java/uk/ac/ebi/subs/ingest/file/ValidationErrorType.java new file mode 100644 index 000000000..f211ec23c --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/file/ValidationErrorType.java @@ -0,0 +1,7 @@ +package uk.ac.ebi.subs.ingest.file; + +public enum ValidationErrorType { + METADATA_ERROR, + FILE_NOT_UPLOADED, + FILE_ERROR +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/file/ValidationJob.java b/src/main/java/uk/ac/ebi/subs/ingest/file/ValidationJob.java new file mode 100644 index 000000000..28b60bcb1 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/file/ValidationJob.java @@ -0,0 +1,16 @@ +package uk.ac.ebi.subs.ingest.file; + +import java.util.UUID; + +import lombok.Data; +import uk.ac.ebi.subs.ingest.core.Checksums; + +@Data +public class ValidationJob { + private UUID validationId; + private Checksums checksums; + private boolean jobCompleted; + private ValidationReport validationReport; + + protected ValidationJob() {} +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/file/ValidationReport.java b/src/main/java/uk/ac/ebi/subs/ingest/file/ValidationReport.java new file mode 100644 index 000000000..27a8ec82c --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/file/ValidationReport.java @@ -0,0 +1,14 @@ +package uk.ac.ebi.subs.ingest.file; + +import java.util.List; + +import lombok.Data; +import uk.ac.ebi.subs.ingest.state.ValidationState; + +@Data +public class ValidationReport { + private ValidationState validationState; + private List validationErrors; + + protected ValidationReport() {} +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/file/web/FileController.java b/src/main/java/uk/ac/ebi/subs/ingest/file/web/FileController.java new file mode 100644 index 000000000..c70aa2338 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/file/web/FileController.java @@ -0,0 +1,229 @@ +package uk.ac.ebi.subs.ingest.file.web; + +import static org.springframework.data.rest.webmvc.RestMediaTypes.TEXT_URI_LIST_VALUE; +import static org.springframework.web.bind.annotation.RequestMethod.POST; +import static org.springframework.web.bind.annotation.RequestMethod.PUT; + +import java.lang.reflect.InvocationTargetException; +import java.net.URISyntaxException; +import java.util.List; + +import javax.validation.ConstraintViolationException; +import javax.validation.Valid; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.rest.webmvc.PersistentEntityResource; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.data.web.PagedResourcesAssembler; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.hateoas.MediaTypes; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.Resources; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.service.*; +import uk.ac.ebi.subs.ingest.file.*; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.security.CheckAllowed; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.exception.NotAllowedDuringSubmissionStateException; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 06/09/17 + */ +@RepositoryRestController +@ExposesResourceFor(File.class) +@RequiredArgsConstructor +@Getter +@Validated +public class FileController { + private final Logger logger = LoggerFactory.getLogger(getClass()); + @NonNull private final FileService fileService; + @NonNull private final FileRepository fileRepository; + @NonNull private final ProcessRepository processRepository; + @NonNull private final PagedResourcesAssembler pagedResourcesAssembler; + @NonNull private final MetadataCrudService metadataCrudService; + @NonNull private final MetadataUpdateService metadataUpdateService; + @Autowired private ValidationStateChangeService validationStateChangeService; + @Autowired private UriToEntityConversionService uriToEntityConversionService; + @Autowired private MetadataLinkingService metadataLinkingService; + + @ExceptionHandler(ConstraintViolationException.class) + @ResponseStatus(HttpStatus.BAD_REQUEST) + ResponseEntity handleConstraintViolationException(final ConstraintViolationException e) { + return new ResponseEntity<>( + "not valid due to validation error: " + e.getMessage(), HttpStatus.BAD_REQUEST); + } + + @CheckAllowed( + value = "#submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @RequestMapping( + path = "/submissionEnvelopes/{sub_id}/files", + method = RequestMethod.POST, + produces = MediaTypes.HAL_JSON_VALUE) + ResponseEntity> createFile( + @PathVariable("sub_id") final SubmissionEnvelope submissionEnvelope, + @RequestBody @Valid final File file, + final PersistentEntityResourceAssembler assembler) { + try { + final File createdFile = fileService.addFileToSubmissionEnvelope(submissionEnvelope, file); + logFileDetails(submissionEnvelope, createdFile); + + return ResponseEntity.accepted().body(assembler.toFullResource(createdFile)); + } catch (FileAlreadyExistsException e) { + throw new IllegalStateException(e); + } + } + + private void logFileDetails(final SubmissionEnvelope submissionEnvelope, final File createdFile) { + logger.info( + "submission uuid {}: created File: id {} uuid {} name {} dataFileUuid {}", + submissionEnvelope.getUuid(), + createdFile.getId(), + createdFile.getUuid(), + createdFile.getFileName(), + createdFile.getDataFileUuid()); + } + + @RequestMapping( + path = "/files/{id}/validationJob", + method = RequestMethod.PUT, + produces = MediaTypes.HAL_JSON_VALUE) + ResponseEntity> addFileValidationJob( + @PathVariable("id") final File file, + @RequestBody final ValidationJob validationJob, + final PersistentEntityResourceAssembler assembler) { + final File entity = fileService.addFileValidationJob(file, validationJob); + final PersistentEntityResource resource = assembler.toFullResource(entity); + + return ResponseEntity.accepted().body(resource); + } + + @CheckAllowed( + value = "#file.submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PatchMapping(path = "/files/{id}") + HttpEntity patchFile( + @PathVariable("id") final File file, + @RequestBody final ObjectNode patch, + final PersistentEntityResourceAssembler assembler) { + final List allowedFields = + List.of( + "content", + "fileName", + "validationJob", + "validationErrors", + "graphValidationErrors", + "fileArchiveResult"); + final ObjectNode validPatch = patch.retain(allowedFields); + final File updatedFile = metadataUpdateService.update(file, validPatch); + final PersistentEntityResource resource = assembler.toFullResource(updatedFile); + + return ResponseEntity.accepted().body(resource); + } + + @CheckAllowed( + value = "#file.submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @RequestMapping( + path = "/files/{id}/inputToProcesses", + method = {PUT, POST}, + consumes = {TEXT_URI_LIST_VALUE}) + HttpEntity linkFileAsInputToProcesses( + @PathVariable("id") final File file, + @RequestBody final Resources incoming, + final HttpMethod requestMethod) + throws URISyntaxException, + InvocationTargetException, + NoSuchMethodException, + IllegalAccessException { + + final List processes = + uriToEntityConversionService.convertLinks(incoming.getLinks(), Process.class); + metadataLinkingService.updateLinks( + file, processes, "inputToProcesses", requestMethod.equals(HttpMethod.PUT)); + + return ResponseEntity.ok().build(); + } + + @CheckAllowed( + value = "#file.submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @RequestMapping( + path = "/files/{id}/derivedByProcesses", + method = {PUT, POST}, + consumes = {TEXT_URI_LIST_VALUE}) + HttpEntity linkFileAsDerivedByProcesses( + @PathVariable("id") final File file, + @RequestBody final Resources incoming, + final HttpMethod requestMethod) + throws URISyntaxException, + InvocationTargetException, + NoSuchMethodException, + IllegalAccessException { + + final List processes = + uriToEntityConversionService.convertLinks(incoming.getLinks(), Process.class); + metadataLinkingService.updateLinks( + file, processes, "derivedByProcesses", requestMethod.equals(HttpMethod.PUT)); + + return ResponseEntity.ok().build(); + } + + @CheckAllowed( + value = "#file.submissionEnvelope.isEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @DeleteMapping(path = "/files/{id}/inputToProcesses/{processId}") + HttpEntity unlinkFileAsInputToProcesses( + @PathVariable("id") final File file, + @PathVariable("processId") final Process process, + final PersistentEntityResourceAssembler assembler) + throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + metadataLinkingService.removeLink(file, process, "inputToProcesses"); + + return ResponseEntity.noContent().build(); + } + + @CheckAllowed( + value = "#file.submissionEnvelope.isEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @DeleteMapping(path = "/files/{id}/derivedByProcesses/{processId}") + HttpEntity unlinkFileAsDerivedByProcesses( + @PathVariable("id") final File file, @PathVariable("processId") final Process process) + throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + metadataLinkingService.removeLink(file, process, "derivedByProcesses"); + + return ResponseEntity.noContent().build(); + } + + @CheckAllowed( + value = "#file.submissionEnvelope.isEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @DeleteMapping(path = "/files/{id}") + ResponseEntity deleteFile(@PathVariable("id") final File file) + throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + metadataLinkingService.removeLinks(file, "inputToProcesses"); + metadataLinkingService.removeLinks(file, "derivedByProcesses"); + metadataCrudService.deleteDocument(file); + + return ResponseEntity.noContent().build(); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/file/web/FileListener.java b/src/main/java/uk/ac/ebi/subs/ingest/file/web/FileListener.java new file mode 100644 index 000000000..93ea98163 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/file/web/FileListener.java @@ -0,0 +1,45 @@ +package uk.ac.ebi.subs.ingest.file.web; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.amqp.AmqpRejectAndDontRequeueException; +import org.springframework.amqp.ImmediateRequeueAmqpException; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.dao.OptimisticLockingFailureException; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import lombok.AllArgsConstructor; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.core.exception.CoreEntityNotFoundException; +import uk.ac.ebi.subs.ingest.file.FileService; +import uk.ac.ebi.subs.ingest.messaging.Constants; + +/** Created by rolando on 07/09/2017. */ +@Component +@AllArgsConstructor +public class FileListener { + private final @NonNull FileService fileService; + private final Logger log = LoggerFactory.getLogger(getClass()); + + @RabbitListener(queues = Constants.Queues.FILE_STAGED_QUEUE) + public void handleFileStagedEvent(FileMessage fileMessage) { + if (!StringUtils.isEmpty(fileMessage.getContentType()) + && fileMessage.getMediaType().isPresent() + && fileMessage.getMediaType().get().equals(FileMediaTypes.HCA_DATA_FILE)) { + try { + fileService.createFileFromFileMessage(fileMessage); + fileService.updateFileFromFileMessage(fileMessage); + } catch (CoreEntityNotFoundException e) { + log.warn(e.getMessage()); + throw new AmqpRejectAndDontRequeueException(e.getMessage()); + } catch (OptimisticLockingFailureException e) { + log.warn("Putting file back on queue: " + e.getMessage()); + throw new ImmediateRequeueAmqpException(e); + } catch (RuntimeException e) { + log.error(e.getMessage()); + throw new AmqpRejectAndDontRequeueException(e.getMessage()); + } + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/file/web/FileMediaTypes.java b/src/main/java/uk/ac/ebi/subs/ingest/file/web/FileMediaTypes.java new file mode 100644 index 000000000..97754ba95 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/file/web/FileMediaTypes.java @@ -0,0 +1,11 @@ +package uk.ac.ebi.subs.ingest.file.web; + +/** Created by rolando on 10/10/2017. */ +public class FileMediaTypes { + public static final String HCA_DATA_FILE = "data"; + public static final String HCA_SAMPLE = "metadata/sample"; + public static final String HCA_PROJECT = "metadata/project"; + public static final String HCA_PROTOCOL = "metadata/protocol"; + public static final String HCA_ASSAY = "metadata/assay"; + public static final String HCA_ANALYSIS = "metadata/analysis"; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/file/web/FileMessage.java b/src/main/java/uk/ac/ebi/subs/ingest/file/web/FileMessage.java new file mode 100644 index 000000000..2d4f3cd21 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/file/web/FileMessage.java @@ -0,0 +1,44 @@ +package uk.ac.ebi.subs.ingest.file.web; + +import java.util.Optional; + +import org.apache.http.entity.ContentType; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import uk.ac.ebi.subs.ingest.core.Checksums; + +/** Created by rolando on 07/09/2017. */ +@Getter +@AllArgsConstructor +@NoArgsConstructor +public class FileMessage { + @JsonProperty("url") + private String cloudUrl; + + @JsonProperty("name") + private String fileName; + + @JsonProperty("upload_area_id") + private String stagingAreaId; + + @JsonProperty("content_type") + private String contentType; + + private Checksums checksums; + private long size; + + /** + * given existence of substring "dcp-type={type}" in this.contentType, extracts {type} + * + * @return the DCP media-type of the file uploaded that triggered this event + */ + @JsonIgnore + public Optional getMediaType() { + return Optional.ofNullable(ContentType.parse(this.getContentType()).getParameter("dcp-type")); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/file/web/FileResourceProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/file/web/FileResourceProcessor.java new file mode 100644 index 000000000..c2e6f6603 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/file/web/FileResourceProcessor.java @@ -0,0 +1,32 @@ +package uk.ac.ebi.subs.ingest.file.web; + +import org.springframework.hateoas.EntityLinks; +import org.springframework.hateoas.Link; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.stereotype.Component; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.file.File; + +@Component +@RequiredArgsConstructor +public class FileResourceProcessor implements ResourceProcessor> { + private final @NonNull EntityLinks entityLinks; + + private Link getCreateValidationJobLink(File file) { + return entityLinks + .linkForSingleResource(file) + .slash(Links.FILE_VALIDATION_JOB_URL) + .withRel(Links.FILE_VALIDATION_JOB_REL); + } + + @Override + public Resource process(Resource resource) { + File fileDocument = resource.getContent(); + resource.add(getCreateValidationJobLink(fileDocument)); + return resource; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/messaging/Constants.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/Constants.java new file mode 100644 index 000000000..78409122c --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/Constants.java @@ -0,0 +1,38 @@ +package uk.ac.ebi.subs.ingest.messaging; + +public class Constants { + public class Queues { + public static final String FILE_STAGED_QUEUE = "ingest.file.create.staged"; + public static final String FILE_VALIDATION_QUEUE = "ingest.file.validation.queue"; + public static final String METADATA_VALIDATION_QUEUE = "ingest.metadata.validation.queue"; + public static final String NOTIFICATIONS_QUEUE = "ingest.notifications.queue"; + public static final String GRAPH_VALIDATION_QUEUE = "ingest.validation.graph.queue"; + } + + public class Exchanges { + public static final String VALIDATION_EXCHANGE = "ingest.validation.exchange"; + public static final String FILE_STAGED_EXCHANGE = "ingest.file.staged.exchange"; + public static final String STATE_TRACKING_EXCHANGE = "ingest.state-tracking.exchange"; + public static final String EXPORTER_EXCHANGE = "ingest.exporter.exchange"; + public static final String UPLOAD_AREA_EXCHANGE = "ingest.upload.area.exchange"; + public static final String NOTIFICATIONS_EXCHANGE = "ingest.notifications.exchange"; + public static final String SPREADSHEET_EXCHANGE = "ingest.spreadsheet.exchange"; + } + + public class Routing { + public static final String ENVELOPE_STATE_UPDATE = + "ingest.state-tracking.envelope.state.update"; + public static final String ENVELOPE_CREATE = "ingest.state-tracking.envelope.create"; + + public static final String MANIFEST_SUBMITTED = "ingest.exporter.manifest.submitted"; + public static final String EXPERIMENT_SUBMITTED = "ingest.exporter.experiment.submitted"; + + public static final String SUBMISSION_SUBMITTED = "ingest.exporter.submission.submitted"; + + public static final String UPLOAD_AREA_CREATE = "ingest.upload.area.create"; + public static final String UPLOAD_AREA_CLEANUP = "ingest.upload.area.cleanup"; + + public static final String NOTIFICATION_NEW = "ingest.notifications.new"; + public static final String SPREADSHEET_GENERATION = "ingest.exporter.spreadsheet.requested"; + } +} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/Message.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/Message.java similarity index 58% rename from src/main/java/org/humancellatlas/ingest/messaging/Message.java rename to src/main/java/uk/ac/ebi/subs/ingest/messaging/Message.java index 624ee38c9..e638c107a 100644 --- a/src/main/java/org/humancellatlas/ingest/messaging/Message.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/Message.java @@ -1,4 +1,4 @@ -package org.humancellatlas.ingest.messaging; +package uk.ac.ebi.subs.ingest.messaging; import lombok.AllArgsConstructor; import lombok.Data; @@ -10,7 +10,7 @@ @NoArgsConstructor @Getter public class Message { - private String exchange; - private String routingKey; - private Object payload; + private String exchange; + private String routingKey; + private Object payload; } diff --git a/src/main/java/uk/ac/ebi/subs/ingest/messaging/MessageRouter.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/MessageRouter.java new file mode 100644 index 000000000..14c923956 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/MessageRouter.java @@ -0,0 +1,127 @@ +package uk.ac.ebi.subs.ingest.messaging; + +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import lombok.NoArgsConstructor; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.core.MetadataDocumentMessageBuilder; +import uk.ac.ebi.subs.ingest.core.web.LinkGenerator; +import uk.ac.ebi.subs.ingest.export.job.ExportJob; +import uk.ac.ebi.subs.ingest.exporter.ExperimentProcess; +import uk.ac.ebi.subs.ingest.messaging.model.MetadataDocumentMessage; +import uk.ac.ebi.subs.ingest.messaging.model.SubmissionEnvelopeMessage; +import uk.ac.ebi.subs.ingest.messaging.model.SubmissionEnvelopeStateUpdateMessage; +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeMessageBuilder; + +@Component +@NoArgsConstructor +public class MessageRouter { + @Autowired private MessageSender messageSender; + @Autowired private LinkGenerator linkGenerator; + + /* messages to validator */ + public boolean routeValidationMessageFor(MetadataDocument document) { + if (document.getValidationState().equals(ValidationState.DRAFT)) { + this.messageSender.queueValidationMessage( + Constants.Exchanges.VALIDATION_EXCHANGE, + Constants.Queues.METADATA_VALIDATION_QUEUE, + messageFor(document), + document.getUpdateDate().toEpochMilli()); + return true; + } else { + return false; + } + } + + /* messages to the exporter */ + + public void sendManifestForExport(ExperimentProcess experimentProcess) { + messageSender.queueNewExportMessage( + Constants.Exchanges.EXPORTER_EXCHANGE, + Constants.Routing.MANIFEST_SUBMITTED, + experimentProcess.toManifestMessage(linkGenerator), + System.currentTimeMillis()); + } + + public void sendExperimentForExport( + ExperimentProcess experimentProcess, ExportJob exportJob, Map context) { + messageSender.queueNewExportMessage( + Constants.Exchanges.EXPORTER_EXCHANGE, + Constants.Routing.EXPERIMENT_SUBMITTED, + experimentProcess.toExportEntityMessage(linkGenerator, exportJob, context), + System.currentTimeMillis()); + } + + public void sendSubmissionForDataExport(ExportJob exportJob, Map context) { + messageSender.queueNewExportMessage( + Constants.Exchanges.EXPORTER_EXCHANGE, + Constants.Routing.SUBMISSION_SUBMITTED, + exportJob.toExportSubmissionMessage(linkGenerator, context), + System.currentTimeMillis()); + } + + /* messages to the upload/staging area manager */ + + public boolean routeRequestUploadAreaCredentials(SubmissionEnvelope envelope) { + this.messageSender.queueUploadManagerMessage( + Constants.Exchanges.UPLOAD_AREA_EXCHANGE, + Constants.Routing.UPLOAD_AREA_CREATE, + messageFor(envelope), + envelope.getUpdateDate().toEpochMilli()); + return true; + } + + public boolean routeRequestUploadAreaCleanup(SubmissionEnvelope envelope) { + this.messageSender.queueUploadManagerMessage( + Constants.Exchanges.UPLOAD_AREA_EXCHANGE, + Constants.Routing.UPLOAD_AREA_CLEANUP, + messageFor(envelope), + envelope.getUpdateDate().toEpochMilli()); + return true; + } + + public void sendGenerateSpreadsheet(ExportJob exportJob, Map context) { + this.messageSender.queueSpreadsheetGenerationMessage( + Constants.Exchanges.EXPORTER_EXCHANGE, + Constants.Routing.SPREADSHEET_GENERATION, + exportJob.toGenerateSubmissionMessage(linkGenerator, context), + System.currentTimeMillis()); + } + + private MetadataDocumentMessage messageFor(MetadataDocument document) { + return MetadataDocumentMessageBuilder.using(linkGenerator).messageFor(document).build(); + } + + private SubmissionEnvelopeMessage messageFor(SubmissionEnvelope envelope) { + return SubmissionEnvelopeMessageBuilder.using(linkGenerator).messageFor(envelope).build(); + } + + private MetadataDocumentMessage documentStateUpdateMessage(MetadataDocument document) { + if (document.getSubmissionEnvelope() == null) { + throw new RuntimeException( + "The metadata document should have a link to a submission envelope."); + } + + String envelopeId = document.getSubmissionEnvelope().getId(); + + return MetadataDocumentMessageBuilder.using(linkGenerator) + .messageFor(document) + .withEnvelopeId(envelopeId) + .withValidationState(document.getValidationState()) + .build(); + } + + private SubmissionEnvelopeStateUpdateMessage messageFor( + SubmissionEnvelope envelope, SubmissionState state) { + SubmissionEnvelopeStateUpdateMessage message = + SubmissionEnvelopeStateUpdateMessage.fromSubmissionEnvelopeMessage(messageFor(envelope)); + message.setRequestedState(state); + return message; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/messaging/MessageSender.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/MessageSender.java new file mode 100644 index 000000000..02477ae8c --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/MessageSender.java @@ -0,0 +1,276 @@ +package uk.ac.ebi.subs.ingest.messaging; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +import java.net.URI; +import java.util.*; +import java.util.concurrent.*; +import java.util.stream.Stream; + +import javax.annotation.Nullable; +import javax.annotation.PostConstruct; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.amqp.rabbit.core.RabbitMessagingTemplate; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.config.ConfigurationService; +import uk.ac.ebi.subs.ingest.messaging.model.MessageProtocol; +import uk.ac.ebi.subs.ingest.messaging.model.MetadataDocumentMessage; +import uk.ac.ebi.subs.ingest.messaging.model.SpreadsheetGenerationMessage; +import uk.ac.ebi.subs.ingest.messaging.model.SubmissionEnvelopeMessage; + +@Service +@Getter +@NoArgsConstructor +public class MessageSender { + + private static final Logger LOGGER = LoggerFactory.getLogger(MessageSender.class); + + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(5); + + private @Autowired @NonNull RabbitMessagingTemplate rabbitMessagingTemplate; + private @Autowired @NonNull ConfigurationService configurationService; + + public void queueValidationMessage( + String exchange, String routingKey, MetadataDocumentMessage payload, long intendedSendTime) { + MessageBuffer.VALIDATION.queueAmqpMessage(exchange, routingKey, payload, intendedSendTime); + } + + public void queueGraphValidationMessage( + String exchange, String routingKey, Object payload, long intendedSendTime) { + MessageBuffer.GRAPH_VALIDATION.queueAmqpMessage( + exchange, routingKey, payload, intendedSendTime); + } + + public void queueNewExportMessage( + String exchange, String routingKey, Object payload, long intendedSendTime) { + MessageBuffer.EXPORT.queueAmqpMessage(exchange, routingKey, payload, intendedSendTime); + } + + public void queueStateTrackingMessage( + String exchange, String routingKey, Object payload, long intendedSendTime) { + MessageBuffer.STATE_TRACKING.queueAmqpMessage(exchange, routingKey, payload, intendedSendTime); + } + + public void queueDocumentStateUpdateMessage(URI uri, Object payload, long intendedSendTime) { + MessageBuffer.STATE_TRACKING.queueHttpMessage(uri, HttpMethod.POST, payload, intendedSendTime); + } + + public void queueDocumentStateDeleteMessage(URI uri, long intendedSendTime) { + MessageBuffer.STATE_TRACKING.queueHttpMessage(uri, HttpMethod.DELETE, null, intendedSendTime); + } + + public void queueUploadManagerMessage( + String exchange, + String routingKey, + SubmissionEnvelopeMessage payload, + long intendedSendTime) { + MessageBuffer.UPLOAD_MANAGER.queueAmqpMessage(exchange, routingKey, payload, intendedSendTime); + } + + public void queueSpreadsheetGenerationMessage( + String exchange, + String routingKey, + SpreadsheetGenerationMessage payload, + long intendedSendTime) { + MessageBuffer.SPREADSHEET_GENERATION.queueAmqpMessage( + exchange, routingKey, payload, intendedSendTime); + } + + @PostConstruct + private void initiateSending() { + List amqpMessageBuffers = + Arrays.asList( + MessageBuffer.ACCESSIONER, + MessageBuffer.EXPORT, + MessageBuffer.UPLOAD_MANAGER, + MessageBuffer.VALIDATION, + MessageBuffer.STATE_TRACKING, + MessageBuffer.GRAPH_VALIDATION, + MessageBuffer.SPREADSHEET_GENERATION); + + amqpMessageBuffers.forEach( + buffer -> + scheduler.scheduleWithFixedDelay( + new AmqpHttpMixinBufferSender(buffer, new RestTemplate(), rabbitMessagingTemplate), + 0, + buffer.getDelayMillis(), + TimeUnit.MILLISECONDS)); + } + + @Data + static class QueuedMessage implements Delayed { + private final MessageProtocol messageProtocol; + private final Object payload; + private String exchange; + private String routingKey; + private URI uri; + private HttpMethod method; + + private final long intendedStartTime; + + public QueuedMessage( + String exchange, String routingKey, Object payload, long intendedStartTime) { + this.messageProtocol = MessageProtocol.AMQP; + this.exchange = exchange; + this.routingKey = routingKey; + this.payload = payload; + this.intendedStartTime = intendedStartTime; + } + + public QueuedMessage( + URI uri, HttpMethod method, @Nullable Object payload, long intendedStartTime) { + this.messageProtocol = MessageProtocol.HTTP; + this.method = method; + this.uri = uri; + this.payload = payload; + this.intendedStartTime = intendedStartTime; + } + + @Override + public long getDelay(TimeUnit unit) { + long delay = intendedStartTime - System.currentTimeMillis(); + return unit.convert(delay, MILLISECONDS); + } + + @Override + public int compareTo(Delayed other) { + long otherDelay = other.getDelay(MILLISECONDS); + return Math.toIntExact(getDelay(TimeUnit.MILLISECONDS) - otherDelay); + } + } + + private enum MessageBuffer { + VALIDATION(SECONDS.toMillis(3)), + EXPORT(SECONDS.toMillis(5)), + UPLOAD_MANAGER(SECONDS.toMillis(1)), + ACCESSIONER(SECONDS.toMillis(2)), + STATE_TRACKING(500L), + GRAPH_VALIDATION(SECONDS.toMillis(5)), + SPREADSHEET_GENERATION(SECONDS.toMillis(5)); + + @Getter private final Long delayMillis; + + private final BlockingQueue messageQueue = new DelayQueue<>(); + + private final Logger log = LoggerFactory.getLogger(getClass()); + + protected Logger getLog() { + return log; + } + + MessageBuffer(Long delayMillis) { + this.delayMillis = delayMillis; + } + + // TODO each enum should already know exchange and routing key + // Why are these part of the contract when they're already defined in Constants? + void queueAmqpMessage( + String exchange, String routingKey, Object payload, long intendedStartTime) { + QueuedMessage message = + new QueuedMessage(exchange, routingKey, payload, intendedStartTime + delayMillis); + try { + messageQueue.add(message); + } catch (IllegalStateException e) { + LOGGER.error(String.format("Failed to queue message: %s", convertToString(message)), e); + throw new RuntimeException(e); + } + } + + void queueHttpMessage( + URI uri, HttpMethod method, @Nullable Object payload, long intendedStartTime) { + QueuedMessage message = + new QueuedMessage(uri, method, payload, intendedStartTime + delayMillis); + try { + messageQueue.add(message); + } catch (IllegalStateException e) { + LOGGER.error(String.format("Failed to queue message: %s", convertToString(message)), e); + throw new RuntimeException(e); + } + } + + public Stream takeAll() { + Queue drainedQueue = + new PriorityQueue<>(Comparator.comparing(QueuedMessage::getIntendedStartTime)); + this.messageQueue.drainTo(drainedQueue); + return Stream.generate(drainedQueue::remove).limit(drainedQueue.size()); + } + + private String convertToString(Object object) { + try { + return new ObjectMapper().writeValueAsString(object); + } catch (JsonProcessingException e) { + LOGGER.debug( + String.format( + "An error in converting message object to string occurred: %s", e.getMessage())); + return ""; + } + } + } + + private static class AmqpHttpMixinBufferSender implements Runnable { + private final MessageBuffer buffer; + private final RestTemplate restTemplate; + private final RabbitMessagingTemplate messagingTemplate; + private final Logger log = + LoggerFactory.getLogger(MessageSender.AmqpHttpMixinBufferSender.class); + + private AmqpHttpMixinBufferSender( + MessageBuffer buffer, + RestTemplate restTemplate, + RabbitMessagingTemplate rabbitMessagingTemplate) { + this.buffer = buffer; + this.restTemplate = restTemplate; + this.messagingTemplate = rabbitMessagingTemplate; + } + + @Override + public void run() { + HttpHeaders headers = uriListHeaders(); + buffer + .takeAll() + .forEach( + message -> { + if (message.getMessageProtocol().equals(MessageProtocol.AMQP)) { + messagingTemplate.convertAndSend( + message.exchange, message.routingKey, message.payload); + } else { + try { + restTemplate.exchange( + message.getUri(), + message.method, + new HttpEntity<>(message.getPayload(), headers), + Object.class); + } catch (Exception e) { + log.error( + String.format( + "error sending HTTP %s message to uri %s with payload %s", + message.method, message.uri, message.payload), + e); + } + } + }); + } + + private HttpHeaders uriListHeaders() { + HttpHeaders headers = new HttpHeaders(); + headers.add("Content-type", "application/json"); + return headers; + } + } +} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/MessageSenderConfig.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/MessageSenderConfig.java similarity index 61% rename from src/main/java/org/humancellatlas/ingest/messaging/MessageSenderConfig.java rename to src/main/java/uk/ac/ebi/subs/ingest/messaging/MessageSenderConfig.java index 8e812e46d..0586f54db 100644 --- a/src/main/java/org/humancellatlas/ingest/messaging/MessageSenderConfig.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/MessageSenderConfig.java @@ -1,4 +1,4 @@ -package org.humancellatlas.ingest.messaging; +package uk.ac.ebi.subs.ingest.messaging; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -10,11 +10,10 @@ @EnableScheduling public class MessageSenderConfig { - @Bean - public TaskScheduler executor() { - ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler(); - scheduler.setPoolSize(7); - return scheduler; - } - + @Bean + public TaskScheduler executor() { + ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler(); + scheduler.setPoolSize(7); + return scheduler; + } } diff --git a/src/main/java/uk/ac/ebi/subs/ingest/messaging/MessageService.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/MessageService.java new file mode 100644 index 000000000..2753f1eb7 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/MessageService.java @@ -0,0 +1,34 @@ +package uk.ac.ebi.subs.ingest.messaging; + +import org.springframework.amqp.rabbit.core.RabbitMessagingTemplate; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.messaging.MessagingException; +import org.springframework.messaging.converter.MessageConversionException; +import org.springframework.stereotype.Service; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +@Getter +public class MessageService { + @Autowired @NonNull private RabbitMessagingTemplate messagingTemplate; + + public void publish(Message message) { + try { + messagingTemplate.convertAndSend( + message.getExchange(), message.getRoutingKey(), message.getPayload()); + } catch (MessageConversionException e) { + throw new IllegalArgumentException( + String.format("Unable to convert payload '%s'", message.getPayload())); + } catch (MessagingException e) { + throw new RuntimeException( + String.format( + "There was a problem sending message '%s' to exchange '%s', with routing key '%s'.", + message.getPayload(), message.getExchange(), message.getRoutingKey())); + } + return; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/messaging/QueueConfig.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/QueueConfig.java new file mode 100644 index 000000000..6543ae624 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/QueueConfig.java @@ -0,0 +1,138 @@ +package uk.ac.ebi.subs.ingest.messaging; + +import org.springframework.amqp.core.*; +import org.springframework.amqp.rabbit.annotation.RabbitListenerConfigurer; +import org.springframework.amqp.rabbit.core.RabbitMessagingTemplate; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistrar; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.messaging.converter.MappingJackson2MessageConverter; +import org.springframework.messaging.converter.MessageConverter; +import org.springframework.messaging.handler.annotation.support.DefaultMessageHandlerMethodFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + +import uk.ac.ebi.subs.ingest.messaging.Constants.Exchanges; +import uk.ac.ebi.subs.ingest.messaging.Constants.Queues; +import uk.ac.ebi.subs.ingest.messaging.Constants.Routing; + +@Configuration +public class QueueConfig implements RabbitListenerConfigurer { + @Bean + Queue queueFileStaged() { + return new Queue(Constants.Queues.FILE_STAGED_QUEUE, false); + } + + @Bean + FanoutExchange fileStagedExchange() { + return new FanoutExchange(Constants.Exchanges.FILE_STAGED_EXCHANGE); + } + + @Bean + Queue queueMetadataValidation() { + return new Queue(Constants.Queues.METADATA_VALIDATION_QUEUE, false); + } + + @Bean + DirectExchange validationExchange() { + return new DirectExchange(Constants.Exchanges.VALIDATION_EXCHANGE); + } + + @Bean + Queue queueGraphValidation() { + return new Queue(Constants.Queues.GRAPH_VALIDATION_QUEUE); + } + + @Bean + TopicExchange stateTrackingExchange() { + return new TopicExchange(Constants.Exchanges.STATE_TRACKING_EXCHANGE); + } + + @Bean + Queue queueNotifications() { + return new Queue(Queues.NOTIFICATIONS_QUEUE, true); + } + + @Bean + TopicExchange notificationExchange() { + return new TopicExchange(Exchanges.NOTIFICATIONS_EXCHANGE); + } + + @Bean + TopicExchange exporterExchange() { + return new TopicExchange(Constants.Exchanges.EXPORTER_EXCHANGE); + } + + @Bean + TopicExchange uploadAreaExchange() { + return new TopicExchange(Constants.Exchanges.UPLOAD_AREA_EXCHANGE); + } + + /* bindings */ + + @Bean + Binding bindingFileStaged(Queue queueFileStaged, FanoutExchange fileStagedExchange) { + return BindingBuilder.bind(queueFileStaged).to(fileStagedExchange); + } + + @Bean + Binding bindingValidation(Queue queueMetadataValidation, DirectExchange validationExchange) { + return BindingBuilder.bind(queueMetadataValidation) + .to(validationExchange) + .with(Constants.Queues.METADATA_VALIDATION_QUEUE); + } + + @Bean + Binding bindingGraphValidation(Queue queueGraphValidation, DirectExchange validationExchange) { + return BindingBuilder.bind(queueGraphValidation) + .to(validationExchange) + .with(Constants.Queues.GRAPH_VALIDATION_QUEUE); + } + + @Bean + Binding bindingNewNotificationQueue( + Queue queueNotifications, TopicExchange notificationExchange) { + return BindingBuilder.bind(queueNotifications) + .to(notificationExchange) + .with(Routing.NOTIFICATION_NEW); + } + + /* rabbit config */ + + @Bean + public MessageConverter messageConverter() { + return jackson2Converter(); + } + + @Bean + public MappingJackson2MessageConverter jackson2Converter() { + ObjectMapper mapper = new ObjectMapper(); + + mapper.registerModule(new JavaTimeModule()); + mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + + return new MappingJackson2MessageConverter(); + } + + @Bean + public DefaultMessageHandlerMethodFactory myHandlerMethodFactory() { + DefaultMessageHandlerMethodFactory factory = new DefaultMessageHandlerMethodFactory(); + factory.setMessageConverter(jackson2Converter()); + return factory; + } + + @Bean + public RabbitMessagingTemplate rabbitMessagingTemplate(RabbitTemplate rabbitTemplate) { + RabbitMessagingTemplate rmt = new RabbitMessagingTemplate(rabbitTemplate); + rmt.setMessageConverter(this.jackson2Converter()); + return rmt; + } + + @Override + public void configureRabbitListeners(RabbitListenerEndpointRegistrar registrar) { + registrar.setMessageHandlerMethodFactory(myHandlerMethodFactory()); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/messaging/ValidationMessage.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/ValidationMessage.java new file mode 100644 index 000000000..573e5bcdb --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/ValidationMessage.java @@ -0,0 +1,17 @@ +package uk.ac.ebi.subs.ingest.messaging; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.core.Uuid; + +/** Created by rolando on 11/09/2017. */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ValidationMessage { + private EntityType entityType; + private Uuid uuid; + private Object content; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/ExportEntityMessage.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/ExportEntityMessage.java new file mode 100644 index 000000000..4d665daa7 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/ExportEntityMessage.java @@ -0,0 +1,23 @@ +package uk.ac.ebi.subs.ingest.messaging.model; + +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public class ExportEntityMessage { + private final String exportJobId; + private final String documentId; + private final String documentUuid; + private final String callbackLink; + private final String documentType; + private final String envelopeId; + private final String envelopeUuid; + private final String projectId; + private final String projectUuid; + private final int index; + private final int total; + private final Map context; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/ExportSubmissionMessage.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/ExportSubmissionMessage.java new file mode 100644 index 000000000..f204a013a --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/ExportSubmissionMessage.java @@ -0,0 +1,16 @@ +package uk.ac.ebi.subs.ingest.messaging.model; + +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public class ExportSubmissionMessage { + private final String exportJobId; + private final String submissionUuid; + private final String projectUuid; + private final String callbackLink; + private final Map context; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/ManifestMessage.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/ManifestMessage.java new file mode 100644 index 000000000..792c6699e --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/ManifestMessage.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.subs.ingest.messaging.model; + +import java.util.UUID; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public class ManifestMessage { + private final UUID bundleUuid; + private final String versionTimestamp; + + private final String documentId; + private final String documentUuid; + private final String callbackLink; + private final String documentType; + private final String envelopeId; + private final String envelopeUuid; + private final int index; + private final int total; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/MessageProtocol.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/MessageProtocol.java new file mode 100644 index 000000000..54a2a0a1a --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/MessageProtocol.java @@ -0,0 +1,6 @@ +package uk.ac.ebi.subs.ingest.messaging.model; + +public enum MessageProtocol { + AMQP, + HTTP +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/MetadataDocumentMessage.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/MetadataDocumentMessage.java new file mode 100644 index 000000000..a1c6dd0e8 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/MetadataDocumentMessage.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.subs.ingest.messaging.model; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import uk.ac.ebi.subs.ingest.state.ValidationState; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 12/09/17 + */ +@Getter +@AllArgsConstructor +public class MetadataDocumentMessage { + private final String documentType; + private final String documentId; + private final String documentUuid; + private final ValidationState validationState; + private final String callbackLink; + private final String envelopeId; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/SpreadsheetGenerationMessage.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/SpreadsheetGenerationMessage.java new file mode 100644 index 000000000..45382e27e --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/SpreadsheetGenerationMessage.java @@ -0,0 +1,16 @@ +package uk.ac.ebi.subs.ingest.messaging.model; + +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +@Getter +public class SpreadsheetGenerationMessage { + private final String exportJobId; + private final String submissionUuid; + private final String projectUuid; + private final String callbackLink; + private final Map context; +} diff --git a/src/main/java/org/humancellatlas/ingest/messaging/model/SubmissionEnvelopeMessage.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/SubmissionEnvelopeMessage.java similarity index 52% rename from src/main/java/org/humancellatlas/ingest/messaging/model/SubmissionEnvelopeMessage.java rename to src/main/java/uk/ac/ebi/subs/ingest/messaging/model/SubmissionEnvelopeMessage.java index 008ae52e4..515c82451 100644 --- a/src/main/java/org/humancellatlas/ingest/messaging/model/SubmissionEnvelopeMessage.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/SubmissionEnvelopeMessage.java @@ -1,4 +1,4 @@ -package org.humancellatlas.ingest.messaging.model; +package uk.ac.ebi.subs.ingest.messaging.model; import lombok.AllArgsConstructor; import lombok.Getter; @@ -10,8 +10,8 @@ @AllArgsConstructor @Getter public class SubmissionEnvelopeMessage { - private final String documentType; - private final String documentId; - private final String documentUuid; - private final String callbackLink; + private final String documentType; + private final String documentId; + private final String documentUuid; + private final String callbackLink; } diff --git a/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/SubmissionEnvelopeStateUpdateMessage.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/SubmissionEnvelopeStateUpdateMessage.java new file mode 100644 index 000000000..50113fc99 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/model/SubmissionEnvelopeStateUpdateMessage.java @@ -0,0 +1,26 @@ +package uk.ac.ebi.subs.ingest.messaging.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import lombok.Getter; +import lombok.Setter; +import uk.ac.ebi.subs.ingest.state.SubmissionState; + +public class SubmissionEnvelopeStateUpdateMessage extends SubmissionEnvelopeMessage { + @Getter @Setter private SubmissionState requestedState; + + public SubmissionEnvelopeStateUpdateMessage( + String documentType, String documentId, String documentUuid, String callbackLink) { + super(documentType, documentId, documentUuid, callbackLink); + } + + @JsonIgnore + public static SubmissionEnvelopeStateUpdateMessage fromSubmissionEnvelopeMessage( + SubmissionEnvelopeMessage message) { + return new SubmissionEnvelopeStateUpdateMessage( + message.getDocumentType(), + message.getDocumentId(), + message.getDocumentUuid(), + message.getCallbackLink()); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/messaging/web/MessagingController.java b/src/main/java/uk/ac/ebi/subs/ingest/messaging/web/MessagingController.java new file mode 100644 index 000000000..622ec946f --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/messaging/web/MessagingController.java @@ -0,0 +1,55 @@ +package uk.ac.ebi.subs.ingest.messaging.web; + +import org.springframework.hateoas.MediaTypes; +import org.springframework.hateoas.Resource; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.messaging.Constants; +import uk.ac.ebi.subs.ingest.messaging.Message; +import uk.ac.ebi.subs.ingest.messaging.MessageService; + +@RestController +@RequiredArgsConstructor +@Getter +public class MessagingController { + @NonNull private final MessageService messageService; + + @PostMapping( + path = "/messaging/fileUploadInfo", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaTypes.HAL_JSON_VALUE) + ResponseEntity> publishFileUploadInfo(@RequestBody ObjectNode uploadInfo) { + Message uploadInfoMessage = + new Message( + Constants.Exchanges.FILE_STAGED_EXCHANGE, + Constants.Queues.FILE_STAGED_QUEUE, + uploadInfo); + getMessageService().publish(uploadInfoMessage); + return new ResponseEntity<>(HttpStatus.OK); + } + + @PostMapping( + path = "/messaging/fileValidationResult", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaTypes.HAL_JSON_VALUE) + ResponseEntity> publishFileValidationResult( + @RequestBody ObjectNode validationResult) { + Message uploadInfoMessage = + new Message( + Constants.Exchanges.VALIDATION_EXCHANGE, + Constants.Queues.FILE_VALIDATION_QUEUE, + validationResult); + getMessageService().publish(uploadInfoMessage); + return new ResponseEntity<>(HttpStatus.OK); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/migrations/MongoChangeLog.java b/src/main/java/uk/ac/ebi/subs/ingest/migrations/MongoChangeLog.java new file mode 100644 index 000000000..b63182157 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/migrations/MongoChangeLog.java @@ -0,0 +1,281 @@ +package uk.ac.ebi.subs.ingest.migrations; + +import static com.mongodb.client.model.Filters.eq; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.function.Consumer; + +import org.bson.Document; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.github.mongobee.changeset.ChangeLog; +import com.github.mongobee.changeset.ChangeSet; +import com.mongodb.MongoCommandException; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Updates; + +@ChangeLog +public class MongoChangeLog { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoChangeLog.class); + + private static final Integer MONGO_INDEX_NOT_FOUND = 27; + + @ChangeSet( + order = "2019-10-30", + id = "featureCompatibilityVersion 3.4", + author = "alexie.staffer@ebi.ac.uk") + public void featureCompatibilityThreeFour(MongoDatabase db) { + if (MongoVersionHelper.featureCompatibilityLessThan(db, "3.4")) + db.runCommand(new Document("setFeatureCompatibilityVersion", "3.4")); + } + + @ChangeSet( + order = "2019-10-31", + id = "featureCompatibilityVersion 3.6", + author = "alexie.staffer@ebi.ac.uk") + public void featureCompatibilityThreeSix(MongoDatabase db) { + if (MongoVersionHelper.featureCompatibilityLessThan(db, "3.6")) + db.runCommand(new Document("setFeatureCompatibilityVersion", "3.6")); + } + + @ChangeSet( + order = "2019-11-01", + id = "featureCompatibilityVersion 4.0", + author = "alexie.staffer@ebi.ac.uk") + public void featureCompatibilityFourZero(MongoDatabase db) { + if (MongoVersionHelper.featureCompatibilityLessThan(db, "4.0")) { + db.runCommand(new Document("setFeatureCompatibilityVersion", "4.0")); + db.runCommand(new Document("setFreeMonitoring", 1).append("action", "disable")); + } + } + + @ChangeSet( + order = "2019-11-02", + id = "featureCompatibilityVersion 4.2", + author = "alexie.staffer@ebi.ac.uk") + public void featureCompatibilityFourTwo(MongoDatabase db) { + if (MongoVersionHelper.featureCompatibilityLessThan(db, "4.2")) + db.runCommand(new Document("setFeatureCompatibilityVersion", "4.2")); + } + + @ChangeSet( + order = "2019-11-03", + id = "singletonSubmissionEnvelope Biomaterial", + author = "alexie.staffer@ebi.ac.uk") + public void singletonSubmissionEnvelopeBiomaterial(MongoDatabase db) { + Document filter = Document.parse("{submissionEnvelopes: {$exists: 1}}"); + List update = new ArrayList<>(); + update.add( + new Document( + "$set", + Document.parse( + "{ submissionEnvelope: { $arrayElemAt: [ \"$submissionEnvelopes\", 0 ] } }"))); + update.add(new Document("$unset", "submissionEnvelopes")); + + db.getCollection("biomaterial").updateMany(filter, update); + } + + @ChangeSet( + order = "2019-11-04", + id = "singletonSubmissionEnvelope Process", + author = "alexie.staffer@ebi.ac.uk") + public void singletonSubmissionEnvelopeProcess(MongoDatabase db) { + Document filter = Document.parse("{submissionEnvelopes: {$exists: 1}}"); + List update = new ArrayList<>(); + update.add( + new Document( + "$set", + Document.parse( + "{ submissionEnvelope: { $arrayElemAt: [ \"$submissionEnvelopes\", 0 ] } }"))); + update.add(new Document("$unset", "submissionEnvelopes")); + + db.getCollection("process").updateMany(filter, update); + } + + @ChangeSet( + order = "2019-11-05", + id = "singletonSubmissionEnvelope Protocol", + author = "alexie.staffer@ebi.ac.uk") + public void singletonSubmissionEnvelopeProtocol(MongoDatabase db) { + Document filter = Document.parse("{submissionEnvelopes: {$exists: 1}}"); + List update = new ArrayList<>(); + update.add( + new Document( + "$set", + Document.parse( + "{ submissionEnvelope: { $arrayElemAt: [ \"$submissionEnvelopes\", 0 ] } }"))); + update.add(new Document("$unset", "submissionEnvelopes")); + + db.getCollection("protocol").updateMany(filter, update); + } + + @ChangeSet( + order = "2019-11-06", + id = "singletonSubmissionEnvelope File", + author = "alexie.staffer@ebi.ac.uk") + public void singletonSubmissionEnvelopeFile(MongoDatabase db) { + Document filter = Document.parse("{submissionEnvelopes: {$exists: 1}}"); + List update = new ArrayList<>(); + update.add( + new Document( + "$set", + Document.parse( + "{ submissionEnvelope: { $arrayElemAt: [ \"$submissionEnvelopes\", 0 ] } }"))); + update.add(new Document("$unset", "submissionEnvelopes")); + + db.getCollection("file").updateMany(filter, update); + } + + @ChangeSet( + order = "2019-11-07", + id = "singletonSubmissionEnvelope Project", + author = "alexie.staffer@ebi.ac.uk") + public void singletonSubmissionEnvelopeProject(MongoDatabase db) { + Document filter = Document.parse("{submissionEnvelopes: {$exists: 1}}"); + List update = new ArrayList<>(); + update.add( + new Document( + "$set", + Document.parse( + "{ submissionEnvelope: { $arrayElemAt: [ \"$submissionEnvelopes\", 0 ] } }"))); + + db.getCollection("project").updateMany(filter, update); + } + + @ChangeSet( + order = "2020-08-11", + id = "Drop Alias Index on archiveEntity", + author = "karoly@ebi.ac.uk") + public void dropAliasIndexOnArchiveEntity(MongoDatabase db) { + try { + db.getCollection("archiveEntity").dropIndex("alias"); + // If the collection does not exist this code will still succeed, + // Which is good because we may change the collection name soon. + } catch (MongoCommandException e) { + if (!MONGO_INDEX_NOT_FOUND.equals(e.getErrorCode())) throw e; + LOGGER.info(e.getErrorMessage()); + } + } + + @ChangeSet( + order = "2020-09-15", + id = "singelton project.dataAccess.type", + author = "alexie.staffer@ebi.ac.uk") + public void singeltonProjectDataAccessType(MongoDatabase db) { + Document filter = Document.parse("{'dataAccess.type': {$type: 'array'}}"); + List update = new ArrayList<>(); + update.add( + new Document( + "$set", + Document.parse("{ 'dataAccess.type': { $arrayElemAt: [ '$dataAccess.type', 0 ] } }"))); + db.getCollection("project").updateMany(filter, update); + } + + @ChangeSet( + order = "2021-05-20", + id = "set default publications info", + author = "alexie.staffer@ebi.ac.uk") + public void setDefaultPublicationsInfo(MongoDatabase db) { + Document filter = Document.parse("{ }"); + List update = new ArrayList<>(); + update.add(new Document("$set", Document.parse("{ 'publicationsInfo': [] }"))); + db.getCollection("project").updateMany(filter, update); + } + + @ChangeSet( + order = "2021-05-27", + id = "Set default isInCatalogue", + author = "alexie.staffer@ebi.ac.uk") + public void setDefaultIsInCatalogue(MongoDatabase db) { + Document filter = Document.parse("{ }"); + List update = new ArrayList<>(); + update.add(new Document("$unset", "publishedToCatalogue")); + update.add(new Document("$set", Document.parse("{ 'isInCatalogue': false }"))); + db.getCollection("project").updateMany(filter, update); + } + + @ChangeSet(order = "2021-07-16", id = "Add index to project", author = "jcbwndsr@ebi.ac.uk") + public void addIndexToProject(MongoDatabase db) { + Document indexQuery = + Document.parse( + "{" + + "'content.project_core.project_title': 'text'," + + "'content.project_core.project_short_name': 'text'," + + "'content.project_core.project_description': 'text'," + + "'content.publications.authors': 'text'," + + "'content.publications.title': 'text'," + + "'content.publications.doi': 'text'," + + "'content.contributors.name': 'text'," + + "'content.insdc_project_accessions': 'text'," + + "'content.ega_accessions': 'text'," + + "'content.dbgap_accessions': 'text'," + + "'content.geo_series_accessions': 'text'," + + "'content.array_express_accessions': 'text'," + + "'content.insdc_study_accessions': 'text'," + + "'content.biostudies_accessions': 'text'," + + "'technology.ontologies.ontology': 'text'," + + "'technology.ontologies.ontology_label': 'text'," + + "'organ.ontologies.ontology': 'text'," + + "'organ.ontologies.ontology_label': 'text'," + + "}"); + db.getCollection("project").createIndex(indexQuery); + } + + @ChangeSet( + order = "2021-11-22", + id = "Set empty graphValidationErrors", + author = "jcbwndsr@ebi.ac.uk") + public void setGraphValidationErrors(MongoDatabase db) { + Document filter = Document.parse("{ }"); + List update = new ArrayList<>(); + update.add(new Document("$set", Document.parse("{ 'graphValidationErrors': [] }"))); + db.getCollection("biomaterial").updateMany(filter, update); + db.getCollection("process").updateMany(filter, update); + db.getCollection("protocol").updateMany(filter, update); + db.getCollection("file").updateMany(filter, update); + } + + @ChangeSet(order = "2021-12-07", id = "Rename submission states", author = "jcbwndsr@ebi.ac.uk") + public void renameSubmissionStates(MongoDatabase db) { + Document filter = Document.parse("{ 'submissionState': 'VALID' }"); + List update = new ArrayList<>(); + update.add(new Document("$set", Document.parse("{ 'submissionState': 'METADATA_VALID' }"))); + db.getCollection("submissionEnvelope").updateMany(filter, update); + + filter = Document.parse("{ 'submissionState': 'VALIDATING' }"); + update = new ArrayList<>(); + update.add( + new Document("$set", Document.parse("{ 'submissionState': 'METADATA_VALIDATING' }"))); + db.getCollection("submissionEnvelope").updateMany(filter, update); + + filter = Document.parse("{ 'submissionState': 'INVALID' }"); + update = new ArrayList<>(); + update.add(new Document("$set", Document.parse("{ 'submissionState': 'METADATA_INVALID' }"))); + db.getCollection("submissionEnvelope").updateMany(filter, update); + } + + @ChangeSet( + order = "2022-05-06", + id = "add missing dataFileUuid for File documents with a unique uuid. dcp-764", + author = "amnon@ebi.ac.uk") + public void addMissingDataFileUuidToFiles(MongoDatabase db) { + MongoCollection files = db.getCollection("file"); + files + .find(eq("dataFileUuid", null)) + .forEach( + (Consumer) + (Document file) -> + files.updateOne( + eq("_id", file.get("_id")), + Updates.set("dataFileUuid", UUID.randomUUID()))); + } + + public void addSubmissionEnvelopeIndexToProcess(MongoDatabase db) { + db.getCollection("process").createIndex(Document.parse("{ \"submissionEnvelope\": 1 }")); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/migrations/MongoVersionHelper.java b/src/main/java/uk/ac/ebi/subs/ingest/migrations/MongoVersionHelper.java new file mode 100644 index 000000000..d6fa95546 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/migrations/MongoVersionHelper.java @@ -0,0 +1,64 @@ +package uk.ac.ebi.subs.ingest.migrations; + +import java.util.ArrayList; +import java.util.List; + +import org.bson.Document; + +import com.mongodb.client.MongoDatabase; +import com.mongodb.connection.ServerVersion; + +class MongoVersionHelper { + private static ServerVersion getVersionFromString(String version) { + List numberList = new ArrayList<>(); + for (String number : version.split("\\.")) { + numberList.add(Integer.parseInt(number)); + } + while (numberList.size() < 3) { + numberList.add(0); + } + return new ServerVersion(numberList); + } + + private static ServerVersion getMajorMinor(ServerVersion version) { + List list = new ArrayList<>(); + list.add(version.getVersionList().get(0)); + list.add(version.getVersionList().get(1)); + list.add(0); + return new ServerVersion(list); + } + + private static String getMajorMinorString(ServerVersion version) { + String major = version.getVersionList().get(0).toString(); + String minor = version.getVersionList().get(1).toString(); + return major + "." + minor; + } + + static ServerVersion getFeatureCompatibilityVersion(MongoDatabase db) { + Document response = + db.runCommand(new Document("getParameter", 1).append("featureCompatibilityVersion", 1)); + if (response.containsKey("ok") && response.containsKey("featureCompatibilityVersion")) { + if (getServerVersion(db).compareTo(getVersionFromString("3.6")) < 0) + return getVersionFromString(response.getString("featureCompatibilityVersion")); + else { + Document featureCompatibilityVersion = + response.get("featureCompatibilityVersion", Document.class); + if (featureCompatibilityVersion.containsKey("version")) + return getVersionFromString(featureCompatibilityVersion.getString("version")); + } + } + throw new UnsupportedOperationException("Could not retrieve featureCompatibilityVersion."); + } + + static ServerVersion getServerVersion(MongoDatabase db) { + Document server_doc = db.runCommand(new Document("buildinfo", 1)); + if (server_doc.containsKey("ok") && server_doc.containsKey("version")) { + return getVersionFromString(server_doc.getString("version")); + } + throw new UnsupportedOperationException("Could not retrieve server version."); + } + + static Boolean featureCompatibilityLessThan(MongoDatabase db, String version) { + return (getFeatureCompatibilityVersion(db).compareTo(getVersionFromString(version)) < 0); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/notifications/NotificationConfiguration.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/NotificationConfiguration.java new file mode 100644 index 000000000..bb6c508db --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/NotificationConfiguration.java @@ -0,0 +1,89 @@ +package uk.ac.ebi.subs.ingest.notifications; + +import java.util.Collection; +import java.util.Collections; + +import org.springframework.amqp.rabbit.core.RabbitMessagingTemplate; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import uk.ac.ebi.subs.ingest.notifications.NotificationConfiguration.NotificationProperties; +import uk.ac.ebi.subs.ingest.notifications.NotificationConfiguration.NotificationProperties.AmqpProperties; +import uk.ac.ebi.subs.ingest.notifications.NotificationConfiguration.NotificationProperties.SmtpProperties; +import uk.ac.ebi.subs.ingest.notifications.processors.NotificationProcessor; +import uk.ac.ebi.subs.ingest.notifications.processors.impl.email.EmailNotificationProcessor; +import uk.ac.ebi.subs.ingest.notifications.processors.impl.email.SMTPConfig; +import uk.ac.ebi.subs.ingest.notifications.sources.NotificationSource; +import uk.ac.ebi.subs.ingest.notifications.sources.impl.rabbit.AmqpConfig; +import uk.ac.ebi.subs.ingest.notifications.sources.impl.rabbit.RabbitNotificationSource; + +@Configuration +@EnableConfigurationProperties({ + NotificationProperties.class, + SmtpProperties.class, + AmqpProperties.class +}) +public class NotificationConfiguration { + + @Bean + public SMTPConfig smtpConfig(SmtpProperties smtpEnvVars) { + return SMTPConfig.builder() + .host(smtpEnvVars.getHost()) + .port(Integer.parseInt(smtpEnvVars.getPort())) + .username(smtpEnvVars.getUsername()) + .password(smtpEnvVars.getPassword()) + .build(); + } + + @Bean + public AmqpConfig amqpConfig(AmqpProperties amqpEnvVars) { + return AmqpConfig.builder() + .sendExchange(amqpEnvVars.getSendExchange()) + .sendRoutingKey(amqpEnvVars.getSendRoutingKey()) + .build(); + } + + @Bean + public Collection notificationProcessors(SMTPConfig smtpConfig) { + EmailNotificationProcessor emailNotificationProcessor = + new EmailNotificationProcessor(smtpConfig); + return Collections.singletonList(emailNotificationProcessor); + } + + @Bean + public NotificationSource notificationSource( + RabbitMessagingTemplate rabbitMessagingTemplate, AmqpConfig amqpConfig) { + return new RabbitNotificationSource(rabbitMessagingTemplate, amqpConfig); + } + + @ConfigurationProperties(prefix = "notifications") + class NotificationProperties { + + @ConfigurationProperties(prefix = "notifications.smtp") + @NoArgsConstructor + @Getter + @Setter + class SmtpProperties { + + private String host = "localhost"; + private String port = "587"; + private String username = "provide username"; + private String password = "provide password"; + } + + @ConfigurationProperties(prefix = "notifications.amqp") + @NoArgsConstructor + @Getter + @Setter + class AmqpProperties { + + private String sendExchange = "provide notifications send exchange"; + private String sendRoutingKey = "provide notifications routing key"; + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/notifications/NotificationCoordinator.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/NotificationCoordinator.java new file mode 100644 index 000000000..3e1dc8bed --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/NotificationCoordinator.java @@ -0,0 +1,123 @@ +package uk.ac.ebi.subs.ingest.notifications; + +import java.util.Collection; +import java.util.Collections; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import lombok.AccessLevel; +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.notifications.exception.ProcessingException; +import uk.ac.ebi.subs.ingest.notifications.model.Notification; +import uk.ac.ebi.subs.ingest.notifications.model.NotificationState; +import uk.ac.ebi.subs.ingest.notifications.processors.NotificationProcessor; +import uk.ac.ebi.subs.ingest.notifications.sources.NotificationSource; + +@RequiredArgsConstructor +@Component +public class NotificationCoordinator { + + private final @NonNull Collection notificationProcessors; + private final @NonNull NotificationSource notificationSource; + private final @NonNull NotificationService notificationService; + + private final Logger log = LoggerFactory.getLogger(getClass()); + + public void queue() { + this.notificationService + .getUnhandledNotifications() + .forEach( + notification -> { + this.notificationService.changeState(notification, NotificationState.QUEUED); + this.notificationSource.supply(Collections.singletonList(notification)); + }); + } + + public void process() { + this.notificationSource.stream() + .forEach( + notification -> { + Notification processingNotification = + this.notificationService.changeState(notification, NotificationState.PROCESSING); + + this.processNotification(processingNotification) + .filter(report -> !report.isSuccessful()) + .findAny() + .ifPresentOrElse( + failedReport -> + this.notificationService.changeState( + processingNotification, NotificationState.FAILED), + () -> + this.notificationService.changeState( + processingNotification, NotificationState.PROCESSED)); + }); + } + + public void cleanup() { + this.notificationService + .getHandledNotifications() + .forEach(notificationService::deleteNotification); + } + + private Stream processNotification(Notification notification) { + Stream processers = this.notificationProcessors.stream(); + + return processers + .filter(notificationProcessor -> notificationProcessor.isEligible(notification)) + .map( + notificationProcessor -> { + try { + notificationProcessor.handle(notification); + return NotificationProcessReport.successReport(notification); + } catch (ProcessingException e) { + log.warn( + String.format( + "Notification processor failed for %s on notification with ID %s", + notificationProcessor.getClass(), notification.getId()), + e); + return NotificationProcessReport.failureReport(notification); + } + }); + } + + @Scheduled(fixedDelay = 20000) + private void scheduledQueue() { + this.queue(); + } + + @Scheduled(fixedDelay = 60000) + private void scheduledProcess() { + this.process(); + } + + @Scheduled(fixedDelay = 300000) + private void scheduledCleanup() { + this.cleanup(); + } + + @Getter + @RequiredArgsConstructor(access = AccessLevel.PRIVATE) + private static class NotificationProcessReport { + + private final Notification notification; + private final NotificationState result; + + public static NotificationProcessReport successReport(Notification notification) { + return new NotificationProcessReport(notification, NotificationState.PROCESSED); + } + + public static NotificationProcessReport failureReport(Notification notification) { + return new NotificationProcessReport(notification, NotificationState.FAILED); + } + + public boolean isSuccessful() { + return this.result.equals(NotificationState.PROCESSED); + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/notifications/NotificationRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/NotificationRepository.java new file mode 100644 index 000000000..ac0bd561f --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/NotificationRepository.java @@ -0,0 +1,29 @@ +package uk.ac.ebi.subs.ingest.notifications; + +import java.util.Optional; +import java.util.stream.Stream; + +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.rest.core.annotation.RestResource; + +import uk.ac.ebi.subs.ingest.notifications.model.Checksum; +import uk.ac.ebi.subs.ingest.notifications.model.Notification; +import uk.ac.ebi.subs.ingest.notifications.model.NotificationState; + +public interface NotificationRepository extends MongoRepository { + + @RestResource(exported = false) + S save(S notification); + + @RestResource(exported = false) + void delete(Notification notification); + + @RestResource(exported = false) + Stream findByStateOrderByNotifyAtDesc(NotificationState state); + + @RestResource(rel = "findByChecksumValue") + Optional findByChecksum_Value(String checksumValue); + + @RestResource(exported = false) + Optional findByChecksum(Checksum checksumValue); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/notifications/NotificationService.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/NotificationService.java new file mode 100644 index 000000000..29f7b8d25 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/NotificationService.java @@ -0,0 +1,78 @@ +package uk.ac.ebi.subs.ingest.notifications; + +import java.util.Optional; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Component; + +import lombok.AllArgsConstructor; +import uk.ac.ebi.subs.ingest.notifications.exception.DuplicateNotification; +import uk.ac.ebi.subs.ingest.notifications.model.Checksum; +import uk.ac.ebi.subs.ingest.notifications.model.Notification; +import uk.ac.ebi.subs.ingest.notifications.model.NotificationRequest; +import uk.ac.ebi.subs.ingest.notifications.model.NotificationState; + +@Component +@AllArgsConstructor +public class NotificationService { + + private final NotificationRepository notificationRepository; + private final Logger log = LoggerFactory.getLogger(getClass()); + + public Notification createNotification(NotificationRequest notificationRequest) { + try { + Notification notification = + Notification.buildNew() + .content(notificationRequest.getContent()) + .metadata(notificationRequest.getMetadata()) + .checksum(notificationRequest.getChecksum()) + .build(); + + return this.notificationRepository.save(notification); + } catch (DuplicateKeyException e) { + String checksumValue = notificationRequest.getChecksum().getValue(); + String id = + this.notificationRepository + .findByChecksum_Value(checksumValue) + .orElseThrow( + () -> { + throw new RuntimeException(e); + }) + .getId(); + + throw new DuplicateNotification( + String.format("Notification checksum value already exists in notification %s", id)); + } + } + + public Optional retrieveForChecksum(Checksum checksum) { + return this.notificationRepository.findByChecksum(checksum); + } + + public Notification changeState(Notification notification, NotificationState toState) { + if (notification.getState().isLegalTransition(toState)) { + notification.setState(toState); + return this.notificationRepository.save(notification); + } else { + throw new IllegalStateException( + String.format( + "Cannot transition notification with ID %s from state %s to %s", + notification.getId(), notification.getState(), toState)); + } + } + + public Stream getUnhandledNotifications() { + return notificationRepository.findByStateOrderByNotifyAtDesc(NotificationState.PENDING); + } + + public Stream getHandledNotifications() { + return notificationRepository.findByStateOrderByNotifyAtDesc(NotificationState.PROCESSED); + } + + public void deleteNotification(Notification notification) { + notificationRepository.delete(notification); + } +} diff --git a/src/main/java/org/humancellatlas/ingest/notifications/exception/DuplicateNotification.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/exception/DuplicateNotification.java similarity index 70% rename from src/main/java/org/humancellatlas/ingest/notifications/exception/DuplicateNotification.java rename to src/main/java/uk/ac/ebi/subs/ingest/notifications/exception/DuplicateNotification.java index 3f8e45860..c1ec2134e 100644 --- a/src/main/java/org/humancellatlas/ingest/notifications/exception/DuplicateNotification.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/exception/DuplicateNotification.java @@ -1,4 +1,4 @@ -package org.humancellatlas.ingest.notifications.exception; +package uk.ac.ebi.subs.ingest.notifications.exception; public class DuplicateNotification extends RuntimeException { public DuplicateNotification(String message) { diff --git a/src/main/java/org/humancellatlas/ingest/notifications/exception/ProcessingException.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/exception/ProcessingException.java similarity index 76% rename from src/main/java/org/humancellatlas/ingest/notifications/exception/ProcessingException.java rename to src/main/java/uk/ac/ebi/subs/ingest/notifications/exception/ProcessingException.java index 9ed36d0d5..36f7125f8 100644 --- a/src/main/java/org/humancellatlas/ingest/notifications/exception/ProcessingException.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/exception/ProcessingException.java @@ -1,4 +1,4 @@ -package org.humancellatlas.ingest.notifications.exception; +package uk.ac.ebi.subs.ingest.notifications.exception; public class ProcessingException extends RuntimeException { public ProcessingException(String message) { diff --git a/src/main/java/org/humancellatlas/ingest/notifications/model/Checksum.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/model/Checksum.java similarity index 80% rename from src/main/java/org/humancellatlas/ingest/notifications/model/Checksum.java rename to src/main/java/uk/ac/ebi/subs/ingest/notifications/model/Checksum.java index e125f3ca0..6c0510004 100644 --- a/src/main/java/org/humancellatlas/ingest/notifications/model/Checksum.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/model/Checksum.java @@ -1,4 +1,4 @@ -package org.humancellatlas.ingest.notifications.model; +package uk.ac.ebi.subs.ingest.notifications.model; import lombok.Data; diff --git a/src/main/java/org/humancellatlas/ingest/notifications/model/Notification.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/model/Notification.java similarity index 50% rename from src/main/java/org/humancellatlas/ingest/notifications/model/Notification.java rename to src/main/java/uk/ac/ebi/subs/ingest/notifications/model/Notification.java index c4308221c..6eaafaf28 100644 --- a/src/main/java/org/humancellatlas/ingest/notifications/model/Notification.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/model/Notification.java @@ -1,36 +1,30 @@ -package org.humancellatlas.ingest.notifications.model; +package uk.ac.ebi.subs.ingest.notifications.model; import java.time.Instant; import java.util.Map; -import lombok.Builder; -import lombok.Data; -import lombok.NonNull; + import org.springframework.data.annotation.Id; import org.springframework.data.mongodb.core.index.Indexed; import org.springframework.data.mongodb.core.mapping.Document; import org.springframework.hateoas.Identifiable; +import lombok.Builder; +import lombok.Data; +import lombok.NonNull; + @Data @Builder @Document public class Notification implements Identifiable { - @Id - private String id; - @NonNull - private final String content; - @NonNull - private final Map metadata; - @NonNull - private final Instant notifyAt; - @NonNull @Indexed - private NotificationState state; - @NonNull @Indexed - private final Checksum checksum; + @Id private String id; + @NonNull private final String content; + @NonNull private final Map metadata; + @NonNull private final Instant notifyAt; + @NonNull @Indexed private NotificationState state; + @NonNull @Indexed private final Checksum checksum; public static NotificationBuilder buildNew() { - return Notification.builder() - .state(NotificationState.PENDING) - .notifyAt(Instant.now()); + return Notification.builder().state(NotificationState.PENDING).notifyAt(Instant.now()); } } diff --git a/src/main/java/org/humancellatlas/ingest/notifications/model/NotificationRequest.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/model/NotificationRequest.java similarity index 78% rename from src/main/java/org/humancellatlas/ingest/notifications/model/NotificationRequest.java rename to src/main/java/uk/ac/ebi/subs/ingest/notifications/model/NotificationRequest.java index 354d8994d..3272ba3f6 100644 --- a/src/main/java/org/humancellatlas/ingest/notifications/model/NotificationRequest.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/model/NotificationRequest.java @@ -1,6 +1,7 @@ -package org.humancellatlas.ingest.notifications.model; +package uk.ac.ebi.subs.ingest.notifications.model; import java.util.Map; + import lombok.Value; @Value diff --git a/src/main/java/org/humancellatlas/ingest/notifications/model/NotificationState.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/model/NotificationState.java similarity index 93% rename from src/main/java/org/humancellatlas/ingest/notifications/model/NotificationState.java rename to src/main/java/uk/ac/ebi/subs/ingest/notifications/model/NotificationState.java index 10708b288..88836690b 100644 --- a/src/main/java/org/humancellatlas/ingest/notifications/model/NotificationState.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/model/NotificationState.java @@ -1,4 +1,5 @@ -package org.humancellatlas.ingest.notifications.model; +package uk.ac.ebi.subs.ingest.notifications.model; + import java.util.List; public enum NotificationState { diff --git a/src/main/java/org/humancellatlas/ingest/notifications/processors/NotificationProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/processors/NotificationProcessor.java similarity index 52% rename from src/main/java/org/humancellatlas/ingest/notifications/processors/NotificationProcessor.java rename to src/main/java/uk/ac/ebi/subs/ingest/notifications/processors/NotificationProcessor.java index 06214cd56..1715a65d5 100644 --- a/src/main/java/org/humancellatlas/ingest/notifications/processors/NotificationProcessor.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/processors/NotificationProcessor.java @@ -1,6 +1,6 @@ -package org.humancellatlas.ingest.notifications.processors; +package uk.ac.ebi.subs.ingest.notifications.processors; -import org.humancellatlas.ingest.notifications.model.Notification; +import uk.ac.ebi.subs.ingest.notifications.model.Notification; public interface NotificationProcessor { diff --git a/src/main/java/org/humancellatlas/ingest/notifications/processors/impl/email/EmailMetadata.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/processors/impl/email/EmailMetadata.java similarity index 78% rename from src/main/java/org/humancellatlas/ingest/notifications/processors/impl/email/EmailMetadata.java rename to src/main/java/uk/ac/ebi/subs/ingest/notifications/processors/impl/email/EmailMetadata.java index 20a6e7bd3..2284e0e95 100644 --- a/src/main/java/org/humancellatlas/ingest/notifications/processors/impl/email/EmailMetadata.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/processors/impl/email/EmailMetadata.java @@ -1,4 +1,4 @@ -package org.humancellatlas.ingest.notifications.processors.impl.email; +package uk.ac.ebi.subs.ingest.notifications.processors.impl.email; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/src/main/java/org/humancellatlas/ingest/notifications/processors/impl/email/EmailNotificationProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/processors/impl/email/EmailNotificationProcessor.java similarity index 72% rename from src/main/java/org/humancellatlas/ingest/notifications/processors/impl/email/EmailNotificationProcessor.java rename to src/main/java/uk/ac/ebi/subs/ingest/notifications/processors/impl/email/EmailNotificationProcessor.java index 5770859d8..e1151d8e3 100644 --- a/src/main/java/org/humancellatlas/ingest/notifications/processors/impl/email/EmailNotificationProcessor.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/processors/impl/email/EmailNotificationProcessor.java @@ -1,15 +1,18 @@ -package org.humancellatlas.ingest.notifications.processors.impl.email; +package uk.ac.ebi.subs.ingest.notifications.processors.impl.email; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Optional; import java.util.Properties; -import org.humancellatlas.ingest.notifications.model.Notification; -import org.humancellatlas.ingest.notifications.processors.NotificationProcessor; -import org.humancellatlas.ingest.notifications.exception.ProcessingException; + import org.springframework.mail.MailSender; import org.springframework.mail.SimpleMailMessage; import org.springframework.mail.javamail.JavaMailSenderImpl; +import com.fasterxml.jackson.databind.ObjectMapper; + +import uk.ac.ebi.subs.ingest.notifications.exception.ProcessingException; +import uk.ac.ebi.subs.ingest.notifications.model.Notification; +import uk.ac.ebi.subs.ingest.notifications.processors.NotificationProcessor; + public class EmailNotificationProcessor implements NotificationProcessor { private final SMTPConfig smtpConfig; private final MailSender mailSender; @@ -22,8 +25,8 @@ public EmailNotificationProcessor(SMTPConfig smtpConfig) { @Override public boolean isEligible(Notification notification) { return Optional.ofNullable(notification.getMetadata()) - .map(metadata -> metadata.containsKey("email")) - .orElse(false); + .map(metadata -> metadata.containsKey("email")) + .orElse(false); } @Override @@ -50,11 +53,13 @@ private static SimpleMailMessage messageFrom(Notification notification) { private static EmailMetadata parseEmailMetadata(Notification notification) { return Optional.ofNullable(notification.getMetadata()) - .map(metadata -> metadata.get("email")) - .map(emailMetadata -> new ObjectMapper().convertValue(emailMetadata, EmailMetadata.class)) - .orElseThrow(() -> { - throw new ProcessingException(String.format("Email metadata empty for notification %s", notification.getId())); - }); + .map(metadata -> metadata.get("email")) + .map(emailMetadata -> new ObjectMapper().convertValue(emailMetadata, EmailMetadata.class)) + .orElseThrow( + () -> { + throw new ProcessingException( + String.format("Email metadata empty for notification %s", notification.getId())); + }); } private static MailSender createMailClient(SMTPConfig smtpConfig) { @@ -70,7 +75,7 @@ private static MailSender createMailClient(SMTPConfig smtpConfig) { props.setProperty("mail.host", "outgoing.ebi.ac.uk"); props.put("mail.smtp.auth", "true"); props.put("mail.smtp.port", "587"); - props.put("mail.smtp.starttls.enable", "true"); //TLS + props.put("mail.smtp.starttls.enable", "true"); // TLS mailSender.setJavaMailProperties(props); diff --git a/src/main/java/org/humancellatlas/ingest/notifications/processors/impl/email/SMTPConfig.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/processors/impl/email/SMTPConfig.java similarity index 74% rename from src/main/java/org/humancellatlas/ingest/notifications/processors/impl/email/SMTPConfig.java rename to src/main/java/uk/ac/ebi/subs/ingest/notifications/processors/impl/email/SMTPConfig.java index 771e8d2c3..6528ccf45 100644 --- a/src/main/java/org/humancellatlas/ingest/notifications/processors/impl/email/SMTPConfig.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/processors/impl/email/SMTPConfig.java @@ -1,4 +1,4 @@ -package org.humancellatlas.ingest.notifications.processors.impl.email; +package uk.ac.ebi.subs.ingest.notifications.processors.impl.email; import lombok.Builder; import lombok.Data; diff --git a/src/main/java/uk/ac/ebi/subs/ingest/notifications/sources/NotificationSource.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/sources/NotificationSource.java new file mode 100644 index 000000000..a00315776 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/sources/NotificationSource.java @@ -0,0 +1,13 @@ +package uk.ac.ebi.subs.ingest.notifications.sources; + +import java.util.List; +import java.util.stream.Stream; + +import uk.ac.ebi.subs.ingest.notifications.model.Notification; + +public interface NotificationSource { + + Stream stream(); + + void supply(List notifications); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/notifications/sources/impl/inmemory/InmemoryNotificationSource.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/sources/impl/inmemory/InmemoryNotificationSource.java new file mode 100644 index 000000000..35f9763b6 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/sources/impl/inmemory/InmemoryNotificationSource.java @@ -0,0 +1,36 @@ +package uk.ac.ebi.subs.ingest.notifications.sources.impl.inmemory; + +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Stream; + +import uk.ac.ebi.subs.ingest.notifications.model.Notification; +import uk.ac.ebi.subs.ingest.notifications.sources.NotificationSource; + +public class InmemoryNotificationSource implements NotificationSource { + + private final Queue queue = new ConcurrentLinkedQueue<>(); + + @Override + public Stream stream() { + return Stream.generate( + () -> { + try { + return queue.remove(); + } catch (NoSuchElementException e) { + return null; + } + }) + .takeWhile(Objects::nonNull); + } + + // ignore IDE suggestion to replace this.queue::add with addAll(); addAll isn't thread safe for + // this particular queue implementation (ConcurrentLinkedQueue) + @Override + public void supply(List notifications) { + notifications.forEach(this.queue::add); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/notifications/sources/impl/rabbit/AmqpConfig.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/sources/impl/rabbit/AmqpConfig.java new file mode 100644 index 000000000..9eb31d1c3 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/sources/impl/rabbit/AmqpConfig.java @@ -0,0 +1,12 @@ +package uk.ac.ebi.subs.ingest.notifications.sources.impl.rabbit; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class AmqpConfig { + + private final String sendExchange; + private final String sendRoutingKey; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/notifications/sources/impl/rabbit/RabbitNotificationSource.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/sources/impl/rabbit/RabbitNotificationSource.java new file mode 100644 index 000000000..b527724bc --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/sources/impl/rabbit/RabbitNotificationSource.java @@ -0,0 +1,70 @@ +package uk.ac.ebi.subs.ingest.notifications.sources.impl.rabbit; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.amqp.rabbit.core.RabbitMessagingTemplate; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.messaging.Constants; +import uk.ac.ebi.subs.ingest.notifications.model.Notification; +import uk.ac.ebi.subs.ingest.notifications.sources.NotificationSource; +import uk.ac.ebi.subs.ingest.notifications.sources.impl.inmemory.InmemoryNotificationSource; + +@Component +@RequiredArgsConstructor +public class RabbitNotificationSource implements NotificationSource { + + private final InmemoryNotificationSource inmemoryNotificationSource = + new InmemoryNotificationSource(); + private final RabbitMessagingTemplate rabbitMessagingTemplate; + private final AmqpConfig amqpConfig; + + private static String jsonString(Notification notification) { + try { + return new ObjectMapper() + .registerModules(new JavaTimeModule()) + .writeValueAsString(notification); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static Notification fromJsonString(String notification) { + try { + return new ObjectMapper() + .registerModules(new JavaTimeModule()) + .readValue(notification, Notification.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @RabbitListener(queues = Constants.Queues.NOTIFICATIONS_QUEUE) + private void listen(String notification) { + this.inmemoryNotificationSource.supply(Collections.singletonList(fromJsonString(notification))); + } + + @Override + public Stream stream() { + return this.inmemoryNotificationSource.stream(); + } + + @Override + public void supply(List notifications) { + notifications.forEach( + notification -> { + this.rabbitMessagingTemplate.convertAndSend( + amqpConfig.getSendExchange(), + amqpConfig.getSendRoutingKey(), + jsonString(notification)); + }); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/notifications/web/NotificationController.java b/src/main/java/uk/ac/ebi/subs/ingest/notifications/web/NotificationController.java new file mode 100644 index 000000000..91f644e0f --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/notifications/web/NotificationController.java @@ -0,0 +1,12 @@ +package uk.ac.ebi.subs.ingest.notifications.web; + +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.hateoas.ExposesResourceFor; + +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.notifications.model.Notification; + +@RepositoryRestController +@ExposesResourceFor(Notification.class) +@RequiredArgsConstructor +public class NotificationController {} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/patch/JsonPatcher.java b/src/main/java/uk/ac/ebi/subs/ingest/patch/JsonPatcher.java new file mode 100644 index 000000000..404b30a85 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/patch/JsonPatcher.java @@ -0,0 +1,35 @@ +package uk.ac.ebi.subs.ingest.patch; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mapping.context.PersistentEntities; +import org.springframework.data.rest.webmvc.json.DomainObjectReader; +import org.springframework.data.rest.webmvc.mapping.Associations; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** Utility class mainly for applying patches to domain objects. */ +@Component +public class JsonPatcher { + + private final DomainObjectReader domainObjectReader; + + private final ObjectMapper objectMapper; + + @Autowired + public JsonPatcher( + PersistentEntities persistentEntities, Associations associations, ObjectMapper objectMapper) { + this.domainObjectReader = new DomainObjectReader(persistentEntities, associations); + this.objectMapper = objectMapper; + } + + /* + Almost the same exact implementation used in {@link org.springframework.data.rest.webmvc.config.JsonPatchHandler} + to merge JSON documents for Spring Data REST. It's copied here because the patch code that Spring uses was made + internal to the framework. + */ + public T merge(ObjectNode patch, T target) { + return domainObjectReader.merge(patch, target, objectMapper); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/patch/Patch.java b/src/main/java/uk/ac/ebi/subs/ingest/patch/Patch.java new file mode 100644 index 000000000..f4e117209 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/patch/Patch.java @@ -0,0 +1,30 @@ +package uk.ac.ebi.subs.ingest.patch; + +import java.util.Map; + +import org.springframework.data.mongodb.core.mapping.DBRef; +import org.springframework.data.mongodb.core.mapping.Document; + +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import uk.ac.ebi.subs.ingest.core.AbstractEntity; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Data +@AllArgsConstructor +@Document +@EqualsAndHashCode(callSuper = true) +public class Patch extends AbstractEntity { + private Map jsonPatch; + private @DBRef SubmissionEnvelope submissionEnvelope; + + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "@class") + private @DBRef T originalDocument; + + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "@class") + private @DBRef T updateDocument; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/patch/PatchRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/patch/PatchRepository.java new file mode 100644 index 000000000..4eb13716b --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/patch/PatchRepository.java @@ -0,0 +1,27 @@ +package uk.ac.ebi.subs.ingest.patch; + +import org.bson.types.ObjectId; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; +import org.springframework.data.rest.core.annotation.RestResource; +import org.springframework.web.bind.annotation.CrossOrigin; + +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@CrossOrigin +public interface PatchRepository extends MongoRepository { + + @RestResource(path = "updatedocument", rel = "WithUpdateDocument") + @Query("{ 'updateDocument.$id': ?0 }") + Patch findByUpdateDocumentId(ObjectId id); + + @RestResource(path = "submissionEnvelope", rel = "WithSubmissionEnvelope") + @Query("{ 'submissionEnvelope.id': ?0 }") + Page> findBySubmissionEnvelopeId(String id, Pageable pageable); + + @RestResource(exported = false) + Long deleteBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/patch/PatchService.java b/src/main/java/uk/ac/ebi/subs/ingest/patch/PatchService.java new file mode 100644 index 000000000..1f57433a6 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/patch/PatchService.java @@ -0,0 +1,36 @@ +package uk.ac.ebi.subs.ingest.patch; + +import java.util.Map; + +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.core.JsonPatch; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.core.service.MetadataDifferService; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Service +@AllArgsConstructor +@Getter +public class PatchService { + private final @NonNull PatchRepository patchRepository; + private final @NonNull MetadataDifferService metadataDifferService; + + public Patch storePatch( + T originalDocument, T updateDocument, SubmissionEnvelope submissionEnvelope) { + JsonPatch patch = metadataDifferService.generatePatch(originalDocument, updateDocument); + Patch patchDocument = + new Patch<>( + new ObjectMapper().convertValue(patch, Map.class), + submissionEnvelope, + originalDocument, + updateDocument); + Patch savedPatch = patchRepository.save(patchDocument); + return savedPatch; + } +} diff --git a/src/main/java/org/humancellatlas/ingest/patch/web/PatchController.java b/src/main/java/uk/ac/ebi/subs/ingest/patch/web/PatchController.java similarity index 69% rename from src/main/java/org/humancellatlas/ingest/patch/web/PatchController.java rename to src/main/java/uk/ac/ebi/subs/ingest/patch/web/PatchController.java index b10af6f7d..a24ed93da 100644 --- a/src/main/java/org/humancellatlas/ingest/patch/web/PatchController.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/patch/web/PatchController.java @@ -1,14 +1,14 @@ -package org.humancellatlas.ingest.patch.web; +package uk.ac.ebi.subs.ingest.patch.web; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.humancellatlas.ingest.patch.Patch; import org.springframework.data.rest.webmvc.RepositoryRestController; import org.springframework.hateoas.ExposesResourceFor; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.patch.Patch; + @RepositoryRestController @RequiredArgsConstructor @ExposesResourceFor(Patch.class) @Getter -public class PatchController { -} +public class PatchController {} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/patch/web/PatchResourceProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/patch/web/PatchResourceProcessor.java new file mode 100644 index 000000000..baecea334 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/patch/web/PatchResourceProcessor.java @@ -0,0 +1,35 @@ +package uk.ac.ebi.subs.ingest.patch.web; + +import org.springframework.hateoas.EntityLinks; +import org.springframework.hateoas.Link; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.stereotype.Component; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.patch.Patch; + +@Component +@RequiredArgsConstructor +public class PatchResourceProcessor implements ResourceProcessor>> { + private final @NonNull EntityLinks entityLinks; + + @Override + public Resource> process(Resource> resource) { + Link originalDocumentLink = + entityLinks + .linkForSingleResource(resource.getContent().getOriginalDocument()) + .withRel("originalDocument"); + + Link updateDocumentLink = + entityLinks + .linkForSingleResource(resource.getContent().getUpdateDocument()) + .withRel("updateDocument"); + + resource.add(originalDocumentLink); + resource.add(updateDocumentLink); + + return resource; + } +} diff --git a/src/main/java/org/humancellatlas/ingest/process/BundleReference.java b/src/main/java/uk/ac/ebi/subs/ingest/process/BundleReference.java similarity index 65% rename from src/main/java/org/humancellatlas/ingest/process/BundleReference.java rename to src/main/java/uk/ac/ebi/subs/ingest/process/BundleReference.java index 2a00152d8..8cfd196a7 100644 --- a/src/main/java/org/humancellatlas/ingest/process/BundleReference.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/process/BundleReference.java @@ -1,9 +1,9 @@ -package org.humancellatlas.ingest.process; - -import lombok.*; +package uk.ac.ebi.subs.ingest.process; import java.util.List; +import lombok.*; + /** * Javadocs go here! * @@ -12,5 +12,5 @@ */ @Data public class BundleReference { - private List bundleUuids; + private List bundleUuids; } diff --git a/src/main/java/org/humancellatlas/ingest/process/InputFileReference.java b/src/main/java/uk/ac/ebi/subs/ingest/process/InputFileReference.java similarity index 53% rename from src/main/java/org/humancellatlas/ingest/process/InputFileReference.java rename to src/main/java/uk/ac/ebi/subs/ingest/process/InputFileReference.java index b87959968..8a5168a74 100644 --- a/src/main/java/org/humancellatlas/ingest/process/InputFileReference.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/process/InputFileReference.java @@ -1,10 +1,10 @@ -package org.humancellatlas.ingest.process; - -import lombok.Data; +package uk.ac.ebi.subs.ingest.process; import java.util.UUID; +import lombok.Data; + @Data public class InputFileReference { - private UUID inputFileUuid; + private UUID inputFileUuid; } diff --git a/src/main/java/uk/ac/ebi/subs/ingest/process/Process.java b/src/main/java/uk/ac/ebi/subs/ingest/process/Process.java new file mode 100644 index 000000000..2bc4185c2 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/process/Process.java @@ -0,0 +1,65 @@ +package uk.ac.ebi.subs.ingest.process; + +import java.util.HashSet; +import java.util.Set; + +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.DBRef; +import org.springframework.data.rest.core.annotation.RestResource; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import uk.ac.ebi.subs.ingest.bundle.BundleManifest; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.protocol.Protocol; + +/** Created by rolando on 16/02/2018. */ +@Getter +@EqualsAndHashCode( + callSuper = true, + exclude = {"project", "projects", "protocols", "inputBundleManifests", "chainedProcesses"}) +public class Process extends MetadataDocument { + @Indexed + private @Setter @DBRef(lazy = true) Project project; + + @RestResource + @DBRef(lazy = true) + private Set projects = new HashSet<>(); + + @RestResource + @DBRef(lazy = true) + private Set protocols = new HashSet<>(); + + @RestResource + @DBRef(lazy = true) + @Indexed + private Set inputBundleManifests = new HashSet<>(); + + private @DBRef Set chainedProcesses = new HashSet<>(); + + @JsonCreator + public Process(@JsonProperty("content") Object content) { + super(EntityType.PROCESS, content); + } + + public Process addInputBundleManifest(BundleManifest bundleManifest) { + this.inputBundleManifests.add(bundleManifest); + return this; + } + + public Process addProtocol(Protocol protocol) { + protocols.add(protocol); + return this; + } + + public Process removeProtocol(Protocol protocol) { + protocols.remove(protocol); + return this; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/process/ProcessRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/process/ProcessRepository.java new file mode 100644 index 000000000..d0a73f3b1 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/process/ProcessRepository.java @@ -0,0 +1,105 @@ +package uk.ac.ebi.subs.ingest.process; + +import java.util.Collection; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.data.rest.core.annotation.RestResource; +import org.springframework.web.bind.annotation.CrossOrigin; + +import uk.ac.ebi.subs.ingest.bundle.BundleManifest; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.security.RowLevelFilterSecurity; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@CrossOrigin +@RowLevelFilterSecurity( + expression = + "(#filterObject.project != null)" + + "? " + + " (" + + " #authentication.authorities.![authority].contains(" + + " 'ROLE_access_' +#filterObject.project.uuid?.toString()) " + + " or " + + " #authentication.authorities.![authority].contains('ROLE_SERVICE') " + + " or " + + " #filterObject.project.content['dataAccess']['type'] " + + " eq T(uk.ac.ebi.subs.ingest.project.DataAccessTypes).OPEN.label" + + " )" + + ":true", + ignoreClasses = {Project.class}) +public interface ProcessRepository extends MongoRepository { + + @RestResource(rel = "findAllByUuid", path = "findAllByUuid") + Page findByUuid(@Param("uuid") Uuid uuid, Pageable pageable); + + @RestResource(rel = "findByUuid", path = "findByUuid") + Optional findByUuidUuidAndIsUpdateFalse(@Param("uuid") UUID uuid); + + Page findByProject(Project project, Pageable pageable); + + @RestResource(exported = false) + Stream findByProject(Project project); + + @RestResource(exported = false) + Stream findByProjectsContaining(Project project); + + Page findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope, Pageable pageable); + + @RestResource(exported = false) + Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + @RestResource(rel = "findBySubmissionAndValidationState") + public Page findBySubmissionEnvelopeAndValidationState( + @Param("envelopeUri") SubmissionEnvelope submissionEnvelope, + @Param("state") ValidationState state, + Pageable pageable); + + @Query( + value = + "{'submissionEnvelope.id': ?0, graphValidationErrors: { $exists: true, $not: {$size: 0} } }") + @RestResource(rel = "findBySubmissionIdWithGraphValidationErrors") + public Page findBySubmissionIdWithGraphValidationErrors( + @Param("envelopeId") String envelopeId, Pageable pageable); + + @RestResource(exported = false) + Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + @RestResource(exported = false) + Long deleteBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + @RestResource(exported = false) + Page findByInputBundleManifestsContaining( + BundleManifest bundleManifest, Pageable pageable); + + @RestResource(exported = false) + public Stream findAllByIdIn(Collection ids); + + @RestResource(exported = false) + Stream findByProtocolsContains(Protocol protocol); + + Stream findByInputBundleManifestsContains(BundleManifest bundleManifest); + + @RestResource(exported = false) + Optional findFirstByProtocolsContains(Protocol protocol); + + long countBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + long countBySubmissionEnvelopeAndValidationState( + SubmissionEnvelope submissionEnvelope, ValidationState validationState); + + @Query( + value = + "{'submissionEnvelope.id': ?0, graphValidationErrors: { $exists: true, $not: {$size: 0} } }", + count = true) + long countBySubmissionEnvelopeAndCountWithGraphValidationErrors(String submissionEnvelopeId); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/process/ProcessService.java b/src/main/java/uk/ac/ebi/subs/ingest/process/ProcessService.java new file mode 100644 index 000000000..9e76db414 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/process/ProcessService.java @@ -0,0 +1,233 @@ +package uk.ac.ebi.subs.ingest.process; + +import java.text.DecimalFormat; +import java.util.*; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.stereotype.Service; + +import lombok.Getter; +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.bundle.BundleManifest; +import uk.ac.ebi.subs.ingest.bundle.BundleManifestRepository; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.state.MetadataDocumentEventHandler; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@Service +@Getter +public class ProcessService { + private final Logger log = LoggerFactory.getLogger(getClass()); + @Autowired private SubmissionEnvelopeRepository submissionEnvelopeRepository; + @Autowired private ProcessRepository processRepository; + @Autowired private FileRepository fileRepository; + @Autowired private BiomaterialRepository biomaterialRepository; + @Autowired private BundleManifestRepository bundleManifestRepository; + @Autowired private ProjectRepository projectRepository; + @Autowired private MetadataCrudService metadataCrudService; + @Autowired private MetadataUpdateService metadataUpdateService; + @Autowired MetadataDocumentEventHandler metadataDocumentEventHandler; + + protected Logger getLog() { + return log; + } + + public Page findInputBiomaterialsForProcess( + final Process process, final Pageable pageable) { + return biomaterialRepository.findByInputToProcessesContaining(process, pageable); + } + + public Page findInputFilesForProcess(final Process process, final Pageable pageable) { + return fileRepository.findByInputToProcessesContaining(process, pageable); + } + + public Page findOutputBiomaterialsForProcess( + final Process process, final Pageable pageable) { + return biomaterialRepository.findByDerivedByProcessesContaining(process, pageable); + } + + public Page findOutputFilesForProcess(final Process process, final Pageable pageable) { + return fileRepository.findByDerivedByProcessesContaining(process, pageable); + } + + public Process addProcessToSubmissionEnvelope( + final SubmissionEnvelope submissionEnvelope, final Process process) { + if (!process.getIsUpdate()) { + projectRepository + .findBySubmissionEnvelopesContains(submissionEnvelope) + .findFirst() + .ifPresent( + project -> { + process.setProject(project); + process.getProjects().add(project); + }); + return metadataCrudService.addToSubmissionEnvelopeAndSave(process, submissionEnvelope); + } else { + return metadataUpdateService.acceptUpdate(process, submissionEnvelope); + } + } + + // TODO Refactor this to use FileService + // Implement logic to have the option to only create and createOrUpdate + public Process addOutputFileToAnalysisProcess(final Process analysis, final File file) { + final SubmissionEnvelope submissionEnvelope = analysis.getSubmissionEnvelope(); + final File targetFile = determineTargetFile(submissionEnvelope, file); + targetFile.addToAnalysis(analysis); + targetFile.setUuid(Uuid.newUuid()); + getFileRepository().save(targetFile); + metadataDocumentEventHandler.handleMetadataDocumentCreate(targetFile); + + return analysis; + } + + public Process addInputFileUuidToProcess(final Process process, final UUID inputFileUuid) { + return fileRepository + .findByUuidUuidAndIsUpdateFalse(inputFileUuid) + .map(inputFile -> addInputFileToProcess(process, inputFile)) + .orElseThrow(ResourceNotFoundException::new); + } + + public Process addInputFileToProcess(final Process process, final File inputFile) { + fileRepository.save(inputFile.addAsInputToProcess(process)); + return process; + } + + private File determineTargetFile(final SubmissionEnvelope submissionEnvelope, final File file) { + final List persistentFiles = + fileRepository.findBySubmissionEnvelopeAndFileName(submissionEnvelope, file.getFileName()); + return persistentFiles.stream().findFirst().orElse(file); + } + + public Process addInputBundleManifest( + final Process analysisProcess, final BundleReference bundleReference) { + for (final String bundleUuid : bundleReference.getBundleUuids()) { + final Optional maybeBundleManifest = + getBundleManifestRepository().findTopByBundleUuidOrderByBundleVersionDesc(bundleUuid); + maybeBundleManifest.ifPresentOrElse( + analysisProcess::addInputBundleManifest, + () -> { + throw new ResourceNotFoundException( + String.format("Could not find bundle with UUID %s", bundleUuid)); + }); + } + return getProcessRepository().save(analysisProcess); + } + + public Page findProcessesByInputBundleUuid( + final UUID bundleUuid, final Pageable pageable) { + final Optional maybeBundleManifest = + bundleManifestRepository.findTopByBundleUuidOrderByBundleVersionDesc(bundleUuid.toString()); + return maybeBundleManifest + .map( + bundleManifest -> + processRepository.findByInputBundleManifestsContaining(bundleManifest, pageable)) + .orElseThrow( + () -> + new ResourceNotFoundException( + String.format("Bundle with UUID %s not found", bundleUuid.toString()))); + } + + /** + * Find all assay process IDs in a submission + * + * @return A collection of IDs of every assay process in a submission + */ + public Set findAssays(final SubmissionEnvelope submissionEnvelope) { + final Set results = new LinkedHashSet<>(); + final long fileStartTime = System.currentTimeMillis(); + + final long fileEndTime = System.currentTimeMillis(); + final float fileQueryTime = ((float) (fileEndTime - fileStartTime)) / 1000; + final String fileQt = new DecimalFormat("#,###.##").format(fileQueryTime); + getLog().info("Retrieving assays: file query time: {} s", fileQt); + final long allBioStartTime = System.currentTimeMillis(); + + fileRepository + .findBySubmissionEnvelope(submissionEnvelope) + .forEach( + derivedFile -> { + for (final Process derivedByProcess : derivedFile.getDerivedByProcesses()) { + biomaterialRepository + .findByInputToProcessesContains(derivedByProcess) + .findAny() + .ifPresent(__ -> results.add(derivedByProcess.getId())); + } + }); + + final long allBioEndTime = System.currentTimeMillis(); + final float allBioQueryTime = ((float) (allBioEndTime - allBioStartTime)) / 1000; + final String allBioQt = new DecimalFormat("#,###.##").format(allBioQueryTime); + getLog().info("Retrieving assays: biomaterial query time: {} s", allBioQt); + return results; + } + + /** + * Find all analysis process IDs in a submission + * + * @return A collection of IDs of every analysis process in a submission + */ + public Set findAnalyses(final SubmissionEnvelope submissionEnvelope) { + final Set results = new LinkedHashSet<>(); + fileRepository + .findBySubmissionEnvelope(submissionEnvelope) + .forEach( + derivedFile -> { + for (final Process derivedByProcess : derivedFile.getDerivedByProcesses()) { + fileRepository + .findByInputToProcessesContains(derivedByProcess) + .findAny() + .ifPresent(__ -> results.add(derivedByProcess.getId())); + } + }); + return results; + } + + public Process resolveBundleReferencesForProcess( + final Process analysis, final BundleReference bundleReference) { + for (final String bundleUuid : bundleReference.getBundleUuids()) { + final Optional maybeBundleManifest = + getBundleManifestRepository().findTopByBundleUuidOrderByBundleVersionDesc(bundleUuid); + maybeBundleManifest.ifPresentOrElse( + bundleManifest -> { + getLog().info("Adding bundle manifest link to process '" + analysis.getId() + "'"); + analysis.addInputBundleManifest(bundleManifest); + final Process savedAnalysis = getProcessRepository().save(analysis); + for (final String fileUuid : bundleManifest.getFileFilesMap().keySet()) { + fileRepository + .findByUuidUuidAndIsUpdateFalse(UUID.fromString(fileUuid)) + .ifPresentOrElse( + analysisInputFile -> { + analysisInputFile.addAsInputToProcess(savedAnalysis); + fileRepository.save(analysisInputFile); + }, + () -> { + throw new ResourceNotFoundException( + String.format("Could not find file with UUID %s", fileUuid)); + }); + } + }, + () -> { + throw new ResourceNotFoundException( + String.format("Could not find bundle with UUID %s", bundleUuid)); + }); + } + return getProcessRepository().save(analysis); + } + + public Stream getProcesses(final Collection processIds) { + return processRepository.findAllByIdIn(processIds); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/process/web/ProcessController.java b/src/main/java/uk/ac/ebi/subs/ingest/process/web/ProcessController.java new file mode 100644 index 000000000..c5cfc863f --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/process/web/ProcessController.java @@ -0,0 +1,269 @@ +package uk.ac.ebi.subs.ingest.process.web; + +import static org.springframework.data.rest.webmvc.RestMediaTypes.TEXT_URI_LIST_VALUE; +import static org.springframework.web.bind.annotation.RequestMethod.POST; +import static org.springframework.web.bind.annotation.RequestMethod.PUT; + +import java.lang.reflect.InvocationTargetException; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.rest.webmvc.PersistentEntityResource; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.data.web.PagedResourcesAssembler; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.Resources; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.service.*; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.process.*; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.security.CheckAllowed; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.exception.NotAllowedDuringSubmissionStateException; + +@RepositoryRestController +@RequiredArgsConstructor +@ExposesResourceFor(Process.class) +@Getter +public class ProcessController { + private final @NonNull ProcessService processService; + private final @NonNull ProcessRepository processRepository; + private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; + private final @NonNull MetadataCrudService metadataCrudService; + private final @NonNull MetadataUpdateService metadataUpdateService; + + private @Autowired final ValidationStateChangeService validationStateChangeService; + private @Autowired final UriToEntityConversionService uriToEntityConversionService; + private @Autowired final MetadataLinkingService metadataLinkingService; + + // Input and Output Resources + @GetMapping("processes/{proc_id}/inputBiomaterials") + ResponseEntity getProcessInputBiomaterials( + @PathVariable("proc_id") final Process process, + final Pageable pageable, + final PersistentEntityResourceAssembler assembler) { + final Page inputBiomaterials = + processService.findInputBiomaterialsForProcess(process, pageable); + return ResponseEntity.ok(pagedResourcesAssembler.toResource(inputBiomaterials, assembler)); + } + + @GetMapping("processes/{proc_id}/inputFiles") + ResponseEntity getProcessInputFiles( + @PathVariable("proc_id") final Process process, + final Pageable pageable, + final PersistentEntityResourceAssembler assembler) { + final Page inputFiles = processService.findInputFilesForProcess(process, pageable); + return ResponseEntity.ok(pagedResourcesAssembler.toResource(inputFiles, assembler)); + } + + @GetMapping("processes/{proc_id}/derivedBiomaterials") + ResponseEntity getProcessOutputBiomaterials( + @PathVariable("proc_id") final Process process, + final Pageable pageable, + final PersistentEntityResourceAssembler assembler) { + final Page outputBiomaterials = + processService.findOutputBiomaterialsForProcess(process, pageable); + return ResponseEntity.ok(pagedResourcesAssembler.toResource(outputBiomaterials, assembler)); + } + + @GetMapping("processes/{proc_id}/derivedFiles") + ResponseEntity getProcessOutputFiles( + @PathVariable("proc_id") final Process process, + final Pageable pageable, + final PersistentEntityResourceAssembler assembler) { + final Page outputFiles = processService.findOutputFilesForProcess(process, pageable); + return ResponseEntity.ok(pagedResourcesAssembler.toResource(outputFiles, assembler)); + } + + // Process Management + @CheckAllowed( + value = "#submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PostMapping("submissionEnvelopes/{sub_id}/processes") + ResponseEntity> addProcessToEnvelope( + @PathVariable("sub_id") final SubmissionEnvelope submissionEnvelope, + @RequestBody final Process process, + @RequestParam("updatingUuid") final Optional updatingUuid, + final PersistentEntityResourceAssembler assembler) { + updatingUuid.ifPresent( + uuid -> { + process.setUuid(new Uuid(uuid.toString())); + process.setIsUpdate(true); + }); + final Process entity = + processService.addProcessToSubmissionEnvelope(submissionEnvelope, process); + final PersistentEntityResource resource = assembler.toFullResource(entity); + return ResponseEntity.accepted().body(resource); + } + + @CheckAllowed( + value = "#submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PutMapping("submissionEnvelopes/{sub_id}/processes/{id}") + ResponseEntity> linkProcessToEnvelope( + @PathVariable("sub_id") final SubmissionEnvelope submissionEnvelope, + @PathVariable("id") final Process process, + final PersistentEntityResourceAssembler assembler) { + final Process entity = + processService.addProcessToSubmissionEnvelope(submissionEnvelope, process); + final PersistentEntityResource resource = assembler.toFullResource(entity); + return ResponseEntity.accepted().body(resource); + } + + @DeleteMapping("processes/{id}") + @CheckAllowed( + value = "#process.submissionEnvelope.isEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + ResponseEntity deleteProcess(@PathVariable("id") final Process process) + throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + metadataLinkingService.removeLinks(process, "protocols"); + metadataCrudService.deleteDocument(process); + return ResponseEntity.noContent().build(); + } + + // Bundle References + @Deprecated + @GetMapping("/processes/{analysis_id}/" + Links.BUNDLE_REF_URL) + ResponseEntity> addBundleReference() { + return ResponseEntity.status(HttpStatus.METHOD_NOT_ALLOWED).build(); + } + + @PutMapping("/processes/{analysis_id}/" + Links.BUNDLE_REF_URL) + @CheckAllowed( + value = "#analysis.submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + ResponseEntity> oldAddBundleReference( + @PathVariable("analysis_id") final Process analysis, + @RequestBody final BundleReference bundleReference, + final PersistentEntityResourceAssembler assembler) { + final Process entity = + processService.resolveBundleReferencesForProcess(analysis, bundleReference); + final PersistentEntityResource resource = assembler.toFullResource(entity); + return ResponseEntity.accepted().body(resource); + } + + @PostMapping("/processes/{analysis_id}/" + Links.BUNDLE_REF_URL) + @CheckAllowed( + value = "#analysis.submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + ResponseEntity> addBundleReference( + @PathVariable("analysis_id") final Process analysis, + @RequestBody final BundleReference bundleReference, + final PersistentEntityResourceAssembler assembler) { + final Process entity = processService.addInputBundleManifest(analysis, bundleReference); + final PersistentEntityResource resource = assembler.toFullResource(entity); + return ResponseEntity.accepted().body(resource); + } + + @PutMapping("/processes/{analysis_id}/" + Links.FILE_REF_URL) + @CheckAllowed( + value = "#analysis.submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + ResponseEntity> addOutputFileReference( + @PathVariable("analysis_id") final Process analysis, + @RequestBody final File file, + final PersistentEntityResourceAssembler assembler) { + final Process result = processService.addOutputFileToAnalysisProcess(analysis, file); + final PersistentEntityResource resource = assembler.toFullResource(result); + return ResponseEntity.accepted().body(resource); + } + + @PostMapping("/processes/{analysis_id}/" + Links.INPUT_FILES_URL) + @CheckAllowed( + value = "#analysis.submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + ResponseEntity> addInputFileReference( + @PathVariable("analysis_id") final Process analysis, + @RequestBody final InputFileReference inputFileReference, + final PersistentEntityResourceAssembler assembler) { + final Process result = + processService.addInputFileUuidToProcess(analysis, inputFileReference.getInputFileUuid()); + final PersistentEntityResource resource = assembler.toFullResource(result); + return ResponseEntity.accepted().body(resource); + } + + // Protocol Management + @CheckAllowed( + value = "#process.submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @RequestMapping( + path = "/processes/{id}/protocols", + method = {PUT, POST}, + consumes = {TEXT_URI_LIST_VALUE}) + HttpEntity linkProtocolsToProcess( + @PathVariable("id") Process process, + @RequestBody Resources incoming, + HttpMethod requestMethod) + throws URISyntaxException, + InvocationTargetException, + NoSuchMethodException, + IllegalAccessException { + List protocols = + uriToEntityConversionService.convertLinks(incoming.getLinks(), Protocol.class); + metadataLinkingService.updateLinks( + process, protocols, "protocols", requestMethod.equals(HttpMethod.PUT)); + return ResponseEntity.ok().build(); + } + + @DeleteMapping(path = "/processes/{id}/protocols/{protocolId}") + @CheckAllowed( + value = "#process.submissionEnvelope.isEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + HttpEntity unlinkProtocolFromProcess( + @PathVariable("id") final Process process, + @PathVariable("protocolId") final Protocol protocol) + throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + metadataLinkingService.removeLink(process, protocol, "protocols"); + return ResponseEntity.noContent().build(); + } + + // Patch Process + @PatchMapping("/processes/{id}") + @CheckAllowed( + value = "#process.submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + HttpEntity patchProcess( + @PathVariable("id") final Process process, + @RequestBody final ObjectNode patch, + final PersistentEntityResourceAssembler assembler) { + final List allowedFields = + List.of("content", "validationErrors", "graphValidationErrors"); + final ObjectNode validPatch = patch.retain(allowedFields); + final Process updatedProcess = metadataUpdateService.update(process, validPatch); + final PersistentEntityResource resource = assembler.toFullResource(updatedProcess); + return ResponseEntity.accepted().body(resource); + } + + // Find Processes by Input Bundle UUID + @GetMapping("/processes/search/findByInputBundleUuid") + ResponseEntity findProcesessByInputBundleUuid( + @RequestParam final String bundleUuid, + final Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + final Page processes = + processService.findProcessesByInputBundleUuid(UUID.fromString(bundleUuid), pageable); + return ResponseEntity.ok(pagedResourcesAssembler.toResource(processes, resourceAssembler)); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/process/web/ProcessResourceProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/process/web/ProcessResourceProcessor.java new file mode 100644 index 000000000..ee7178e15 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/process/web/ProcessResourceProcessor.java @@ -0,0 +1,91 @@ +package uk.ac.ebi.subs.ingest.process.web; + +import org.springframework.hateoas.EntityLinks; +import org.springframework.hateoas.Link; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.stereotype.Component; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.process.Process; + +@Component +@RequiredArgsConstructor +public class ProcessResourceProcessor implements ResourceProcessor> { + private final @NonNull EntityLinks entityLinks; + + private Link getInputBiomaterialsLink(Process process) { + return entityLinks + .linkForSingleResource(process) + .slash(Links.INPUT_BIOMATERIALS_URL) + .withRel(Links.INPUT_BIOMATERIALS_REL); + } + + private Link getDerivedBiomaterialsLink(Process process) { + return entityLinks + .linkForSingleResource(process) + .slash(Links.DERIVED_BY_BIOMATERIALS_URL) + .withRel(Links.DERIVED_BY_BIOMATERIALS_REL); + } + + private Link getInputFilesLink(Process process) { + return entityLinks + .linkForSingleResource(process) + .slash(Links.INPUT_FILES_URL) + .withRel(Links.INPUT_FILES_REL); + } + + private Link getDerivedFilesLink(Process process) { + return entityLinks + .linkForSingleResource(process) + .slash(Links.DERIVED_BY_FILES_URL) + .withRel(Links.DERIVED_BY_FILES_REL); + } + + private Link getBundleReferencesLink(Process process) { + return entityLinks + .linkForSingleResource(process) + .slash(Links.BUNDLE_REF_URL) + .withRel(Links.BUNDLE_REF_REL); + } + + private Link getFileReferencesLink(Process process) { + return entityLinks + .linkForSingleResource(process) + .slash(Links.FILE_REF_URL) + .withRel(Links.FILE_REF_REL); + } + + @Deprecated + private Link getOldEvilBundleReferencesLink(Process process) { + return entityLinks + .linkForSingleResource(process) + .slash(Links.BUNDLE_REF_URL) + .withRel(Links.BUNDLE_REF_OLD_EVIL_REL); + } + + @Deprecated + private Link getOldEvilFileReferencesLink(Process process) { + return entityLinks + .linkForSingleResource(process) + .slash(Links.FILE_REF_URL) + .withRel(Links.FILE_REF_OLD_EVIL_REL); + } + + @Override + public Resource process(Resource resource) { + Process process = resource.getContent(); + resource.add( + getInputBiomaterialsLink(process), + getDerivedBiomaterialsLink(process), + getInputFilesLink(process), + getDerivedFilesLink(process), + getBundleReferencesLink(process), + getFileReferencesLink(process), + getOldEvilBundleReferencesLink(process), + getOldEvilFileReferencesLink(process)); + return resource; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/process/web/ProcessSearchProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/process/web/ProcessSearchProcessor.java new file mode 100644 index 000000000..6f44d7263 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/process/web/ProcessSearchProcessor.java @@ -0,0 +1,26 @@ +package uk.ac.ebi.subs.ingest.process.web; + +import static org.springframework.hateoas.mvc.ControllerLinkBuilder.linkTo; +import static org.springframework.hateoas.mvc.ControllerLinkBuilder.methodOn; + +import org.springframework.data.rest.webmvc.RepositorySearchesResource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.stereotype.Component; + +import uk.ac.ebi.subs.ingest.process.Process; + +/** Created by rolando on 29/06/2018. */ +@Component +public class ProcessSearchProcessor implements ResourceProcessor { + + @Override + public RepositorySearchesResource process(RepositorySearchesResource resource) { + if (resource.getDomainType().equals(Process.class)) { + resource.add( + linkTo(methodOn(ProcessController.class).findProcesessByInputBundleUuid(null, null, null)) + .withRel("findByInputBundleUuid")); + } + + return resource; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/BuilderHelper.java b/src/main/java/uk/ac/ebi/subs/ingest/project/BuilderHelper.java new file mode 100644 index 000000000..9dbd6eb18 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/BuilderHelper.java @@ -0,0 +1,71 @@ +package uk.ac.ebi.subs.ingest.project; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class BuilderHelper { + private final B builderInstance; + private final List builderOperationalFields = List.of("builderHelper"); + + public BuilderHelper(B builderInstance) { + this.builderInstance = builderInstance; + } + + private static String toSetterName(String fieldName) { + String capitalisedFieldName = fieldName.substring(0, 1).toUpperCase() + fieldName.substring(1); + return "set" + capitalisedFieldName; + } + + public void copyFieldsFromBuilder(T target, List ignoreFieldsList) { + Class targetClass = target.getClass(); + Map targetFieldsMap = + Stream.iterate(targetClass, c -> c.getSuperclass() != null, Class::getSuperclass) + .map(Class::getDeclaredFields) + .flatMap(Arrays::stream) + .filter(f -> !ignoreFieldsList.contains(f.getName())) + .collect(Collectors.toMap(Field::getName, Function.identity())); + Arrays.stream(builderInstance.getClass().getDeclaredFields()) + .filter(f -> !ignoreFieldsList.contains(f.getName())) + .filter(f -> !builderOperationalFields.contains(f.getName())) + .forEach( + builderField -> { + try { + Field targetField = targetFieldsMap.get(builderField.getName()); + String fieldName = targetField.getName(); + Method setter = getMethod(target, fieldName, targetField); + setter.invoke(target, builderField.get(this.builderInstance)); + } catch (IllegalAccessException + | NoSuchMethodException + | InvocationTargetException e) { + throw new RuntimeException(e); + } + }); + } + + private static Method getMethod(T target, String fieldName, Field projectField) + throws NoSuchMethodException { + return target.getClass().getMethod(toSetterName(fieldName), projectField.getType()); + } + + public Map asMap(List excludeList) { + T target = null; + try { + Method buildMethod = this.builderInstance.getClass().getMethod("build"); + target = (T) buildMethod.invoke(this.builderInstance); + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new RuntimeException(e); + } + return new ObjectToMapConverter().asMap(target, excludeList); + } + + public static Map asMap(T target) { + return new ObjectToMapConverter().asMap(target); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccess.java b/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccess.java new file mode 100644 index 000000000..60eea9ed9 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccess.java @@ -0,0 +1,19 @@ +package uk.ac.ebi.subs.ingest.project; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import lombok.*; + +@Getter +@Setter +@ToString +@RequiredArgsConstructor +@EqualsAndHashCode +public class DataAccess { + @JsonSerialize(using = DataAccessTypesJsonSerializer.class) + @JsonDeserialize(using = DataAccessTypesJsonDeserializer.class) + private final @NonNull DataAccessTypes type; + + String notes; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccessTypes.java b/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccessTypes.java new file mode 100644 index 000000000..d46c9d4c5 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccessTypes.java @@ -0,0 +1,31 @@ +package uk.ac.ebi.subs.ingest.project; + +import java.util.Arrays; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.Getter; + +public enum DataAccessTypes { + @JsonProperty("OPEN") + OPEN("All fully open"), + @JsonProperty("MANAGED") + MANAGED("All managed access"), + @JsonProperty("MIXTURE") + MIXTURE("A mixture of open and managed"), + @JsonProperty("COMPLICATED") + COMPLICATED("It's complicated"); + + @Getter final String label; + + DataAccessTypes(String label) { + this.label = label; + } + + public static DataAccessTypes fromLabel(String label) { + return Arrays.stream(DataAccessTypes.values()) + .filter(s -> s.getLabel().equals(label)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(label)); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccessTypesJsonDeserializer.java b/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccessTypesJsonDeserializer.java new file mode 100644 index 000000000..263c37f63 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccessTypesJsonDeserializer.java @@ -0,0 +1,28 @@ +package uk.ac.ebi.subs.ingest.project; + +import java.io.IOException; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +/** Deserialises {@link DataAccessTypes} JSON value. to enum Used when reading json input. */ +public class DataAccessTypesJsonDeserializer extends StdDeserializer { + public DataAccessTypesJsonDeserializer() { + this(null); + } + + public DataAccessTypesJsonDeserializer(Class t) { + super(t); + } + + @Override + public DataAccessTypes deserialize(JsonParser p, DeserializationContext ctxt) + throws IOException, JsonProcessingException { + // p.nextToken(); + // p.nextToken(); + String source = p.getText(); + return DataAccessTypes.fromLabel(source); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccessTypesJsonSerializer.java b/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccessTypesJsonSerializer.java new file mode 100644 index 000000000..1ffa7c374 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccessTypesJsonSerializer.java @@ -0,0 +1,30 @@ +package uk.ac.ebi.subs.ingest.project; + +import java.io.IOException; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; + +/** + * Serialises {@link DataAccessTypes} enum to a JSON value. Used when converting an object to JSON + * using {@link com.fasterxml.jackson.databind.ObjectMapper} + */ +public class DataAccessTypesJsonSerializer extends StdSerializer { + public DataAccessTypesJsonSerializer() { + this(null); + } + + public DataAccessTypesJsonSerializer(Class t) { + super(t); + } + + @Override + public void serialize(DataAccessTypes value, JsonGenerator gen, SerializerProvider provider) + throws IOException { + // gen.writeStartObject(); + // gen.writeFieldName("type"); + gen.writeString(value.getLabel()); + // gen.writeEndObject(); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccessTypesReadConverter.java b/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccessTypesReadConverter.java new file mode 100644 index 000000000..6bfbdca51 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccessTypesReadConverter.java @@ -0,0 +1,16 @@ +package uk.ac.ebi.subs.ingest.project; + +import org.springframework.core.convert.converter.Converter; +import org.springframework.data.convert.ReadingConverter; +import org.springframework.lang.Nullable; + +/** Used to deserialize data types when reading from Mongo */ +@ReadingConverter() +public class DataAccessTypesReadConverter implements Converter { + + @Override + @Nullable + public DataAccessTypes convert(String source) { + return DataAccessTypes.fromLabel(source); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccessTypesWriteConverter.java b/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccessTypesWriteConverter.java new file mode 100644 index 000000000..44e0295a0 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/DataAccessTypesWriteConverter.java @@ -0,0 +1,16 @@ +package uk.ac.ebi.subs.ingest.project; + +import org.springframework.core.convert.converter.Converter; +import org.springframework.data.convert.WritingConverter; +import org.springframework.lang.Nullable; + +/** Used to deserialize data types when reading from Mongo */ +@WritingConverter() +public class DataAccessTypesWriteConverter implements Converter { + + @Override + @Nullable + public String convert(DataAccessTypes source) { + return source.getLabel(); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/ObjectToMapConverter.java b/src/main/java/uk/ac/ebi/subs/ingest/project/ObjectToMapConverter.java new file mode 100644 index 000000000..fb7baf726 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/ObjectToMapConverter.java @@ -0,0 +1,30 @@ +package uk.ac.ebi.subs.ingest.project; + +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; + +@AllArgsConstructor +@NoArgsConstructor +public class ObjectToMapConverter { + + private ObjectMapper objectMapper; + + public Map asMap(T target, List excludeList) { + if (objectMapper == null) { + objectMapper = new ObjectMapper(); + } + Map projectAsMap = objectMapper.convertValue(target, new TypeReference<>() {}); + excludeList.forEach(projectAsMap::remove); + return projectAsMap; + } + + public Map asMap(T target) { + return asMap(target, List.of()); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/Project.java b/src/main/java/uk/ac/ebi/subs/ingest/project/Project.java new file mode 100644 index 000000000..f783cb86a --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/Project.java @@ -0,0 +1,121 @@ +package uk.ac.ebi.subs.ingest.project; + +import java.time.Instant; +import java.util.*; +import java.util.stream.Collectors; + +import javax.validation.constraints.NotNull; + +import org.springframework.data.mongodb.core.mapping.DBRef; +import org.springframework.data.rest.core.annotation.RestResource; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.dataset.Dataset; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 30/08/17 + */ +@Getter +@EqualsAndHashCode( + callSuper = true, + exclude = {"supplementaryFiles", "submissionEnvelopes"}) +public class Project extends MetadataDocument { + @RestResource + @JsonIgnore + @DBRef(lazy = true) + private Set supplementaryFiles = new HashSet<>(); + + // A project can have multiple datasets + @RestResource private Set datasets = new HashSet<>(); + + // A project may have 1 or more submissions related to it. + @JsonIgnore + private @DBRef(lazy = true) Set submissionEnvelopes = new HashSet<>(); + + @Setter private Instant releaseDate; + + @Setter private Instant accessionDate; + + @Setter private Object technology; + + @Setter private Object organ; + + @Setter private Integer cellCount; + + @Setter @Deprecated private DataAccess dataAccess; + + @Setter private Object identifyingOrganisms; + + @Setter private String primaryWrangler; + + @Setter private String secondaryWrangler; + + @Setter private WranglingState wranglingState; + + @Setter private Integer wranglingPriority; + + @Setter private String wranglingNotes; + + @Setter private Boolean isInCatalogue; + + @Setter private Instant cataloguedDate; + + @Setter private List publicationsInfo; + + @Setter private Integer dcpReleaseNumber; + + @Setter private List projectLabels; + + @Setter private List projectNetworks; + + @JsonCreator + public Project(@JsonProperty("content") Object content) { + super(EntityType.PROJECT, content); + } + + public void addToSubmissionEnvelopes(@NotNull SubmissionEnvelope submissionEnvelope) { + this.submissionEnvelopes.add(submissionEnvelope); + } + + // ToDo: Find a better way of ensuring that DBRefs to deleted objects aren't returned. + @JsonIgnore + public List getOpenSubmissionEnvelopes() { + return this.submissionEnvelopes.stream() + .filter(Objects::nonNull) + .filter(env -> env.getSubmissionState() != null) + .filter(SubmissionEnvelope::isOpen) + .collect(Collectors.toList()); + } + + public Boolean getHasOpenSubmission() { + return !getOpenSubmissionEnvelopes().isEmpty(); + } + + @JsonIgnore + public Boolean isEditable() { + return this.submissionEnvelopes.stream() + .filter(Objects::nonNull) + .allMatch(SubmissionEnvelope::isEditable); + } + + public static ProjectBuilder builder() { + return new ProjectBuilder(); + } + + public void addDataset(final Dataset dataset) { + datasets.add(dataset); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectBuilder.java b/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectBuilder.java new file mode 100644 index 000000000..e22af4eed --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectBuilder.java @@ -0,0 +1,59 @@ +package uk.ac.ebi.subs.ingest.project; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import uk.ac.ebi.subs.ingest.core.Uuid; + +/** + * generic builder that uses reflection. Might not be suitable for production due to slower + * performance. Currently used only for testing. + */ +public class ProjectBuilder { + public final BuilderHelper builderHelper = new BuilderHelper(this); + Map content = new HashMap<>(); + + Uuid uuid = Uuid.newUuid(); + + public ProjectBuilder emptyProject() { + return this; + } + + public ProjectBuilder withManagedAccess() { + return withDataAccess(new DataAccess(DataAccessTypes.MANAGED)); + } + + public ProjectBuilder withOpenAccess() { + return withDataAccess(new DataAccess(DataAccessTypes.OPEN)); + } + + public ProjectBuilder withDataAccess(DataAccess dataAccess) { + content.put("dataAccess", new ObjectToMapConverter().asMap(dataAccess)); + return this; + } + + public ProjectBuilder withUuid(String uuid) { + this.uuid = new Uuid(uuid); + return this; + } + + public ProjectBuilder withShortName(String shortName) { + Map projectCore = + (Map) + content.computeIfAbsent("project_core", k -> new HashMap()); + projectCore.put("project_short_name", shortName); + return this; + } + + public Project build() { + Project project = new Project(content); + List constructorFields = List.of("content"); + builderHelper.copyFieldsFromBuilder(project, constructorFields); + return project; + } + + public Map asMap() { + return builderHelper.asMap(List.of("contentLastModified")); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectChangeListener.java b/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectChangeListener.java new file mode 100644 index 000000000..2a7264251 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectChangeListener.java @@ -0,0 +1,29 @@ +package uk.ac.ebi.subs.ingest.project; + +import org.springframework.data.mongodb.core.mapping.event.AbstractMongoEventListener; +import org.springframework.data.mongodb.core.mapping.event.AfterSaveEvent; +import org.springframework.data.mongodb.core.mapping.event.BeforeSaveEvent; +import org.springframework.stereotype.Component; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.exception.MultipleOpenSubmissionsException; + +@Component +@RequiredArgsConstructor +@Getter +public class ProjectChangeListener extends AbstractMongoEventListener { + private final ProjectEventHandler projectEventHandler; + + @Override + public void onBeforeSave(BeforeSaveEvent event) { + Project project = event.getSource(); + if (project.getOpenSubmissionEnvelopes().size() > 1) + throw new MultipleOpenSubmissionsException("A project can't have multiple open submissions."); + } + + @Override + public void onAfterSave(AfterSaveEvent event) { + // do nothing + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectEventHandler.java b/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectEventHandler.java new file mode 100644 index 000000000..e8d03c160 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectEventHandler.java @@ -0,0 +1,142 @@ +package uk.ac.ebi.subs.ingest.project; + +import static org.springframework.util.DigestUtils.md5DigestAsHex; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.notifications.NotificationService; +import uk.ac.ebi.subs.ingest.notifications.exception.DuplicateNotification; +import uk.ac.ebi.subs.ingest.notifications.model.Checksum; +import uk.ac.ebi.subs.ingest.notifications.model.Notification; +import uk.ac.ebi.subs.ingest.notifications.model.NotificationRequest; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.user.IdentityService; + +@Component +@RequiredArgsConstructor +public class ProjectEventHandler { + + private final NotificationService notificationService; + private final Environment environment; + private final IdentityService identityService; + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private static String objectToString(Object object) { + try { + return new ObjectMapper().writeValueAsString(object); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static String objectToPrettyString(Object object) { + try { + return new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(object); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public Notification registeredProject(Project project) { + String message = "A new project [" + project.getUuid() + "] was registered."; + String header = "project-registered"; + Checksum checksum = + new Checksum(header, md5DigestAsHex((header + ":" + project.getUuid()).getBytes())); + return notifyWranglersByEmail(message, checksum); + } + + public Notification editedProjectMetadata(Project project) { + String notificationContent = + String.format( + "Project %s was updated:\n\nNew content:\n\n%s", + project.getUuid().getUuid().toString(), objectToPrettyString(project.getContent())); + Checksum checksum = editedProjectChecksum(project); + + return notifyWranglersByEmail(notificationContent, checksum); + } + + public Notification deletedProject(Project project) { + String notificationContent = + String.format( + "Project %s was deleted:\n\n%s", + project.getUuid().getUuid().toString(), objectToPrettyString(project.getContent())); + + Checksum checksum = deletedProjectChecksum(project); + + return notifyWranglersByEmail(notificationContent, checksum); + } + + public Optional validatedProject(Project project) { + if (project.getValidationState().equals(ValidationState.VALID)) { + String notificationContent = + String.format( + "Project %s has been validated:\n\n%s", + project.getUuid().getUuid().toString(), objectToPrettyString(project.getContent())); + + Checksum checksum = validProjectChecksum(project); + + return Optional.of(notifyWranglersByEmail(notificationContent, checksum)); + } else { + return Optional.empty(); + } + } + + private Notification notifyWranglersByEmail( + String notificationContent, Checksum notificationChecksum) { + var notificationMetadata = new HashMap(); + var emailMetadata = new HashMap(); + emailMetadata.put("to", this.emailNotificationsFromAddress()); + emailMetadata.put("from", identityService.wranglerEmail()); + emailMetadata.put("subject", "HCA DCP project update"); + emailMetadata.put("body", notificationContent); + notificationMetadata.put("email", emailMetadata); + + NotificationRequest notificationRequest = + new NotificationRequest(notificationContent, notificationMetadata, notificationChecksum); + try { + return this.notificationService.createNotification(notificationRequest); + } catch (DuplicateNotification e) { + return this.notificationService + .retrieveForChecksum(notificationChecksum) + .orElseThrow( + () -> { + log.error("Duplicate notification for non-existent checksum"); + throw new RuntimeException(e); + }); + } + } + + private Checksum editedProjectChecksum(Project project) { + String checksumInput = String.format("%s:%s", "project-edited", project.getUuid().getUuid()); + return new Checksum("project-edited", md5DigestAsHex(checksumInput.getBytes())); + } + + private Checksum deletedProjectChecksum(Project project) { + String checksumInput = String.format("%s:%s", "project-deleted", project.getUuid().getUuid()); + return new Checksum("project-deleted", md5DigestAsHex(checksumInput.getBytes())); + } + + private Checksum validProjectChecksum(Project project) { + String checksumInput = + String.format( + "%s:%s:%s", + "project-validated", project.getUuid().getUuid(), objectToString(project.getContent())); + return new Checksum("project-validated", md5DigestAsHex(checksumInput.getBytes())); + } + + private String emailNotificationsFromAddress() { + return environment.getProperty( + "PROJECT_NOTIFICATIONS_FROM_ADDRESS", "hca-notifications-test@ebi.ac.uk"); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectLinkChangeListener.java b/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectLinkChangeListener.java new file mode 100644 index 000000000..ccb00b837 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectLinkChangeListener.java @@ -0,0 +1,48 @@ +package uk.ac.ebi.subs.ingest.project; + +import java.util.Collection; +import java.util.Objects; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.rest.core.annotation.*; +import org.springframework.stereotype.Component; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Component +@RepositoryEventHandler +@RequiredArgsConstructor +public class ProjectLinkChangeListener { + + @Autowired ProjectService projectService; + private final @NonNull Logger log = LoggerFactory.getLogger(getClass()); + + /** + * The `linked` parameter, due to a bug in Spring, is passed to the handler as a {@link + * java.lang.reflect.Proxy} object. This is a proxy to a {@link Collection}. Since we need to + * respond to associations of {@link SubmissionEnvelope}s and not other properties of {@link + * Project}, we need to filter the contents of the `linked` Collection. + * + * @link Stack + * Overflow ticket + */ + @HandleBeforeLinkSave + public void beforeLinkSave(Project project, Object linked) { + Stream.of(linked) + .map(Collection.class::cast) + .flatMap(Collection::stream) + .filter(Objects::nonNull) + .filter(SubmissionEnvelope.class::isInstance) + .findAny() + .ifPresent( + o -> { + projectService.updateWranglingState(project, WranglingState.IN_PROGRESS); + }); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectQueryBuilder.java b/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectQueryBuilder.java new file mode 100644 index 000000000..c2ea7bbb6 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectQueryBuilder.java @@ -0,0 +1,136 @@ +package uk.ac.ebi.subs.ingest.project; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.CriteriaDefinition; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.core.query.TextCriteria; + +import uk.ac.ebi.subs.ingest.project.web.SearchFilter; +import uk.ac.ebi.subs.ingest.project.web.SearchType; + +public class ProjectQueryBuilder { + + public static Query buildProjectsQuery(SearchFilter searchFilter) { + List criteriaList = new ArrayList<>(); + criteriaList.add(Criteria.where("isUpdate").is(false)); + addIsCriterionForAttribute(criteriaList, "wranglingState", searchFilter.getWranglingState()); + addIsCriterionForAttribute(criteriaList, "primaryWrangler", searchFilter.getPrimaryWrangler()); + addIsCriterionForAttribute( + criteriaList, "wranglingPriority", searchFilter.getWranglingPriority()); + addIsCriterionForAttribute(criteriaList, "isInCatalogue", searchFilter.getHcaCatalogue()); + addLTECriterionForAttribute(criteriaList, "cellCount", searchFilter.getMaxCellCount()); + addGTECriterionForAttribute(criteriaList, "cellCount", searchFilter.getMinCellCount()); + addInCriterionForAttribute( + criteriaList, "identifyingOrganisms", searchFilter.getIdentifyingOrganism()); + addInCriterionForAttribute( + criteriaList, "dcpReleaseNumber", searchFilter.getDcpReleaseNumber()); + addInCriterionForAttribute(criteriaList, "projectLabels", searchFilter.getProjectLabels()); + addInCriterionForAttribute(criteriaList, "projectNetworks", searchFilter.getProjectNetworks()); + + if (searchFilter.getDataAccess() != null) { + addIsCriterionForAttribute( + criteriaList, "content.dataAccess.type", searchFilter.getDataAccess().getLabel()); + } + + Optional.ofNullable(searchFilter.getHasOfficialHcaPublication()) + .map( + value -> + Criteria.where("content.publications") + .elemMatch(Criteria.where("official_hca_publication").is(value))) + .ifPresent(criteriaList::add); + + Optional.ofNullable(searchFilter.getOrganOntology()) + .map( + value -> + Criteria.where("organ.ontologies").elemMatch(Criteria.where("ontology").is(value))) + .ifPresent(criteriaList::add); + + Criteria queryCriteria = + new Criteria().andOperator(criteriaList.toArray(new Criteria[criteriaList.size()])); + Query query = new Query().addCriteria(queryCriteria); + addKeywordSearchCriteria(searchFilter).ifPresent(query::addCriteria); + return query; + } + + private static Optional addKeywordSearchCriteria(SearchFilter searchFilter) { + try { + return buildUuidCriteria(searchFilter); + } catch (IllegalArgumentException e) { + return buildTextCriteria(searchFilter); + } + } + + private static Optional buildTextCriteria(SearchFilter searchFilter) { + return Optional.ofNullable(searchFilter.getSearch()) + .map(search -> ProjectQueryBuilder.formatSearchString(searchFilter)) + .map(search -> TextCriteria.forDefaultLanguage().matching(String.valueOf(search))); + } + + private static Optional buildUuidCriteria(SearchFilter searchFilter) { + return Optional.ofNullable(searchFilter.getSearch()) + .map(UUID::fromString) + .map(uuid -> Criteria.where("uuid.uuid").is(uuid)); + } + + private static void addIsCriterionForAttribute( + List criteria_list, String attributeName, Object attributeValue) { + Optional.ofNullable(attributeValue) + .map(value -> Criteria.where(attributeName).is(value)) + .ifPresent(criteria_list::add); + } + + private static void addLTECriterionForAttribute( + List criteria_list, String attributeName, Integer attributeValue) { + Optional.ofNullable(attributeValue) + .map(value -> Criteria.where(attributeName).lte(value)) + .ifPresent(criteria_list::add); + } + + private static void addGTECriterionForAttribute( + List criteria_list, String attributeName, Integer attributeValue) { + Optional.ofNullable(attributeValue) + .map(value -> Criteria.where(attributeName).gte(value)) + .ifPresent(criteria_list::add); + } + + private static void addInCriterionForAttribute( + List criteria_list, String attributeName, Object attributeValue) { + Optional.ofNullable(attributeValue) + .map(value -> Criteria.where(attributeName).in(value)) + .ifPresent(criteria_list::add); + } + + private static final Map> keywordFormatterMap = + Map.of( + SearchType.ExactMatch, + (SearchFilter searchFilter) -> encloseInQuotes(searchFilter.getSearch()), + SearchType.AllKeywords, + (SearchFilter searchFilter) -> + Stream.of(splitBySpace(searchFilter)) + .map(ProjectQueryBuilder::encloseInQuotes) + .collect(Collectors.joining(" "))); + + public static String formatSearchString(SearchFilter searchFilter) { + if (searchFilter.getSearch() != null && searchFilter.getSearch().contains("\"")) { + return searchFilter.getSearch(); + } + return Optional.ofNullable(searchFilter) + .map(SearchFilter::getSearchType) + .map(keywordFormatterMap::get) + .map(formatterFunction -> formatterFunction.apply(searchFilter)) + .orElse(searchFilter.getSearch()); + } + + private static String[] splitBySpace(SearchFilter searchFilter) { + return searchFilter.getSearch().split(" +"); + } + + private static String encloseInQuotes(String search) { + return "\"" + search + "\""; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectRepository.java new file mode 100644 index 000000000..32493730b --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectRepository.java @@ -0,0 +1,72 @@ +package uk.ac.ebi.subs.ingest.project; + +import java.util.Collection; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.repository.query.Param; +import org.springframework.data.rest.core.annotation.RestResource; +import org.springframework.web.bind.annotation.CrossOrigin; + +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 31/08/17 + */ +@CrossOrigin +public interface ProjectRepository extends MongoRepository { + + @RestResource(rel = "findAllByUuid", path = "findAllByUuid") + Page findByUuid(@Param("uuid") Uuid uuid, Pageable pageable); + + @RestResource(rel = "findByUuid", path = "findByUuid") + Optional findByUuidUuidAndIsUpdateFalse(@Param("uuid") UUID uuid); + + @RestResource(path = "findByUser", rel = "findByUser") + Page findByUser(@Param(value = "user") String user, Pageable pageable); + + @RestResource(rel = "findByUserAndPrimaryWrangler") + Page findByUserOrPrimaryWrangler( + @Param(value = "user") String user, + @Param(value = "primaryWrangler") String primaryWrangler, + Pageable pageable); + + Page findBySubmissionEnvelopesContaining( + SubmissionEnvelope submissionEnvelope, Pageable pageable); + + @RestResource(exported = false) + Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + @RestResource(rel = "findBySubmissionAndValidationState") + public Page findBySubmissionEnvelopeAndValidationState( + @Param("envelopeUri") SubmissionEnvelope submissionEnvelope, + @Param("state") ValidationState state, + Pageable pageable); + + long countByUser(String user); + + @RestResource(exported = false) + Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + @RestResource(exported = false) + Stream findBySupplementaryFilesContains(File file); + + @RestResource(exported = false) + Stream findBySubmissionEnvelopesContains(SubmissionEnvelope submissionEnvelope); + + @RestResource(exported = false) + Stream findByUuid(Uuid uuid); + + @RestResource(rel = "catalogue", path = "catalogue") + Page findByIsInCatalogueTrue(Pageable pageable); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectService.java b/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectService.java new file mode 100644 index 000000000..c994af612 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/ProjectService.java @@ -0,0 +1,259 @@ +package uk.ac.ebi.subs.ingest.project; + +import static java.lang.String.format; +import static java.util.Objects.isNull; +import static java.util.stream.Collectors.toSet; + +import java.time.Instant; +import java.util.*; + +import javax.validation.constraints.NotNull; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.audit.AuditEntry; +import uk.ac.ebi.subs.ingest.audit.AuditEntryService; +import uk.ac.ebi.subs.ingest.audit.AuditType; +import uk.ac.ebi.subs.ingest.bundle.BundleManifest; +import uk.ac.ebi.subs.ingest.bundle.BundleManifestRepository; +import uk.ac.ebi.subs.ingest.bundle.BundleType; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.dataset.Dataset; +import uk.ac.ebi.subs.ingest.dataset.DatasetRepository; +import uk.ac.ebi.subs.ingest.project.exception.NonEmptyProject; +import uk.ac.ebi.subs.ingest.project.web.SearchFilter; +import uk.ac.ebi.subs.ingest.schemas.SchemaService; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@Service +@RequiredArgsConstructor +@Getter +public class ProjectService { + @Autowired private final MongoTemplate mongoTemplate; + + // Helper class for capturing copies of a Project and all Submission Envelopes related to them. + private static class ProjectBag { + + private final Set projects; + private final Set submissionEnvelopes; + + public ProjectBag(Set projects, Set submissionEnvelopes) { + this.projects = projects; + this.submissionEnvelopes = submissionEnvelopes; + } + } + + private final @NonNull SubmissionEnvelopeRepository submissionEnvelopeRepository; + private final @NonNull ProjectRepository projectRepository; + private final @NotNull DatasetRepository datasetRepository; + private final @NonNull MetadataCrudService metadataCrudService; + private final @NonNull MetadataUpdateService metadataUpdateService; + private final @NonNull SchemaService schemaService; + private final @NonNull BundleManifestRepository bundleManifestRepository; + private final @NonNull AuditEntryService auditEntryService; + + private final @NonNull ProjectEventHandler projectEventHandler; + + private final Logger log = LoggerFactory.getLogger(getClass()); + + protected Logger getLog() { + return log; + } + + public Project register(final Project project) { + project.setCataloguedDate(null); + if (!isNull(project.getIsInCatalogue()) && project.getIsInCatalogue()) { + project.setCataloguedDate(Instant.now()); + } + Project persistentProject = projectRepository.save(project); + projectEventHandler.registeredProject(persistentProject); + return persistentProject; + } + + public Project createSuggestedProject(final ObjectNode suggestion) { + Map content = createBaseContentForProject(); + Project suggestedProject = new Project(content); + suggestedProject.setWranglingState(WranglingState.NEW_SUGGESTION); + var notes = + String.format( + "DOI: %s \nName: %s \nEmail: %s \nComments: %s", + suggestion.get("doi"), + suggestion.get("name"), + suggestion.get("email"), + suggestion.get("comments")); + suggestedProject.setWranglingNotes(notes); + return this.register(suggestedProject); + } + + public Project update(final Project project, ObjectNode patch, Boolean sendNotification) { + if (patch.has("isInCatalogue") + && patch.get("isInCatalogue").asBoolean() + && project.getCataloguedDate() == null) { + project.setCataloguedDate(Instant.now()); + } + + updateWranglingState(project, patch); + Project updatedProject = metadataUpdateService.update(project, patch); + + if (sendNotification) { + projectEventHandler.editedProjectMetadata(updatedProject); + } + + return updatedProject; + } + + private void updateWranglingState(Project project, ObjectNode patch) { + Optional.ofNullable(patch.get("wranglingState")) + .map(JsonNode::asText) + .map(WranglingState::getName) + .ifPresent(newWranglingState -> updateWranglingState(project, newWranglingState)); + } + + public void updateWranglingState(Project project, @NonNull WranglingState newWranglingState) { + WranglingState currentWranglingState = project.getWranglingState(); + if (currentWranglingState != newWranglingState) { + log.info( + "setting project {} from {} to {}", + project.getId(), + currentWranglingState, + newWranglingState); + project.setWranglingState(newWranglingState); + projectRepository.save(project); + AuditEntry wranglingStateUpdate = + new AuditEntry( + AuditType.STATUS_UPDATED, currentWranglingState, newWranglingState, project); + auditEntryService.addAuditEntry(wranglingStateUpdate); + } + } + + public Project addProjectToSubmissionEnvelope( + SubmissionEnvelope submissionEnvelope, Project project) { + if (!project.getIsUpdate()) { + return metadataCrudService.addToSubmissionEnvelopeAndSave(project, submissionEnvelope); + } else { + return metadataUpdateService.acceptUpdate(project, submissionEnvelope); + } + } + + public Project linkProjectSubmissionEnvelope( + SubmissionEnvelope submissionEnvelope, Project project) { + final String projectId = project.getId(); + project.addToSubmissionEnvelopes(submissionEnvelope); + projectRepository.save(project); + + projectRepository + .findByUuidUuidAndIsUpdateFalse(project.getUuid().getUuid()) + .ifPresent( + projectByUuid -> { + if (!projectByUuid.getId().equals(projectId)) { + projectByUuid.addToSubmissionEnvelopes(submissionEnvelope); + projectRepository.save(projectByUuid); + } + }); + return project; + } + + public Page findBundleManifestsByProjectUuidAndBundleType( + Uuid projectUuid, BundleType bundleType, Pageable pageable) { + return this.projectRepository + .findByUuidUuidAndIsUpdateFalse(projectUuid.getUuid()) + .map( + project -> + bundleManifestRepository.findBundleManifestsByProjectAndBundleType( + project, bundleType, pageable)) + .orElseThrow( + () -> { + throw new ResourceNotFoundException( + format("Project with UUID %s not found", projectUuid.getUuid().toString())); + }); + } + + public Set getSubmissionEnvelopes(Project project) { + return gather(project).submissionEnvelopes; + } + + public void delete(Project project) throws NonEmptyProject { + ProjectBag projectBag = gather(project); + if (projectBag.submissionEnvelopes.isEmpty()) { + projectBag.projects.forEach( + _project -> { + metadataCrudService.deleteDocument(_project); + projectEventHandler.deletedProject(_project); + }); + } else { + throw new NonEmptyProject(); + } + } + + private Map createBaseContentForProject() { + Map content = new HashMap<>(); + final String entityType = "project"; + final String highLevelEntity = "type"; + content.put( + "describedBy", + schemaService.getLatestSchemaByEntityType(highLevelEntity, entityType).getSchemaUri()); + content.put("schema_type", entityType); + return content; + } + + private ProjectBag gather(Project project) { + Set envelopes = new HashSet<>(); + Set projects = this.projectRepository.findByUuid(project.getUuid()).collect(toSet()); + projects.forEach( + copy -> { + envelopes.addAll(copy.getSubmissionEnvelopes()); + envelopes.add(copy.getSubmissionEnvelope()); + }); + + // ToDo: Find a better way of ensuring that DBRefs to deleted objects aren't returned. + envelopes.removeIf(env -> env == null || env.getSubmissionState() == null); + return new ProjectBag(projects, envelopes); + } + + public Page filterProjects(SearchFilter searchFilter, Pageable pageable) { + Query query = ProjectQueryBuilder.buildProjectsQuery(searchFilter); + log.debug("Project Search query: " + query); + + List projects = mongoTemplate.find(query.with(pageable), Project.class); + long count = mongoTemplate.count(query, Project.class); + return new PageImpl<>(projects, pageable, count); + } + + public List getProjectAuditEntries(Project project) { + return auditEntryService.getAuditEntriesForAbstractEntity(project); + } + + public final Project linkDatasetToProject(final Project project, final Dataset dataset) { + final String projectId = project.getId(); + final String datasetId = dataset.getId(); + + projectRepository + .findById(projectId) + .orElseThrow(() -> new ResourceNotFoundException("Project: " + projectId)); + datasetRepository + .findById(datasetId) + .orElseThrow(() -> new ResourceNotFoundException("Dataset: " + datasetId)); + + project.addDataset(dataset); + + return projectRepository.save(project); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/WranglingState.java b/src/main/java/uk/ac/ebi/subs/ingest/project/WranglingState.java new file mode 100644 index 000000000..01937957c --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/WranglingState.java @@ -0,0 +1,37 @@ +package uk.ac.ebi.subs.ingest.project; + +import com.fasterxml.jackson.annotation.JsonValue; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@JsonSerialize(using = WranglingStateSerializer.class) +public enum WranglingState { + NEW("New"), + ELIGIBLE("Eligible"), + NOT_ELIGIBLE("Not eligible"), + IN_PROGRESS("In progress"), + STALLED("Stalled"), + SUBMITTED("Submitted"), + PUBLISHED_IN_DCP("Published in DCP"), + DELETED("Deleted"), + NEW_SUGGESTION("New Suggestion"); + + protected String value; + + WranglingState(String value) { + this.value = value; + } + + @JsonValue + public String getValue() { + return this.value; + } + + public static WranglingState getName(String value) { + for (WranglingState wranglingState : values()) { + if (wranglingState.value.equals(value)) { + return wranglingState; + } + } + return null; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/WranglingStateSerializer.java b/src/main/java/uk/ac/ebi/subs/ingest/project/WranglingStateSerializer.java new file mode 100644 index 000000000..58036595f --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/WranglingStateSerializer.java @@ -0,0 +1,16 @@ +package uk.ac.ebi.subs.ingest.project; + +import java.io.IOException; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +public class WranglingStateSerializer extends JsonSerializer { + @Override + public void serialize( + WranglingState value, JsonGenerator generator, SerializerProvider serializers) + throws IOException { + generator.writeString(value.value); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/exception/NonEmptyProject.java b/src/main/java/uk/ac/ebi/subs/ingest/project/exception/NonEmptyProject.java new file mode 100644 index 000000000..06820b63f --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/exception/NonEmptyProject.java @@ -0,0 +1,8 @@ +package uk.ac.ebi.subs.ingest.project.exception; + +public class NonEmptyProject extends Exception { + + public NonEmptyProject() { + super("Operation cannot be carried out on non-empty Project."); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/exception/NotAllowedWithSubmissionInStateException.java b/src/main/java/uk/ac/ebi/subs/ingest/project/exception/NotAllowedWithSubmissionInStateException.java new file mode 100644 index 000000000..dad3483c4 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/exception/NotAllowedWithSubmissionInStateException.java @@ -0,0 +1,9 @@ +package uk.ac.ebi.subs.ingest.project.exception; + +import uk.ac.ebi.subs.ingest.security.exception.NotAllowedException; + +public class NotAllowedWithSubmissionInStateException extends NotAllowedException { + public NotAllowedWithSubmissionInStateException() { + super("Operation not allowed while the project has a submission in a non-editable state."); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/web/ProjectController.java b/src/main/java/uk/ac/ebi/subs/ingest/project/web/ProjectController.java new file mode 100644 index 000000000..a9cc532a3 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/web/ProjectController.java @@ -0,0 +1,288 @@ +package uk.ac.ebi.subs.ingest.project.web; + +import java.util.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; +import org.springframework.data.rest.webmvc.PersistentEntityResource; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.data.web.PagedResourcesAssembler; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.hateoas.PagedResources; +import org.springframework.hateoas.Resource; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.annotation.Secured; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.*; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.bundle.BundleManifest; +import uk.ac.ebi.subs.ingest.bundle.BundleType; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.core.service.ValidationStateChangeService; +import uk.ac.ebi.subs.ingest.dataset.Dataset; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.project.ProjectEventHandler; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.project.ProjectService; +import uk.ac.ebi.subs.ingest.project.exception.NonEmptyProject; +import uk.ac.ebi.subs.ingest.project.exception.NotAllowedWithSubmissionInStateException; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; +import uk.ac.ebi.subs.ingest.security.CheckAllowed; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.exception.NotAllowedDuringSubmissionStateException; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 05/09/17 + */ +@RepositoryRestController +@ExposesResourceFor(Project.class) +@RequiredArgsConstructor +@Getter +public class ProjectController { + + private static final Logger LOGGER = LoggerFactory.getLogger(ProjectController.class); + + private final @NonNull ProjectService projectService; + + private final @NonNull ValidationStateChangeService validationStateChangeService; + + private final @NonNull ProjectEventHandler projectEventHandler; + + private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; + + private final @NonNull ProjectRepository projectRepository; + private final @NonNull BiomaterialRepository biomaterialRepository; + private final @NonNull ProcessRepository processRepository; + private final @NonNull ProtocolRepository protocolRepository; + private final @NonNull FileRepository fileRepository; + + private final @NonNull MetadataUpdateService metadataUpdateService; + + @PostMapping("/projects") + ResponseEntity> register( + @RequestBody final Project project, final PersistentEntityResourceAssembler assembler) { + Project result = projectService.register(project); + return ResponseEntity.ok().body(assembler.toFullResource(result)); + } + + @PostMapping("/projects/suggestion") + ResponseEntity> suggest( + @RequestBody final ObjectNode suggestion, final PersistentEntityResourceAssembler assembler) { + Project suggestedProject = projectService.createSuggestedProject(suggestion); + return ResponseEntity.ok().body(assembler.toFullResource(suggestedProject)); + } + + @CheckAllowed( + value = "#project.isEditable()", + exception = NotAllowedWithSubmissionInStateException.class) + @PatchMapping("/projects/{id}") + ResponseEntity> update( + @PathVariable("id") final Project project, + @RequestParam(value = "partial", defaultValue = "false") Boolean partial, + @RequestBody final ObjectNode patch, + final PersistentEntityResourceAssembler assembler) { + + List allowedFields = + List.of( + "accessionDate", + "cellCount", + "content", + "identifyingOrganisms", + "isInCatalogue", + "organ", + "primaryWrangler", + "publicationsInfo", + "releaseDate", + "secondaryWrangler", + "technology", + "validationErrors", + "wranglingState", + "wranglingPriority", + "wranglingNotes", + "dcpReleaseNumber", + "projectLabels", + "projectNetworks"); + + ObjectNode validPatch = patch.retain(allowedFields); + Project updatedProject = projectService.update(project, validPatch, !partial); + return ResponseEntity.ok().body(assembler.toFullResource(updatedProject)); + } + + @PreAuthorize("hasAnyRole('ROLE_CONTRIBUTOR', 'ROLE_WRANGLER', 'ROLE_SERVICE')") + @CheckAllowed( + value = "#submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PostMapping(path = "submissionEnvelopes/{sub_id}/projects") + ResponseEntity> addProjectToEnvelope( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + @RequestBody Project project, + @RequestParam("updatingUuid") Optional updatingUuid, + PersistentEntityResourceAssembler assembler) { + updatingUuid.ifPresent( + uuid -> { + project.setUuid(new Uuid(uuid.toString())); + project.setIsUpdate(true); + }); + Project entity = + getProjectService().addProjectToSubmissionEnvelope(submissionEnvelope, project); + PersistentEntityResource resource = assembler.toFullResource(entity); + return ResponseEntity.accepted().body(resource); + } + + @GetMapping(path = "/projects/{id}/bundleManifests") + ResponseEntity>> getBundleManifests( + @PathVariable("id") Project project, + @RequestParam("bundleType") Optional bundleType, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page bundleManifests = + projectService + .getBundleManifestRepository() + .findBundleManifestsByProjectAndBundleType(project, bundleType.orElse(null), pageable); + return ResponseEntity.ok( + pagedResourcesAssembler.toResource(bundleManifests, resourceAssembler)); + } + + @PreAuthorize( + "hasAnyRole('ROLE_access_'+#project.uuid, 'ROLE_SERVICE')" + + "or #project['content']['dataAccess']['type'] eq T(uk.ac.ebi.subs.ingest.project.DataAccessTypes).OPEN.label") + @GetMapping(path = "/projects/{id}/submissionEnvelopes") + ResponseEntity>> getProjectSubmissionEnvelopes( + @PathVariable("id") Project project, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + var envelopes = projectService.getSubmissionEnvelopes(project); + var resultPage = new PageImpl<>(new ArrayList<>(envelopes), pageable, envelopes.size()); + return ResponseEntity.ok(pagedResourcesAssembler.toResource(resultPage, resourceAssembler)); + } + + @PreAuthorize( + "hasAnyRole('ROLE_access_'+#project.uuid, 'ROLE_SERVICE')" + + "or #project['content']['dataAccess']['type'] eq T(uk.ac.ebi.subs.ingest.project.DataAccessTypes).OPEN.label") + @RequestMapping(path = "/projects/{project_id}/biomaterials", method = RequestMethod.GET) + ResponseEntity getBiomaterials( + @PathVariable("project_id") Project project, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page biomaterials = getBiomaterialRepository().findByProject(project, pageable); + return ResponseEntity.ok( + getPagedResourcesAssembler().toResource(biomaterials, resourceAssembler)); + } + + @PreAuthorize( + "hasAnyRole('ROLE_access_'+#project.uuid, 'ROLE_SERVICE')" + + "or #project['content']['dataAccess']['type'] eq T(uk.ac.ebi.subs.ingest.project.DataAccessTypes).OPEN.label") + @RequestMapping(path = "/projects/{project_id}/processes", method = RequestMethod.GET) + ResponseEntity getProcesses( + @PathVariable("project_id") Project project, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page processes = getProcessRepository().findByProject(project, pageable); + return ResponseEntity.ok(getPagedResourcesAssembler().toResource(processes, resourceAssembler)); + } + + @PreAuthorize( + "hasAnyRole('ROLE_access_'+#project.uuid, 'ROLE_SERVICE')" + + "or #project['content']['dataAccess']['type'] " + + " eq T(uk.ac.ebi.subs.ingest.project.DataAccessTypes).OPEN.label") + @RequestMapping(path = "/projects/{project_id}/protocols", method = RequestMethod.GET) + ResponseEntity getProtocols( + @PathVariable("project_id") Project project, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page protocols = getProtocolRepository().findByProject(project, pageable); + return ResponseEntity.ok(getPagedResourcesAssembler().toResource(protocols, resourceAssembler)); + } + + @PreAuthorize( + "hasAnyRole('ROLE_access_'+#project.uuid, 'ROLE_SERVICE')" + + "or #project['content']['dataAccess']['type'] " + + " eq T(uk.ac.ebi.subs.ingest.project.DataAccessTypes).OPEN.label") + @RequestMapping(path = "/projects/{project_id}/files", method = RequestMethod.GET) + ResponseEntity getFiles( + @PathVariable("project_id") Project project, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page files = getFileRepository().findByProject(project, pageable); + return ResponseEntity.ok(getPagedResourcesAssembler().toResource(files, resourceAssembler)); + } + + @CheckAllowed( + value = "#submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PutMapping(path = "projects/{proj_id}/submissionEnvelopes/{sub_id}") + ResponseEntity> linkSubmissionToProject( + @PathVariable("proj_id") Project project, + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + PersistentEntityResourceAssembler assembler) { + Project savedProject = + getProjectService().linkProjectSubmissionEnvelope(submissionEnvelope, project); + PersistentEntityResource projectResource = assembler.toFullResource(savedProject); + return ResponseEntity.accepted().body(projectResource); + } + + @CheckAllowed( + value = "#project.isEditable()", + exception = NotAllowedWithSubmissionInStateException.class) + @DeleteMapping(path = "projects/{id}") + public ResponseEntity delete(@PathVariable("id") Project project) { + try { + projectService.delete(project); + return ResponseEntity.noContent().build(); + } catch (NonEmptyProject nonEmptyProject) { + String message = nonEmptyProject.getMessage(); + LOGGER.debug(message); + Map errorResponse = Map.of("message", message); + return ResponseEntity.status(HttpStatus.CONFLICT).body(errorResponse); + } + } + + @GetMapping(path = "projects/filter") + @Secured({"ROLE_WRANGLER", "ROLE_SERVICE"}) + public ResponseEntity>> filterProjects( + @ModelAttribute SearchFilter searchFilter, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + var projects = projectService.filterProjects(searchFilter, pageable); + return ResponseEntity.ok(pagedResourcesAssembler.toResource(projects, resourceAssembler)); + } + + @GetMapping(path = "projects/{id}/auditLogs") + public ResponseEntity getProjectAuditLogs(@PathVariable("id") Project project) { + if (project == null) { + return ResponseEntity.notFound().build(); + } + + return ResponseEntity.ok(projectService.getProjectAuditEntries(project)); + } + + @PutMapping(path = "projects/{project_id}/datasets/{dataset_id}") + public ResponseEntity> linkDatasetToProject( + @PathVariable("project_id") final Project project, + @PathVariable("dataset_id") final Dataset dataset, + final PersistentEntityResourceAssembler assembler) { + return ResponseEntity.accepted() + .body(assembler.toFullResource(getProjectService().linkDatasetToProject(project, dataset))); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/web/ProjectResourceProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/project/web/ProjectResourceProcessor.java new file mode 100644 index 000000000..7fd5221dc --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/web/ProjectResourceProcessor.java @@ -0,0 +1,41 @@ +package uk.ac.ebi.subs.ingest.project.web; + +import org.springframework.hateoas.EntityLinks; +import org.springframework.hateoas.Link; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.stereotype.Component; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.project.Project; + +@Component +@RequiredArgsConstructor +public class ProjectResourceProcessor implements ResourceProcessor> { + private final @NonNull EntityLinks entityLinks; + + @Override + public Resource process(Resource resource) { + Project project = resource.getContent(); + resource.add(getBundleManifestsLink(project)); + resource.add(getAuditLogsLink(project)); + + return resource; + } + + private Link getBundleManifestsLink(Project project) { + return entityLinks + .linkForSingleResource(project) + .slash(Links.BUNDLE_MANIFESTS_URL) + .withRel(Links.BUNDLE_MANIFESTS_REL); + } + + private Link getAuditLogsLink(Project project) { + return entityLinks + .linkForSingleResource(project) + .slash(Links.AUDIT_LOGS_URL) + .withRel(Links.AUDIT_LOGS_REL); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/web/SearchFilter.java b/src/main/java/uk/ac/ebi/subs/ingest/project/web/SearchFilter.java new file mode 100644 index 000000000..d35906f59 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/web/SearchFilter.java @@ -0,0 +1,29 @@ +package uk.ac.ebi.subs.ingest.project.web; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.ToString; +import uk.ac.ebi.subs.ingest.project.DataAccessTypes; + +@AllArgsConstructor +@Builder +@ToString +public class SearchFilter { + @Getter String search; + @Getter String wranglingState; + @Getter String primaryWrangler; + @Getter Integer wranglingPriority; + @Getter Boolean hasOfficialHcaPublication; + @Getter String identifyingOrganism; + @Getter String organOntology; + @Getter Integer minCellCount; + @Getter Integer maxCellCount; + @Getter Integer dcpReleaseNumber; + @Getter DataAccessTypes dataAccess; + @Getter String projectLabels; + @Getter String projectNetworks; + @Getter Boolean hcaCatalogue; + + @Builder.Default @Getter SearchType searchType = SearchType.AllKeywords; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/project/web/SearchType.java b/src/main/java/uk/ac/ebi/subs/ingest/project/web/SearchType.java new file mode 100644 index 000000000..afa8a272d --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/project/web/SearchType.java @@ -0,0 +1,7 @@ +package uk.ac.ebi.subs.ingest.project.web; + +public enum SearchType { + AnyKeyword, + AllKeywords, + ExactMatch +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/protocol/Protocol.java b/src/main/java/uk/ac/ebi/subs/ingest/protocol/Protocol.java new file mode 100644 index 000000000..0a42e94db --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/protocol/Protocol.java @@ -0,0 +1,46 @@ +package uk.ac.ebi.subs.ingest.protocol; + +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.DBRef; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.project.Project; + +@Getter +@EqualsAndHashCode( + callSuper = true, + exclude = {"project"}) +@NoArgsConstructor +public class Protocol extends MetadataDocument { + @Indexed + private @Setter @DBRef(lazy = true) Project project; + + private boolean linked = false; + + @JsonCreator + public Protocol(@JsonProperty("content") Object content) { + super(EntityType.PROTOCOL, content); + } + + public boolean isLinked() { + return linked; + } + + /* TODO + This method was originally made as simple as possible to only support the orphaned entity use case. However, + this can be enhanced further to add a full-fledged component that enables back linking to all Processes that refer + to this Protocol. In that case, the isLinked implementation will need to be changed to check if the list of + processes is empty or not. This approach was not initially chosen because it would have required data migration. + */ + public void markAsLinked() { + this.linked = true; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/protocol/ProtocolRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/protocol/ProtocolRepository.java new file mode 100644 index 000000000..9079be819 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/protocol/ProtocolRepository.java @@ -0,0 +1,92 @@ +package uk.ac.ebi.subs.ingest.protocol; + +import java.util.Collection; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.data.rest.core.annotation.RestResource; +import org.springframework.web.bind.annotation.CrossOrigin; + +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.security.RowLevelFilterSecurity; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 31/08/17 + */ +@CrossOrigin +@RowLevelFilterSecurity( + expression = + "(#filterObject.project != null)" + + "? " + + " (" + + " #authentication.authorities.![authority].contains(" + + " 'ROLE_access_' +#filterObject.project.uuid?.toString()) " + + " or " + + " #authentication.authorities.![authority].contains('ROLE_SERVICE') " + + " or " + + " #filterObject.project.content['dataAccess']['type'] " + + " eq T(uk.ac.ebi.subs.ingest.project.DataAccessTypes).OPEN.label" + + " )" + + ":true", + ignoreClasses = {Project.class}) +public interface ProtocolRepository extends MongoRepository { + + public Page findBySubmissionEnvelope( + SubmissionEnvelope submissionEnvelope, Pageable pageable); + + public Page findByProject(Project project, Pageable pageable); + + @RestResource(exported = false) + Stream findByProject(Project project); + + @RestResource(exported = false) + public Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + @RestResource(exported = false) + Long deleteBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + @RestResource(rel = "findBySubmissionAndValidationState") + public Page findBySubmissionEnvelopeAndValidationState( + @Param("envelopeUri") SubmissionEnvelope submissionEnvelope, + @Param("state") ValidationState state, + Pageable pageable); + + @Query( + value = + "{'submissionEnvelope.id': ?0, graphValidationErrors: { $exists: true, $not: {$size: 0} } }") + @RestResource(rel = "findBySubmissionIdWithGraphValidationErrors") + public Page findBySubmissionIdWithGraphValidationErrors( + @Param("envelopeId") String envelopeId, Pageable pageable); + + @RestResource(exported = false) + Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + @RestResource(rel = "findAllByUuid", path = "findAllByUuid") + Page findByUuid(@Param("uuid") Uuid uuid, Pageable pageable); + + @RestResource(rel = "findByUuid", path = "findByUuid") + Optional findByUuidUuidAndIsUpdateFalse(@Param("uuid") UUID uuid); + + long countBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + long countBySubmissionEnvelopeAndValidationState( + SubmissionEnvelope submissionEnvelope, ValidationState validationState); + + @Query( + value = + "{'submissionEnvelope.id': ?0, graphValidationErrors: { $exists: true, $not: {$size: 0} } }", + count = true) + long countBySubmissionEnvelopeAndCountWithGraphValidationErrors(String submissionEnvelopeId); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/protocol/ProtocolService.java b/src/main/java/uk/ac/ebi/subs/ingest/protocol/ProtocolService.java new file mode 100644 index 000000000..de84f817a --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/protocol/ProtocolService.java @@ -0,0 +1,67 @@ +package uk.ac.ebi.subs.ingest.protocol; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 05/09/17 + */ +@Service +@RequiredArgsConstructor +@Getter +public class ProtocolService { + + private final @NonNull MetadataCrudService metadataCrudService; + private final @NonNull MetadataUpdateService metadataUpdateService; + + private final @NonNull SubmissionEnvelopeRepository submissionEnvelopeRepository; + private final @NonNull ProjectRepository projectRepository; + private final @NonNull ProtocolRepository protocolRepository; + private final @NonNull ProcessRepository processRepository; + + private final Logger log = LoggerFactory.getLogger(getClass()); + + protected Logger getLog() { + return log; + } + + public Protocol addProtocolToSubmissionEnvelope( + SubmissionEnvelope submissionEnvelope, Protocol protocol) { + if (!protocol.getIsUpdate()) { + projectRepository + .findBySubmissionEnvelopesContains(submissionEnvelope) + .findFirst() + .ifPresent(protocol::setProject); + return metadataCrudService.addToSubmissionEnvelopeAndSave(protocol, submissionEnvelope); + } else { + return metadataUpdateService.acceptUpdate(protocol, submissionEnvelope); + } + } + + public Page retrieve(SubmissionEnvelope submission, Pageable pageable) { + Page protocols = protocolRepository.findBySubmissionEnvelope(submission, pageable); + protocols.forEach( + protocol -> { + processRepository + .findFirstByProtocolsContains(protocol) + .ifPresent(it -> protocol.markAsLinked()); + }); + return protocols; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/protocol/web/ProtocolController.java b/src/main/java/uk/ac/ebi/subs/ingest/protocol/web/ProtocolController.java new file mode 100644 index 000000000..f7a2a22e4 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/protocol/web/ProtocolController.java @@ -0,0 +1,113 @@ +package uk.ac.ebi.subs.ingest.protocol.web; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import org.springframework.data.rest.webmvc.PersistentEntityResource; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.data.web.PagedResourcesAssembler; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.hateoas.Resource; +import org.springframework.http.HttpEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.patch.JsonPatcher; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; +import uk.ac.ebi.subs.ingest.protocol.ProtocolService; +import uk.ac.ebi.subs.ingest.security.CheckAllowed; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.exception.NotAllowedDuringSubmissionStateException; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 05/09/17 + */ +@RepositoryRestController +@ExposesResourceFor(Protocol.class) +@RequiredArgsConstructor +@Getter +public class ProtocolController { + private final @NonNull ProtocolService protocolService; + private final @NonNull ProtocolRepository protocolRepository; + + private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; + + private final @NonNull JsonPatcher jsonPatcher; + + private final @NonNull MetadataCrudService metadataCrudService; + private final @NonNull MetadataUpdateService metadataUpdateService; + + @CheckAllowed( + value = "#submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @RequestMapping(path = "/submissionEnvelopes/{sub_id}/protocols", method = RequestMethod.POST) + ResponseEntity> addProtocolToEnvelope( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + @RequestBody Protocol protocol, + @RequestParam("updatingUuid") Optional updatingUuid, + PersistentEntityResourceAssembler assembler) { + updatingUuid.ifPresent( + uuid -> { + protocol.setUuid(new Uuid(uuid.toString())); + protocol.setIsUpdate(true); + }); + Protocol entity = + getProtocolService().addProtocolToSubmissionEnvelope(submissionEnvelope, protocol); + PersistentEntityResource resource = assembler.toFullResource(entity); + return ResponseEntity.accepted().body(resource); + } + + @CheckAllowed( + value = "#submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @RequestMapping( + path = "/submissionEnvelopes/{sub_id}/protocols/{protocol_id}", + method = RequestMethod.PUT) + ResponseEntity> linkProtocolToEnvelope( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + @PathVariable("id") Protocol protocol, + PersistentEntityResourceAssembler assembler) { + Protocol entity = + getProtocolService().addProtocolToSubmissionEnvelope(submissionEnvelope, protocol); + PersistentEntityResource resource = assembler.toFullResource(entity); + return ResponseEntity.accepted().body(resource); + } + + @CheckAllowed( + value = "#protocol.submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @RequestMapping(path = "/protocols/{id}", method = RequestMethod.PATCH) + HttpEntity patchProtocol( + @PathVariable("id") Protocol protocol, + @RequestBody final ObjectNode patch, + PersistentEntityResourceAssembler assembler) { + List allowedFields = List.of("content", "validationErrors", "graphValidationErrors"); + ObjectNode validPatch = patch.retain(allowedFields); + Protocol updatedProtocol = metadataUpdateService.update(protocol, validPatch); + PersistentEntityResource resource = assembler.toFullResource(updatedProtocol); + return ResponseEntity.accepted().body(resource); + } + + @CheckAllowed( + value = "#protocol.submissionEnvelope.isEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @DeleteMapping(path = "/protocols/{id}") + ResponseEntity deleteProtocol(@PathVariable("id") Protocol protocol) { + metadataCrudService.deleteDocument(protocol); + return ResponseEntity.noContent().build(); + } +} diff --git a/src/main/java/org/humancellatlas/ingest/query/MetadataCriteria.java b/src/main/java/uk/ac/ebi/subs/ingest/query/MetadataCriteria.java similarity index 66% rename from src/main/java/org/humancellatlas/ingest/query/MetadataCriteria.java rename to src/main/java/uk/ac/ebi/subs/ingest/query/MetadataCriteria.java index 050f08e59..2b7192843 100644 --- a/src/main/java/org/humancellatlas/ingest/query/MetadataCriteria.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/query/MetadataCriteria.java @@ -1,4 +1,4 @@ -package org.humancellatlas.ingest.query; +package uk.ac.ebi.subs.ingest.query; import lombok.AllArgsConstructor; import lombok.Data; @@ -10,7 +10,7 @@ @NoArgsConstructor @Getter public class MetadataCriteria { - String field; - Operator operator; - Object value; + String field; + Operator operator; + Object value; } diff --git a/src/main/java/uk/ac/ebi/subs/ingest/query/MetadataQueryService.java b/src/main/java/uk/ac/ebi/subs/ingest/query/MetadataQueryService.java new file mode 100644 index 000000000..32e9a0836 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/query/MetadataQueryService.java @@ -0,0 +1,57 @@ +package uk.ac.ebi.subs.ingest.query; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.stereotype.Service; + +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.protocol.Protocol; + +/** Created by prabhat on 02/11/2020. */ +@Service +public class MetadataQueryService { + @Autowired private MongoTemplate mongoTemplate; + + @Autowired private QueryBuilder queryBuilder; + + public Page findByCriteria( + EntityType metadataType, + List criteriaList, + Boolean andCriteria, + Pageable pageable) { + Query query = queryBuilder.build(criteriaList, andCriteria); + List result = mongoTemplate.find(query.with(pageable), getEntityClass(metadataType)); + long count = mongoTemplate.count(query, getEntityClass(metadataType)); + + return new PageImpl<>(result, pageable, count); + } + ; + + Class getEntityClass(EntityType metadataType) { + switch (metadataType) { + case BIOMATERIAL: + return Biomaterial.class; + case PROTOCOL: + return Protocol.class; + case PROJECT: + return Project.class; + case PROCESS: + return Process.class; + case FILE: + return File.class; + default: + throw new ResourceNotFoundException(); + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/query/Operator.java b/src/main/java/uk/ac/ebi/subs/ingest/query/Operator.java new file mode 100644 index 000000000..8baa59424 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/query/Operator.java @@ -0,0 +1,13 @@ +package uk.ac.ebi.subs.ingest.query; + +public enum Operator { + GT, + GTE, + LT, + LTE, + IN, + NIN, + IS, + NE, + REGEX +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/query/QueryBuilder.java b/src/main/java/uk/ac/ebi/subs/ingest/query/QueryBuilder.java new file mode 100644 index 000000000..53c5d332e --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/query/QueryBuilder.java @@ -0,0 +1,72 @@ +package uk.ac.ebi.subs.ingest.query; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.springframework.data.mongodb.InvalidMongoDbApiUsageException; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.stereotype.Component; + +@Component +public class QueryBuilder { + public Query build(List criteriaList, Boolean andCriteria) { + Query query = new Query(); + + List criterias = new ArrayList<>(); + + for (MetadataCriteria metadataCriteria : criteriaList) { + String field = metadataCriteria.getField(); + Criteria criteria = Criteria.where(field); + switch (metadataCriteria.getOperator()) { + case IS: + try { + criteria = criteria.is(metadataCriteria.getValue()); + } catch (InvalidMongoDbApiUsageException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } + break; + case NE: + criteria = criteria.ne(metadataCriteria.getValue()); + break; + case GT: + criteria = criteria.gt(metadataCriteria.getValue()); + break; + case GTE: + criteria = criteria.gte(metadataCriteria.getValue()); + break; + case LT: + criteria = criteria.lt(metadataCriteria.getValue()); + break; + case LTE: + criteria = criteria.lte(metadataCriteria.getValue()); + break; + case IN: + criteria = criteria.in((Collection) metadataCriteria.getValue()); + break; + case NIN: + criteria = criteria.nin((Collection) metadataCriteria.getValue()); + break; + case REGEX: + criteria = criteria.regex((String) metadataCriteria.getValue(), "i"); + break; + default: + throw new IllegalArgumentException( + String.format( + "MetadataCriteria %s is not supported.", metadataCriteria.getOperator())); + } + criterias.add(criteria); + } + + if (andCriteria) { + query.addCriteria( + new Criteria().andOperator(criterias.toArray(new Criteria[criterias.size()]))); + } else { + query.addCriteria( + new Criteria().orOperator(criterias.toArray(new Criteria[criterias.size()]))); + } + + return query; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/schemas/Schema.java b/src/main/java/uk/ac/ebi/subs/ingest/schemas/Schema.java new file mode 100644 index 000000000..9357c9926 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/schemas/Schema.java @@ -0,0 +1,105 @@ +package uk.ac.ebi.subs.ingest.schemas; + +import static java.lang.String.format; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import lombok.Getter; +import uk.ac.ebi.subs.ingest.core.AbstractEntity; + +@Getter +public class Schema extends AbstractEntity implements Comparable { + + private String highLevelEntity; + private String schemaVersion; + private String domainEntity; + private String subDomainEntity; + private String concreteEntity; + + private String compoundKeys; + + @JsonIgnore private String schemaUri; + + public Schema( + String highLevelEntity, + String schemaVersion, + String domainEntity, + String subDomainEntity, + String concreteEntity, + String schemaUri) { + this.highLevelEntity = highLevelEntity; + this.schemaVersion = schemaVersion; + this.domainEntity = domainEntity; + this.subDomainEntity = subDomainEntity; + this.concreteEntity = concreteEntity; + this.schemaUri = schemaUri; + this.compoundKeys = concatenateKeys(); + } + + private String concatenateKeys() { + return format("%s/%s/%s/%s", highLevelEntity, domainEntity, subDomainEntity, concreteEntity); + } + + @Override + public int compareTo(Schema other) { + int difference = compoundKeys.compareTo(other.compoundKeys); + if (difference == 0) { + SemanticVersion otherSchemaVersion = SemanticVersion.parse(other.schemaVersion); + difference = SemanticVersion.parse(schemaVersion).compareTo(otherSchemaVersion); + } + return difference; + } + + private static class SemanticVersion implements Comparable { + + private static final Pattern VERSION_PATTERN = + Pattern.compile( + "(?\\p{Digit}+)" + "(.(?\\p{Digit}+))??(.(?\\p{Digit}+))??"); + + final int major; + final int minor; + int patch; + + SemanticVersion(int major, int minor, int patch) { + this.major = major; + this.minor = minor; + this.patch = patch; + } + + static SemanticVersion parse(String version) { + Matcher match = VERSION_PATTERN.matcher(version); + if (match.matches()) { + int major = parseVersionSegment(match, "major"); + int minor = parseVersionSegment(match, "minor"); + int patch = parseVersionSegment(match, "patch"); + return new SemanticVersion(major, minor, patch); + } else { + throw new RuntimeException("Invalid version format."); + } + } + + private static int parseVersionSegment(Matcher match, String versionSegment) { + String segmentValue = match.group(versionSegment); + int value = 0; + if (segmentValue != null) { + value = Integer.parseInt(segmentValue); + } + return value; + } + + @Override + public int compareTo(SemanticVersion other) { + int difference = major - other.major; + if (difference == 0) { + difference = minor - other.minor; + if (difference == 0) { + difference = patch - other.patch; + } + } + return difference; + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/schemas/SchemaRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/schemas/SchemaRepository.java new file mode 100644 index 000000000..f6dd9596e --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/schemas/SchemaRepository.java @@ -0,0 +1,56 @@ +package uk.ac.ebi.subs.ingest.schemas; + +import java.util.List; +import java.util.stream.Stream; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.data.rest.core.annotation.RestResource; +import org.springframework.web.bind.annotation.CrossOrigin; + +import uk.ac.ebi.subs.ingest.core.Uuid; + +/** Created by rolando on 18/04/2018. */ +@CrossOrigin +public interface SchemaRepository extends MongoRepository { + @RestResource(exported = false) + S save(S schema); + + @RestResource(exported = false) + List save(Iterable schemas); + + @RestResource(exported = false) + void delete(Schema schema); + + @RestResource(exported = false) + List findByUuidEquals(Uuid uuid); + + @RestResource + Page findBySchemaVersionAfter( + @Param("schema-version-range") String schemaVersionRange, Pageable pageable); + + @RestResource(rel = "querySchemas") + @Query( + "{$and :[" + + "?#{ [0] == null ? { $where : 'true'} : { 'highLevelEntity' : {'$regex' : [0]} } }," + + "?#{ [1] == null ? { $where : 'true'} : { 'concreteEntity' : {'$regex' : [1]} } }," + + "?#{ [2] == null ? { $where : 'true'} : { 'domainEntity' : {'$regex' : [2]} } }," + + "?#{ [3] == null ? { $where : 'true'} : { 'subDomainEntity' : {'$regex' : [3]} } }," + + "?#{ [4] == null ? { $where : 'true'} : { 'schemaVersion' : {'$regex' : [4]} } }" + + "]}") + Page querySchemas( + @Param("high-level-entity") String highLevelEntity, + @Param("concrete-entity") String concreteEntity, + @Param("domain-entity") String domainEntity, + @Param("sub-domain-entity") String subDomainEntity, + @Param("schema-version") String schemaVersion, + Pageable pageable); + + @RestResource(exported = false) + Stream findAllByOrderBySchemaVersionDesc(); + + Page findAllByOrderBySchemaVersionDesc(Pageable pageable); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/schemas/SchemaService.java b/src/main/java/uk/ac/ebi/subs/ingest/schemas/SchemaService.java new file mode 100644 index 000000000..e3fcb35e4 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/schemas/SchemaService.java @@ -0,0 +1,190 @@ +package uk.ac.ebi.subs.ingest.schemas; + +import java.net.URI; +import java.util.*; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.schemas.schemascraper.SchemaScraper; +import uk.ac.ebi.subs.ingest.schemas.schemascraper.impl.SchemaScrapeException; + +@Service +@Getter +@NoArgsConstructor +@AllArgsConstructor +public class SchemaService { + + @Autowired private SchemaRepository schemaRepository; + + @Autowired private SchemaScraper schemaScraper; + + @Autowired private Environment environment; + + private static final int EVERY_24_HOURS = 1000 * 60 * 60 * 24; + + public List filterLatestSchemas(String highLevelEntity) { + return getLatestSchemas().stream() + .filter(schema -> schema.getHighLevelEntity().matches(highLevelEntity)) + .collect(Collectors.toList()); + } + + public List getLatestSchemas() { + List allSchemas = schemaRepository.findAll(); + allSchemas.sort(Collections.reverseOrder()); + + Set latestSchemas = new LinkedHashSet<>(); + allSchemas.stream().map(LatestSchema::new).forEach(latestSchemas::add); + + return latestSchemas.stream().map(LatestSchema::getSchema).collect(Collectors.toList()); + } + + public Schema getLatestSchemaByEntityType(String highLevelEntity, String entityType) { + List allLatestSchema = + filterLatestSchemas(highLevelEntity).stream() + .filter(schema -> schema.getConcreteEntity().matches(entityType)) + .collect(Collectors.toList()); + + return allLatestSchema.size() > 0 ? allLatestSchema.get(0) : null; + } + + @Scheduled(fixedDelay = EVERY_24_HOURS) + public void updateSchemasCollection() { + String schemaBaseUri = getSchemaBaseUri(); + + if (schemaBaseUri == null) + throw new SchemaScrapeException("SCHEMA_BASE_URI environmental variable should not be null."); + + if (schemaBaseUri.endsWith("/")) { + schemaBaseUri = schemaBaseUri.substring(0, schemaBaseUri.length() - 1); + } + + // TODO Find a way how to neatly exclude the files + schemaScraper.getAllSchemaURIs(URI.create(schemaBaseUri)).stream() + .filter( + schemaUri -> + !schemaUri.toString().contains("index.html") + && !schemaUri.toString().contains("property_migrations")) + .forEach(this::doUpdate); + } + + public String getSchemaBaseUri() { + return environment.getProperty("SCHEMA_BASE_URI"); + } + + private void doUpdate(URI schemaUri) { + Schema schemaDocument = schemaDescriptionFromSchemaUri(schemaUri); + + UUID schemaUuid = UUID.nameUUIDFromBytes(schemaUri.toString().getBytes()); + schemaDocument.setUuid(new Uuid(schemaUuid.toString())); + + deleteMatchingSchemas(schemaUuid); + schemaRepository.save(schemaDocument); + } + + private void deleteMatchingSchemas(UUID schemaUuid) { + Collection matchingSchemas = + schemaRepository.findByUuidEquals(new Uuid(schemaUuid.toString())); + schemaRepository.deleteAll(matchingSchemas); + } + + public Collection schemaDescriptionFromSchemaUris(Collection schemaUris) { + return schemaUris.stream() + .map(this::schemaDescriptionFromSchemaUri) + .collect(Collectors.toList()); + } + + private Schema schemaDescriptionFromSchemaUri(URI schemaUri) { + String[] splitString = schemaUri.toString().split("/"); + String schemaFullUri = environment.getProperty("SCHEMA_BASE_URI") + schemaUri; + + String highLevelEntity = splitString[0]; + String version = null; + String domainEntity = ""; + String subDomainEntity = ""; + String concreteEntity = ""; + + try { + // Handle "bundle" schemas specifically + if ("bundle".equals(highLevelEntity) && splitString.length == 3) { + version = splitString[1]; + concreteEntity = splitString[2]; + return new Schema( + highLevelEntity, version, domainEntity, subDomainEntity, concreteEntity, schemaFullUri); + } + + for (int i = 0; i < splitString.length; i++) { + if (isVersion(splitString[i])) { + version = splitString[i]; + if (i + 1 < splitString.length) { + domainEntity = splitString[i + 1]; + if (i + 2 < splitString.length) { + concreteEntity = splitString[i + 2]; + } + } + break; + } + } + + if (version == null) { + throw new SchemaScrapeException("Couldn't find a valid version in URI: " + schemaFullUri); + } + + if (highLevelEntity != null && version != null && concreteEntity != null) { + return new Schema( + highLevelEntity, version, domainEntity, subDomainEntity, concreteEntity, schemaFullUri); + } else { + throw new SchemaScrapeException( + "Couldn't construct a Schema document from URI: " + schemaFullUri); + } + } catch (Exception e) { + System.err.println("Error while processing URI: " + schemaUri); + e.printStackTrace(); + throw e; + } + } + + private boolean isVersion(String str) { + return str.matches("\\d+\\.\\d+\\.\\d+"); // Matches versions like 1.0.0, 10.1.2, etc. + } + + /** + * A wrapper for Schema documents used to define a looser equals()/hashCode() to determine + * equivalence of Schemas based only on a Schema's high level entity, type, etc. + */ + private class LatestSchema { + @Getter private final Schema schema; + + LatestSchema(Schema schema) { + this.schema = schema; + } + + @Override + public boolean equals(Object to) { + if (to == this) return true; + if (!(to instanceof LatestSchema)) { + return false; + } + + LatestSchema schema = (LatestSchema) to; + return schema.hashCode() == this.hashCode(); + } + + @Override + public int hashCode() { + int result = 17; + result = 31 * result + this.schema.getConcreteEntity().hashCode(); + result = 31 * result + this.schema.getHighLevelEntity().hashCode(); + result = 31 * result + this.schema.getDomainEntity().hashCode(); + result = 31 * result + this.schema.getSubDomainEntity().hashCode(); + return result; + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/schemas/schemascraper/SchemaScraper.java b/src/main/java/uk/ac/ebi/subs/ingest/schemas/schemascraper/SchemaScraper.java new file mode 100644 index 000000000..044165dc0 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/schemas/schemascraper/SchemaScraper.java @@ -0,0 +1,13 @@ +package uk.ac.ebi.subs.ingest.schemas.schemascraper; + +import java.net.URI; +import java.util.Collection; + +/** + * Created by rolando on 19/04/2018. + * + *

Collects schemas from schema.humancellatlas.org (or some other schema bucket) + */ +public interface SchemaScraper { + Collection getAllSchemaURIs(URI schemaBucketLocation); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/schemas/schemascraper/impl/ListBucketResult.java b/src/main/java/uk/ac/ebi/subs/ingest/schemas/schemascraper/impl/ListBucketResult.java new file mode 100644 index 000000000..3953a76ba --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/schemas/schemascraper/impl/ListBucketResult.java @@ -0,0 +1,32 @@ +package uk.ac.ebi.subs.ingest.schemas.schemascraper.impl; + +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; + +/** Created by rolando on 19/04/2018. */ +public class ListBucketResult { + public ListBucketResult() {} + + @JacksonXmlProperty(localName = "Name") + public String name; + + @JacksonXmlProperty(localName = "Contents") + public List contents = new ArrayList<>(); + + static class Content { + Content() {} + + @JacksonXmlProperty(localName = "Key") + public String key; + + public void setKey(String key) { + this.key = key; + } + + public String getKey() { + return this.key; + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/schemas/schemascraper/impl/S3BucketSchemaScraper.java b/src/main/java/uk/ac/ebi/subs/ingest/schemas/schemascraper/impl/S3BucketSchemaScraper.java new file mode 100644 index 000000000..6251bf631 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/schemas/schemascraper/impl/S3BucketSchemaScraper.java @@ -0,0 +1,54 @@ +package uk.ac.ebi.subs.ingest.schemas.schemascraper.impl; + +import java.io.IOException; +import java.net.URI; +import java.util.Collection; +import java.util.stream.Collectors; + +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.dataformat.xml.JacksonXmlModule; +import com.fasterxml.jackson.dataformat.xml.XmlMapper; + +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.schemas.schemascraper.SchemaScraper; + +/** + * Created by rolando on 19/04/2018. + * + *

Scrapes schemas from an s3 bucket's default XML file-list page + */ +@Service +public class S3BucketSchemaScraper implements SchemaScraper { + private final @NonNull RestTemplate restTemplate; + private final @NonNull XmlMapper xmlMapper; + + public S3BucketSchemaScraper() { + this.restTemplate = new RestTemplate(new HttpComponentsClientHttpRequestFactory()); + + JacksonXmlModule xmlModule = new JacksonXmlModule(); + xmlModule.setDefaultUseWrapper(false); + XmlMapper xmlMapper = new XmlMapper(xmlModule); + xmlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + this.xmlMapper = xmlMapper; + } + + @Override + public Collection getAllSchemaURIs(URI schemaBucketLocation) { + String bucketListingXmlString = + this.restTemplate.getForObject(schemaBucketLocation, String.class); + try { + ListBucketResult listBucketResult = + xmlMapper.readValue(bucketListingXmlString, ListBucketResult.class); + return listBucketResult.contents.stream() + .map(content -> URI.create(content.getKey())) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new SchemaScrapeException( + "Failed to parse schema bucket xml at URL: " + schemaBucketLocation, e); + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/schemas/schemascraper/impl/SchemaScrapeException.java b/src/main/java/uk/ac/ebi/subs/ingest/schemas/schemascraper/impl/SchemaScrapeException.java new file mode 100644 index 000000000..170e0d051 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/schemas/schemascraper/impl/SchemaScrapeException.java @@ -0,0 +1,12 @@ +package uk.ac.ebi.subs.ingest.schemas.schemascraper.impl; + +/** Created by rolando on 19/04/2018. */ +public class SchemaScrapeException extends RuntimeException { + public SchemaScrapeException(String message) { + super(message); + } + + public SchemaScrapeException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/schemas/web/SchemaController.java b/src/main/java/uk/ac/ebi/subs/ingest/schemas/web/SchemaController.java new file mode 100644 index 000000000..2be9e9dfd --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/schemas/web/SchemaController.java @@ -0,0 +1,69 @@ +package uk.ac.ebi.subs.ingest.schemas.web; + +import java.util.List; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.data.web.PagedResourcesAssembler; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.schemas.Schema; +import uk.ac.ebi.subs.ingest.schemas.SchemaService; + +/** Created by rolando on 19/04/2018. */ +@RepositoryRestController +@ExposesResourceFor(Schema.class) +@RequiredArgsConstructor +public class SchemaController { + private final @NonNull SchemaService schemaService; + private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; + + @RequestMapping(path = "/schemas/update", method = RequestMethod.POST) + ResponseEntity triggerSchemasUpdate() { + schemaService.updateSchemasCollection(); + return new ResponseEntity<>(HttpStatus.ACCEPTED); + } + + @RequestMapping(path = "/schemas/search/latestSchemas", method = RequestMethod.GET) + ResponseEntity latestSchemas( + Pageable pageable, final PersistentEntityResourceAssembler resourceAssembler) { + List latestSchemas = schemaService.getLatestSchemas(); + Page latestSchemasPage = generatePageFromSchemaList(pageable, latestSchemas); + return ResponseEntity.ok( + pagedResourcesAssembler.toResource(latestSchemasPage, resourceAssembler)); + } + + @RequestMapping(path = "/schemas/search/filterLatestSchemas", method = RequestMethod.GET) + ResponseEntity filterLatestSchemas( + @RequestParam String highLevelEntity, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + List latestSchemas = schemaService.filterLatestSchemas(highLevelEntity); + Page latestSchemasPage = generatePageFromSchemaList(pageable, latestSchemas); + return ResponseEntity.ok( + pagedResourcesAssembler.toResource(latestSchemasPage, resourceAssembler)); + } + + private Page generatePageFromSchemaList(Pageable pageable, List schemaList) { + List latestSchemasSubList = + schemaList.subList( + (int) (pageable.getOffset()), + (int) + (pageable.getOffset() + + Math.min( + pageable.getOffset() + pageable.getPageSize(), + schemaList.size() - pageable.getOffset()))); + + return new PageImpl<>(latestSchemasSubList, pageable, schemaList.size()); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/schemas/web/SchemaResourceProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/schemas/web/SchemaResourceProcessor.java new file mode 100644 index 000000000..d2635645a --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/schemas/web/SchemaResourceProcessor.java @@ -0,0 +1,18 @@ +package uk.ac.ebi.subs.ingest.schemas.web; + +import org.springframework.hateoas.Link; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.stereotype.Component; + +import uk.ac.ebi.subs.ingest.schemas.Schema; + +/** Created by rolando on 19/04/2018. */ +@Component +public class SchemaResourceProcessor implements ResourceProcessor> { + + public Resource process(Resource resource) { + resource.add(new Link(resource.getContent().getSchemaUri(), "json-schema")); + return resource; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/schemas/web/SchemaSearchProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/schemas/web/SchemaSearchProcessor.java new file mode 100644 index 000000000..1c140ac79 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/schemas/web/SchemaSearchProcessor.java @@ -0,0 +1,29 @@ +package uk.ac.ebi.subs.ingest.schemas.web; + +import static org.springframework.hateoas.mvc.ControllerLinkBuilder.linkTo; +import static org.springframework.hateoas.mvc.ControllerLinkBuilder.methodOn; + +import org.springframework.data.rest.webmvc.RepositorySearchesResource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.stereotype.Component; + +import uk.ac.ebi.subs.ingest.schemas.Schema; + +/** Created by rolando on 23/04/2018. */ +@Component +public class SchemaSearchProcessor implements ResourceProcessor { + + @Override + public RepositorySearchesResource process(RepositorySearchesResource searchesResource) { + if (searchesResource.getDomainType().equals(Schema.class)) { + searchesResource.add( + linkTo(methodOn(SchemaController.class).latestSchemas(null, null)) + .withRel("latestSchemas")); + searchesResource.add( + linkTo(methodOn(SchemaController.class).filterLatestSchemas(null, null, null)) + .withRel("filterLatestSchemas")); + } + + return searchesResource; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/Account.java b/src/main/java/uk/ac/ebi/subs/ingest/security/Account.java new file mode 100644 index 000000000..a138a3a3c --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/Account.java @@ -0,0 +1,76 @@ +package uk.ac.ebi.subs.ingest.security; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.Document; + +import lombok.Getter; +import lombok.Setter; + +@Document +public class Account { + + public static final Account GUEST = new GuestAccount(); + public static final Account SERVICE = new ServiceAccount(); + + /** A Null Object subclass of Account that represents an unregistered Guest. */ + private static class GuestAccount extends Account { + + private static final String EMPTY = ""; + + private GuestAccount() { + super(EMPTY, EMPTY); + setName(EMPTY); + addRole(Role.GUEST); + } + } + + /** A Null Object subclass of Account that represents a service. */ + private static class ServiceAccount extends Account { + + private static final String EMPTY = ""; + + private ServiceAccount() { + super(EMPTY, EMPTY); + setName(EMPTY); + addRole(Role.SERVICE); + } + } + + @Id @Getter private String id; + + @Indexed(unique = true) + @Getter + private String providerReference; + + @Getter @Setter private String name; + + private Set roles = new HashSet<>(); + + // needed for reflection used by frameworks + private Account() {} + + public Account(String providerReference) { + this.providerReference = providerReference; + } + + public Account(String id, String providerReference) { + this.id = id; + this.providerReference = providerReference; + } + + /** + * @return an unmodifiable Set of Roles. + */ + public Set getRoles() { + return Collections.unmodifiableSet(roles); + } + + public void addRole(Role role) { + roles.add(role); + } +} diff --git a/src/main/java/org/humancellatlas/ingest/security/AccountRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/security/AccountRepository.java similarity index 62% rename from src/main/java/org/humancellatlas/ingest/security/AccountRepository.java rename to src/main/java/uk/ac/ebi/subs/ingest/security/AccountRepository.java index 08b83a5fe..e34132ea1 100644 --- a/src/main/java/org/humancellatlas/ingest/security/AccountRepository.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/AccountRepository.java @@ -1,16 +1,16 @@ -package org.humancellatlas.ingest.security; +package uk.ac.ebi.subs.ingest.security; + +import java.util.List; import org.springframework.data.mongodb.repository.MongoRepository; import org.springframework.data.rest.core.annotation.RestResource; import org.springframework.stereotype.Repository; -import java.util.List; - @Repository -@RestResource(exported=false) +@RestResource(exported = false) public interface AccountRepository extends MongoRepository { - Account findByProviderReference(String providerReference); + Account findByProviderReference(String providerReference); - List findAccountByRoles(Role role); + List findAccountByRoles(Role role); } diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/AccountService.java b/src/main/java/uk/ac/ebi/subs/ingest/security/AccountService.java new file mode 100644 index 000000000..d7541eb7b --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/AccountService.java @@ -0,0 +1,6 @@ +package uk.ac.ebi.subs.ingest.security; + +public interface AccountService { + + Account register(Account account); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/CheckAllowed.java b/src/main/java/uk/ac/ebi/subs/ingest/security/CheckAllowed.java new file mode 100644 index 000000000..9f18ebd6a --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/CheckAllowed.java @@ -0,0 +1,30 @@ +package uk.ac.ebi.subs.ingest.security; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import uk.ac.ebi.subs.ingest.security.exception.NotAllowedException; + +/** + * Annotation for allowing a method to proceed based on the evaluation of a given SpEL input. See: + * https://docs.spring.io/spring-framework/docs/3.1.0.M1/spring-framework-reference/html/expressions.html + * for SpEL docs + * + *

Allows a value (SpEL input) and an optional custom exception + * + *

E.g.: @CheckAllowed("#foo.bar") public void myMethod(Object foo) { ... } + * + *

or: @CheckAllowed(value = "#foo.bar", exception = MyCustomException.class) public void + * myMethod(Object foo) { ... } + * + *

See SecurityAspect for the aspect and advice that use this. + */ +@Target({ElementType.METHOD, ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +public @interface CheckAllowed { + String value(); + + Class exception() default NotAllowedException.class; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/CorsConfig.java b/src/main/java/uk/ac/ebi/subs/ingest/security/CorsConfig.java new file mode 100644 index 000000000..c31b975a6 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/CorsConfig.java @@ -0,0 +1,20 @@ +package uk.ac.ebi.subs.ingest.security; + +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.CorsRegistry; +import org.springframework.web.servlet.config.annotation.EnableWebMvc; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +@Configuration +@EnableWebMvc +public class CorsConfig implements WebMvcConfigurer { + + @Override + public void addCorsMappings(CorsRegistry registry) { + registry + .addMapping("/**") + .allowedMethods("GET", "PATCH", "POST", "PUT", "DELETE") + .allowedOrigins("*") + .allowedHeaders("*"); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/DefaultAccountService.java b/src/main/java/uk/ac/ebi/subs/ingest/security/DefaultAccountService.java new file mode 100644 index 000000000..8e54e9a1e --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/DefaultAccountService.java @@ -0,0 +1,26 @@ +package uk.ac.ebi.subs.ingest.security; + +import org.springframework.stereotype.Component; + +import uk.ac.ebi.subs.ingest.security.exception.DuplicateAccount; + +@Component +public class DefaultAccountService implements AccountService { + + private final AccountRepository accountRepository; + + public DefaultAccountService(AccountRepository accountRepository) { + this.accountRepository = accountRepository; + } + + @Override + public Account register(Account account) { + account.addRole(Role.CONTRIBUTOR); + Account persistentAccount = + accountRepository.findByProviderReference(account.getProviderReference()); + if (persistentAccount != null) { + throw new DuplicateAccount(); + } + return this.accountRepository.save(account); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/ElixirConfig.java b/src/main/java/uk/ac/ebi/subs/ingest/security/ElixirConfig.java new file mode 100644 index 000000000..1dcdf6289 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/ElixirConfig.java @@ -0,0 +1,27 @@ +package uk.ac.ebi.subs.ingest.security; + +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import uk.ac.ebi.subs.ingest.security.authn.provider.elixir.ElixirJwkVault; +import uk.ac.ebi.subs.ingest.security.common.jwk.JwtVerifierResolver; +import uk.ac.ebi.subs.ingest.security.common.jwk.UrlJwkProviderResolver; + +@Configuration +public class ElixirConfig { + + public static final String ELIXIR = "elixir"; + + @Value("${AUTH_ISSUER}") + private String issuer; + + @Bean + @Qualifier(ELIXIR) + public JwtVerifierResolver elixirJwtVerifierResolver() { + var urlJwkProviderResolver = new UrlJwkProviderResolver(issuer + "/jwk"); + ElixirJwkVault jwkVault = new ElixirJwkVault(urlJwkProviderResolver); + return new JwtVerifierResolver(jwkVault, null, issuer); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/GcpConfig.java b/src/main/java/uk/ac/ebi/subs/ingest/security/GcpConfig.java new file mode 100644 index 000000000..7af720834 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/GcpConfig.java @@ -0,0 +1,38 @@ +package uk.ac.ebi.subs.ingest.security; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.authentication.AuthenticationProvider; + +import uk.ac.ebi.subs.ingest.security.authn.provider.gcp.GcpDomainWhiteList; +import uk.ac.ebi.subs.ingest.security.authn.provider.gcp.GcpJwkVault; +import uk.ac.ebi.subs.ingest.security.authn.provider.gcp.GoogleServiceJwtAuthenticationProvider; +import uk.ac.ebi.subs.ingest.security.common.jwk.JwtVerifierResolver; +import uk.ac.ebi.subs.ingest.security.common.jwk.UrlJwkProviderResolver; + +@Configuration +public class GcpConfig { + + public static final String GCP = "gcp"; + + @Value("${GCP_JWK_PROVIDER_BASE_URL}") + private String googleJwkProviderBaseUrl; + + @Value(value = "${SVC_AUTH_AUDIENCE}") + private String serviceAudience; + + @Value(value = "#{('${GCP_PROJECT_WHITELIST}').split(',')}") + private List projectWhitelist; + + @Bean(name = GCP) + public AuthenticationProvider gcpAuthenticationProvider() { + var urlJwkProviderResolver = new UrlJwkProviderResolver(googleJwkProviderBaseUrl); + var googleJwkVault = new GcpJwkVault(urlJwkProviderResolver); + var googleJwtVerifierResolver = new JwtVerifierResolver(googleJwkVault, serviceAudience, null); + return new GoogleServiceJwtAuthenticationProvider( + new GcpDomainWhiteList(projectWhitelist), googleJwtVerifierResolver); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/Role.java b/src/main/java/uk/ac/ebi/subs/ingest/security/Role.java new file mode 100644 index 000000000..68ad4a0c9 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/Role.java @@ -0,0 +1,15 @@ +package uk.ac.ebi.subs.ingest.security; + +import org.springframework.security.core.GrantedAuthority; + +public enum Role implements GrantedAuthority { + WRANGLER, + CONTRIBUTOR, + GUEST, + SERVICE; + + @Override + public String getAuthority() { + return name(); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/RowLevelFilterSecurity.java b/src/main/java/uk/ac/ebi/subs/ingest/security/RowLevelFilterSecurity.java new file mode 100644 index 000000000..648684449 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/RowLevelFilterSecurity.java @@ -0,0 +1,12 @@ +package uk.ac.ebi.subs.ingest.security; + +import java.lang.annotation.*; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Inherited +public @interface RowLevelFilterSecurity { + String expression(); + + Class[] ignoreClasses() default {}; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/RowLevelFilterSecurityAspect.java b/src/main/java/uk/ac/ebi/subs/ingest/security/RowLevelFilterSecurityAspect.java new file mode 100644 index 000000000..0ab2fb331 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/RowLevelFilterSecurityAspect.java @@ -0,0 +1,184 @@ +package uk.ac.ebi.subs.ingest.security; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Pointcut; +import org.aspectj.lang.reflect.MethodSignature; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; +import org.springframework.security.access.AccessDeniedException; +import org.springframework.security.authentication.AuthenticationCredentialsNotFoundException; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Component; + +import uk.ac.ebi.subs.ingest.core.MetadataDocument; + +/** + * @see Spring 2 AOP + * Docs + */ +@Aspect +@Component +public class RowLevelFilterSecurityAspect { + + // TODO: implement "Before" protection for data modification functions + @Pointcut("execution(* uk.ac.ebi.subs.ingest.*.*Repository.find*(..))") + public void repositoryFindFunctions() {} + + @Pointcut( + "execution(* org.springframework.data.repository.PagingAndSortingRepository.find*(..)) ") + public void repositoryInheritedFindFunctions() {} + + @Pointcut( + "target(uk.ac.ebi.subs.ingest.file.FileRepository) " + + "|| target(uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository) " + + "|| target(uk.ac.ebi.subs.ingest.process.ProcessRepository) " + + "|| target(uk.ac.ebi.subs.ingest.protocol.ProtocolRepository) ") + public void ingestRepositoriesAreTheTarget() {} + + @Around("(ingestRepositoriesAreTheTarget() " + "&& repositoryInheritedFindFunctions())") + public Object applyRowLevelSecurity(ProceedingJoinPoint joinPoint) throws Throwable { + Object queryResult = joinPoint.proceed(); + try { + return new RowlevelSecurityAdviceHelper(joinPoint).filterResult(queryResult); + } catch (AccessDeniedException | AuthenticationCredentialsNotFoundException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException( + String.format( + "problem during advice for %s: %s", + joinPoint.getSignature().getName(), e.getMessage()), + e); + } + } + + class RowlevelSecurityAdviceHelper { + private final MethodSignature signature; + private final ProceedingJoinPoint joinPoint; + private final Method method; + private final RowLevelFilterSecurity rowLevelFilterSecurity; + private final SpelHelper spelHelper = new SpelHelper(); + private final Authentication authentication; + + public RowlevelSecurityAdviceHelper(ProceedingJoinPoint joinPoint) { + this.joinPoint = joinPoint; + this.signature = (MethodSignature) this.joinPoint.getSignature(); + this.method = signature.getMethod(); + + this.rowLevelFilterSecurity = + Optional.ofNullable(method.getClass().getAnnotation(RowLevelFilterSecurity.class)) + .orElseGet(() -> readAnnotationFromSuperInterface(joinPoint)); + this.authentication = + Optional.ofNullable(SecurityContextHolder.getContext().getAuthentication()) + .orElseThrow(() -> new AuthenticationCredentialsNotFoundException("unauthorized")); + } + + private RowLevelFilterSecurity readAnnotationFromSuperInterface(ProceedingJoinPoint joinPoint) { + return (RowLevelFilterSecurity) + Arrays.stream(joinPoint.getThis().getClass().getInterfaces()) + .filter(o -> o.getName().contains("ingest")) + .flatMap(o -> Arrays.stream(o.getAnnotations())) + .filter( + a -> a.annotationType().getName().equals(RowLevelFilterSecurity.class.getName())) + .findFirst() + .get(); + } + + private Object filterDocumentList(List documentList) { + List retainedDocuments = + documentList.stream() + .filter(this::evaluateDocumentExpression) + .collect(Collectors.toList()); + return retainedDocuments; + } + + private Boolean evaluateDocumentExpression(MetadataDocument document) { + List variableNames = buildVariableNames(method); + List variableValues = buildVariableValues(document, authentication); + String spelExpression = rowLevelFilterSecurity.expression(); + return spelHelper.parseExpression(variableNames, variableValues, spelExpression); + } + + private List buildVariableValues( + MetadataDocument document, Authentication authentication) { + List variableValues = new ArrayList<>(); + variableValues.add(document); + variableValues.add(authentication); + variableValues.addAll(List.of(joinPoint.getArgs())); + return variableValues; + } + + private List buildVariableNames(Method method) { + List variableNames = new ArrayList<>(); + variableNames.add("filterObject"); + variableNames.add("authentication"); + Arrays.stream(method.getParameters()).forEach(p -> variableNames.add(p.getName())); + return variableNames; + } + + public Object filterResult(Object queryResult) { + Object result; + if (queryResult instanceof Page) { + result = filterPage((Page) queryResult); + } else if (queryResult instanceof List) { + result = filterList((List) queryResult); + } else if (queryResult instanceof Optional) { + result = filterOptional((Optional) queryResult); + } else { + throw new IllegalArgumentException( + "only supports filtering results of type Page, List, Optional. Type given: " + + queryResult.getClass()); + } + return result; + } + + private Object filterOptional(Optional queryResult) { + // if on ignore list - return object + if (isResultInIgnoreList(queryResult)) { + return queryResult; + } + List variableNames = buildVariableNames(method); + List variableValues = + buildVariableValues((MetadataDocument) queryResult.get(), authentication); + if (spelHelper.parseExpression( + variableNames, variableValues, rowLevelFilterSecurity.expression())) { + return queryResult; + } + throw new AccessDeniedException("access denied"); + } + + private boolean isResultInIgnoreList(Optional queryResult) { + return Arrays.stream(rowLevelFilterSecurity.ignoreClasses()) + .anyMatch(queryResult.get().getClass()::isInstance); + } + + private Object filterList(List queryResult) { + Object result; + List documentList = queryResult; + if (documentList.size() == 0) { + result = documentList; + } else { + result = filterDocumentList((List) documentList); + } + return result; + } + + private Object filterPage(Page queryResult) { + Page page = queryResult; + Pageable pageable = page.getPageable(); + List documentList = page.getContent(); + List filteredDocumentList = (List) filterDocumentList(documentList); + return new PageImpl<>(filteredDocumentList, pageable, filteredDocumentList.size()); + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/SecurityAspect.java b/src/main/java/uk/ac/ebi/subs/ingest/security/SecurityAspect.java new file mode 100644 index 000000000..ab33daf60 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/SecurityAspect.java @@ -0,0 +1,42 @@ +package uk.ac.ebi.subs.ingest.security; + +import java.lang.reflect.Method; + +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Before; +import org.aspectj.lang.reflect.MethodSignature; +import org.springframework.stereotype.Component; + +import uk.ac.ebi.subs.ingest.security.exception.NotAllowedException; + +@Aspect +@Component +public class SecurityAspect { + private final SpelHelper spelHelper = new SpelHelper(); + + /** + * Advice that runs before any method with the CheckAllowed annotation. Parses the given SpEL in + * the annotation and throws error if the result returns False. + * + * @param joinPoint + * @throws Throwable + */ + @Before("@annotation(uk.ac.ebi.subs.ingest.security.CheckAllowed) && execution(* *(..))") + public void checkAllowed(JoinPoint joinPoint) throws Throwable { + MethodSignature signature = (MethodSignature) joinPoint.getSignature(); + Method method = signature.getMethod(); + CheckAllowed annotation = method.getAnnotation(CheckAllowed.class); + Boolean isAllowed = + spelHelper.parseExpression( + signature.getParameterNames(), joinPoint.getArgs(), annotation.value()); + Class exceptionClass = annotation.exception(); + if (!isAllowed) { + throw exceptionClass.getDeclaredConstructor().newInstance(); + } + } + + private Boolean parseExpression(String[] params, Object[] args, String expression) { + return spelHelper.parseExpression(params, args, expression); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/SecurityConfig.java b/src/main/java/uk/ac/ebi/subs/ingest/security/SecurityConfig.java new file mode 100644 index 000000000..31b03b4ae --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/SecurityConfig.java @@ -0,0 +1,156 @@ +package uk.ac.ebi.subs.ingest.security; + +import static java.util.stream.Collectors.toList; +import static org.springframework.http.HttpMethod.*; +import static uk.ac.ebi.subs.ingest.security.ElixirConfig.ELIXIR; +import static uk.ac.ebi.subs.ingest.security.GcpConfig.GCP; +import static uk.ac.ebi.subs.ingest.security.Role.*; + +import java.util.*; +import java.util.stream.Stream; + +import javax.servlet.http.HttpServletRequest; + +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.HttpMethod; +import org.springframework.security.authentication.AuthenticationProvider; +import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; +import org.springframework.security.config.http.SessionCreationPolicy; +import org.springframework.security.web.util.matcher.AntPathRequestMatcher; + +import com.auth0.spring.security.api.BearerSecurityContextRepository; +import com.auth0.spring.security.api.JwtAuthenticationEntryPoint; + +@EnableWebSecurity +@EnableGlobalMethodSecurity(prePostEnabled = true) +@Configuration +public class SecurityConfig extends WebSecurityConfigurerAdapter { + private static final String FORWARDED_HOST = "x-forwarded-host"; + private static final List SECURED_ANT_PATHS = setupSecuredAntPaths(); + private static final List SECURED_WRANGLER_ANT_PATHS = + setupWranglerAntPaths(); + + // The following endpoints are only secured when accessed from the outside the cluster + + private static List setupSecuredAntPaths() { + List antPathMatchers = new ArrayList<>(); + antPathMatchers.addAll(defineAntPathMatchers(POST, "/**")); + antPathMatchers.addAll(defineAntPathMatchers(PATCH, "/projects/*")); + return Collections.unmodifiableList(antPathMatchers); + } + + private static List setupWranglerAntPaths() { + List antPathMatchers = new ArrayList<>(); + antPathMatchers.addAll(defineAntPathMatchers(GET, "/bundleManifests")); + antPathMatchers.addAll(defineAntPathMatchers(GET, "/submissionManifests")); + + antPathMatchers.addAll(defineAntPathMatchers(GET, "/submissionEnvelopes")); + antPathMatchers.addAll(defineAntPathMatchers(GET, "/biomaterials")); + antPathMatchers.addAll(defineAntPathMatchers(GET, "/files")); + antPathMatchers.addAll(defineAntPathMatchers(GET, "/processes")); + antPathMatchers.addAll(defineAntPathMatchers(GET, "/protocols")); + + antPathMatchers.addAll(defineAntPathMatchers(GET, "/projects")); + antPathMatchers.addAll(defineAntPathMatchers(PUT, "/**")); + antPathMatchers.addAll(defineAntPathMatchers(PATCH, "/**")); + antPathMatchers.addAll(defineAntPathMatchers(DELETE, "/**")); + return Collections.unmodifiableList(antPathMatchers); + } + + private static List defineAntPathMatchers( + HttpMethod method, String... patterns) { + return Stream.of(patterns) + .map(pattern -> new AntPathRequestMatcher(pattern, method.name())) + .collect(toList()); + } + + private final AuthenticationProvider gcpAuthenticationProvider; + private final AuthenticationProvider elixirAuthenticationProvider; + private final AuthenticationProvider awsCognitoAuthenticationProvider; + + public SecurityConfig( + @Qualifier(GCP) AuthenticationProvider gcp, + @Qualifier(ELIXIR) AuthenticationProvider elixir, + @Qualifier("COGNITO") AuthenticationProvider awsCognito) { + this.gcpAuthenticationProvider = gcp; + this.elixirAuthenticationProvider = elixir; + this.awsCognitoAuthenticationProvider = awsCognito; + } + + @Override + protected void configure(HttpSecurity http) throws Exception { + http.authenticationProvider(elixirAuthenticationProvider) + .authenticationProvider(gcpAuthenticationProvider) + .authenticationProvider(awsCognitoAuthenticationProvider) + .securityContext() + .securityContextRepository(new BearerSecurityContextRepository()) + .and() + .exceptionHandling() + .authenticationEntryPoint(new JwtAuthenticationEntryPoint()) + .and() + .httpBasic() + .disable() + .csrf() + .disable() + .sessionManagement() + .sessionCreationPolicy(SessionCreationPolicy.STATELESS) + .and() + .cors() + .and() + .authorizeRequests() + .antMatchers(GET, "/") + .permitAll() + .antMatchers(GET, "/schemas/**") + .permitAll() + .antMatchers(GET, "/health") + .permitAll() + .antMatchers(GET, "/info") + .permitAll() + .antMatchers(GET, "/prometheus") + .permitAll() + .antMatchers(GET, "/browser/**") + .permitAll() + .antMatchers(POST, "/submissionEnvelopes") + .authenticated() + .antMatchers(POST, "/submissionEnvelopes/**") + .authenticated() + .antMatchers(POST, "/projects") + .authenticated() + .antMatchers(POST, "/studies") + .authenticated() + .antMatchers(POST, "/projects/suggestion") + .permitAll() + .antMatchers(POST, "/projects/catalogue") + .permitAll() + .antMatchers(GET, "/user/**") + .authenticated() + .antMatchers(GET, "/auth/account") + .authenticated() + .antMatchers(POST, "/auth/registration") + .hasAuthority(GUEST.name()) + .requestMatchers(SecurityConfig::isSecuredEndpointFromOutside) + .authenticated() + .requestMatchers(SecurityConfig::isSecuredWranglerEndpointFromOutside) + .hasAnyAuthority(WRANGLER.name(), SERVICE.name()) + .antMatchers(GET, "/**") + .permitAll(); + } + + private static Boolean isSecuredEndpointFromOutside(HttpServletRequest request) { + return SECURED_ANT_PATHS.stream().anyMatch(matcher -> matcher.matches(request)) + && SecurityConfig.isRequestOutsideProxy(request); + } + + private static Boolean isSecuredWranglerEndpointFromOutside(HttpServletRequest request) { + return SECURED_WRANGLER_ANT_PATHS.stream().anyMatch(matcher -> matcher.matches(request)) + && SecurityConfig.isRequestOutsideProxy(request); + } + + private static Boolean isRequestOutsideProxy(HttpServletRequest request) { + return Optional.ofNullable(request.getHeader(FORWARDED_HOST)).isPresent(); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/SpelHelper.java b/src/main/java/uk/ac/ebi/subs/ingest/security/SpelHelper.java new file mode 100644 index 000000000..0f01dff40 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/SpelHelper.java @@ -0,0 +1,26 @@ +package uk.ac.ebi.subs.ingest.security; + +import java.util.List; + +import org.springframework.expression.ExpressionParser; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.expression.spel.support.StandardEvaluationContext; + +public class SpelHelper { + public SpelHelper() {} + + public Boolean parseExpression(List params, List args, String expression) { + return parseExpression((String[]) params.toArray(new String[0]), args.toArray(), expression); + } + + Boolean parseExpression(String[] params, Object[] args, String expression) { + ExpressionParser parser = new SpelExpressionParser(); + StandardEvaluationContext context = new StandardEvaluationContext(); + + for (int i = 0; i < params.length; i++) { + context.setVariable(params[i], args[i]); + } + + return parser.parseExpression(expression).getValue(context, Boolean.class); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/UserAuditing.java b/src/main/java/uk/ac/ebi/subs/ingest/security/UserAuditing.java new file mode 100644 index 000000000..5374f413b --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/UserAuditing.java @@ -0,0 +1,34 @@ +package uk.ac.ebi.subs.ingest.security; + +import java.util.Optional; + +import org.springframework.data.domain.AuditorAware; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Component; + +/* + * User auditing to get the current user to put into the database: Based on https://docs.spring.io/spring-data/jpa/docs/current/reference/html/#auditing + */ + +@Component +public class UserAuditing implements AuditorAware { + + @Override + public Optional getCurrentAuditor() { + + Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); + if (authentication == null || !authentication.isAuthenticated()) { + return Optional.empty(); + } + + Object principal = authentication.getPrincipal(); + + if (Account.class.isAssignableFrom(principal.getClass())) { + Account account = (Account) authentication.getPrincipal(); + return Optional.of(account.getId() != null ? account.getId() : account.getName()); + } else { + return Optional.ofNullable(principal.toString()); + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/authn/oidc/OpenIdAuthentication.java b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/oidc/OpenIdAuthentication.java new file mode 100644 index 000000000..7866023da --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/oidc/OpenIdAuthentication.java @@ -0,0 +1,80 @@ +package uk.ac.ebi.subs.ingest.security.authn.oidc; + +import java.util.Collection; + +import org.springframework.security.core.Authentication; +import org.springframework.security.core.GrantedAuthority; + +import uk.ac.ebi.subs.ingest.security.Account; + +public class OpenIdAuthentication implements Authentication { + + private Account account; + private UserInfo userInfo; + + private boolean authenticated = false; + + public OpenIdAuthentication(final Account principal) { + account = Account.GUEST; + if (principal != null) { + account = principal; + } + } + + public OpenIdAuthentication(Account principal, UserInfo credentials) { + this(principal); + authenticateWith(credentials); + } + + public OpenIdAuthentication(UserInfo credentials) { + this(null, credentials); + } + + @Override + public Object getPrincipal() { + return account; + } + + @Override + public Object getCredentials() { + return userInfo; + } + + @Override + public Collection getAuthorities() { + return account.getRoles(); + } + + @Override + public String getName() { + return account.getProviderReference(); + } + + @Override + public Object getDetails() { + return userInfo; + } + + @Override + public boolean isAuthenticated() { + return authenticated; + } + + @Override + public void setAuthenticated(boolean isAuthenticated) throws IllegalArgumentException { + throw new IllegalArgumentException( + "Operation not supported. Use authenticateWith to set status."); + } + + public void authenticateWith(UserInfo credentials) { + this.userInfo = credentials; + if (credentials == null) { + authenticated = false; + return; + } + authenticated = + account == Account.GUEST + || account == Account.SERVICE + || credentials.getSubjectId().equalsIgnoreCase(account.getProviderReference()); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/authn/oidc/UserInfo.java b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/oidc/UserInfo.java new file mode 100644 index 000000000..7ee1b339c --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/oidc/UserInfo.java @@ -0,0 +1,40 @@ +package uk.ac.ebi.subs.ingest.security.authn.oidc; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.*; +import uk.ac.ebi.subs.ingest.security.Account; + +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter(AccessLevel.PROTECTED) +public class UserInfo { + + @JsonProperty("sub") + private String subjectId; + + private String name; + + @JsonProperty("preferred_username") + private String preferredUsername; + + @JsonProperty("given_name") + private String givenName; + + @JsonProperty("family_name") + private String familyName; + + private String email; + + public UserInfo(String subjectId, String name) { + this.subjectId = subjectId; + this.name = name; + } + + public Account toAccount() { + Account account = new Account(subjectId); + account.setName(name); + return account; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/auth0/UserJwt.java b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/auth0/UserJwt.java new file mode 100644 index 000000000..e011ad343 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/auth0/UserJwt.java @@ -0,0 +1,18 @@ +package uk.ac.ebi.subs.ingest.security.authn.provider.auth0; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.interfaces.DecodedJWT; +import com.auth0.spring.security.api.authentication.JwtAuthentication; + +public class UserJwt { + private DecodedJWT token; + + public UserJwt(JwtAuthentication jwtAuthentication) { + this.token = JWT.decode(jwtAuthentication.getToken()); + } + + public String getGroup() { + String claimName = "https://auth.data.humancellatlas.org/group"; + return token.getClaim(claimName).asString(); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/auth0/UserJwtAuthenticationProvider.java b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/auth0/UserJwtAuthenticationProvider.java new file mode 100644 index 000000000..f2bea8be2 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/auth0/UserJwtAuthenticationProvider.java @@ -0,0 +1,47 @@ +package uk.ac.ebi.subs.ingest.security.authn.provider.auth0; + +import static java.util.Optional.ofNullable; + +import javax.annotation.Nullable; + +import org.springframework.security.authentication.AuthenticationProvider; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.AuthenticationException; + +import com.auth0.spring.security.api.JwtAuthenticationProvider; +import com.auth0.spring.security.api.authentication.JwtAuthentication; + +import uk.ac.ebi.subs.ingest.security.exception.InvalidUserGroup; + +public class UserJwtAuthenticationProvider implements AuthenticationProvider { + private final JwtAuthenticationProvider delegate; + + public UserJwtAuthenticationProvider(JwtAuthenticationProvider delegate) { + this.delegate = delegate; + } + + @Override + public Authentication authenticate(Authentication authentication) throws AuthenticationException { + Authentication jwtAuthentication = delegate.authenticate(authentication); + ofNullable(jwtAuthentication) + .ifPresent( + auth -> { + JwtAuthentication jwt = (JwtAuthentication) authentication; + UserJwt user = new UserJwt(jwt); + verifyUser(user); + }); + return jwtAuthentication; + } + + private void verifyUser(UserJwt user) { + String group = user.getGroup(); + if (group == null || !group.toLowerCase().equals("hca")) { + throw new InvalidUserGroup(group); + } + } + + @Override + public boolean supports(@Nullable Class authentication) { + return delegate.supports(authentication); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/aws/AwsCognitoServiceAuthenticationProvider.java b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/aws/AwsCognitoServiceAuthenticationProvider.java new file mode 100644 index 000000000..bd3930c27 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/aws/AwsCognitoServiceAuthenticationProvider.java @@ -0,0 +1,139 @@ +package uk.ac.ebi.subs.ingest.security.authn.provider.aws; + +import java.util.Date; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Lazy; +import org.springframework.core.env.Environment; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.security.authentication.AuthenticationProvider; +import org.springframework.security.authentication.AuthenticationServiceException; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.AuthenticationException; +import org.springframework.stereotype.Component; +import org.springframework.web.reactive.function.client.WebClient; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.interfaces.DecodedJWT; +import com.auth0.spring.security.api.authentication.JwtAuthentication; + +import lombok.extern.slf4j.Slf4j; +import uk.ac.ebi.subs.ingest.security.Account; +import uk.ac.ebi.subs.ingest.security.authn.oidc.OpenIdAuthentication; +import uk.ac.ebi.subs.ingest.security.authn.oidc.UserInfo; +import uk.ac.ebi.subs.ingest.security.exception.UnlistedJwtIssuer; + +@Component +@Qualifier("COGNITO") +@Lazy +@Slf4j +public class AwsCognitoServiceAuthenticationProvider implements AuthenticationProvider { + private final Environment environment; + private static final String AWS_COGNITO_NON_OPENID_SCOPE = "aws.cognito.signin.user.admin"; + private final WebClient webClient; + public final String awsCognitoDomainUrl; + + @Autowired + public AwsCognitoServiceAuthenticationProvider( + final Environment environment, final WebClient.Builder webClientBuilder) { + this.environment = environment; + this.awsCognitoDomainUrl = this.environment.getProperty("AWS_COGNITO_DOMAIN"); + this.webClient = + webClientBuilder + .baseUrl(this.awsCognitoDomainUrl) + .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) + .build(); + } + + @Override + public Authentication authenticate(final Authentication authentication) + throws AuthenticationException { + if (!supports(authentication.getClass())) { + return null; + } + + final JwtAuthentication jwt = (JwtAuthentication) authentication; + final String accessToken = jwt.getToken(); + final DecodedJWT decodedJWT = JWT.decode(accessToken); + final String issuer = decodedJWT.getIssuer(); + final String scope = decodedJWT.getClaim("scope").asString(); + + verifyIssuer(issuer); + + if (scope == null) { + throw new AuthenticationServiceException("Invalid user information"); + } + + if (scope.equalsIgnoreCase(AWS_COGNITO_NON_OPENID_SCOPE)) { + return authenticateWithNonOpenIdScope(decodedJWT); + } + + return authenticateWithUserInfoEndpoint(accessToken); + } + + private Authentication authenticateWithNonOpenIdScope(final DecodedJWT decodedJWT) { + final String userName = decodedJWT.getClaim("username").asString(); + final String sub = decodedJWT.getClaim("sub").asString(); + final Date expiresAt = decodedJWT.getExpiresAt(); + + // Check if the token is expired + if (expiresAt != null && expiresAt.before(new Date())) { + throw new AuthenticationServiceException("Token is expired"); + } + + if (userName == null || sub == null) { + throw new AuthenticationServiceException("Invalid user information"); + } + + final UserInfo userInfo = new UserInfo(sub, userName); + final Account account = userInfo.toAccount(); + + account.setName(userInfo.getName()); + + final OpenIdAuthentication openIdAuth = new OpenIdAuthentication(account); + openIdAuth.authenticateWith(userInfo); + + return openIdAuth; + } + + private Authentication authenticateWithUserInfoEndpoint(final String accessToken) { + try { + final String userInfoUrl = awsCognitoDomainUrl + "/userinfo"; + final UserInfo userInfo = + webClient + .get() + .uri(userInfoUrl) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken) + .retrieve() + .bodyToMono(UserInfo.class) + .block(); + + if (userInfo != null && userInfo.getEmail() != null) { + final Account account = userInfo.toAccount(); + account.setName(userInfo.getEmail()); + + final OpenIdAuthentication openIdAuth = new OpenIdAuthentication(account); + openIdAuth.authenticateWith(userInfo); + + return openIdAuth; + } else { + throw new AuthenticationServiceException("Invalid user information"); + } + } catch (final Exception e) { + throw new AuthenticationServiceException("Error authenticating with Cognito", e); + } + } + + private void verifyIssuer(final String issuer) { + if (!issuer.contains("cognito")) { + throw new UnlistedJwtIssuer(String.format("Not a Cognito issued token: %s", issuer), issuer); + } + } + + @Override + public boolean supports(final Class authentication) { + return JwtAuthentication.class.isAssignableFrom(authentication); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/elixir/ElixirAaiAuthenticationProvider.java b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/elixir/ElixirAaiAuthenticationProvider.java new file mode 100644 index 000000000..e1891a2cd --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/elixir/ElixirAaiAuthenticationProvider.java @@ -0,0 +1,102 @@ +package uk.ac.ebi.subs.ingest.security.authn.provider.elixir; + +import static org.apache.http.HttpHeaders.AUTHORIZATION; +import static uk.ac.ebi.subs.ingest.security.ElixirConfig.ELIXIR; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.security.authentication.AuthenticationProvider; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.AuthenticationException; +import org.springframework.stereotype.Component; +import org.springframework.web.reactive.function.client.WebClient; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.exceptions.JWTVerificationException; +import com.auth0.jwt.exceptions.TokenExpiredException; +import com.auth0.jwt.interfaces.JWTVerifier; +import com.auth0.spring.security.api.authentication.JwtAuthentication; + +import uk.ac.ebi.subs.ingest.security.Account; +import uk.ac.ebi.subs.ingest.security.AccountRepository; +import uk.ac.ebi.subs.ingest.security.authn.oidc.OpenIdAuthentication; +import uk.ac.ebi.subs.ingest.security.authn.oidc.UserInfo; +import uk.ac.ebi.subs.ingest.security.common.jwk.DelegatingJwtAuthentication; +import uk.ac.ebi.subs.ingest.security.common.jwk.JwtVerifierResolver; +import uk.ac.ebi.subs.ingest.security.exception.JwtVerificationFailed; +import uk.ac.ebi.subs.ingest.security.exception.UnlistedJwtIssuer; + +@Component +@Qualifier(ELIXIR) +public class ElixirAaiAuthenticationProvider implements AuthenticationProvider { + private static final Logger LOGGER = + LoggerFactory.getLogger(ElixirAaiAuthenticationProvider.class); + + private final JwtVerifierResolver jwtVerifierResolver; + + private final AccountRepository accountRepository; + + private final WebClient webClient; + + public ElixirAaiAuthenticationProvider( + @Qualifier(ELIXIR) JwtVerifierResolver jwtVerifierResolver, + AccountRepository accountRepository, + WebClient.Builder webCliBuilder) { + this.jwtVerifierResolver = jwtVerifierResolver; + this.accountRepository = accountRepository; + webClient = webCliBuilder.build(); + } + + @Override + public Authentication authenticate(Authentication authentication) throws AuthenticationException { + if (!supports(authentication.getClass())) { + return null; + } + try { + JwtAuthentication jwt = (JwtAuthentication) authentication; + String token = jwt.getToken(); + String issuer = JWT.decode(token).getIssuer(); + verifyIssuer(issuer); + + JWTVerifier jwtVerifier = jwtVerifierResolver.resolve(jwt.getToken()); + DelegatingJwtAuthentication verifiedAuth = + DelegatingJwtAuthentication.delegate(jwt, jwtVerifier); + + token = verifiedAuth.getToken(); + UserInfo userInfo = retrieveUserInfo(token); + + Account account = accountRepository.findByProviderReference(userInfo.getSubjectId()); + OpenIdAuthentication openIdAuth = new OpenIdAuthentication(account); + openIdAuth.authenticateWith(userInfo); + return openIdAuth; + } catch (TokenExpiredException e) { + throw new JwtVerificationFailed(e); + } catch (JWTVerificationException e) { + LOGGER.error("JWT verification failed: {}", e.getMessage()); + throw new JwtVerificationFailed(e); + } + } + + private UserInfo retrieveUserInfo(String token) { + return webClient + .get() + .uri(String.format("%s/userinfo", jwtVerifierResolver.getIssuer())) + .header(AUTHORIZATION, String.format("Bearer %s", token)) + .retrieve() + .bodyToMono(UserInfo.class) + .block(); + } + + private void verifyIssuer(String issuer) { + if (!issuer.contains("elixir")) { + throw new UnlistedJwtIssuer( + String.format("Not an Elxir AAI issued token: %s", issuer), issuer); + } + } + + @Override + public boolean supports(Class authentication) { + return JwtAuthentication.class.isAssignableFrom(authentication); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/elixir/ElixirJwkVault.java b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/elixir/ElixirJwkVault.java new file mode 100644 index 000000000..884204ed2 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/elixir/ElixirJwkVault.java @@ -0,0 +1,30 @@ +package uk.ac.ebi.subs.ingest.security.authn.provider.elixir; + +import java.security.PublicKey; + +import com.auth0.jwk.JwkException; +import com.auth0.jwk.UrlJwkProvider; +import com.auth0.jwt.interfaces.DecodedJWT; + +import uk.ac.ebi.subs.ingest.security.common.jwk.JwkVault; +import uk.ac.ebi.subs.ingest.security.common.jwk.UrlJwkProviderResolver; + +public class ElixirJwkVault implements JwkVault { + + private final UrlJwkProviderResolver urlJwkProviderResolver; + + public ElixirJwkVault(UrlJwkProviderResolver urlJwkProviderResolver) { + this.urlJwkProviderResolver = urlJwkProviderResolver; + } + + @Override + public PublicKey getPublicKey(DecodedJWT jwt) { + UrlJwkProvider jwkProvider = urlJwkProviderResolver.resolve(); + try { + var jwk = jwkProvider.get(jwt.getKeyId()); + return jwk.getPublicKey(); + } catch (JwkException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/gcp/GcpDomainWhiteList.java b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/gcp/GcpDomainWhiteList.java new file mode 100644 index 000000000..ddcd032a4 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/gcp/GcpDomainWhiteList.java @@ -0,0 +1,24 @@ +package uk.ac.ebi.subs.ingest.security.authn.provider.gcp; + +import static java.lang.String.format; +import static java.util.Arrays.asList; + +import java.util.Collections; +import java.util.List; + +public class GcpDomainWhiteList { + + private final List domains; + + public GcpDomainWhiteList(List domains) { + this.domains = Collections.unmodifiableList(domains); + } + + public GcpDomainWhiteList(String... domains) { + this(asList(domains)); + } + + public boolean lists(String email) { + return domains.stream().map(domain -> format("@%s", domain)).anyMatch(email::endsWith); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/gcp/GcpJwkVault.java b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/gcp/GcpJwkVault.java new file mode 100644 index 000000000..3717c4fd7 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/gcp/GcpJwkVault.java @@ -0,0 +1,31 @@ +package uk.ac.ebi.subs.ingest.security.authn.provider.gcp; + +import java.security.PublicKey; + +import com.auth0.jwk.JwkException; +import com.auth0.jwk.UrlJwkProvider; +import com.auth0.jwt.interfaces.DecodedJWT; + +import uk.ac.ebi.subs.ingest.security.common.jwk.JwkVault; +import uk.ac.ebi.subs.ingest.security.common.jwk.UrlJwkProviderResolver; + +public class GcpJwkVault implements JwkVault { + + private final UrlJwkProviderResolver urlJwkProviderResolver; + + public GcpJwkVault(UrlJwkProviderResolver urlJwkProviderResolver) { + this.urlJwkProviderResolver = urlJwkProviderResolver; + } + + @Override + public PublicKey getPublicKey(DecodedJWT jwt) { + var issuer = jwt.getIssuer(); + UrlJwkProvider jwkProvider = urlJwkProviderResolver.resolve(issuer); + try { + var jwk = jwkProvider.get(jwt.getKeyId()); + return jwk.getPublicKey(); + } catch (JwkException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/gcp/GoogleServiceJwtAuthenticationProvider.java b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/gcp/GoogleServiceJwtAuthenticationProvider.java new file mode 100644 index 000000000..b2a8497cc --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/authn/provider/gcp/GoogleServiceJwtAuthenticationProvider.java @@ -0,0 +1,75 @@ +package uk.ac.ebi.subs.ingest.security.authn.provider.gcp; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.security.authentication.AuthenticationProvider; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.AuthenticationException; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.exceptions.JWTVerificationException; +import com.auth0.jwt.interfaces.DecodedJWT; +import com.auth0.jwt.interfaces.JWTVerifier; +import com.auth0.spring.security.api.authentication.JwtAuthentication; + +import uk.ac.ebi.subs.ingest.security.Account; +import uk.ac.ebi.subs.ingest.security.authn.oidc.OpenIdAuthentication; +import uk.ac.ebi.subs.ingest.security.authn.oidc.UserInfo; +import uk.ac.ebi.subs.ingest.security.common.jwk.DelegatingJwtAuthentication; +import uk.ac.ebi.subs.ingest.security.common.jwk.JwtVerifierResolver; +import uk.ac.ebi.subs.ingest.security.exception.JwtVerificationFailed; +import uk.ac.ebi.subs.ingest.security.exception.UnlistedJwtIssuer; + +public class GoogleServiceJwtAuthenticationProvider implements AuthenticationProvider { + + private static Logger logger = + LoggerFactory.getLogger(GoogleServiceJwtAuthenticationProvider.class); + + private final JwtVerifierResolver jwtVerifierResolver; + + private final GcpDomainWhiteList projectWhitelist; + + public GoogleServiceJwtAuthenticationProvider( + GcpDomainWhiteList projectWhitelist, JwtVerifierResolver jwtVerifierResolver) { + this.jwtVerifierResolver = jwtVerifierResolver; + this.projectWhitelist = projectWhitelist; + } + + @Override + public boolean supports(Class authentication) { + return JwtAuthentication.class.isAssignableFrom(authentication); + } + + @Override + public Authentication authenticate(Authentication authentication) throws AuthenticationException { + if (!supports(authentication.getClass())) { + return null; + } + try { + JwtAuthentication jwt = (JwtAuthentication) authentication; + verifyIssuer(jwt); + + JWTVerifier jwtVerifier = jwtVerifierResolver.resolve(jwt.getToken()); + Authentication jwtAuth = DelegatingJwtAuthentication.delegate(jwt, jwtVerifier); + + logger.info("Authenticated with jwt with scopes {}", jwtAuth.getAuthorities()); + + Account account = Account.SERVICE; + UserInfo serviceUser = new UserInfo(JWT.decode(jwt.getToken()).getSubject(), "password"); + OpenIdAuthentication openIdAuth = new OpenIdAuthentication(account, serviceUser); + return openIdAuth; + } catch (JWTVerificationException e) { + logger.error("JWT verification failed: {}", e.getMessage()); + throw new JwtVerificationFailed(e); + } + } + + private void verifyIssuer(JwtAuthentication jwt) { + DecodedJWT token = JWT.decode(jwt.getToken()); + String issuer = token.getIssuer(); + + if (!projectWhitelist.lists(issuer)) { + throw UnlistedJwtIssuer.notWhitelisted(issuer); + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/Auth0JwtAuthentication.java b/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/Auth0JwtAuthentication.java new file mode 100644 index 000000000..3e2390c79 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/Auth0JwtAuthentication.java @@ -0,0 +1,15 @@ +package uk.ac.ebi.subs.ingest.security.common.jwk; + +import org.springframework.security.core.Authentication; + +/** + * A custom Authentication interface based on Auth0's JwtAuthentication without the frustrating + * verify method that takes concrete JWTVerifier instead of the JWTVerifier interface (yeah, + * confusing). + */ +public interface Auth0JwtAuthentication extends Authentication { + + String getToken(); + + String getKeyId(); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/DelegatingJwtAuthentication.java b/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/DelegatingJwtAuthentication.java new file mode 100644 index 000000000..fa16e8331 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/DelegatingJwtAuthentication.java @@ -0,0 +1,76 @@ +package uk.ac.ebi.subs.ingest.security.common.jwk; + +import java.util.Collection; + +import org.springframework.security.core.Authentication; +import org.springframework.security.core.GrantedAuthority; + +import com.auth0.jwt.interfaces.DecodedJWT; +import com.auth0.jwt.interfaces.JWTVerifier; +import com.auth0.spring.security.api.authentication.JwtAuthentication; + +public class DelegatingJwtAuthentication implements Auth0JwtAuthentication { + + private Authentication authentication; + + private DecodedJWT token; + + public static DelegatingJwtAuthentication delegate( + JwtAuthentication source, JWTVerifier verifier) { + var authentication = source.verify(null); + DecodedJWT token = verifier.verify(source.getToken()); + return new DelegatingJwtAuthentication(authentication, token); + } + + private DelegatingJwtAuthentication(Authentication authentication, DecodedJWT token) { + this.authentication = authentication; + this.token = token; + } + + @Override + public String getToken() { + return token.getToken(); + } + + @Override + public String getKeyId() { + return token.getKeyId(); + } + + @Override + public Collection getAuthorities() { + return authentication.getAuthorities(); + } + + @Override + public Object getCredentials() { + return authentication.getCredentials(); + } + + @Override + public Object getDetails() { + return authentication.getDetails(); + } + + @Override + public Object getPrincipal() { + return authentication.getPrincipal(); + } + + @Override + public boolean isAuthenticated() { + // The construction of this object would only succeed if the token has first + // been successfully verified. + return true; + } + + @Override + public void setAuthenticated(boolean isAuthenticated) throws IllegalArgumentException { + throw new IllegalArgumentException("Authenticate through delegation to a new instance."); + } + + @Override + public String getName() { + return authentication.getName(); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/DelegatingJwtVerifier.java b/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/DelegatingJwtVerifier.java new file mode 100644 index 000000000..d49c696ad --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/DelegatingJwtVerifier.java @@ -0,0 +1,84 @@ +package uk.ac.ebi.subs.ingest.security.common.jwk; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.algorithms.Algorithm; +import com.auth0.jwt.exceptions.JWTVerificationException; +import com.auth0.jwt.interfaces.DecodedJWT; +import com.auth0.jwt.interfaces.JWTVerifier; +import com.auth0.jwt.interfaces.Verification; + +/** + * A {@link JWTVerifier} that delegates to Auth0's {@link com.auth0.jwt.JWTVerifier} (which is + * another implementation of the interface). Yes, the interface and the default implementing class + * are named the same, which illustrates how confusing Auth0's library for processing JWTs can be. + * Hence, we have all these wrapper classes to hopefully help us deal with that. + */ +public class DelegatingJwtVerifier implements JWTVerifier { + + public static class Builder { + + private Verification verification; + + private String audience; + private String issuer; + + private Builder(Verification verification) { + this.verification = verification; + } + + public static Builder require(Algorithm algorithm) { + return new Builder(JWT.require(algorithm)); + } + + public Builder withAudience(String audience) { + this.audience = audience; + verification.withAudience(audience); + return this; + } + + public Builder withIssuer(String issuer) { + this.issuer = issuer; + verification.withIssuer(issuer); + return this; + } + + public JWTVerifier build() { + DelegatingJwtVerifier verifier = new DelegatingJwtVerifier(verification.build()); + verifier.audience = audience; + verifier.issuer = issuer; + return verifier; + } + } + + private final JWTVerifier delegate; + + private String audience; + private String issuer; + + /** Effectively an alias for {@link Builder#require(Algorithm)}. */ + public static Builder require(Algorithm algorithm) { + return Builder.require(algorithm); + } + + private DelegatingJwtVerifier(JWTVerifier delegate) { + this.delegate = delegate; + } + + public String getAudience() { + return audience; + } + + public String getIssuer() { + return issuer; + } + + @Override + public DecodedJWT verify(String token) throws JWTVerificationException { + return delegate.verify(token); + } + + @Override + public DecodedJWT verify(DecodedJWT jwt) throws JWTVerificationException { + return delegate.verify(jwt); + } +} diff --git a/src/main/java/org/humancellatlas/ingest/security/common/jwk/JwkVault.java b/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/JwkVault.java similarity index 52% rename from src/main/java/org/humancellatlas/ingest/security/common/jwk/JwkVault.java rename to src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/JwkVault.java index 7e0100917..c465632ae 100644 --- a/src/main/java/org/humancellatlas/ingest/security/common/jwk/JwkVault.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/JwkVault.java @@ -1,11 +1,10 @@ -package org.humancellatlas.ingest.security.common.jwk; - -import com.auth0.jwt.interfaces.DecodedJWT; +package uk.ac.ebi.subs.ingest.security.common.jwk; import java.security.PublicKey; -public interface JwkVault { +import com.auth0.jwt.interfaces.DecodedJWT; - PublicKey getPublicKey(DecodedJWT jwt); +public interface JwkVault { + PublicKey getPublicKey(DecodedJWT jwt); } diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/JwtVerifierResolver.java b/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/JwtVerifierResolver.java new file mode 100644 index 000000000..7d96db16d --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/JwtVerifierResolver.java @@ -0,0 +1,43 @@ +package uk.ac.ebi.subs.ingest.security.common.jwk; + +import java.security.interfaces.RSAPublicKey; +import java.util.Optional; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.algorithms.Algorithm; +import com.auth0.jwt.interfaces.DecodedJWT; +import com.auth0.jwt.interfaces.JWTVerifier; + +/** + * Helper class whose main purpose is to create a {@link JWTVerifier} instance given a JWT string. + * + *

This is part of the wrapper subsystem to help compartmentalise the area of the application + * that relies on Auth0's library for processing JWTs. + */ +public class JwtVerifierResolver { + + private final JwkVault jwkVault; + + private final Optional audience; + private final Optional issuer; + + public JwtVerifierResolver(JwkVault jwkVault, String audience, String issuer) { + this.jwkVault = jwkVault; + this.audience = Optional.ofNullable(audience); + this.issuer = Optional.ofNullable(issuer); + } + + public String getIssuer() { + return issuer.orElse(null); + } + + public JWTVerifier resolve(String jwt) { + DecodedJWT token = JWT.decode(jwt); + RSAPublicKey publicKey = (RSAPublicKey) jwkVault.getPublicKey(token); + DelegatingJwtVerifier.Builder builder = + DelegatingJwtVerifier.require(Algorithm.RSA256(publicKey, null)); + audience.ifPresent(builder::withAudience); + builder.withIssuer(issuer.orElse(token.getIssuer())); + return builder.build(); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/RemoteJwkProvider.java b/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/RemoteJwkProvider.java new file mode 100644 index 000000000..4ea18b8d1 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/RemoteJwkProvider.java @@ -0,0 +1,20 @@ +package uk.ac.ebi.subs.ingest.security.common.jwk; + +import java.net.URL; + +import com.auth0.jwk.UrlJwkProvider; + +/** A UrlJwkProvider that allows inspection of its internal URL. */ +public class RemoteJwkProvider extends UrlJwkProvider { + + private final URL url; + + public RemoteJwkProvider(URL url) { + super(url); + this.url = url; + } + + public URL getUrl() { + return url; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/UrlJwkProviderResolver.java b/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/UrlJwkProviderResolver.java new file mode 100644 index 000000000..665698610 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/common/jwk/UrlJwkProviderResolver.java @@ -0,0 +1,33 @@ +package uk.ac.ebi.subs.ingest.security.common.jwk; + +import java.net.MalformedURLException; +import java.net.URL; + +import com.auth0.jwk.UrlJwkProvider; + +public class UrlJwkProviderResolver { + + private final URL baseUrl; + + public UrlJwkProviderResolver(String baseUrl) { + try { + this.baseUrl = new URL(baseUrl); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + // TODO cache providers based on relative path + public UrlJwkProvider resolve(String relativePath) { + try { + var providerUrl = new URL(baseUrl, relativePath); + return new RemoteJwkProvider(providerUrl); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + public UrlJwkProvider resolve() { + return new RemoteJwkProvider(this.baseUrl); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/exception/DuplicateAccount.java b/src/main/java/uk/ac/ebi/subs/ingest/security/exception/DuplicateAccount.java new file mode 100644 index 000000000..983ca51c3 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/exception/DuplicateAccount.java @@ -0,0 +1,8 @@ +package uk.ac.ebi.subs.ingest.security.exception; + +public class DuplicateAccount extends RuntimeException { + + public DuplicateAccount() { + super("Operation failed due to Account duplication."); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/exception/InvalidUserGroup.java b/src/main/java/uk/ac/ebi/subs/ingest/security/exception/InvalidUserGroup.java new file mode 100644 index 000000000..a5b253df6 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/exception/InvalidUserGroup.java @@ -0,0 +1,10 @@ +package uk.ac.ebi.subs.ingest.security.exception; + +import org.springframework.security.core.AuthenticationException; + +public class InvalidUserGroup extends AuthenticationException { + + public InvalidUserGroup(String group) { + super(String.format("Invalid user group, %s", group)); + } +} diff --git a/src/main/java/org/humancellatlas/ingest/security/exception/JwtVerificationFailed.java b/src/main/java/uk/ac/ebi/subs/ingest/security/exception/JwtVerificationFailed.java similarity index 52% rename from src/main/java/org/humancellatlas/ingest/security/exception/JwtVerificationFailed.java rename to src/main/java/uk/ac/ebi/subs/ingest/security/exception/JwtVerificationFailed.java index b8a69e349..5b8392bd8 100644 --- a/src/main/java/org/humancellatlas/ingest/security/exception/JwtVerificationFailed.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/exception/JwtVerificationFailed.java @@ -1,12 +1,12 @@ -package org.humancellatlas.ingest.security.exception; +package uk.ac.ebi.subs.ingest.security.exception; -import com.auth0.jwt.exceptions.JWTVerificationException; import org.springframework.security.core.AuthenticationException; -public class JwtVerificationFailed extends AuthenticationException { +import com.auth0.jwt.exceptions.JWTVerificationException; - public JwtVerificationFailed(JWTVerificationException cause) { - super("JWT verification failed.", cause); - } +public class JwtVerificationFailed extends AuthenticationException { + public JwtVerificationFailed(JWTVerificationException cause) { + super("JWT verification failed.", cause); + } } diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/exception/NotAllowedException.java b/src/main/java/uk/ac/ebi/subs/ingest/security/exception/NotAllowedException.java new file mode 100644 index 000000000..64274348c --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/exception/NotAllowedException.java @@ -0,0 +1,11 @@ +package uk.ac.ebi.subs.ingest.security.exception; + +public class NotAllowedException extends RuntimeException { + public NotAllowedException() { + super("Operation not allowed."); + } + + public NotAllowedException(String customMessage) { + super(customMessage); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/exception/UnlistedJwtIssuer.java b/src/main/java/uk/ac/ebi/subs/ingest/security/exception/UnlistedJwtIssuer.java new file mode 100644 index 000000000..d73196172 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/exception/UnlistedJwtIssuer.java @@ -0,0 +1,24 @@ +package uk.ac.ebi.subs.ingest.security.exception; + +import static java.lang.String.format; + +import org.springframework.security.core.AuthenticationException; + +public class UnlistedJwtIssuer extends AuthenticationException { + + private final String issuer; + + public UnlistedJwtIssuer(String issuer, String message) { + super(message); + this.issuer = issuer; + } + + public static UnlistedJwtIssuer notWhitelisted(String issuer) { + return new UnlistedJwtIssuer( + format("Issuer [%s] is not specified in the whitelist.", issuer), issuer); + } + + public String getIssuer() { + return issuer; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/security/web/AuthenticationController.java b/src/main/java/uk/ac/ebi/subs/ingest/security/web/AuthenticationController.java new file mode 100644 index 000000000..7cef5a65b --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/security/web/AuthenticationController.java @@ -0,0 +1,51 @@ +package uk.ac.ebi.subs.ingest.security.web; + +import static org.springframework.http.MediaType.APPLICATION_JSON_UTF8_VALUE; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.Authentication; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; + +import uk.ac.ebi.subs.ingest.security.Account; +import uk.ac.ebi.subs.ingest.security.AccountService; +import uk.ac.ebi.subs.ingest.security.Role; +import uk.ac.ebi.subs.ingest.security.authn.oidc.OpenIdAuthentication; +import uk.ac.ebi.subs.ingest.security.authn.oidc.UserInfo; +import uk.ac.ebi.subs.ingest.security.exception.DuplicateAccount; + +@Controller +@RequestMapping("/auth") +public class AuthenticationController { + + private final AccountService accountService; + + public AuthenticationController(AccountService accountService) { + this.accountService = accountService; + } + + @PostMapping(path = "/registration", produces = APPLICATION_JSON_UTF8_VALUE) + public ResponseEntity register(Authentication authentication) { + var openIdAuthentication = (OpenIdAuthentication) authentication; + var userInfo = (UserInfo) openIdAuthentication.getCredentials(); + try { + Account account = userInfo.toAccount(); + Account persistentAccount = accountService.register(account); + return ResponseEntity.ok().body(persistentAccount); + } catch (DuplicateAccount duplicateAccount) { + return ResponseEntity.status(HttpStatus.CONFLICT).build(); + } + } + + @GetMapping(path = "/account", produces = APPLICATION_JSON_UTF8_VALUE) + public ResponseEntity getAccount(Authentication authentication) { + if (authentication.getAuthorities().contains(Role.GUEST)) { + return ResponseEntity.notFound().build(); + } + Account account = (Account) authentication.getPrincipal(); + return ResponseEntity.ok().body(account); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/StagingJob.java b/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/StagingJob.java new file mode 100644 index 000000000..d3bc00094 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/StagingJob.java @@ -0,0 +1,54 @@ +package uk.ac.ebi.subs.ingest.stagingjob; + +import java.time.Instant; +import java.util.UUID; + +import org.springframework.data.annotation.CreatedDate; +import org.springframework.data.annotation.Id; +import org.springframework.data.annotation.PersistenceConstructor; +import org.springframework.data.mongodb.core.index.CompoundIndex; +import org.springframework.data.mongodb.core.index.CompoundIndexes; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.hateoas.Identifiable; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.*; + +@Getter +@CompoundIndexes({ + @CompoundIndex( + name = "stagingAreaUuidAndFileName", + def = "{'stagingAreaUuid' : 1, 'stagingAreaFileName' : 1}", + unique = true) +}) +@Document +@EqualsAndHashCode +@RequiredArgsConstructor +public class StagingJob implements Identifiable { + + @Id private String id; + + @CreatedDate private Instant createdDate; + + @Indexed private final UUID stagingAreaUuid; + + private final String stagingAreaFileName; + + private String metadataUuid; + + @Setter private String stagingAreaFileUri; + + @JsonCreator + @PersistenceConstructor + public StagingJob( + @JsonProperty(value = "stagingAreaUuid") UUID stagingAreaUuid, + @JsonProperty(value = "metadataUuid") String metadataUuid, + @JsonProperty(value = "stagingAreaFileName") String stagingAreaFileName) { + this.stagingAreaUuid = stagingAreaUuid; + this.metadataUuid = metadataUuid; + this.stagingAreaFileName = stagingAreaFileName; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/StagingJobRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/StagingJobRepository.java new file mode 100644 index 000000000..472bffe80 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/StagingJobRepository.java @@ -0,0 +1,25 @@ +package uk.ac.ebi.subs.ingest.stagingjob; + +import java.util.UUID; + +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.repository.query.Param; +import org.springframework.data.rest.core.annotation.RestResource; +import org.springframework.web.bind.annotation.CrossOrigin; + +@CrossOrigin +public interface StagingJobRepository extends MongoRepository { + @RestResource(exported = false) + T save(T stagingJob); + + @RestResource + void delete(StagingJob stagingJob); + + @RestResource(exported = false) + void deleteAllByStagingAreaUuid(UUID stagingAreaUuid); + + @RestResource(rel = "findByStagingAreaAndFileName") + T findByStagingAreaUuidAndStagingAreaFileName( + @Param("stagingAreaUuid") UUID stagingAreaUuid, + @Param("stagingAreaFileName") String stagingAreaFileName); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/StagingJobService.java b/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/StagingJobService.java new file mode 100644 index 000000000..2be322adf --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/StagingJobService.java @@ -0,0 +1,54 @@ +package uk.ac.ebi.subs.ingest.stagingjob; + +import java.util.UUID; + +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Service; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class StagingJobService { + + @NonNull private final StagingJobRepository stagingJobRepository; + + public StagingJob register(StagingJob stagingJob) { + try { + return stagingJobRepository.save(stagingJob); + } catch (DuplicateKeyException e) { + throw new JobAlreadyRegisteredException( + stagingJob.getStagingAreaUuid(), stagingJob.getStagingAreaFileName()); + } + } + + @Deprecated + public StagingJob registerNewJob(UUID stagingAreaUuid, String stagingAreaFileName) { + try { + StagingJob stagingJob = new StagingJob(stagingAreaUuid, stagingAreaFileName); + return stagingJobRepository.save(stagingJob); + } catch (DuplicateKeyException e) { + throw new JobAlreadyRegisteredException(stagingAreaUuid, stagingAreaFileName); + } + } + + public StagingJob completeJob(StagingJob stagingJob, String stagingAreaUri) { + stagingJob.setStagingAreaFileUri(stagingAreaUri); + return stagingJobRepository.save(stagingJob); + } + + public void deleteJobsForStagingArea(UUID stagingAreaUuid) { + stagingJobRepository.deleteAllByStagingAreaUuid(stagingAreaUuid); + } + + public static class JobAlreadyRegisteredException extends IllegalStateException { + + public JobAlreadyRegisteredException(UUID stagingAreaUuid, String fileName) { + super( + String.format( + "Staging job request already exists for file %s at upload area %s", + fileName, stagingAreaUuid)); + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobCompleteRequest.java b/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobCompleteRequest.java new file mode 100644 index 000000000..8b210a9ea --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobCompleteRequest.java @@ -0,0 +1,8 @@ +package uk.ac.ebi.subs.ingest.stagingjob.web; + +import lombok.Data; + +@Data +public class StagingJobCompleteRequest { + private String stagingAreaFileUri; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobController.java b/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobController.java new file mode 100644 index 000000000..231d589dd --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobController.java @@ -0,0 +1,50 @@ +package uk.ac.ebi.subs.ingest.stagingjob.web; + +import java.util.UUID; + +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.stagingjob.StagingJob; +import uk.ac.ebi.subs.ingest.stagingjob.StagingJobService; + +@RepositoryRestController +@ExposesResourceFor(StagingJob.class) +@RequiredArgsConstructor +@RequestMapping("/stagingJobs") +public class StagingJobController { + + private final @NonNull StagingJobService stagingJobService; + + @PostMapping + public ResponseEntity createStagingJob( + @RequestBody StagingJob stagingJob, PersistentEntityResourceAssembler resourceAssembler) { + StagingJob persistentJob = stagingJobService.register(stagingJob); + return ResponseEntity.ok(resourceAssembler.toFullResource(persistentJob)); + } + + @PatchMapping(path = "/{stagingJob}" + Links.COMPLETE_STAGING_JOB_URL) + ResponseEntity completeStagingJob( + @PathVariable("stagingJob") StagingJob stagingJob, + @RequestBody StagingJobCompleteRequest stagingJobCompleteRequest, + final PersistentEntityResourceAssembler resourceAssembler) { + StagingJob completedStagingJob = + stagingJobService.completeJob( + stagingJob, stagingJobCompleteRequest.getStagingAreaFileUri()); + + return ResponseEntity.ok(resourceAssembler.toFullResource(completedStagingJob)); + } + + @DeleteMapping + ResponseEntity deleteStagingJobs(@RequestParam("stagingAreaUuid") UUID stagingAreaUuid) { + stagingJobService.deleteJobsForStagingArea(stagingAreaUuid); + return new ResponseEntity(HttpStatus.NO_CONTENT); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobCreateRequest.java b/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobCreateRequest.java new file mode 100644 index 000000000..73feeba35 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobCreateRequest.java @@ -0,0 +1,11 @@ +package uk.ac.ebi.subs.ingest.stagingjob.web; + +import java.util.UUID; + +import lombok.Data; + +@Data +public class StagingJobCreateRequest { + private UUID stagingAreaUuid; + private String stagingAreaFileName; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobResourceProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobResourceProcessor.java new file mode 100644 index 000000000..5beb665ff --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobResourceProcessor.java @@ -0,0 +1,34 @@ +package uk.ac.ebi.subs.ingest.stagingjob.web; + +import org.springframework.hateoas.EntityLinks; +import org.springframework.hateoas.Link; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.stereotype.Component; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.stagingjob.StagingJob; + +@RequiredArgsConstructor +@Component +public class StagingJobResourceProcessor implements ResourceProcessor> { + private final @NonNull EntityLinks entityLinks; + + private Link getCompleteStagingJobLink(StagingJob stagingJob) { + return entityLinks + .linkForSingleResource(stagingJob) + .slash(Links.COMPLETE_STAGING_JOB_URL) + .withRel(Links.COMPLETE_STAGING_JOB_REL); + } + + @Override + public Resource process(Resource stagingJobResource) { + StagingJob stagingJob = stagingJobResource.getContent(); + + stagingJobResource.add(getCompleteStagingJobLink(stagingJob)); + + return stagingJobResource; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobResourcesProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobResourcesProcessor.java new file mode 100644 index 000000000..9690dd43f --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobResourcesProcessor.java @@ -0,0 +1,29 @@ +package uk.ac.ebi.subs.ingest.stagingjob.web; + +import static org.springframework.hateoas.mvc.ControllerLinkBuilder.*; + +import org.springframework.hateoas.Link; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.hateoas.Resources; +import org.springframework.stereotype.Component; + +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.stagingjob.StagingJob; + +@Component +@RequiredArgsConstructor +public class StagingJobResourcesProcessor + implements ResourceProcessor>> { + + private Link getDeleteByStagingAreaLink() { + return linkTo(methodOn(StagingJobController.class).deleteStagingJobs(null)) + .withRel("delete-staging-jobs"); + } + + @Override + public Resources> process(Resources> resources) { + resources.add(getDeleteByStagingAreaLink()); + return resources; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/state/MetadataDocumentEventHandler.java b/src/main/java/uk/ac/ebi/subs/ingest/state/MetadataDocumentEventHandler.java new file mode 100644 index 000000000..b0d3d1798 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/state/MetadataDocumentEventHandler.java @@ -0,0 +1,27 @@ +package uk.ac.ebi.subs.ingest.state; + +import org.springframework.data.rest.core.annotation.HandleAfterCreate; +import org.springframework.data.rest.core.annotation.RepositoryEventHandler; +import org.springframework.stereotype.Component; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; + +@RepositoryEventHandler +@Component +@RequiredArgsConstructor +public class MetadataDocumentEventHandler { + private final @NonNull MessageRouter messageRouter; + + @HandleAfterCreate + public void metadataDocumentAfterCreate(MetadataDocument document) { + this.handleMetadataDocumentCreate(document); + } + + public void handleMetadataDocumentCreate(MetadataDocument document) { + messageRouter.routeValidationMessageFor(document); + // messageRouter.routeStateTrackingUpdateMessageFor(document); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/state/MetadataStateChangeListener.java b/src/main/java/uk/ac/ebi/subs/ingest/state/MetadataStateChangeListener.java new file mode 100644 index 000000000..b9f408b42 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/state/MetadataStateChangeListener.java @@ -0,0 +1,55 @@ +package uk.ac.ebi.subs.ingest.state; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.mongodb.core.mapping.event.AbstractMongoEventListener; +import org.springframework.data.mongodb.core.mapping.event.AfterSaveEvent; +import org.springframework.data.mongodb.core.mapping.event.BeforeConvertEvent; +import org.springframework.stereotype.Component; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 12/09/17 + */ +@Component +@RequiredArgsConstructor +@Getter +public class MetadataStateChangeListener extends AbstractMongoEventListener { + private final MessageRouter messageRouter; + private final Logger log = LoggerFactory.getLogger(getClass()); + + protected Logger getLog() { + return log; + } + + @Override + public void onAfterSave(AfterSaveEvent event) { + MetadataDocument document = event.getSource(); + messageRouter.routeValidationMessageFor(document); + } + + @Override + public void onBeforeConvert(BeforeConvertEvent event) { + MetadataDocument document = event.getSource(); + + // TODO Ideally, this should be being set when the submission is submitted. + // The exporter could set this. Putting this back here for now for convenience. + if (!Optional.ofNullable(document.getDcpVersion()).isPresent()) { + document.setDcpVersion(document.getSubmissionDate()); + } + + if (!Optional.ofNullable(document.getUuid()).isPresent()) { + document.setUuid(Uuid.newUuid()); + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/state/SubmissionState.java b/src/main/java/uk/ac/ebi/subs/ingest/state/SubmissionState.java new file mode 100644 index 000000000..ca9aa6380 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/state/SubmissionState.java @@ -0,0 +1,22 @@ +package uk.ac.ebi.subs.ingest.state; + +/** + * @author Simon Jupp + * @date 04/09/2017 Samples, Phenotypes and Ontologies Team, EMBL-EBI + */ +public enum SubmissionState { + PENDING, + DRAFT, + METADATA_VALID, + METADATA_INVALID, + GRAPH_VALID, + GRAPH_INVALID, + SUBMITTED, + PROCESSING, + ARCHIVING, + ARCHIVED, + EXPORTING, + EXPORTED, + CLEANUP, + COMPLETE +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/state/SubmissionStateChangeListener.java b/src/main/java/uk/ac/ebi/subs/ingest/state/SubmissionStateChangeListener.java new file mode 100644 index 000000000..0af828423 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/state/SubmissionStateChangeListener.java @@ -0,0 +1,38 @@ +package uk.ac.ebi.subs.ingest.state; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mongodb.core.mapping.event.AbstractMongoEventListener; +import org.springframework.data.mongodb.core.mapping.event.AfterSaveEvent; +import org.springframework.stereotype.Component; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Component +@RequiredArgsConstructor +@Getter +public class SubmissionStateChangeListener extends AbstractMongoEventListener { + @Autowired @NonNull private final MessageRouter messageRouter; + + private final Logger log = LoggerFactory.getLogger(getClass()); + + protected Logger getLog() { + return log; + } + + @Override + public void onAfterSave(AfterSaveEvent event) { + SubmissionEnvelope submissionEnvelope = event.getSource(); + + if (submissionEnvelope.getSubmissionState().equals(SubmissionState.CLEANUP)) { + log.info( + String.format("Requesting cleanup for envelope with ID %s", submissionEnvelope.getId())); + this.messageRouter.routeRequestUploadAreaCleanup(submissionEnvelope); + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/state/SubmitAction.java b/src/main/java/uk/ac/ebi/subs/ingest/state/SubmitAction.java new file mode 100644 index 000000000..88e40de16 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/state/SubmitAction.java @@ -0,0 +1,8 @@ +package uk.ac.ebi.subs.ingest.state; + +public enum SubmitAction { + ARCHIVE, + EXPORT, + CLEANUP, + EXPORT_METADATA +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/state/ValidationState.java b/src/main/java/uk/ac/ebi/subs/ingest/state/ValidationState.java new file mode 100644 index 000000000..7fa3c7d8d --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/state/ValidationState.java @@ -0,0 +1,18 @@ +package uk.ac.ebi.subs.ingest.state; + +import com.fasterxml.jackson.annotation.JsonCreator; + +/** Created by rolando on 07/09/2017. */ +public enum ValidationState { + DRAFT, + VALIDATING, + VALID, + INVALID, + PROCESSING, + COMPLETE; + + @JsonCreator + public static ValidationState fromString(String key) { + return key == null ? null : ValidationState.valueOf(key.toUpperCase()); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/state/ValidationStateChangeListener.java b/src/main/java/uk/ac/ebi/subs/ingest/state/ValidationStateChangeListener.java new file mode 100644 index 000000000..f9697b2c9 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/state/ValidationStateChangeListener.java @@ -0,0 +1,28 @@ +package uk.ac.ebi.subs.ingest.state; + +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Component; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.core.ValidationEvent; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.project.ProjectEventHandler; + +@Component +@RequiredArgsConstructor +public class ValidationStateChangeListener implements ApplicationListener { + private final @NonNull ProjectEventHandler projectEventHandler; + + @Override + public void onApplicationEvent(ValidationEvent event) { + MetadataDocument document = (MetadataDocument) event.getSource(); + // messageRouter.routeStateTrackingUpdateMessageFor(document); + + if (document.getType().equals(EntityType.PROJECT)) { + projectEventHandler.validatedProject((Project) document); + } + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/state/ValidationStateEventPublisher.java b/src/main/java/uk/ac/ebi/subs/ingest/state/ValidationStateEventPublisher.java new file mode 100644 index 000000000..81a621a50 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/state/ValidationStateEventPublisher.java @@ -0,0 +1,21 @@ +package uk.ac.ebi.subs.ingest.state; + +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Component; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.core.ValidationEvent; + +@Component +@RequiredArgsConstructor +public class ValidationStateEventPublisher { + private final @NonNull ApplicationEventPublisher applicationEventPublisher; + + public void publishValidationStateChangeEventFor(MetadataDocument metadataDocument) { + ValidationEvent validationEvent = + new ValidationEvent(metadataDocument, metadataDocument.getValidationState().toString()); + applicationEventPublisher.publishEvent(validationEvent); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/study/Study.java b/src/main/java/uk/ac/ebi/subs/ingest/study/Study.java new file mode 100644 index 000000000..6de3497f1 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/study/Study.java @@ -0,0 +1,121 @@ +package uk.ac.ebi.subs.ingest.study; + +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.validation.constraints.NotNull; + +import org.springframework.data.mongodb.core.mapping.DBRef; +import org.springframework.data.mongodb.core.mapping.Field; +import org.springframework.data.rest.core.annotation.RestResource; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import uk.ac.ebi.subs.ingest.core.DescriptiveSchema; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.dataset.Dataset; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Getter +@EqualsAndHashCode( + callSuper = true, + exclude = {"submissionEnvelopes"}) +@JsonIgnoreProperties({"firstDcpVersion", "dcpVersion"}) +public class Study extends MetadataDocument implements DescriptiveSchema { + // A study may have 1 or more submissions related to it. + @JsonIgnore + private @DBRef(lazy = true) Set submissionEnvelopes = new HashSet<>(); + + // A study can have multiple datasets + @RestResource private Set datasets = new HashSet<>(); + + @Field("described_by") + private String describedBy; + + @Field("schema_version") + private String schemaVersion; + + @Field("schema_type") + private String schemaType; + + @JsonCreator + public Study( + @JsonProperty("described_by") String describedBy, + @JsonProperty("schema_version") String schemaVersion, + @JsonProperty("schema_type") String schemaType, + @JsonProperty("content") Object content) { + super(EntityType.STUDY, content); + this.describedBy = describedBy; + this.schemaVersion = schemaVersion; + this.schemaType = schemaType; + } + + public void addToSubmissionEnvelopes(@NotNull SubmissionEnvelope submissionEnvelope) { + this.submissionEnvelopes.add(submissionEnvelope); + } + + // Override MorphicDescriptiveSchema methods + @Override + public String getDescribedBy() { + return describedBy; + } + + @Override + public void setDescribedBy(String describedBy) { + this.describedBy = describedBy; + } + + @Override + public String getSchemaVersion() { + return schemaVersion; + } + + @Override + public void setSchemaVersion(String schemaVersion) { + this.schemaVersion = schemaVersion; + } + + @Override + public String getSchemaType() { + return schemaType; + } + + @Override + public void setSchemaType(String schemaType) { + this.schemaType = schemaType; + } + + public void addDataset(final Dataset dataset) { + datasets.add(dataset); + } + + // ToDo: Find a better way of ensuring that DBRefs to deleted objects aren't returned. + @JsonIgnore + public List getOpenSubmissionEnvelopes() { + return this.submissionEnvelopes.stream() + .filter(Objects::nonNull) + .filter(env -> env.getSubmissionState() != null) + .filter(SubmissionEnvelope::isOpen) + .collect(Collectors.toList()); + } + + public Boolean getHasOpenSubmission() { + return !getOpenSubmissionEnvelopes().isEmpty(); + } + + @JsonIgnore + public Boolean isEditable() { + return this.submissionEnvelopes.stream() + .filter(Objects::nonNull) + .allMatch(SubmissionEnvelope::isEditable); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/study/StudyEventHandler.java b/src/main/java/uk/ac/ebi/subs/ingest/study/StudyEventHandler.java new file mode 100644 index 000000000..6e239f288 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/study/StudyEventHandler.java @@ -0,0 +1,26 @@ +package uk.ac.ebi.subs.ingest.study; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import lombok.RequiredArgsConstructor; + +@Component +@RequiredArgsConstructor +public class StudyEventHandler { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + public void registeredStudy(Study study) { + log.info("A new study [" + study.getUuid() + "] was registered."); + } + + public void updatedStudy(Study study) { + log.info("Updated study: {}", study); + } + + public void deletedStudy(String id) { + log.info("Deleted study with ID: {}", id); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/study/StudyRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/study/StudyRepository.java new file mode 100644 index 000000000..18c813466 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/study/StudyRepository.java @@ -0,0 +1,38 @@ +package uk.ac.ebi.subs.ingest.study; + +import java.util.Collection; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.repository.query.Param; +import org.springframework.data.rest.core.annotation.RestResource; +import org.springframework.web.bind.annotation.CrossOrigin; + +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@CrossOrigin +public interface StudyRepository extends MongoRepository { + + @RestResource(rel = "findAllByUuid", path = "findAllByUuid") + Page findByUuid(@Param("uuid") Uuid uuid, Pageable pageable); + + @RestResource(exported = false) + Stream findByUuid(Uuid uuid); + + @RestResource(rel = "findByUuid", path = "findByUuid") + Optional findByUuidUuidAndIsUpdateFalse(@Param("uuid") UUID uuid); + + Page findBySubmissionEnvelopesContaining( + SubmissionEnvelope submissionEnvelope, Pageable pageable); + + @RestResource(exported = false) + Stream findBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); + + @RestResource(exported = false) + Collection findAllBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/study/StudyService.java b/src/main/java/uk/ac/ebi/subs/ingest/study/StudyService.java new file mode 100644 index 000000000..f93b77e94 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/study/StudyService.java @@ -0,0 +1,177 @@ +package uk.ac.ebi.subs.ingest.study; + +import static java.util.stream.Collectors.toSet; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +import javax.validation.constraints.NotNull; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.server.ResponseStatusException; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.dataset.Dataset; +import uk.ac.ebi.subs.ingest.dataset.DatasetRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Service +@RequiredArgsConstructor +@Getter +public class StudyService { + protected final Logger getLog() { + return log; + } + + private static class StudyBag { + private final Set studies; + private final Set submissionEnvelopes; + + public StudyBag(final Set studies, final Set submissionEnvelopes) { + this.studies = Collections.unmodifiableSet(new HashSet<>(studies)); + this.submissionEnvelopes = Collections.unmodifiableSet(new HashSet<>(submissionEnvelopes)); + } + } + + private final MongoTemplate mongoTemplate; + private final @NonNull StudyRepository studyRepository; + private final @NotNull DatasetRepository datasetRepository; + private final @NonNull MetadataCrudService metadataCrudService; + private final @NonNull MetadataUpdateService metadataUpdateService; + private final @NonNull StudyEventHandler studyEventHandler; + private final ObjectMapper objectMapper = new ObjectMapper(); + private final Logger log = LoggerFactory.getLogger(getClass()); + + public final Study register(final Study study) { + final Study persistentStudy = studyRepository.save(study); + studyEventHandler.registeredStudy(persistentStudy); + + return persistentStudy; + } + + public final Study update(final Study study, final ObjectNode patch) { + final String studyId = study.getId(); + final Optional existingStudyOptional = studyRepository.findById(studyId); + + if (existingStudyOptional.isEmpty()) { + log.warn("Attempted to update study with ID: {} but not found.", studyId); + + throw new ResponseStatusException(HttpStatus.NOT_FOUND); + } + + final Study existingStudy = existingStudyOptional.get(); + final Study updatedStudy = metadataUpdateService.update(existingStudy, patch); + + studyEventHandler.updatedStudy(updatedStudy); + + return updatedStudy; + } + + public Study findById(String studyId) { + Optional studyOptional = studyRepository.findById(studyId); + return studyOptional.orElseThrow( + () -> + new ResponseStatusException( + HttpStatus.NOT_FOUND, "Study not found with ID: " + studyId)); + } + + public final Study replace(final String studyId, final Study updatedStudy) { + final Optional existingStudyOptional = studyRepository.findById(studyId); + + if (existingStudyOptional.isEmpty()) { + log.warn("Study not found with ID: {}", studyId); + throw new ResponseStatusException(HttpStatus.NOT_FOUND); + } + + studyRepository.save(updatedStudy); + studyEventHandler.updatedStudy(updatedStudy); + + return updatedStudy; + } + + public final void delete(final String studyId) { + final Optional deleteStudyOptional = studyRepository.findById(studyId); + + if (deleteStudyOptional.isEmpty()) { + log.warn("Attempted to delete study with ID: {} but not found.", studyId); + throw new ResponseStatusException(HttpStatus.NOT_FOUND); + } + + final Study deleteStudy = deleteStudyOptional.get(); + + metadataCrudService.deleteDocument(deleteStudy); + studyEventHandler.deletedStudy(studyId); + } + + public final Study addStudyToSubmissionEnvelope( + final SubmissionEnvelope submissionEnvelope, final Study study) { + if (!study.getIsUpdate()) { + return metadataCrudService.addToSubmissionEnvelopeAndSave(study, submissionEnvelope); + } else { + return metadataUpdateService.acceptUpdate(study, submissionEnvelope); + } + } + + public final Study linkStudyToSubmissionEnvelope( + final SubmissionEnvelope submissionEnvelope, final Study study) { + final String studyId = study.getId(); + + studyRepository + .findById(studyId) + .orElseThrow(() -> new ResourceNotFoundException("Study: " + studyId)); + study.addToSubmissionEnvelopes(submissionEnvelope); + studyRepository.save(study); + + return study; + } + + public final Study linkDatasetToStudy(final Study study, final Dataset dataset) { + final String studyId = study.getId(); + final String datasetId = dataset.getId(); + + studyRepository + .findById(studyId) + .orElseThrow(() -> new ResourceNotFoundException("Study: " + studyId)); + datasetRepository + .findById(datasetId) + .orElseThrow(() -> new ResourceNotFoundException("Dataset: " + datasetId)); + + study.addDataset(dataset); + + return studyRepository.save(study); + } + + public final Set getSubmissionEnvelopes(final Study study) { + return gather(study).submissionEnvelopes; + } + + private StudyService.StudyBag gather(final Study study) { + final Set envelopes = new HashSet<>(); + final Set studies = this.studyRepository.findByUuid(study.getUuid()).collect(toSet()); + + studies.forEach( + copy -> { + envelopes.addAll(copy.getSubmissionEnvelopes()); + envelopes.add(copy.getSubmissionEnvelope()); + }); + + // ToDo: Find a better way of ensuring that DBRefs to deleted objects aren't returned. + envelopes.removeIf(env -> env == null || env.getSubmissionState() == null); + + return new StudyService.StudyBag(studies, envelopes); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/study/web/StudyController.java b/src/main/java/uk/ac/ebi/subs/ingest/study/web/StudyController.java new file mode 100644 index 000000000..6f906efbe --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/study/web/StudyController.java @@ -0,0 +1,109 @@ +package uk.ac.ebi.subs.ingest.study.web; + +import java.util.Optional; +import java.util.UUID; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.rest.webmvc.PersistentEntityResource; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.hateoas.Resource; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.dataset.Dataset; +import uk.ac.ebi.subs.ingest.security.CheckAllowed; +import uk.ac.ebi.subs.ingest.study.Study; +import uk.ac.ebi.subs.ingest.study.StudyRepository; +import uk.ac.ebi.subs.ingest.study.StudyService; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.exception.NotAllowedDuringSubmissionStateException; + +@RepositoryRestController +@ExposesResourceFor(Study.class) +@RequiredArgsConstructor +@Getter +/* +Controller for studies + */ +public class StudyController { + private static final Logger LOGGER = LoggerFactory.getLogger(StudyController.class); + private final @NonNull StudyService studyService; + private final @NonNull StudyRepository studyRepository; + + @PatchMapping("/studies/{studyId}") + public ResponseEntity> updateStudy( + @PathVariable("studyId") final String studyId, // Change to String + @RequestBody final ObjectNode patch, + final PersistentEntityResourceAssembler assembler) { + + // Fetch the Study object in the service layer based on studyId + Study study = studyService.findById(studyId); // New service method to get Study + return ResponseEntity.ok().body(assembler.toFullResource(studyService.update(study, patch))); + } + + @DeleteMapping("/studies/{studyId}") + public ResponseEntity deleteStudy(@PathVariable final String studyId) { + studyService.delete(studyId); + + return ResponseEntity.noContent().build(); + } + + // TODO: merge add and link, no reason to have both + // @PreAuthorize("hasAnyRole('ROLE_CONTRIBUTOR', 'ROLE_WRANGLER', 'ROLE_SERVICE')") + @CheckAllowed( + value = "#submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PostMapping(path = "submissionEnvelopes/{sub_id}/studies") + public ResponseEntity> addStudyToEnvelopeAndLink( + @PathVariable("sub_id") final SubmissionEnvelope submissionEnvelope, + @RequestBody final Study study, + @RequestParam("updatingUuid") final Optional updatingUuid, + final PersistentEntityResourceAssembler assembler) { + updatingUuid.ifPresent( + uuid -> { + study.setUuid(new Uuid(uuid.toString())); + study.setIsUpdate(true); + }); + + final Study savedStudy = + getStudyService().addStudyToSubmissionEnvelope(submissionEnvelope, study); + final PersistentEntityResource resource = + assembler.toFullResource( + getStudyService().linkStudyToSubmissionEnvelope(submissionEnvelope, savedStudy)); + + return ResponseEntity.accepted().body(resource); + } + + @CheckAllowed( + value = "#submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PutMapping(path = "submissionEnvelopes/{sub_id}/studies/{stud_id}/") + public ResponseEntity> linkSubmissionToStudy( + @PathVariable("sub_id") final SubmissionEnvelope submissionEnvelope, + @PathVariable("stud_id") final Study study, + final PersistentEntityResourceAssembler assembler) { + final Study savedStudy = + getStudyService().linkStudyToSubmissionEnvelope(submissionEnvelope, study); + final PersistentEntityResource studyResource = assembler.toFullResource(savedStudy); + + return ResponseEntity.accepted().body(studyResource); + } + + @PutMapping(path = "studies/{stud_id}/datasets/{dataset_id}") + public ResponseEntity> linkDatasetToStudy( + @PathVariable("stud_id") final Study study, + @PathVariable("dataset_id") final Dataset dataset, + final PersistentEntityResourceAssembler assembler) { + return ResponseEntity.accepted() + .body(assembler.toFullResource(getStudyService().linkDatasetToStudy(study, dataset))); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/SpreadsheetGenerationJob.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/SpreadsheetGenerationJob.java new file mode 100644 index 000000000..fd1111eea --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/SpreadsheetGenerationJob.java @@ -0,0 +1,11 @@ +package uk.ac.ebi.subs.ingest.submission; + +import java.time.Instant; + +import lombok.Data; + +@Data +public class SpreadsheetGenerationJob { + private Instant finishedDate; + private Instant createdDate; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/StagingDetails.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/StagingDetails.java new file mode 100644 index 000000000..62ad9d94c --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/StagingDetails.java @@ -0,0 +1,16 @@ +package uk.ac.ebi.subs.ingest.submission; + +import lombok.Data; +import uk.ac.ebi.subs.ingest.core.Uuid; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 15/09/17 + */ +@Data +class StagingDetails { + private Uuid stagingAreaUuid; + private StagingUrn stagingAreaLocation; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/StagingUrn.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/StagingUrn.java new file mode 100644 index 000000000..0900fecc6 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/StagingUrn.java @@ -0,0 +1,31 @@ +package uk.ac.ebi.subs.ingest.submission; + +import java.net.URI; + +import com.fasterxml.jackson.annotation.JsonCreator; + +import lombok.Data; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 15/09/17 + */ +@Data +class StagingUrn { + private URI value; + + @JsonCreator + public StagingUrn(String name) { + this.value = URI.create(name); + + // test this uri is a URN + if (!value.isOpaque()) { + throw new IllegalArgumentException( + String.format("Staging URN is malformed: %s", value.toString())); + } + } + + StagingUrn() {} +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelope.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelope.java new file mode 100644 index 000000000..7ec0d7b74 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelope.java @@ -0,0 +1,127 @@ +package uk.ac.ebi.subs.ingest.submission; + +import java.util.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.mongodb.core.index.CompoundIndex; +import org.springframework.data.mongodb.core.mapping.Document; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import uk.ac.ebi.subs.ingest.core.AbstractEntity; +import uk.ac.ebi.subs.ingest.core.EntityType; +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.state.SubmitAction; + +@Getter +@Document +/* +Used as a workaround to inheritance issue. +Not proper to annotate uuid in parent class as we don't want uuid index for all subtypes. +*/ +@CompoundIndex(def = "{ 'uuid': 1 }", unique = true) +@EqualsAndHashCode(callSuper = true) +public class SubmissionEnvelope extends AbstractEntity { + private static final Logger log = LoggerFactory.getLogger(SubmissionEnvelope.class); + private @Setter StagingDetails stagingDetails; + private @Setter Boolean triggersAnalysis; + private @Setter Boolean isUpdate; + private @Setter Set submitActions; + private @Setter SpreadsheetGenerationJob lastSpreadsheetGenerationJob; + private SubmissionState submissionState; + + public SubmissionEnvelope() { + super(EntityType.SUBMISSION); + this.submissionState = SubmissionState.PENDING; + this.triggersAnalysis = true; + this.isUpdate = false; + this.submitActions = new HashSet<>(); + } + + public static List allowedSubmissionStateTransitions(SubmissionState fromState) { + List allowedStates = new ArrayList<>(); + + switch (fromState) { + case PENDING: + case METADATA_INVALID: + case GRAPH_INVALID: + allowedStates.add(SubmissionState.DRAFT); + break; + case DRAFT: + allowedStates.add(SubmissionState.METADATA_VALID); + allowedStates.add(SubmissionState.METADATA_INVALID); + break; + case METADATA_VALID: + allowedStates.add(SubmissionState.DRAFT); + allowedStates.add(SubmissionState.METADATA_INVALID); + allowedStates.add(SubmissionState.GRAPH_VALID); + allowedStates.add(SubmissionState.GRAPH_INVALID); + break; + case GRAPH_VALID: + allowedStates.add(SubmissionState.SUBMITTED); + allowedStates.add(SubmissionState.DRAFT); + break; + case SUBMITTED: + allowedStates.add(SubmissionState.PROCESSING); + allowedStates.add(SubmissionState.EXPORTING); + break; + case PROCESSING: + allowedStates.add(SubmissionState.ARCHIVING); + break; + case ARCHIVING: + allowedStates.add(SubmissionState.ARCHIVED); + break; + case ARCHIVED: + allowedStates.add(SubmissionState.EXPORTING); + break; + case EXPORTED: + allowedStates.add(SubmissionState.CLEANUP); + break; + case CLEANUP: + allowedStates.add(SubmissionState.COMPLETE); + break; + default: + break; + } + + return allowedStates; + } + + public List allowedSubmissionStateTransitions() { + return allowedSubmissionStateTransitions(getSubmissionState()); + } + + public void enactStateTransition(SubmissionState targetState) { + if (this.submissionState != targetState) { + this.submissionState = targetState; + } + } + + public boolean isOpen() { + List states = Arrays.asList(SubmissionState.values()); + return states.indexOf(this.getSubmissionState()) < states.indexOf(SubmissionState.SUBMITTED); + } + + private List getNonEditableStates() { + return Arrays.asList( + /*SubmissionState.PENDING,*/ + SubmissionState.EXPORTING, + SubmissionState.PROCESSING, + SubmissionState.CLEANUP, + SubmissionState.ARCHIVED, + SubmissionState.SUBMITTED); + } + + public boolean isEditable() { + return !this.getNonEditableStates().contains(this.submissionState); + } + + public boolean isSystemEditable() { + return this.getNonEditableStates().stream() + .filter(state -> state != SubmissionState.CLEANUP) + .filter(state -> state != SubmissionState.PENDING) + .noneMatch(state -> state == this.submissionState); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeCreateHandler.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeCreateHandler.java new file mode 100644 index 000000000..4613b5d8d --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeCreateHandler.java @@ -0,0 +1,30 @@ +package uk.ac.ebi.subs.ingest.submission; + +import org.springframework.data.rest.core.annotation.HandleBeforeCreate; +import org.springframework.data.rest.core.annotation.RepositoryEventHandler; +import org.springframework.stereotype.Component; + +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.Uuid; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 15/09/17 + */ +@Component +@RepositoryEventHandler +@RequiredArgsConstructor +public class SubmissionEnvelopeCreateHandler { + @HandleBeforeCreate + public boolean submissionEnvelopeBeforeCreate(SubmissionEnvelope submissionEnvelope) { + this.setUuid(submissionEnvelope); + return true; + } + + public SubmissionEnvelope setUuid(SubmissionEnvelope submissionEnvelope) { + submissionEnvelope.setUuid(Uuid.newUuid()); + return submissionEnvelope; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeMessageBuilder.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeMessageBuilder.java new file mode 100644 index 000000000..761d6a426 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeMessageBuilder.java @@ -0,0 +1,66 @@ +package uk.ac.ebi.subs.ingest.submission; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import uk.ac.ebi.subs.ingest.core.web.LinkGenerator; +import uk.ac.ebi.subs.ingest.messaging.model.SubmissionEnvelopeMessage; + +public class SubmissionEnvelopeMessageBuilder { + public static SubmissionEnvelopeMessageBuilder using(LinkGenerator linkGenerator) { + return new SubmissionEnvelopeMessageBuilder(linkGenerator); + } + + private LinkGenerator linkGenerator; + + private Class documentType; + private String submissionEnvelopeId; + private String submissionEnvelopeUuid; + + private final Logger log = LoggerFactory.getLogger(getClass()); + + protected Logger getLog() { + return log; + } + + private SubmissionEnvelopeMessageBuilder(LinkGenerator linkGenerator) { + this.linkGenerator = linkGenerator; + } + + public SubmissionEnvelopeMessageBuilder messageFor(SubmissionEnvelope submissionEnvelope) { + withDocumentType(submissionEnvelope.getClass()) + .withId(submissionEnvelope.getId()) + .withUuid(submissionEnvelope.getUuid().getUuid().toString()); + + return this; + } + + private SubmissionEnvelopeMessageBuilder withDocumentType( + Class documentClass) { + this.documentType = documentClass; + + return this; + } + + private SubmissionEnvelopeMessageBuilder withId(String metadataDocId) { + this.submissionEnvelopeId = metadataDocId; + + return this; + } + + private SubmissionEnvelopeMessageBuilder withUuid(String uuid) { + this.submissionEnvelopeUuid = uuid; + + return this; + } + + public SubmissionEnvelopeMessage build() { + + String callbackLink = linkGenerator.createCallback(documentType, submissionEnvelopeId); + return new SubmissionEnvelopeMessage( + documentType.getSimpleName().toLowerCase(), + submissionEnvelopeId, + submissionEnvelopeUuid, + callbackLink); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeRepository.java new file mode 100644 index 000000000..158113ea7 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeRepository.java @@ -0,0 +1,37 @@ +package uk.ac.ebi.subs.ingest.submission; + +import java.util.UUID; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.repository.query.Param; +import org.springframework.data.rest.core.annotation.RestResource; +import org.springframework.web.bind.annotation.CrossOrigin; + +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.state.SubmissionState; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 31/08/17 + */ +@CrossOrigin +public interface SubmissionEnvelopeRepository extends MongoRepository { + + @RestResource(exported = false) + SubmissionEnvelope findByUuid(Uuid uuid); + + @RestResource(rel = "findByUuid") + SubmissionEnvelope findByUuidUuid(@Param("uuid") UUID uuid); + + @RestResource(path = "findByUser", rel = "findByUser") + Page findByUser(@Param(value = "user") String user, Pageable pageable); + + Page findBySubmissionState( + @Param("submissionState") SubmissionState submissionState, Pageable pageable); + + long countBySubmissionStateAndUser(SubmissionState submissionState, String user); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeService.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeService.java new file mode 100644 index 000000000..7b8649eff --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeService.java @@ -0,0 +1,363 @@ +package uk.ac.ebi.subs.ingest.submission; + +import java.text.DecimalFormat; +import java.time.Instant; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.dao.OptimisticLockingFailureException; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.stereotype.Service; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.bundle.BundleManifestRepository; +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.core.exception.StateTransitionNotAllowed; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.errors.SubmissionErrorRepository; +import uk.ac.ebi.subs.ingest.exporter.Exporter; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.patch.PatchRepository; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.project.ProjectService; +import uk.ac.ebi.subs.ingest.project.WranglingState; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.state.SubmitAction; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submissionmanifest.SubmissionManifestRepository; + +@Service +@RequiredArgsConstructor +public class SubmissionEnvelopeService { + @NonNull private final Logger log = LoggerFactory.getLogger(getClass()); + @NonNull private final MetadataCrudService metadataCrudService; + @NonNull private final MessageRouter messageRouter; + @NonNull private final Exporter exporter; + @NonNull private final ExecutorService executorService = Executors.newFixedThreadPool(5); + @NonNull private final SubmissionEnvelopeRepository submissionEnvelopeRepository; + @NonNull private final SubmissionEnvelopeCreateHandler submissionEnvelopeCreateHandler; + @NonNull private final SubmissionManifestRepository submissionManifestRepository; + @NonNull private BundleManifestRepository bundleManifestRepository; + @NonNull private ProjectRepository projectRepository; + @NonNull private ProjectService projectService; + @NonNull private ProcessRepository processRepository; + @NonNull private ProtocolRepository protocolRepository; + @NonNull private FileRepository fileRepository; + @NonNull private BiomaterialRepository biomaterialRepository; + @NonNull private PatchRepository patchRepository; + @NonNull private SubmissionErrorRepository submissionErrorRepository; + + public void handleSubmitRequest(SubmissionEnvelope envelope, List submitActions) { + getProject(envelope) + .ifPresentOrElse( + project -> { + if (!project.getValidationState().equals(ValidationState.VALID)) { + throw new StateTransitionNotAllowed( + String.format( + "Envelope with id %s cannot be submitted when the project is invalid.", + envelope.getId())); + } + }, + () -> { + throw new StateTransitionNotAllowed( + String.format( + "Envelope with id %s cannot be submitted without a project.", + envelope.getId())); + }); + + if (envelope.getSubmissionState() != SubmissionState.GRAPH_VALID) { + throw new StateTransitionNotAllowed( + String.format( + "Envelope with id %s cannot be submitted without a graph valid state", + envelope.getId())); + } + + if (isSubmitAction(submitActions)) { + envelope.setSubmitActions(new HashSet<>(submitActions)); + submissionEnvelopeRepository.save(envelope); + } else { + throw new IllegalArgumentException( + String.format( + "Envelope with id %s and state %s is submitted without the required submit actions", + envelope.getId(), envelope.getSubmissionState())); + } + handleEnvelopeStateUpdateRequest(envelope, SubmissionState.SUBMITTED); + } + + public void handleEnvelopeStateUpdateRequest(SubmissionEnvelope envelope, SubmissionState state) { + if (envelope.getSubmissionState() == state) { + log.info( + String.format( + "No Need to transition submissionEnvelope: %s already in state: %s", + envelope.getId(), envelope.getSubmissionState())); + } else if (!envelope.allowedSubmissionStateTransitions().contains(state)) { + throw new StateTransitionNotAllowed( + String.format( + "Envelope with id %s cannot be transitioned from state %s to state %s", + envelope.getId(), envelope.getSubmissionState(), state)); + } else { + /*messageRouter.routeStateTrackingUpdateMessageForEnvelopeEvent(envelope, state); + + if (state == SubmissionState.GRAPH_VALIDATION_REQUESTED) { + removeGraphValidationErrors(envelope); + }*/ + + envelope.enactStateTransition(state); + submissionEnvelopeRepository.save(envelope); + } + } + + public void handleCommitSubmit(SubmissionEnvelope envelope) { + Set submitActions = envelope.getSubmitActions(); + + if (submitActions.isEmpty()) { + log.info( + String.format( + "No Submit Actions for submission: %s in state: %s", + envelope.getId(), envelope.getSubmissionState())); + } else if (submitActions.contains(SubmitAction.ARCHIVE)) { + handleEnvelopeStateUpdateRequest(envelope, SubmissionState.PROCESSING); + archiveSubmission(envelope); + } else { + handleCommitArchived(envelope); + } + } + + public void handleCommitArchived(SubmissionEnvelope envelope) { + Set submitActions = envelope.getSubmitActions(); + + if (submitActions.contains(SubmitAction.EXPORT)) { + handleEnvelopeStateUpdateRequest(envelope, SubmissionState.EXPORTING); + exportData(envelope); + } else if (submitActions.contains(SubmitAction.EXPORT_METADATA)) { + handleEnvelopeStateUpdateRequest(envelope, SubmissionState.EXPORTING); + generateSpreadsheet(envelope); + } else { + handleCommitExported(envelope); + } + } + + public void handleCommitExported(SubmissionEnvelope envelope) { + getProject(envelope) + .ifPresent( + project -> projectService.updateWranglingState(project, WranglingState.SUBMITTED)); + + if (envelope.getSubmitActions().contains(SubmitAction.CLEANUP)) { + cleanupSubmission(envelope); + } + } + + private void archiveSubmission(SubmissionEnvelope envelope) { + submit(exporter::exportManifests, envelope, "Archive Submission"); + } + + public void generateSpreadsheet(SubmissionEnvelope envelope) { + submit(exporter::generateSpreadsheet, envelope, "Generate Spreadsheet"); + } + + public void exportData(SubmissionEnvelope envelope) { + submit(exporter::exportData, envelope, "Export Data"); + } + + public void cleanupSubmission(SubmissionEnvelope envelope) { + try { + handleEnvelopeStateUpdateRequest(envelope, SubmissionState.CLEANUP); + } catch (Exception e) { + log.error( + String.format( + "Uncaught Exception sending message to cleanup upload area for submission %s", + envelope.getId()), + e); + } + } + + public SubmissionEnvelope createUpdateSubmissionEnvelope() { + SubmissionEnvelope updateSubmissionEnvelope = new SubmissionEnvelope(); + + submissionEnvelopeCreateHandler.setUuid(updateSubmissionEnvelope); + updateSubmissionEnvelope.setIsUpdate(true); + + return submissionEnvelopeRepository.insert(updateSubmissionEnvelope); + } + + public void deleteSubmission(SubmissionEnvelope submissionEnvelope, boolean forceDelete) { + if (!(submissionEnvelope.isOpen() || forceDelete)) + throw new UnsupportedOperationException( + "Cannot delete submission if it is already submitted!"); + + RetryTemplate retry = + RetryTemplate.builder() + .maxAttempts(5) + .fixedBackoff(75) + .retryOn(OptimisticLockingFailureException.class) + .build(); + retry.execute( + context -> { + cleanupLinksToSubmissionMetadata(submissionEnvelope); + return null; + }); + + biomaterialRepository.deleteBySubmissionEnvelope(submissionEnvelope); + processRepository.deleteBySubmissionEnvelope(submissionEnvelope); + protocolRepository.deleteBySubmissionEnvelope(submissionEnvelope); + fileRepository.deleteBySubmissionEnvelope(submissionEnvelope); + bundleManifestRepository.deleteByEnvelopeUuid( + submissionEnvelope.getUuid().getUuid().toString()); + patchRepository.deleteBySubmissionEnvelope(submissionEnvelope); + submissionManifestRepository.deleteBySubmissionEnvelope(submissionEnvelope); + submissionErrorRepository.deleteBySubmissionEnvelope(submissionEnvelope); + submissionEnvelopeRepository.delete(submissionEnvelope); + + this.messageRouter.routeRequestUploadAreaCleanup(submissionEnvelope); + } + + /** + * Ensures that any links to metadata in the submission are removed. + * + * @param submissionEnvelope + */ + private void cleanupLinksToSubmissionMetadata(SubmissionEnvelope submissionEnvelope) { + long startTime = System.currentTimeMillis(); + + processRepository + .findBySubmissionEnvelope(submissionEnvelope) + .forEach(metadataCrudService::removeLinksToDocument); + + protocolRepository + .findBySubmissionEnvelope(submissionEnvelope) + .forEach(metadataCrudService::removeLinksToDocument); + + bundleManifestRepository + .findByEnvelopeUuid(submissionEnvelope.getUuid().getUuid().toString()) + .forEach( + bundleManifest -> + processRepository + .findByInputBundleManifestsContains(bundleManifest) + .forEach( + process -> { + process.getInputBundleManifests().remove(bundleManifest); + processRepository.save(process); + })); + + fileRepository + .findBySubmissionEnvelope(submissionEnvelope) + .forEach(metadataCrudService::removeLinksToDocument); + + // project cleanup + projectRepository + .findBySubmissionEnvelope(submissionEnvelope) + .forEach( + project -> { + project.setSubmissionEnvelope( + null); // TODO: address this; we should implement project containers that aren't + // deleted as part of deleteSubmission() + projectRepository.save(project); + }); + + projectRepository + .findBySubmissionEnvelopesContains(submissionEnvelope) + .forEach( + project -> { + project.getSubmissionEnvelopes().remove(submissionEnvelope); + projectRepository.save(project); + }); + + long endTime = System.currentTimeMillis(); + float duration = ((float) (endTime - startTime)) / 1000; + String durationStr = new DecimalFormat("#,###.##").format(duration); + log.info("cleanup link time: {} s", durationStr); + } + + private boolean isSubmitAction(List submitActions) { + return submitActions.contains(SubmitAction.ARCHIVE) + || submitActions.contains(SubmitAction.EXPORT) + || submitActions.contains(SubmitAction.EXPORT_METADATA); + } + + private void removeGraphValidationErrors(SubmissionEnvelope submissionEnvelope) { + biomaterialRepository.saveAll( + biomaterialRepository + .findBySubmissionEnvelope(submissionEnvelope) + .peek(biomaterial -> biomaterial.setGraphValidationErrors(new ArrayList<>())) + .collect(Collectors.toList())); + + processRepository.saveAll( + processRepository + .findBySubmissionEnvelope(submissionEnvelope) + .peek(process -> process.setGraphValidationErrors(new ArrayList<>())) + .collect(Collectors.toList())); + + protocolRepository.saveAll( + protocolRepository + .findBySubmissionEnvelope(submissionEnvelope) + .peek(protocol -> protocol.setGraphValidationErrors(new ArrayList<>())) + .collect(Collectors.toList())); + + fileRepository.saveAll( + fileRepository + .findBySubmissionEnvelope(submissionEnvelope) + .peek(file -> file.setGraphValidationErrors(new ArrayList<>())) + .collect(Collectors.toList())); + } + + public Optional getSubmissionContentLastUpdated(SubmissionEnvelope submissionEnvelope) { + PageRequest request = PageRequest.of(0, 1, new Sort(Sort.Direction.DESC, "updateDate")); + List projects = + projectRepository + .findBySubmissionEnvelopesContaining(submissionEnvelope, request) + .getContent(); + List biomaterials = + biomaterialRepository.findBySubmissionEnvelope(submissionEnvelope, request).getContent(); + List protocols = + protocolRepository.findBySubmissionEnvelope(submissionEnvelope, request).getContent(); + List processes = + processRepository.findBySubmissionEnvelope(submissionEnvelope, request).getContent(); + List files = + fileRepository.findBySubmissionEnvelope(submissionEnvelope, request).getContent(); + + return Stream.of(projects, biomaterials, protocols, processes, files) + .flatMap(List::stream) + .map(MetadataDocument::getUpdateDate) + .max(Instant::compareTo); + } + + public Optional getProject(SubmissionEnvelope submissionEnvelope) { + return projectRepository.findBySubmissionEnvelopesContains(submissionEnvelope).findFirst(); + } + + private void submit( + Consumer submissionAction, + SubmissionEnvelope submission, + String actionName) { + executorService.submit( + () -> { + try { + submissionAction.accept(submission); + } catch (Exception e) { + log.error( + String.format( + "Uncaught Exception sending message %s for Submission %s", + actionName, submission.getId()), + e); + } + }); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionStateMachineService.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionStateMachineService.java new file mode 100644 index 000000000..3bed2df31 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/SubmissionStateMachineService.java @@ -0,0 +1,58 @@ +package uk.ac.ebi.subs.ingest.submission; + +import java.net.URI; +import java.util.Map; +import java.util.UUID; + +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestOperations; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +import lombok.AllArgsConstructor; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.config.ConfigurationService; +import uk.ac.ebi.subs.ingest.state.ValidationState; + +/** Created by rolando on 05/09/2018. */ +@Service +@AllArgsConstructor +public class SubmissionStateMachineService { + private final @NonNull RestOperations restOperations = new RestTemplate(); + private final @NonNull ConfigurationService configurationService; + + private static HttpEntity DEFAULT_HTTP_ENTITY = defaultHttpEntity(); + + public Map documentStatesForEnvelope( + SubmissionEnvelope submissionEnvelope) { + UUID envelopeUuid = submissionEnvelope.getUuid().getUuid(); + ParameterizedTypeReference> documentStatesType = + new ParameterizedTypeReference>() {}; + URI documentStatesUri = + UriComponentsBuilder.newInstance() + .scheme(configurationService.getStateTrackerScheme()) + .host(configurationService.getStateTrackerHost()) + .port(configurationService.getStateTrackerPort()) + .pathSegment(configurationService.getDocumentStatesPath(), envelopeUuid.toString()) + .build() + .toUri(); + + return restOperations + .exchange(documentStatesUri, HttpMethod.GET, DEFAULT_HTTP_ENTITY, documentStatesType) + .getBody(); + } + + private static HttpEntity defaultHttpEntity() { + return new HttpEntity<>(null, uriListHeaders()); + } + + private static HttpHeaders uriListHeaders() { + HttpHeaders headers = new HttpHeaders(); + headers.add("Content-type", "application/json"); + return headers; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/exception/NotAllowedDuringSubmissionStateException.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/exception/NotAllowedDuringSubmissionStateException.java new file mode 100644 index 000000000..b0b465b24 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/exception/NotAllowedDuringSubmissionStateException.java @@ -0,0 +1,17 @@ +package uk.ac.ebi.subs.ingest.submission.exception; + +import uk.ac.ebi.subs.ingest.security.exception.NotAllowedException; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +public class NotAllowedDuringSubmissionStateException extends NotAllowedException { + public NotAllowedDuringSubmissionStateException() { + super("Operation not allowed during the current submission state for the envelope."); + } + + public NotAllowedDuringSubmissionStateException(SubmissionEnvelope submissionEnvelope) { + super( + String.format( + "Operation not allowed during the current submission state %s for the envelope %s", + submissionEnvelope.getSubmissionState(), submissionEnvelope.getUuid())); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/web/MongoAggregationUtils.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/web/MongoAggregationUtils.java new file mode 100644 index 000000000..6726ce3f0 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/web/MongoAggregationUtils.java @@ -0,0 +1,18 @@ +package uk.ac.ebi.subs.ingest.submission.web; + +import java.util.List; +import java.util.stream.Collectors; + +import org.bson.Document; +import org.springframework.data.mongodb.core.aggregation.Aggregation; +import org.springframework.data.mongodb.core.aggregation.AggregationOperation; + +public class MongoAggregationUtils { + public static Aggregation aggregationPipelineFromStrings(List jsonStages) { + return Aggregation.newAggregation( + jsonStages.stream() + .map(Document::parse) + .map((Document d) -> (AggregationOperation) context -> context.getMappedObject(d)) + .collect(Collectors.toList())); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionController.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionController.java new file mode 100644 index 000000000..a816430a8 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionController.java @@ -0,0 +1,534 @@ +package uk.ac.ebi.subs.ingest.submission.web; + +import java.time.Instant; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.data.web.PagedResourcesAssembler; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.http.HttpEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.bundle.BundleManifest; +import uk.ac.ebi.subs.ingest.bundle.BundleManifestRepository; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.dataset.Dataset; +import uk.ac.ebi.subs.ingest.dataset.DatasetRepository; +import uk.ac.ebi.subs.ingest.exporter.Exporter; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.process.ProcessService; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; +import uk.ac.ebi.subs.ingest.protocol.ProtocolService; +import uk.ac.ebi.subs.ingest.security.CheckAllowed; +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.state.SubmitAction; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.study.Study; +import uk.ac.ebi.subs.ingest.study.StudyRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeService; +import uk.ac.ebi.subs.ingest.submission.SubmissionStateMachineService; +import uk.ac.ebi.subs.ingest.submission.exception.NotAllowedDuringSubmissionStateException; +import uk.ac.ebi.subs.ingest.submissionmanifest.SubmissionManifest; +import uk.ac.ebi.subs.ingest.submissionmanifest.SubmissionManifestRepository; + +/** + * Spring controller that will handle submission events on a {@link SubmissionEnvelope} + * + * @author Tony Burdett + * @date 31/08/17 + */ +@RepositoryRestController +@ExposesResourceFor(SubmissionEnvelope.class) +@RequiredArgsConstructor +@Getter +public class SubmissionController { + private final @NonNull Logger log = LoggerFactory.getLogger(getClass()); + private final @NonNull Exporter exporter; + private final @NonNull SubmissionEnvelopeService submissionEnvelopeService; + private final @NonNull SubmissionStateMachineService submissionStateMachineService; + private final @NonNull ProcessService processService; + private final @NonNull ProtocolService protocolService; + private final @NonNull SubmissionEnvelopeRepository submissionEnvelopeRepository; + private final @NonNull FileRepository fileRepository; + private final @NonNull ProjectRepository projectRepository; + private final @NonNull StudyRepository studyRepository; + private final @NonNull DatasetRepository datasetRepository; + private final @NonNull ProtocolRepository protocolRepository; + private final @NonNull BiomaterialRepository biomaterialRepository; + private final @NonNull ProcessRepository processRepository; + private final @NonNull BundleManifestRepository bundleManifestRepository; + private final @NonNull SubmissionManifestRepository submissionManifestRepository; + private final MessageRouter messageRouter; + private final @NonNull PagedResourcesAssembler pagedResourcesAssembler; + + @PostMapping("/submissionEnvelopes" + Links.UPDATE_SUBMISSION_URL) + ResponseEntity createUpdateSubmission( + final PersistentEntityResourceAssembler resourceAssembler) { + SubmissionEnvelope updateSubmission = + getSubmissionEnvelopeService().createUpdateSubmissionEnvelope(); + + return ResponseEntity.ok(resourceAssembler.toFullResource(updateSubmission)); + } + + @GetMapping({ + "/submissionEnvelopes/{sub_id}" + Links.PROJECTS_URL, + "/submissionEnvelopes/{sub_id}" + Links.SUBMISSION_RELATED_PROJECTS_URL + }) + ResponseEntity getProjects( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page projects = + getProjectRepository().findBySubmissionEnvelopesContaining(submissionEnvelope, pageable); + + return ResponseEntity.ok(getPagedResourcesAssembler().toResource(projects, resourceAssembler)); + } + + @GetMapping({ + "/submissionEnvelopes/{sub_id}" + Links.STUDIES_URL, + "/submissionEnvelopes/{sub_id}" + Links.SUBMISSION_RELATED_STUDIES_URL + }) + ResponseEntity getStudies( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page studies = + getStudyRepository().findBySubmissionEnvelopesContaining(submissionEnvelope, pageable); + + return ResponseEntity.ok(getPagedResourcesAssembler().toResource(studies, resourceAssembler)); + } + + @GetMapping({ + "/submissionEnvelopes/{sub_id}" + Links.DATASETS_URL, + "/submissionEnvelopes/{sub_id}" + Links.SUBMISSION_RELATED_DATASETS_URL + }) + ResponseEntity getDatasets( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page datasets = + getDatasetRepository().findBySubmissionEnvelope(submissionEnvelope, pageable); + + return ResponseEntity.ok(getPagedResourcesAssembler().toResource(datasets, resourceAssembler)); + } + + @GetMapping("/submissionEnvelopes/{sub_id}/biomaterials") + ResponseEntity getBiomaterials( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page biomaterials = + getBiomaterialRepository().findBySubmissionEnvelope(submissionEnvelope, pageable); + + return ResponseEntity.ok( + getPagedResourcesAssembler().toResource(biomaterials, resourceAssembler)); + } + + @GetMapping("/submissionEnvelopes/{sub_id}/processes") + ResponseEntity getProcesses( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page processes = + getProcessRepository().findBySubmissionEnvelope(submissionEnvelope, pageable); + + return ResponseEntity.ok(getPagedResourcesAssembler().toResource(processes, resourceAssembler)); + } + + @GetMapping("/submissionEnvelopes/{sub_id}/protocols") + ResponseEntity getProtocols( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page protocols = protocolService.retrieve(submissionEnvelope, pageable); + + return ResponseEntity.ok(getPagedResourcesAssembler().toResource(protocols, resourceAssembler)); + } + + @GetMapping("/submissionEnvelopes/{sub_id}/files") + ResponseEntity getFiles( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page files = getFileRepository().findBySubmissionEnvelope(submissionEnvelope, pageable); + + return ResponseEntity.ok(getPagedResourcesAssembler().toResource(files, resourceAssembler)); + } + + @GetMapping("/submissionEnvelopes/{sub_id}/bundleManifests") + ResponseEntity getBundleManifests( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page bundleManifests = + getBundleManifestRepository() + .findByEnvelopeUuid(submissionEnvelope.getUuid().getUuid().toString(), pageable); + + return ResponseEntity.ok( + getPagedResourcesAssembler().toResource(bundleManifests, resourceAssembler)); + } + + @GetMapping("/submissionEnvelopes/{sub_id}/submissionManifest") + ResponseEntity getSubmissionManifests( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + Optional submissionManifest = + Optional.ofNullable( + getSubmissionManifestRepository() + .findBySubmissionEnvelopeId(submissionEnvelope.getId())); + if (submissionManifest.isPresent()) { + return ResponseEntity.ok(resourceAssembler.toFullResource(submissionManifest.get())); + } else { + return ResponseEntity.notFound().build(); + } + } + + @GetMapping("/submissionEnvelopes/{sub_id}/biomaterials/{state}") + ResponseEntity getSamplesWithValidationState( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + @PathVariable("state") String state, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page biomaterials = + getBiomaterialRepository() + .findBySubmissionEnvelopeAndValidationState( + submissionEnvelope, ValidationState.valueOf(state.toUpperCase()), pageable); + + return ResponseEntity.ok( + getPagedResourcesAssembler().toResource(biomaterials, resourceAssembler)); + } + + @GetMapping("/submissionEnvelopes/{sub_id}/processes/{state}") + ResponseEntity getProcessesWithValidationState( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + @PathVariable("state") String state, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page processes = + getProcessRepository() + .findBySubmissionEnvelopeAndValidationState( + submissionEnvelope, ValidationState.valueOf(state.toUpperCase()), pageable); + + return ResponseEntity.ok(getPagedResourcesAssembler().toResource(processes, resourceAssembler)); + } + + @GetMapping("/submissionEnvelopes/{sub_id}/protocols/{state}") + ResponseEntity getProtocolsWithValidationState( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + @PathVariable("state") String state, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page protocols = + getProtocolRepository() + .findBySubmissionEnvelopeAndValidationState( + submissionEnvelope, ValidationState.valueOf(state.toUpperCase()), pageable); + + return ResponseEntity.ok(getPagedResourcesAssembler().toResource(protocols, resourceAssembler)); + } + + @GetMapping("/submissionEnvelopes/{sub_id}/files/{state}") + ResponseEntity getFilesWithValidationState( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + @PathVariable("state") String state, + Pageable pageable, + final PersistentEntityResourceAssembler resourceAssembler) { + Page files = + getFileRepository() + .findBySubmissionEnvelopeAndValidationState( + submissionEnvelope, ValidationState.valueOf(state.toUpperCase()), pageable); + + return ResponseEntity.ok(getPagedResourcesAssembler().toResource(files, resourceAssembler)); + } + + @CheckAllowed( + value = "#submissionEnvelope.isEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PutMapping("/submissionEnvelopes/{id}" + Links.SUBMIT_URL) + HttpEntity submitEnvelopeRequest( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + @RequestBody(required = false) List submitActionParam, + final PersistentEntityResourceAssembler resourceAssembler) { + List submitActions = + Optional.of( + submitActionParam.stream() + .map(submitAction -> SubmitAction.valueOf(submitAction.toUpperCase())) + .collect(Collectors.toList())) + .orElse(List.of(SubmitAction.ARCHIVE, SubmitAction.EXPORT, SubmitAction.CLEANUP)); + + submissionEnvelopeService.handleSubmitRequest(submissionEnvelope, submitActions); + + return ResponseEntity.accepted().body(resourceAssembler.toFullResource(submissionEnvelope)); + } + + @CheckAllowed( + value = "#submissionEnvelope.isEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PutMapping("/submissionEnvelopes/{id}" + Links.ARCHIVED_URL) + HttpEntity completeArchivingEnvelopeRequest( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + submissionEnvelopeService.handleEnvelopeStateUpdateRequest( + submissionEnvelope, SubmissionState.ARCHIVED); + + return ResponseEntity.accepted().body(resourceAssembler.toFullResource(submissionEnvelope)); + } + + @CheckAllowed( + value = "#submissionEnvelope.isEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PutMapping("/submissionEnvelopes/{id}" + Links.EXPORT_URL) + HttpEntity exportEnvelopeRequest( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + submissionEnvelopeService.exportData(submissionEnvelope); + + return ResponseEntity.accepted().body(resourceAssembler.toFullResource(submissionEnvelope)); + } + + @CheckAllowed( + value = "#submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PutMapping("/submissionEnvelopes/{id}" + Links.CLEANUP_URL) + HttpEntity cleanupEnvelopeRequest( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + submissionEnvelopeService.handleEnvelopeStateUpdateRequest( + submissionEnvelope, SubmissionState.CLEANUP); + + return ResponseEntity.accepted().body(resourceAssembler.toFullResource(submissionEnvelope)); + } + + @CheckAllowed( + value = "#submissionEnvelope.isSystemEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PutMapping("/submissionEnvelopes/{id}" + Links.COMPLETE_URL) + HttpEntity completeEnvelopeRequest( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + submissionEnvelopeService.handleEnvelopeStateUpdateRequest( + submissionEnvelope, SubmissionState.COMPLETE); + + return ResponseEntity.accepted().body(resourceAssembler.toFullResource(submissionEnvelope)); + } + + private HttpEntity enactStateTransition( + SubmissionState state, + SubmissionEnvelope envelope, + final PersistentEntityResourceAssembler resourceAssembler) { + envelope.enactStateTransition(state); + getSubmissionEnvelopeRepository().save(envelope); + + return ResponseEntity.accepted().body(resourceAssembler.toFullResource(envelope)); + } + + @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_DRAFT_URL) + public HttpEntity enactDraftEnvelope( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + return this.enactStateTransition(SubmissionState.DRAFT, submissionEnvelope, resourceAssembler); + } + + @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_METADATA_INVALID_URL) + HttpEntity enactInvalidEnvelope( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + return this.enactStateTransition( + SubmissionState.METADATA_INVALID, submissionEnvelope, resourceAssembler); + } + + @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_METADATA_VALID_URL) + HttpEntity enactValidEnvelope( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + return this.enactStateTransition( + SubmissionState.METADATA_VALID, submissionEnvelope, resourceAssembler); + } + + @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_SUBMIT_URL) + public HttpEntity enactSubmitEnvelope( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + HttpEntity response = + this.enactStateTransition(SubmissionState.SUBMITTED, submissionEnvelope, resourceAssembler); + log.info( + String.format("Submission envelope with ID %s was submitted.", submissionEnvelope.getId())); + submissionEnvelopeService.handleCommitSubmit(submissionEnvelope); + + return response; + } + + @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_PROCESSING_URL) + HttpEntity enactProcessEnvelope( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + return this.enactStateTransition( + SubmissionState.PROCESSING, submissionEnvelope, resourceAssembler); + } + + @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_ARCHIVING_URL) + HttpEntity enactArchivingEnvelope( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + return this.enactStateTransition( + SubmissionState.ARCHIVING, submissionEnvelope, resourceAssembler); + } + + @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_ARCHIVED_URL) + HttpEntity enactArchivedEnvelope( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + HttpEntity response = + this.enactStateTransition(SubmissionState.ARCHIVED, submissionEnvelope, resourceAssembler); + log.info( + String.format("Submission envelope with ID %s was archived.", submissionEnvelope.getId())); + submissionEnvelopeService.handleCommitArchived(submissionEnvelope); + + return response; + } + + @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_EXPORTING_URL) + HttpEntity enactExportingEnvelope( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + return this.enactStateTransition( + SubmissionState.EXPORTING, submissionEnvelope, resourceAssembler); + } + + @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_EXPORTED_URL) + HttpEntity enactExportedEnvelope( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + HttpEntity response = + this.enactStateTransition(SubmissionState.EXPORTED, submissionEnvelope, resourceAssembler); + log.info( + String.format("Submission envelope with ID %s was exported.", submissionEnvelope.getId())); + submissionEnvelopeService.handleCommitExported(submissionEnvelope); + + return response; + } + + @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_CLEANUP_URL) + HttpEntity enactCleanupEnvelope( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + return this.enactStateTransition( + SubmissionState.CLEANUP, submissionEnvelope, resourceAssembler); + } + + @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_COMPLETE_URL) + HttpEntity enactCompleteEnvelope( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + return this.enactStateTransition( + SubmissionState.COMPLETE, submissionEnvelope, resourceAssembler); + } + + @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_GRAPH_VALID_URL) + HttpEntity enactGraphValid( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + return this.enactStateTransition( + SubmissionState.GRAPH_VALID, submissionEnvelope, resourceAssembler); + } + + @PutMapping("/submissionEnvelopes/{id}" + Links.COMMIT_GRAPH_INVALID_URL) + HttpEntity enactGraphInvalid( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + return this.enactStateTransition( + SubmissionState.GRAPH_INVALID, submissionEnvelope, resourceAssembler); + } + + private HttpEntity performStateUpdateRequest( + SubmissionState state, + SubmissionEnvelope envelope, + final PersistentEntityResourceAssembler resourceAssembler) { + submissionEnvelopeService.handleEnvelopeStateUpdateRequest(envelope, state); + return ResponseEntity.accepted().body(resourceAssembler.toFullResource(envelope)); + } + + /*@CheckAllowed( + value = "#submissionEnvelope.isEditable()", + exception = NotAllowedDuringSubmissionStateException.class) + @PutMapping("/submissionEnvelopes/{id}" + Links.GRAPH_VALIDATION_REQUESTED_URL) + HttpEntity requestGraphValidation( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + // Used by the user (UI) to start the validation process + return this.performStateUpdateRequest( + SubmissionState.GRAPH_VALIDATION_REQUESTED, submissionEnvelope, resourceAssembler); + }*/ + + @PutMapping("/submissionEnvelopes/{id}" + Links.GRAPH_VALID_URL) + HttpEntity requestGraphValid( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + // used by ingest-graph-validator to notify that graph is valid + return this.performStateUpdateRequest( + SubmissionState.GRAPH_VALID, submissionEnvelope, resourceAssembler); + } + + @PutMapping("/submissionEnvelopes/{id}" + Links.GRAPH_INVALID_URL) + HttpEntity requestGraphInvalid( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + final PersistentEntityResourceAssembler resourceAssembler) { + // used by ingest-graph-validator to notify that graph is invalid + return this.performStateUpdateRequest( + SubmissionState.GRAPH_INVALID, submissionEnvelope, resourceAssembler); + } + + @GetMapping("/submissionEnvelopes/{id}" + Links.SUBMISSION_DOCUMENTS_SM_URL) + ResponseEntity getDocumentStateMachineReport( + @PathVariable("id") SubmissionEnvelope submissionEnvelope) { + return ResponseEntity.ok( + getSubmissionStateMachineService().documentStatesForEnvelope(submissionEnvelope)); + } + + @GetMapping("/submissionEnvelopes/{id}/sync") + HttpEntity forceStateCheck(@PathVariable("id") SubmissionEnvelope submissionEnvelope) { + // TODO: if really needed, modify this method to ask the state tracker component for an update + return ResponseEntity.noContent().build(); + } + + @DeleteMapping("/submissionEnvelopes/{id}") + public HttpEntity forceDeleteSubmission( + @PathVariable("id") SubmissionEnvelope submissionEnvelope, + @RequestParam(name = "force", required = false, defaultValue = "false") boolean forceDelete) { + getSubmissionEnvelopeService().deleteSubmission(submissionEnvelope, forceDelete); + + return ResponseEntity.accepted().build(); + } + + @GetMapping("/submissionEnvelopes/{id}" + Links.SUBMISSION_CONTENT_LAST_UPDATED_URL) + ResponseEntity getContentLastUpdated( + @PathVariable("id") SubmissionEnvelope submissionEnvelope) { + Optional lastUpdateDate = + submissionEnvelopeService.getSubmissionContentLastUpdated(submissionEnvelope); + + return ResponseEntity.ok( + Objects.requireNonNull(lastUpdateDate.map(Instant::toString).orElse(null))); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionEnvelopeCollectionResourceProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionEnvelopeCollectionResourceProcessor.java new file mode 100644 index 000000000..85873da75 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionEnvelopeCollectionResourceProcessor.java @@ -0,0 +1,30 @@ +package uk.ac.ebi.subs.ingest.submission.web; + +import org.springframework.data.rest.webmvc.RepositoryLinksResource; +import org.springframework.hateoas.*; +import org.springframework.stereotype.Component; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@Component +@RequiredArgsConstructor +public class SubmissionEnvelopeCollectionResourceProcessor + implements ResourceProcessor { + private final @NonNull EntityLinks entityLinks; + + private Link getUpdateSubmissionsLink() { + return entityLinks + .linkFor(SubmissionEnvelope.class) + .slash(Links.UPDATE_SUBMISSION_URL) + .withRel(Links.UPDATE_SUBMISSION_REL); + } + + @Override + public RepositoryLinksResource process(RepositoryLinksResource resource) { + resource.add(getUpdateSubmissionsLink()); + return resource; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionEnvelopeResourceProcessor.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionEnvelopeResourceProcessor.java new file mode 100644 index 000000000..7e769a710 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionEnvelopeResourceProcessor.java @@ -0,0 +1,352 @@ +package uk.ac.ebi.subs.ingest.submission.web; + +import java.util.Arrays; +import java.util.Optional; + +import org.springframework.hateoas.EntityLinks; +import org.springframework.hateoas.Link; +import org.springframework.hateoas.Resource; +import org.springframework.hateoas.ResourceProcessor; +import org.springframework.stereotype.Component; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submissionmanifest.SubmissionManifest; +import uk.ac.ebi.subs.ingest.submissionmanifest.SubmissionManifestRepository; + +/** + * Javadocs go here! + * + * @author Tony Burdett + * @date 31/08/17 + */ +@Component +@RequiredArgsConstructor +public class SubmissionEnvelopeResourceProcessor + implements ResourceProcessor> { + private final @NonNull EntityLinks entityLinks; + private final @NonNull SubmissionManifestRepository submissionManifestRepository; + + private Link getBiomaterialsLink(SubmissionEnvelope submissionEnvelope) { + return entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(Links.BIOMATERIALS_URL) + .withRel(Links.BIOMATERIALS_REL); + } + + private Link getProcessesLink(SubmissionEnvelope submissionEnvelope) { + return entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(Links.PROCESSES_URL) + .withRel(Links.PROCESSES_REL); + } + + private Link getFilesLink(SubmissionEnvelope submissionEnvelope) { + return entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(Links.FILES_URL) + .withRel(Links.FILES_REL); + } + + private Link getProjectsLink(SubmissionEnvelope submissionEnvelope) { + return entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(Links.PROJECTS_URL) + .withRel(Links.PROJECTS_REL); + } + + private Link getStudiesLink(SubmissionEnvelope submissionEnvelope) { + return entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(Links.STUDIES_URL) + .withRel(Links.STUDIES_REL); + } + + private Link getDatasetsLink(SubmissionEnvelope submissionEnvelope) { + return entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(Links.DATASETS_URL) + .withRel(Links.DATASETS_REL); + } + + private Link getProtocolsLink(SubmissionEnvelope submissionEnvelope) { + return entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(Links.PROTOCOLS_URL) + .withRel(Links.PROTOCOLS_REL); + } + + private Link getBundleManifestsLink(SubmissionEnvelope submissionEnvelope) { + return entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(Links.BUNDLE_MANIFESTS_URL) + .withRel(Links.BUNDLE_MANIFESTS_REL); + } + + private Link getSubmissionManifestsLink(SubmissionEnvelope submissionEnvelope) { + return entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(Links.SUBMISSION_MANIFEST_URL) + .withRel(Links.SUBMISSION_MANIFEST_REL); + } + + private Link getExportJobsLink(SubmissionEnvelope submissionEnvelope) { + return entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(Links.EXPORT_JOBS_URL) + .withRel(Links.EXPORT_JOBS_REL); + } + + private Link getSubmissionErrorsLink(SubmissionEnvelope submissionEnvelope) { + return entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(Links.SUBMISSION_ERRORS_URL) + .withRel(Links.SUBMISSION_ERRORS_REL); + } + + private Link getSubmissionSummary(SubmissionEnvelope submissionEnvelope) { + return entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(Links.SUBMISSION_SUMMARY_URL) + .withRel(Links.SUBMISSION_SUMMARY_REL); + } + + private Link getSubmissionLinkingMap(SubmissionEnvelope submissionEnvelope) { + return entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(Links.SUBMISSION_LINKING_MAP_URL) + .withRel(Links.SUBMISSION_LINKING_MAP_REL); + } + + private Link getSubmissionContentLastUpdatedLink(SubmissionEnvelope submissionEnvelope) { + return entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(Links.SUBMISSION_CONTENT_LAST_UPDATED_URL) + .withRel(Links.SUBMISSION_CONTENT_LAST_UPDATED_REL); + } + + private Link getSubmissionRelatedProjectLink(SubmissionEnvelope submissionEnvelope) { + return entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(Links.SUBMISSION_RELATED_PROJECTS_URL) + .withRel(Links.SUBMISSION_RELATED_PROJECTS_REL); + } + + private Link getSubmissionDocumentStateLink(SubmissionEnvelope submissionEnvelope) { + return entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(Links.SUBMISSION_DOCUMENTS_SM_URL) + .withRel(Links.SUBMISSION_DOCUMENTS_SM_REL); + } + + private Optional getStateTransitionLink( + SubmissionEnvelope submissionEnvelope, SubmissionState targetState) { + Optional transitionResourceName = + getSubresourceNameForRequestSubmissionState(submissionEnvelope, targetState); + if (transitionResourceName.isPresent()) { + Optional rel = getRelNameForRequestSubmissionState(targetState); + if (rel.isPresent()) { + return Optional.of( + entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(transitionResourceName.get()) + .withRel(rel.get())); + } else { + throw new RuntimeException( + String.format( + "Unexpected link/rel mismatch exception (link = '%s', rel = " + "'%s')", + transitionResourceName.toString(), rel.toString())); + } + } else { + return Optional.empty(); + } + } + + private Optional getCommitStateTransitionLink( + SubmissionEnvelope submissionEnvelope, SubmissionState targetState) { + Optional transitionResourceName = + getSubresourceNameForCommitSubmissionState(targetState); + if (transitionResourceName.isPresent()) { + Optional rel = getRelNameForCommitSubmissionState(targetState); + if (rel.isPresent()) { + return Optional.of( + entityLinks + .linkForSingleResource(submissionEnvelope) + .slash(transitionResourceName.get()) + .withRel(rel.get())); + } else { + throw new RuntimeException( + String.format( + "Unexpected link/rel mismatch exception (link = '%s', rel = " + "'%s')", + transitionResourceName.toString(), rel.toString())); + } + } else { + return Optional.empty(); + } + } + + private Optional getRelNameForRequestSubmissionState(SubmissionState submissionState) { + switch (submissionState) { + case GRAPH_VALID: + return Optional.of(Links.GRAPH_VALID_REL); + case GRAPH_INVALID: + return Optional.of(Links.GRAPH_INVALID_REL); + case SUBMITTED: + return Optional.of(Links.SUBMIT_REL); + case ARCHIVED: + return Optional.of(Links.ARCHIVED_REL); + case EXPORTING: + return Optional.of(Links.EXPORT_REL); + case CLEANUP: + return Optional.of(Links.CLEANUP_REL); + case COMPLETE: + return Optional.of(Links.COMPLETE_REL); + default: + // default returns no links (not expecting external user interaction) + return Optional.empty(); + } + } + + private Optional getSubmitLink(SubmissionEnvelope submissionEnvelope) { + SubmissionManifest submissionManifest = + this.submissionManifestRepository.findBySubmissionEnvelopeId(submissionEnvelope.getId()); + + if (submissionManifest == null) return Optional.of(Links.SUBMIT_URL); + else if (submissionManifest.getExpectedLinks() != null + && submissionManifest.getExpectedLinks().equals(submissionManifest.getActualLinks())) { + return Optional.of(Links.SUBMIT_URL); + } else { + return Optional.empty(); + } + } + + private Optional getSubresourceNameForRequestSubmissionState( + SubmissionEnvelope submissionEnvelope, SubmissionState submissionState) { + switch (submissionState) { + case SUBMITTED: + return this.getSubmitLink(submissionEnvelope); + case ARCHIVED: + return Optional.of(Links.ARCHIVED_URL); + case EXPORTING: + return Optional.of(Links.EXPORT_URL); + case CLEANUP: + return Optional.of(Links.CLEANUP_URL); + case COMPLETE: + return Optional.of(Links.COMPLETE_URL); + default: + // default returns no subresource name (not expecting external user interaction) + return Optional.empty(); + } + } + + private Optional getRelNameForCommitSubmissionState(SubmissionState submissionState) { + switch (submissionState) { + case DRAFT: + return Optional.of(Links.COMMIT_DRAFT_REL); + case METADATA_INVALID: + return Optional.of(Links.COMMIT_METADATA_INVALID_REL); + case METADATA_VALID: + return Optional.of(Links.COMMIT_METADATA_VALID_REL); + case GRAPH_VALID: + return Optional.of(Links.COMMIT_GRAPH_VALID_REL); + case GRAPH_INVALID: + return Optional.of(Links.COMMIT_GRAPH_INVALID_REL); + case SUBMITTED: + return Optional.of(Links.COMMIT_SUBMIT_REL); + case PROCESSING: + return Optional.of(Links.COMMIT_PROCESSING_REL); + case ARCHIVING: + return Optional.of(Links.COMMIT_ARCHIVING_REL); + case ARCHIVED: + return Optional.of(Links.COMMIT_ARCHIVED_REL); + case EXPORTING: + return Optional.of(Links.COMMIT_EXPORTING_REL); + case EXPORTED: + return Optional.of(Links.COMMIT_EXPORTED_REL); + case CLEANUP: + return Optional.of(Links.COMMIT_CLEANUP_REL); + case COMPLETE: + return Optional.of(Links.COMMIT_COMPLETE_REL); + default: + // default returns no links (not expecting external user interaction) + return Optional.empty(); + } + } + + private Optional getSubresourceNameForCommitSubmissionState( + SubmissionState submissionState) { + switch (submissionState) { + case DRAFT: + return Optional.of(Links.COMMIT_DRAFT_URL); + case METADATA_INVALID: + return Optional.of(Links.COMMIT_METADATA_INVALID_URL); + case METADATA_VALID: + return Optional.of(Links.COMMIT_METADATA_VALID_URL); + case GRAPH_VALID: + return Optional.of(Links.COMMIT_GRAPH_VALID_URL); + case GRAPH_INVALID: + return Optional.of(Links.COMMIT_GRAPH_INVALID_URL); + case SUBMITTED: + return Optional.of(Links.COMMIT_SUBMIT_URL); + case PROCESSING: + return Optional.of(Links.COMMIT_PROCESSING_URL); + case ARCHIVING: + return Optional.of(Links.COMMIT_ARCHIVING_URL); + case ARCHIVED: + return Optional.of(Links.COMMIT_ARCHIVED_URL); + case EXPORTING: + return Optional.of(Links.COMMIT_EXPORTING_URL); + case EXPORTED: + return Optional.of(Links.COMMIT_EXPORTED_URL); + case CLEANUP: + return Optional.of(Links.COMMIT_CLEANUP_URL); + case COMPLETE: + return Optional.of(Links.COMMIT_COMPLETE_URL); + default: + // default returns no subresource name (not expecting external user interaction) + return Optional.empty(); + } + } + + @Override + public Resource process(Resource resource) { + SubmissionEnvelope submissionEnvelope = resource.getContent(); + + // add subresource links for each type of metadata document in a submission envelope + resource.add(getBiomaterialsLink(submissionEnvelope)); + resource.add(getProcessesLink(submissionEnvelope)); + resource.add(getFilesLink(submissionEnvelope)); + resource.add(getProjectsLink(submissionEnvelope)); + resource.add(getStudiesLink(submissionEnvelope)); + resource.add(getDatasetsLink(submissionEnvelope)); + resource.add(getProtocolsLink(submissionEnvelope)); + resource.add(getBundleManifestsLink(submissionEnvelope)); + resource.add(getSubmissionManifestsLink(submissionEnvelope)); + resource.add(getExportJobsLink(submissionEnvelope)); + resource.add(getSubmissionErrorsLink(submissionEnvelope)); + resource.add(getSubmissionDocumentStateLink(submissionEnvelope)); + resource.add(getSubmissionSummary(submissionEnvelope)); + resource.add(getSubmissionLinkingMap(submissionEnvelope)); + resource.add(getSubmissionContentLastUpdatedLink(submissionEnvelope)); + resource.add(getSubmissionRelatedProjectLink(submissionEnvelope)); + + // add subresource links for allowed state transition requests + submissionEnvelope.allowedSubmissionStateTransitions().stream() + .map(submissionState -> getStateTransitionLink(submissionEnvelope, submissionState)) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(resource::add); + + // add subresource links for state tracker to commit state transitions + Arrays.stream(SubmissionState.values()) + .map(submissionState -> getCommitStateTransitionLink(submissionEnvelope, submissionState)) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(resource::add); + + return resource; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionLinkMapController.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionLinkMapController.java new file mode 100644 index 000000000..2aa619398 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionLinkMapController.java @@ -0,0 +1,161 @@ +package uk.ac.ebi.subs.ingest.submission.web; + +import java.util.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.core.web.Links; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@RestController +public class SubmissionLinkMapController { + + @Autowired BiomaterialRepository biomaterialRepository; + @Autowired FileRepository fileRepository; + @Autowired ProcessRepository processRepository; + @Autowired ProtocolRepository protocolRepository; + + @Autowired SubmissionLinkMapRepository submissionLinkMapRepository; + + @NonNull private final Logger log = LoggerFactory.getLogger(getClass()); + + @RequestMapping( + path = "/submissionEnvelopes/{sub_id}" + Links.SUBMISSION_LINKING_MAP_URL, + method = RequestMethod.GET) + @ResponseBody + public SubmissionLinkingMap getSubmissionLinkMap( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope) { + return new SubmissionLinkingMap(submissionEnvelope); + } + + @Getter + public class SubmissionLinkingMap { + final Map processes = new Hashtable<>(); + final Map protocols = new Hashtable<>(); + final Map biomaterials = new Hashtable<>(); + final Map files = new Hashtable<>(); + + public SubmissionLinkingMap(SubmissionEnvelope submissionEnvelope) { + log.info("before processes"); + getProcessLinksUsingAggregation(submissionEnvelope); + log.info("found {} processes", processes.size()); + log.info("before biomaterials"); + findBiomaterialsLinkedProcessesForSubmission(submissionEnvelope); + log.info("found {} biomaterials", biomaterials.size()); + log.info("before files"); + findFilesLinkedProcessesForSubmission(submissionEnvelope); + log.info("found {} files", files.size()); + } + + private void findBiomaterialsLinkedProcessesForSubmission( + SubmissionEnvelope submissionEnvelope) { + submissionLinkMapRepository + .findLinkedProcessesByEntityTypeAndSubmission(submissionEnvelope, "biomaterial") + .forEach( + bioMaterialsAndProcesses -> + this.biomaterials.compute( + bioMaterialsAndProcesses.entityId, + (_processId, plm) -> { + BiomaterialLinkingMap biomaterialLinkingMap = + Optional.ofNullable(plm).orElse(new BiomaterialLinkingMap()); + biomaterialLinkingMap.inputToProcesses.addAll( + bioMaterialsAndProcesses.inputToProcesses); + biomaterialLinkingMap.derivedByProcesses.addAll( + bioMaterialsAndProcesses.derivedByProcesses); + return biomaterialLinkingMap; + })); + } + + private void findFilesLinkedProcessesForSubmission(SubmissionEnvelope submissionEnvelope) { + submissionLinkMapRepository + .findLinkedProcessesByEntityTypeAndSubmission(submissionEnvelope, "file") + .forEach( + filesAndProcesses -> + this.files.compute( + filesAndProcesses.entityId, + (_processId, plm) -> { + FileLinkingMap fileLinkingMap = + Optional.ofNullable(plm).orElse(new FileLinkingMap()); + fileLinkingMap.inputToProcesses.addAll(filesAndProcesses.inputToProcesses); + fileLinkingMap.derivedByProcesses.addAll( + filesAndProcesses.derivedByProcesses); + return fileLinkingMap; + })); + } + + private void getProcessLinksUsingAggregation(SubmissionEnvelope submissionEnvelope) { + submissionLinkMapRepository + .findProcessInputBiomaterials(submissionEnvelope) + .forEach( + processAndInputBiomaterials -> + this.processes.compute( + processAndInputBiomaterials.processId, + (_processId, plm) -> { + ProcessLinkingMap processLinkingMap = + Optional.ofNullable(plm).orElse(new ProcessLinkingMap()); + processLinkingMap.inputBiomaterials.addAll( + processAndInputBiomaterials.inputBiomaterials); + return processLinkingMap; + })); + submissionLinkMapRepository + .findProcessInputFiles(submissionEnvelope) + .forEach( + (ProcessAndInputFiles processAndInputFiles) -> + this.processes.compute( + processAndInputFiles.processId, + (_processId, plm) -> { + ProcessLinkingMap processLinkingMap = + Optional.ofNullable(plm).orElse(new ProcessLinkingMap()); + processLinkingMap.inputFiles.addAll(processAndInputFiles.inputFiles); + return processLinkingMap; + })); + log.info("processes: before protocols"); + submissionLinkMapRepository + .findProcessProtocols(submissionEnvelope) + .forEach( + process -> + this.processes.compute( + process.entityId, + (_processId, plm) -> { + ProcessLinkingMap processLinkingMap = + Optional.ofNullable(plm).orElse(new ProcessLinkingMap()); + processLinkingMap.protocols.addAll(process.protocols); + return processLinkingMap; + })); + log.info("processes: finished protocols"); + } + } + + @Getter + @AllArgsConstructor + public static class ProcessLinkingMap { + final Collection protocols = new HashSet<>(); + final Collection inputBiomaterials = new HashSet<>(); + final Collection inputFiles = new HashSet<>(); + } + + @Getter + @NoArgsConstructor + public static class BiomaterialLinkingMap { + final Collection derivedByProcesses = new HashSet<>(); + final Collection inputToProcesses = new HashSet<>(); + } + + @Getter + @NoArgsConstructor + public static class FileLinkingMap { + final Collection derivedByProcesses = new HashSet<>(); + final Collection inputToProcesses = new HashSet<>(); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionLinkMapRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionLinkMapRepository.java new file mode 100644 index 000000000..26cbbc613 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionLinkMapRepository.java @@ -0,0 +1,199 @@ +package uk.ac.ebi.subs.ingest.submission.web; + +import static org.springframework.data.mongodb.core.aggregation.Aggregation.*; +import static org.springframework.data.mongodb.core.aggregation.ArrayOperators.ArrayElemAt.arrayOf; +import static org.springframework.data.mongodb.core.aggregation.Fields.UNDERSCORE_ID; +import static org.springframework.data.mongodb.core.aggregation.ObjectOperators.ObjectToArray.valueOfToArray; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Sort; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.aggregation.Aggregation; +import org.springframework.data.mongodb.core.aggregation.AggregationResults; +import org.springframework.data.mongodb.core.aggregation.ConvertOperators.ToString; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.stereotype.Repository; + +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +class ProcessAndInputBiomaterials { + String processId; + List inputBiomaterials; +} + +class ProcessAndInputFiles { + String processId; + List inputFiles; +} + +class EntityWithInputsAndDerivedBy { + String entityId; + List inputToProcesses; + List derivedByProcesses; +} + +class EntityWithProtocols { + String entityId; + List protocols; +} + +@Repository +public class SubmissionLinkMapRepository { + + @Autowired MongoTemplate mongoTemplate; + + List findProcessInputBiomaterials( + SubmissionEnvelope submissionEnvelope) { + String entity_type = "biomaterial"; + Aggregation agg = + buildAggregationQueryForProcessInputs(submissionEnvelope, entity_type, "inputBiomaterials"); + AggregationResults processAndInputBiomaterials = + mongoTemplate.aggregate(agg, entity_type, ProcessAndInputBiomaterials.class); + return processAndInputBiomaterials.getMappedResults(); + } + + private static Aggregation buildAggregationQueryForProcessInputs( + SubmissionEnvelope submissionEnvelope, String entity_type, String inputBiomaterials) { + return newAggregation( + project("inputToProcesses", UNDERSCORE_ID) + .and(arrayOf(valueOfToArray("submissionEnvelope")).elementAt(1)) + .as("submission_id"), + project("inputToProcesses", UNDERSCORE_ID) + .and(ToString.toString("$submission_id.v")) + .as("submission_id"), + match(Criteria.where("submission_id").is(submissionEnvelope.getId())), + unwind("inputToProcesses"), + project(UNDERSCORE_ID) + .and(arrayOf(valueOfToArray("inputToProcesses")).elementAt(1)) + .as("process_id"), + project("process_id").and(UNDERSCORE_ID).as(entity_type + "_id"), + group("$process_id.v") + .addToSet(ToString.toString("$" + entity_type + "_id")) + .as(inputBiomaterials), + project(inputBiomaterials).and(ToString.toString("$_id")).as("processId"), + sort(Sort.Direction.DESC, "processId")); + } + + List findProcessInputFiles(SubmissionEnvelope submissionEnvelope) { + String entity_type = "file"; + Aggregation agg = + buildAggregationQueryForProcessInputs(submissionEnvelope, entity_type, "inputFiles"); + AggregationResults processAndInputFiles = + mongoTemplate.aggregate(agg, entity_type, ProcessAndInputFiles.class); + return processAndInputFiles.getMappedResults(); + } + + List findLinkedProcessesByEntityTypeAndSubmission( + SubmissionEnvelope submissionEnvelope, String entity_type) { + List jsonStages = + List.of( + " {\n" + + " $project: {\n" + + " submission_id: { $arrayElemAt: [{ $objectToArray: \"$submissionEnvelope\" }, 1] },\n" + + " inputToProcesses: 1,\n" + + " \"derivedByProcesses\": 1,\n" + + " }\n" + + " }", + " {\n" + + " $project: {\n" + + " submission_id: { $toString: '$submission_id.v' },\n" + + " inputToProcesses: 1,\n" + + " \"derivedByProcesses\": 1,\n" + + " }\n" + + " }", + String.format( + "{ \"$match\": { \"submission_id\": \"%s\", } }", submissionEnvelope.getId()), + "{\n" + + " $project: {\n" + + " \"inputToProcesses\": {\n" + + " $map: {\n" + + " input: \"$inputToProcesses\",\n" + + " as: \"process_id\",\n" + + " in: { $arrayElemAt: [{ $objectToArray: \"$$process_id\" }, 1] }\n" + + " }\n" + + " },\n" + + " \"derivedByProcesses\": {\n" + + " $map: {\n" + + " input: \"$derivedByProcesses\",\n" + + " as: \"process_id\",\n" + + " in: { $arrayElemAt: [{ $objectToArray: \"$$process_id\" }, 1] }\n" + + " }\n" + + " },\n" + + " }\n" + + " }", + "{\n" + + " $project: {\n" + + " _id: 0" + + " entityId: {$toString:\"$_id\"}," + + " \"inputToProcesses\": {\n" + + " $map: {\n" + + " input: \"$inputToProcesses\",\n" + + " as: \"process_id\",\n" + + " in: {$toString: \"$$process_id.v\"}\n" + + " }\n" + + " },\n" + + " \"derivedByProcesses\": {\n" + + " $map: {\n" + + " input: \"$derivedByProcesses\",\n" + + " as: \"process_id\",\n" + + " in: {$toString: \"$$process_id.v\"}\n" + + " }\n" + + " },\n" + + " }\n" + + " }"); + Aggregation aggregation = MongoAggregationUtils.aggregationPipelineFromStrings(jsonStages); + return mongoTemplate + .aggregate(aggregation, entity_type, EntityWithInputsAndDerivedBy.class) + .getMappedResults(); + } + + public List findProcessProtocols(SubmissionEnvelope submissionEnvelope) { + List jsonStages = + List.of( + "{\n" + + " $project: {\n" + + " submission_id: { $arrayElemAt: [{ $objectToArray: \"$submissionEnvelope\" }, 1] },\n" + + " protocols: 1,\n" + + " }\n" + + " },\n", + " {\n" + + " $project: {\n" + + " submission_id: { $toString: '$submission_id.v' },\n" + + " protocols: 1,\n" + + " }\n" + + " },\n", + String.format( + " { \"$match\": { \"submission_id\": \"%s\", } },\n", + submissionEnvelope.getId()), + " {\n" + + " $project: {\n" + + " \"protocols\": {\n" + + " $map: {\n" + + " input: \"$protocols\",\n" + + " as: \"protocol_id\",\n" + + " in: { $arrayElemAt: [{ $objectToArray: \"$$protocol_id\" }, 1] }\n" + + " }\n" + + " },\n" + + " }\n" + + " },\n", + " {\n" + + " $project: {\n" + + " _id:0,\n" + + " entityId: {$toString:\"$_id\"},\n" + + " \"protocols\": {\n" + + " $map: {\n" + + " input: \"$protocols\",\n" + + " as: \"protocol_id\",\n" + + " in: {$toString: \"$$protocol_id.v\"}\n" + + " }\n" + + " },\n" + + " }\n" + + " }\n"); + Aggregation aggregation = MongoAggregationUtils.aggregationPipelineFromStrings(jsonStages); + return mongoTemplate + .aggregate(aggregation, "process", EntityWithProtocols.class) + .getMappedResults(); + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionSummaryController.java b/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionSummaryController.java new file mode 100644 index 000000000..6a34bc8bc --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionSummaryController.java @@ -0,0 +1,128 @@ +package uk.ac.ebi.subs.ingest.submission.web; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestController; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.file.ValidationErrorType; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@RestController +public class SubmissionSummaryController { + + @Autowired BiomaterialRepository biomaterialRepository; + @Autowired FileRepository fileRepository; + @Autowired ProcessRepository processRepository; + @Autowired ProtocolRepository protocolRepository; + + @RequestMapping(path = "/submissionEnvelopes/{sub_id}/summary", method = RequestMethod.GET) + @ResponseBody + public SubmissionSummary submissionSummary( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope) { + SubmissionSummary summary = new SubmissionSummary(); + summary.setUuid(submissionEnvelope.getUuid()); + summary.setTotalBiomaterials( + biomaterialRepository.countBySubmissionEnvelope(submissionEnvelope)); + summary.setTotalFiles(fileRepository.countBySubmissionEnvelope(submissionEnvelope)); + summary.setTotalProcesses(processRepository.countBySubmissionEnvelope(submissionEnvelope)); + summary.setTotalProtocols(protocolRepository.countBySubmissionEnvelope(submissionEnvelope)); + + long invalidBiomaterials = + biomaterialRepository.countBySubmissionEnvelopeAndValidationState( + submissionEnvelope, ValidationState.INVALID); + // Setting a special graphInvalid[type] property until dcp-546 is done + // This allows us to filter by graph invalid entities until we consolidate the + // graphValidationState into + // validationState + long graphInvalidBiomaterials = + biomaterialRepository.countBySubmissionEnvelopeAndCountWithGraphValidationErrors( + submissionEnvelope.getId()); + + long invalidFiles = + fileRepository.countBySubmissionEnvelopeAndValidationState( + submissionEnvelope, ValidationState.INVALID); + long graphInvalidFiles = + fileRepository.countBySubmissionEnvelopeAndCountWithGraphValidationErrors( + submissionEnvelope.getId()); + long fileMetadataErrors = + fileRepository.countBySubmissionEnvelopeIdAndErrorType( + submissionEnvelope.getId(), ValidationErrorType.METADATA_ERROR.name()); + long missingFiles = + fileRepository.countBySubmissionEnvelopeIdAndErrorType( + submissionEnvelope.getId(), ValidationErrorType.FILE_NOT_UPLOADED.name()); + long fileErrors = + fileRepository.countBySubmissionEnvelopeIdAndErrorType( + submissionEnvelope.getId(), ValidationErrorType.FILE_ERROR.name()); + + long invalidProcesses = + processRepository.countBySubmissionEnvelopeAndValidationState( + submissionEnvelope, ValidationState.INVALID); + long graphInvalidProcesses = + processRepository.countBySubmissionEnvelopeAndCountWithGraphValidationErrors( + submissionEnvelope.getId()); + long invalidProtocols = + protocolRepository.countBySubmissionEnvelopeAndValidationState( + submissionEnvelope, ValidationState.INVALID); + long graphInvalidProtocols = + protocolRepository.countBySubmissionEnvelopeAndCountWithGraphValidationErrors( + submissionEnvelope.getId()); + + long totalInvalid = + invalidBiomaterials + + graphInvalidBiomaterials + + (fileMetadataErrors + missingFiles + fileErrors + graphInvalidFiles) + + invalidProcesses + + graphInvalidProcesses + + invalidProtocols + + graphInvalidProtocols; + + summary.setInvalidBiomaterials(invalidBiomaterials); + summary.setGraphInvalidBiomaterials(graphInvalidBiomaterials); + + summary.setInvalidFiles(invalidFiles); + summary.setGraphInvalidFiles(graphInvalidFiles); + summary.setFileMetadataErrors(fileMetadataErrors); + summary.setMissingFiles(missingFiles); + summary.setFileErrors(fileErrors); + + summary.setInvalidProcesses(invalidProcesses); + summary.setGraphInvalidProcesses(graphInvalidProcesses); + + summary.setInvalidProtocols(invalidProtocols); + summary.setGraphInvalidProtocols(graphInvalidProtocols); + + summary.setTotalInvalid(totalInvalid); + + return summary; + } + + @Getter + @Setter + @NoArgsConstructor + public class SubmissionSummary { + + private Uuid uuid; + private Long totalBiomaterials, invalidBiomaterials, graphInvalidBiomaterials; + private Long totalFiles, + invalidFiles, + graphInvalidFiles, + fileMetadataErrors, + missingFiles, + fileErrors; + private Long totalProcesses, invalidProcesses, graphInvalidProcesses; + private Long totalProtocols, invalidProtocols, graphInvalidProtocols; + private Long totalInvalid; + } +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submissionmanifest/SubmissionManifest.java b/src/main/java/uk/ac/ebi/subs/ingest/submissionmanifest/SubmissionManifest.java new file mode 100644 index 000000000..ee44da4a5 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submissionmanifest/SubmissionManifest.java @@ -0,0 +1,28 @@ +package uk.ac.ebi.subs.ingest.submissionmanifest; + +import org.springframework.data.mongodb.core.mapping.DBRef; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; +import uk.ac.ebi.subs.ingest.core.AbstractEntity; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +/** Created by rolando on 30/05/2018. */ +@AllArgsConstructor +@Getter +public class SubmissionManifest extends AbstractEntity { + private final Integer expectedBiomaterials; + private final Integer expectedProcesses; + private final Integer expectedFiles; + private final Integer expectedProtocols; + private final Integer expectedProjects; + + private @Setter Integer actualLinks = 0; + private final Integer expectedLinks; + + private final Integer totalCount; + + @Setter + private @DBRef(lazy = true) SubmissionEnvelope submissionEnvelope; +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submissionmanifest/SubmissionManifestRepository.java b/src/main/java/uk/ac/ebi/subs/ingest/submissionmanifest/SubmissionManifestRepository.java new file mode 100644 index 000000000..b63440736 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submissionmanifest/SubmissionManifestRepository.java @@ -0,0 +1,14 @@ +package uk.ac.ebi.subs.ingest.submissionmanifest; + +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.web.bind.annotation.CrossOrigin; + +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +/** Created by rolando on 30/05/2018. */ +@CrossOrigin +public interface SubmissionManifestRepository extends MongoRepository { + S findBySubmissionEnvelopeId(String envelopeId); + + Long deleteBySubmissionEnvelope(SubmissionEnvelope submissionEnvelope); +} diff --git a/src/main/java/uk/ac/ebi/subs/ingest/submissionmanifest/web/SubmissionManifestController.java b/src/main/java/uk/ac/ebi/subs/ingest/submissionmanifest/web/SubmissionManifestController.java new file mode 100644 index 000000000..44fd8b010 --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/submissionmanifest/web/SubmissionManifestController.java @@ -0,0 +1,41 @@ +package uk.ac.ebi.subs.ingest.submissionmanifest.web; + +import org.springframework.data.rest.webmvc.PersistentEntityResource; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.rest.webmvc.RepositoryRestController; +import org.springframework.hateoas.ExposesResourceFor; +import org.springframework.hateoas.Resource; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submissionmanifest.SubmissionManifest; +import uk.ac.ebi.subs.ingest.submissionmanifest.SubmissionManifestRepository; + +/** Created by rolando on 30/05/2018. */ +@RepositoryRestController +@ExposesResourceFor(SubmissionManifest.class) +@RequiredArgsConstructor +@Getter +public class SubmissionManifestController { + private final @NonNull SubmissionManifestRepository submissionManifestRepository; + + @RequestMapping( + path = "submissionEnvelopes/{sub_id}/submissionManifest", + method = RequestMethod.POST) + ResponseEntity> addManifestToEnvelope( + @PathVariable("sub_id") SubmissionEnvelope submissionEnvelope, + @RequestBody SubmissionManifest submissionManifest, + PersistentEntityResourceAssembler assembler) { + submissionManifest.setSubmissionEnvelope(submissionEnvelope); + SubmissionManifest manifest = submissionManifestRepository.save(submissionManifest); + PersistentEntityResource resource = assembler.toFullResource(manifest); + return ResponseEntity.accepted().body(resource); + } +} diff --git a/src/main/java/org/humancellatlas/ingest/user/IdentityService.java b/src/main/java/uk/ac/ebi/subs/ingest/user/IdentityService.java similarity index 61% rename from src/main/java/org/humancellatlas/ingest/user/IdentityService.java rename to src/main/java/uk/ac/ebi/subs/ingest/user/IdentityService.java index 7660c105c..540732322 100644 --- a/src/main/java/org/humancellatlas/ingest/user/IdentityService.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/user/IdentityService.java @@ -1,4 +1,4 @@ -package org.humancellatlas.ingest.user; +package uk.ac.ebi.subs.ingest.user; public interface IdentityService { String wranglerEmail(); diff --git a/src/main/java/org/humancellatlas/ingest/user/PrototypeIdentityService.java b/src/main/java/uk/ac/ebi/subs/ingest/user/PrototypeIdentityService.java similarity index 66% rename from src/main/java/org/humancellatlas/ingest/user/PrototypeIdentityService.java rename to src/main/java/uk/ac/ebi/subs/ingest/user/PrototypeIdentityService.java index 978225cf9..6dccf3b23 100644 --- a/src/main/java/org/humancellatlas/ingest/user/PrototypeIdentityService.java +++ b/src/main/java/uk/ac/ebi/subs/ingest/user/PrototypeIdentityService.java @@ -1,9 +1,10 @@ -package org.humancellatlas.ingest.user; +package uk.ac.ebi.subs.ingest.user; -import lombok.RequiredArgsConstructor; import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; +import lombok.RequiredArgsConstructor; + @Component @RequiredArgsConstructor public class PrototypeIdentityService implements IdentityService { @@ -11,8 +12,6 @@ public class PrototypeIdentityService implements IdentityService { @Override public String wranglerEmail() { - return environment.getProperty("WRANGLER_EMAILS", - "hca-notifications-test@ebi.ac.uk"); - + return environment.getProperty("WRANGLER_EMAILS", "hca-notifications-test@ebi.ac.uk"); } } diff --git a/src/main/java/uk/ac/ebi/subs/ingest/user/UserController.java b/src/main/java/uk/ac/ebi/subs/ingest/user/UserController.java new file mode 100644 index 000000000..e564037fd --- /dev/null +++ b/src/main/java/uk/ac/ebi/subs/ingest/user/UserController.java @@ -0,0 +1,149 @@ +package uk.ac.ebi.subs.ingest.user; + +import static org.springframework.http.MediaType.APPLICATION_JSON_UTF8_VALUE; + +import java.util.Optional; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.rest.core.support.SelfLinkProvider; +import org.springframework.data.rest.webmvc.RepositoryLinksResource; +import org.springframework.data.web.PagedResourcesAssembler; +import org.springframework.hateoas.*; +import org.springframework.hateoas.mvc.ControllerLinkBuilder; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestController; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.security.Account; +import uk.ac.ebi.subs.ingest.security.AccountRepository; +import uk.ac.ebi.subs.ingest.security.Role; +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; +import uk.ac.ebi.subs.ingest.submission.web.SubmissionEnvelopeResourceProcessor; + +@RestController +@CrossOrigin +@RequestMapping("/user") +public class UserController implements ResourceProcessor { + + @Autowired SubmissionEnvelopeRepository submissionEnvelopeRepository; + + @Autowired ProjectRepository projectRepository; + + @Autowired AccountRepository accountRepository; + + @Autowired + private PagedResourcesAssembler submissionEnvelopePagedResourcesAssembler; + + @Autowired private PagedResourcesAssembler projectPagedResourcesAssembler; + + @Autowired private SubmissionEnvelopeResourceProcessor submissionEnvelopeResourceProcessor; + + @Autowired private SelfLinkProvider linkProvider; + + @Autowired private EntityLinks entityLinks; + + @RequestMapping(value = "/summary") + @ResponseBody + public Summary summary() { + String user = getCurrentAccount().getId(); + long pendingSubmissions = + submissionEnvelopeRepository.countBySubmissionStateAndUser(SubmissionState.PENDING, user); + long draftSubmissions = + submissionEnvelopeRepository.countBySubmissionStateAndUser(SubmissionState.DRAFT, user); + long completedSubmissions = + submissionEnvelopeRepository.countBySubmissionStateAndUser(SubmissionState.COMPLETE, user); + long projects = projectRepository.countByUser(user); + return new Summary(pendingSubmissions, draftSubmissions, completedSubmissions, projects); + } + + @RequestMapping(value = "/submissionEnvelopes") + public PagedResources> getUserSubmissionEnvelopes( + Pageable pageable) { + Page submissionEnvelopes = + submissionEnvelopeRepository.findByUser(getCurrentAccount().getId(), pageable); + PagedResources> pagedResources = + submissionEnvelopePagedResourcesAssembler.toResource(submissionEnvelopes); + for (Resource resource : pagedResources) { + resource.add(entityLinks.linkForSingleResource(resource.getContent()).withRel(Link.REL_SELF)); + submissionEnvelopeResourceProcessor.process(resource); + } + return pagedResources; + } + + @RequestMapping(value = "/projects") + public PagedResources> getUserProjects(Pageable pageable) { + Page projects = Page.empty(); + + if (getCurrentAccount().getRoles().contains(Role.WRANGLER)) { + projects = + projectRepository.findByUserOrPrimaryWrangler( + getCurrentAccount().getId(), getCurrentAccount().getId(), pageable); + } else { + projects = projectRepository.findByUser(getCurrentAccount().getId(), pageable); + } + + PagedResources> pagedResources = + projectPagedResourcesAssembler.toResource(projects); + for (Resource resource : pagedResources) { + resource.add(entityLinks.linkForSingleResource(resource.getContent()).withRel(Link.REL_SELF)); + } + return pagedResources; + } + + private Account getCurrentAccount() { + Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); + Account account = (Account) authentication.getPrincipal(); + return account; + } + + @GetMapping(path = "/list", produces = APPLICATION_JSON_UTF8_VALUE) + ResponseEntity listUsers( + Authentication authentication, @RequestParam("role") Optional role) { + if (!authentication.getAuthorities().contains(Role.WRANGLER)) + return new ResponseEntity<>("Unauthorized", HttpStatus.UNAUTHORIZED); + + return role.map(value -> ResponseEntity.ok(accountRepository.findAccountByRoles(value))) + .orElseGet(() -> ResponseEntity.ok(accountRepository.findAll())); + } + + @Override + public RepositoryLinksResource process(RepositoryLinksResource resource) { + resource.add(ControllerLinkBuilder.linkTo(UserController.class).withRel("user")); + return resource; + } + + @Getter + @Setter + @NoArgsConstructor + public class Summary { + + private Long draftSubmissions = 0L; + private Long pendingSubmissions = 0L; + private Long completedSubmissions = 0L; + private Long projects = 0L; + + public Summary( + long pendingSubmissions, long draftSubmissions, long completedSubmissions, long projects) { + this.pendingSubmissions = pendingSubmissions; + this.draftSubmissions = draftSubmissions; + this.completedSubmissions = completedSubmissions; + this.projects = projects; + } + } +} diff --git a/src/main/resources/application-local.properties b/src/main/resources/application-local.properties index 62577d741..d595ff981 100644 --- a/src/main/resources/application-local.properties +++ b/src/main/resources/application-local.properties @@ -1,15 +1,19 @@ # use this application properties profile for running locally # with a local mongodb and rabbitmq # i.e. -Dspring.profile.active=local - AUTH_ISSUER=https://login.elixir-czech.org/oidc/ GCP_JWK_PROVIDER_BASE_URL=https://www.googleapis.com/service_accounts/v1/jwk/ GCP_PROJECT_WHITELIST=hca-dcp-production.iam.gserviceaccount.com,human-cell-atlas-travis-test.iam.gserviceaccount.com,broad-dsde-mint-dev.iam.gserviceaccount.com,broad-dsde-mint-test.iam.gserviceaccount.com,broad-dsde-mint-staging.iam.gserviceaccount.com -SCHEMA_BASE_URI=https://schema.humancellatlas.org/ +SCHEMA_BASE_URI=https://dev.schema.morphic.bio/ SVC_AUTH_AUDIENCE=https://dev.data.humancellatlas.org/ - USR_AUTH_AUDIENCE=https://dev.data.humancellatlas.org/ -schema.base-uri=https://schema.humancellatlas.org/ -spring.data.mongodb.uri=mongodb://localhost:27017/admin +schema.base-uri=https://dev.schema.morphic.bio/ +spring.data.mongodb.uri=mongodb+srv://morphic-dev:mDev12345@morphicdevcluster.jduoa8p.mongodb.net/MorphicDev?retryWrites=true&w=majority spring.rabbitmq.host=localhost spring.rabbitmq.port=5672 +# display mongo queries +#logging.level.org.springframework.data.mongodb.core.MongoTemplate=DEBUG +#logging.level.org.springframework.data.mongodb.repository.query=DEBUG +# display security messages +logging.level.org.springframework.security.access.expression.method=DEBUG +logging.level.org.springframework.security=DEBUG diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 30f206a84..846effa09 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -2,28 +2,22 @@ server.port=8080 server.connection-timeout=120000 server.use-forward-headers=true - # REST spring.data.rest.enable-enum-translation=true - #Jackson spring.jackson.mapper.infer-property-mutators=false spring.jackson.serialization.write_dates_as_timestamps=false - # security - management.endpoints.web.base-path=/ management.endpoints.web.exposure.include=health,info,jolokia,prometheus management.endpoint.health.show-details=always management.endpoint.jolokia.enabled=false management.endpoint.prometheus.enabled=true - # logging -logging.level.org.humancellatlas.ingest=INFO -logging.level.org.springframework.amqp=INFO - -logging.level.org.humancellatlas.ingest.security= ERROR -logging.level.com.auth0.spring.security.api.JwtAuthenticationProvider= WARN - - -spring.aop.proxy-target-class=false \ No newline at end of file +logging.level.uk.ac.ebi.subs.ingest=INFO +logging.level.org.springframework.amqp=OFF +logging.level.uk.ac.ebi.subs.ingest.security=ERROR +logging.level.com.auth0.spring.security.api.JwtAuthenticationProvider=WARN +spring.aop.proxy-target-class=false +#biosamples +post.samples.ebibiosamples=true diff --git a/src/test/java/org/humancellatlas/ingest/ProjectJson.java b/src/test/java/org/humancellatlas/ingest/ProjectJson.java deleted file mode 100644 index 12c2383a2..000000000 --- a/src/test/java/org/humancellatlas/ingest/ProjectJson.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.humancellatlas.ingest; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; - -import java.util.Map; - -public class ProjectJson { - String title; - - public static ProjectJson fromTitle(String title){ - ProjectJson project = new ProjectJson(); - project.title = title; - return project; - } - - public ObjectNode toObjectNode() { - ObjectMapper mapper = new ObjectMapper(); - ObjectNode content = mapper.createObjectNode(); - ObjectNode projectCore0 = content.putObject("project_core"); - projectCore0.put("project_title", this.title); - - ObjectNode metadata = mapper.createObjectNode(); - metadata.set("content", content); - - return metadata; - } - - public Map toMap() { - ObjectMapper mapper = new ObjectMapper(); - ObjectNode project = this.toObjectNode(); - return mapper.convertValue(project, new TypeReference>(){}); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/audit/AuditTypeTest.java b/src/test/java/org/humancellatlas/ingest/audit/AuditTypeTest.java deleted file mode 100644 index 71561b240..000000000 --- a/src/test/java/org/humancellatlas/ingest/audit/AuditTypeTest.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.humancellatlas.ingest.audit; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.json.JsonTest; - -import java.io.IOException; - -import static junit.framework.TestCase.assertEquals; - -@JsonTest -public class AuditTypeTest { - @Autowired - private ObjectMapper objectMapper; - - @Test - public void testDeserialize() throws IOException { - assertEquals(objectMapper.readValue("\"Status updated\"", AuditType.class), AuditType.STATUS_UPDATED); - } - - @Test - public void testSerialize() throws JsonProcessingException { - assertEquals(objectMapper.writeValueAsString(AuditType.STATUS_UPDATED),"\"Status updated\""); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/core/MetadataDocumentTest.java b/src/test/java/org/humancellatlas/ingest/core/MetadataDocumentTest.java deleted file mode 100644 index 4d9781cce..000000000 --- a/src/test/java/org/humancellatlas/ingest/core/MetadataDocumentTest.java +++ /dev/null @@ -1,93 +0,0 @@ -package org.humancellatlas.ingest.core; - -import lombok.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = { - MetadataDocument.class, -}) -public class MetadataDocumentTest { - @Test - @DisplayName("Is Equal with matching Static Members") - public void testSameStatics(){ - //given - var map1 = Map.of("Key1","Value1", "Key2", "Value2"); - var map2 = Map.of("Key1","Value1", "Key2", "Value2"); - - MetadataDocument doc1 = new DocumentTest(EntityType.PROJECT, "Identifier", map1); - doc1.setUuid(Uuid.newUuid()); - MetadataDocument doc2 = new DocumentTest(EntityType.PROJECT, "Identifier", map2); - doc2.setUuid(doc1.getUuid()); - - assertThat(doc1).isEqualTo(doc2); - } - - @Test - @DisplayName("Is Equal with similar content") - public void testSimilarContent(){ - //given - var map1 = Map.of("Key1","Value1", "Key2", "Value2"); - var map2 = Map.of("Key2", "Value2", "Key1","Value1"); - MetadataDocument doc1 = new DocumentTest(EntityType.PROJECT, "Identifier", map1); - MetadataDocument doc2 = new DocumentTest(EntityType.PROJECT, "Identifier", map2); - - assertThat(doc1).isEqualTo(doc2); - } - - @Test - @DisplayName("Is Not Equal with different id") - public void testDifferentID(){ - //given - var map1 = Map.of("Key1","Value1", "Key2", "Value2"); - var map2 = Map.of("Key1","Value1", "Key2", "Value2"); - MetadataDocument doc1 = new DocumentTest(EntityType.PROJECT, "Identifier-One", map1); - MetadataDocument doc2 = new DocumentTest(EntityType.PROJECT, "Identifier-Two", map2); - - assertThat(doc1).isNotEqualTo(doc2); - } - - @Test - @DisplayName("Is Not Equal with different content") - public void testDifferentContent(){ - //given - var map1 = Map.of("Key1","Value1", "Key2", "Value2"); - var map2 = Map.of("Key1","Value1", "Key3", "Value3"); - MetadataDocument doc1 = new DocumentTest(EntityType.PROJECT, "Identifier", map1); - MetadataDocument doc2 = new DocumentTest(EntityType.PROJECT, "Identifier", map2); - - assertThat(doc1).isNotEqualTo(doc2); - } - - @Test - @DisplayName("Is Not Equal with different content") - public void testDifferentUUIDs(){ - //given - var map1 = Map.of("Key1","Value1", "Key2", "Value2"); - var map2 = Map.of("Key1","Value1", "Key2", "Value2"); - MetadataDocument doc1 = new DocumentTest(EntityType.PROJECT, "Identifier", map1); - doc1.setUuid(Uuid.newUuid()); - MetadataDocument doc2 = new DocumentTest(EntityType.PROJECT, "Identifier", map2); - doc2.setUuid(Uuid.newUuid()); - - assertThat(doc1).isNotEqualTo(doc2); - } - - @Getter - @EqualsAndHashCode(callSuper = true) - static - class DocumentTest extends MetadataDocument{ - DocumentTest(EntityType type, String id, Object content) { - super(type, content); - this.id = id; - } - } -} diff --git a/src/test/java/org/humancellatlas/ingest/core/service/MetadataCrudServiceTest.java b/src/test/java/org/humancellatlas/ingest/core/service/MetadataCrudServiceTest.java deleted file mode 100644 index 5c58baeac..000000000 --- a/src/test/java/org/humancellatlas/ingest/core/service/MetadataCrudServiceTest.java +++ /dev/null @@ -1,80 +0,0 @@ -package org.humancellatlas.ingest.core.service; - -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.core.service.strategy.impl.*; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.util.stream.Stream; - -import static org.mockito.Mockito.*; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = { - MetadataCrudService.class, - BiomaterialCrudStrategy.class, - FileCrudStrategy.class, - ProcessCrudStrategy.class, - ProjectCrudStrategy.class, - ProtocolCrudStrategy.class -}) -public class MetadataCrudServiceTest { - @Autowired private MetadataCrudService crudService; - @Autowired private BiomaterialCrudStrategy biomaterialCrudStrategy; - @Autowired private FileCrudStrategy fileCrudStrategy; - @Autowired private ProcessCrudStrategy processCrudStrategy; - @Autowired private ProjectCrudStrategy projectCrudStrategy; - @Autowired private ProtocolCrudStrategy protocolCrudStrategy; - - @MockBean private MessageRouter messageRouter; - @MockBean private BiomaterialRepository biomaterialRepository; - @MockBean private FileRepository fileRepository; - @MockBean private ProcessRepository processRepository; - @MockBean private ProjectRepository projectRepository; - @MockBean private ProtocolRepository protocolRepository; - - private static Stream providedTestDocuments() { - return Stream.of( - Arguments.of(new Biomaterial(null)), - Arguments.of(new File(null, "fileName")), - Arguments.of(new Process(null)), - Arguments.of(new Project(null)), - Arguments.of(new Protocol(null)) - ); - } - - @ParameterizedTest - @MethodSource("providedTestDocuments") - public void removeLinksSendsMessageToStateTracker(MetadataDocument document) { - // when - crudService.removeLinksToDocument(document); - // then - verify(messageRouter, times(1)).routeStateTrackingDeleteMessageFor(document); - } - - @ParameterizedTest - @MethodSource("providedTestDocuments") - public void deleteSendsMessageToStateTracker(MetadataDocument document) { - // when - crudService.deleteDocument(document); - // then - verify(messageRouter, times(1)).routeStateTrackingDeleteMessageFor(document); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/core/service/MetadataLinkingServiceTest.java b/src/test/java/org/humancellatlas/ingest/core/service/MetadataLinkingServiceTest.java deleted file mode 100644 index f8e70b342..000000000 --- a/src/test/java/org/humancellatlas/ingest/core/service/MetadataLinkingServiceTest.java +++ /dev/null @@ -1,147 +0,0 @@ -package org.humancellatlas.ingest.core.service; - - -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.state.SubmissionState; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.dao.OptimisticLockingFailureException; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.lang.reflect.InvocationTargetException; -import java.util.List; -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.*; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = {MetadataLinkingService.class}) -public class MetadataLinkingServiceTest { - @Autowired - private MetadataLinkingService service; - - @MockBean - private ValidationStateChangeService validationStateChangeService; - - @MockBean - private MongoTemplate mongoTemplate; - - Protocol protocol; - Protocol protocol2; - Process process; - SubmissionEnvelope submission; - - @BeforeEach - void setUp() { - submission = new SubmissionEnvelope(); - submission.enactStateTransition(SubmissionState.GRAPH_VALID); - protocol = spy(new Protocol(null)); - doReturn("protocol1").when(protocol).getId(); - protocol2 = spy(new Protocol(null)); - doReturn("protocol2").when(protocol2).getId(); - process = spy(new Process(null)); - doReturn("process").when(process).getId(); - process.setSubmissionEnvelope(submission); - } - - @Test - public void testReplaceLink() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { - process.addProtocol(protocol); - - // when - service.replaceLinks(process, List.of(protocol2), "protocols"); - - // then - assertThat(process.getProtocols().size()).isEqualTo(1); - assertThat(process.getProtocols().contains(protocol2)).isTrue(); - - verify(validationStateChangeService, times(0)).changeValidationState(protocol.getType(), protocol.getId(), ValidationState.DRAFT); - verify(validationStateChangeService, times(0)).changeValidationState(protocol2.getType(), protocol2.getId(), ValidationState.DRAFT); - verify(validationStateChangeService).changeValidationState(process.getType(), process.getId(), ValidationState.DRAFT); - verify(mongoTemplate).save(process); - - } - - @Test - public void testReplaceLinkNotGraphValid() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { - process.addProtocol(protocol); - submission.enactStateTransition(SubmissionState.METADATA_VALID); - - // when - service.replaceLinks(process, List.of(protocol2), "protocols"); - - // then - assertThat(process.getProtocols().size()).isEqualTo(1); - assertThat(process.getProtocols().contains(protocol2)).isTrue(); - - verify(validationStateChangeService, times(0)).changeValidationState(protocol.getType(), protocol.getId(), ValidationState.DRAFT); - verify(validationStateChangeService, times(0)).changeValidationState(protocol2.getType(), protocol2.getId(), ValidationState.DRAFT); - verify(validationStateChangeService, times(0)).changeValidationState(process.getType(), process.getId(), ValidationState.DRAFT); - verify(mongoTemplate).save(process); - - } - - @Test - public void testAddLink() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { - - // when - service.addLinks(process, List.of(protocol), "protocols"); - - // then - assertThat(process.getProtocols().size()).isEqualTo(1); - assertThat(process.getProtocols().contains(protocol)).isTrue(); - verify(validationStateChangeService, times(0)).changeValidationState(protocol.getType(), protocol.getId(), ValidationState.DRAFT); - verify(validationStateChangeService).changeValidationState(process.getType(), process.getId(), ValidationState.DRAFT); - verify(mongoTemplate).save(process); - - } - - @Test - public void testAddLinkNotGraphValid() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { - submission.enactStateTransition(SubmissionState.METADATA_VALID); - - // when - service.addLinks(process, List.of(protocol), "protocols"); - - // then - assertThat(process.getProtocols().size()).isEqualTo(1); - assertThat(process.getProtocols().contains(protocol)).isTrue(); - verify(validationStateChangeService, times(0)).changeValidationState(protocol.getType(), protocol.getId(), ValidationState.DRAFT); - verify(validationStateChangeService, times(0)).changeValidationState(process.getType(), process.getId(), ValidationState.DRAFT); - verify(mongoTemplate).save(process); - - } - - - - @Test - public void testRetry() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { - // given - - when(validationStateChangeService.changeValidationState(process.getType(), process.getId(), ValidationState.DRAFT)) - .thenThrow(new OptimisticLockingFailureException("Error")) - .thenThrow(new OptimisticLockingFailureException("Error")) - .thenReturn(process); - - - // when - service.addLinks(process, List.of(protocol), "protocols"); - - // then - assertThat(process.getProtocols().size()).isEqualTo(1); - assertThat(process.getProtocols().contains(protocol)).isTrue(); - verify(validationStateChangeService,times(0)).changeValidationState(protocol.getType(), protocol.getId(), ValidationState.DRAFT); - verify(validationStateChangeService, times(3)).changeValidationState(process.getType(), process.getId(), ValidationState.DRAFT); - verify(mongoTemplate).save(process); - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/core/service/MetadataUpdateServiceTest.java b/src/test/java/org/humancellatlas/ingest/core/service/MetadataUpdateServiceTest.java deleted file mode 100644 index a81cb59ec..000000000 --- a/src/test/java/org/humancellatlas/ingest/core/service/MetadataUpdateServiceTest.java +++ /dev/null @@ -1,155 +0,0 @@ -package org.humancellatlas.ingest.core.service; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.humancellatlas.ingest.ProjectJson; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.patch.JsonPatcher; -import org.humancellatlas.ingest.patch.PatchService; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.state.ValidationState; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.data.mapping.context.PersistentEntities; -import org.springframework.data.rest.webmvc.mapping.Associations; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = {MetadataUpdateService.class, JsonPatcher.class, ObjectMapper.class}) -public class MetadataUpdateServiceTest { - @Autowired - private MetadataUpdateService service; - - @MockBean - private MetadataDifferService metadataDifferService; - - @MockBean - private MetadataCrudService metadataCrudService; - - @Autowired - private JsonPatcher jsonPatcher; - - - @MockBean - private PatchService patchService; - - @MockBean - private ValidationStateChangeService validationStateChangeService; - - @MockBean - PersistentEntities persistentEntities; - - @MockBean - private Associations associations; - - @Test - public void testUpdateShouldSaveAndReturnUpdatedMetadata() { - //given: - JsonNode content = ProjectJson.fromTitle("Old Project Title").toObjectNode().get("content"); - Project project = new Project(content); - - ObjectNode patch = ProjectJson.fromTitle("New Project Title").toObjectNode(); - - when(metadataCrudService.save(any())).thenReturn(project); - - //when: - Project updatedProject = service.update(project, patch); - - //then: - assertThat(updatedProject).isEqualTo(project); - verify(metadataCrudService).save(project); - } - - @Test - public void testUpdateShouldSetStateToDraftWhenContentChanged() { - //given: - JsonNode content = ProjectJson.fromTitle("Old Project Title").toObjectNode().get("content"); - Project project = new Project(content); - - ObjectNode patch = ProjectJson.fromTitle("New Project Title").toObjectNode(); - - when(metadataCrudService.save(any())).thenReturn(project); - - //when: - Project updatedProject = service.update(project, patch); - - //then: - assertThat(updatedProject).isEqualTo(project); - verify(metadataCrudService).save(project); - verify(validationStateChangeService).changeValidationState(project.getType(), project.getId(), ValidationState.DRAFT); - } - - @Test - public void testUpdateShouldNotSetStateWhenContentIsUnchanged() { - //given: - JsonNode content = ProjectJson.fromTitle("Old Project Title").toObjectNode().get("content"); - Project project = new Project(content); - - ObjectNode patch = ProjectJson.fromTitle("Old Project Title").toObjectNode(); - - when(metadataCrudService.save(any())).thenReturn(project); - - //when: - Project updatedProject = service.update(project, patch); - - //then: - assertThat(updatedProject).isEqualTo(project); - verify(metadataCrudService).save(project); - verify(validationStateChangeService, never()).changeValidationState(project.getType(), project.getId(), ValidationState.DRAFT); - } - - @Test - public void testUpdateShouldNotSetStateWhenNoContent() { - //given: - JsonNode content = ProjectJson.fromTitle("Old Project Title").toObjectNode().get("content"); - Project project = new Project(content); - - ObjectMapper mapper = new ObjectMapper(); - ObjectNode patch = mapper.createObjectNode(); - patch.put("isInCatalogue", false); - - when(metadataCrudService.save(any())).thenReturn(project); - - //when: - Project updatedProject = service.update(project, patch); - - //then: - assertThat(updatedProject).isEqualTo(project); - verify(metadataCrudService).save(project); - verify(validationStateChangeService, never()).changeValidationState(project.getType(), project.getId(), ValidationState.DRAFT); - } - - @Test - public void testUpdateProjectWithSupplementaryFileShouldNotThrowRecursionError() { - //given: - JsonNode content = ProjectJson.fromTitle("Project with supplementary file").toObjectNode().get("content"); - Project project = new Project(content); - - File supplementaryFile = new File(null, "fileName"); - supplementaryFile.setProject(project); - project.getSupplementaryFiles().add(supplementaryFile); - - ObjectNode patch = ProjectJson.fromTitle("Updated project with supplementary file").toObjectNode(); - - when(metadataCrudService.save(any())).thenReturn(project); - - //when: - Project updatedProject = service.update(project, patch); - - //then: - assertThat(updatedProject).isEqualTo(project); - verify(metadataCrudService).save(project); - verify(validationStateChangeService).changeValidationState(project.getType(), project.getId(), ValidationState.DRAFT); - } - - -} diff --git a/src/test/java/org/humancellatlas/ingest/core/service/strategy/BiomaterialCrudStrategyTest.java b/src/test/java/org/humancellatlas/ingest/core/service/strategy/BiomaterialCrudStrategyTest.java deleted file mode 100644 index 277e73ff5..000000000 --- a/src/test/java/org/humancellatlas/ingest/core/service/strategy/BiomaterialCrudStrategyTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.humancellatlas.ingest.core.service.strategy; - -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.core.service.strategy.impl.BiomaterialCrudStrategy; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import static org.mockito.Mockito.verify; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = {BiomaterialCrudStrategy.class}) -public class BiomaterialCrudStrategyTest { - @Autowired private BiomaterialCrudStrategy biomaterialCrudStrategy; - - @MockBean private BiomaterialRepository biomaterialRepository; - @MockBean private MessageRouter messageRouter; - - private Biomaterial testBiomaterial; - - @BeforeEach - void setUp() { - testBiomaterial = new Biomaterial(null); - } - - @Test - public void testDeleteBiomaterial() { - //when - biomaterialCrudStrategy.deleteDocument(testBiomaterial); - //then - verify(biomaterialRepository).delete(testBiomaterial); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/core/service/strategy/FileCrudStrategyTest.java b/src/test/java/org/humancellatlas/ingest/core/service/strategy/FileCrudStrategyTest.java deleted file mode 100644 index 08fb32c41..000000000 --- a/src/test/java/org/humancellatlas/ingest/core/service/strategy/FileCrudStrategyTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.humancellatlas.ingest.core.service.strategy; - -import org.humancellatlas.ingest.core.service.strategy.impl.FileCrudStrategy; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.*; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = {FileCrudStrategy.class}) -public class FileCrudStrategyTest { - @Autowired private FileCrudStrategy fileCrudStrategy; - - @MockBean private FileRepository fileRepository; - @MockBean private ProjectRepository projectRepository; - @MockBean private MessageRouter messageRouter; - - private File testFile; - - @BeforeEach - void setUp() { - testFile = new File(null, "fileName"); - } - - @Test - public void testRemoveLinksFile() { - //given - Project projectWithFile = new Project(null); - projectWithFile.getSupplementaryFiles().add(testFile); - when(projectRepository.findBySupplementaryFilesContains(testFile)).thenReturn(Stream.of(projectWithFile)); - - // when - fileCrudStrategy.removeLinksToDocument(testFile); - - // then - assertThat(projectWithFile.getSupplementaryFiles()).isEmpty(); - verify(projectRepository).save(projectWithFile); - } - - @Test - public void testDeleteFile() { - //when - fileCrudStrategy.deleteDocument(testFile); - //then - verify(fileRepository).delete(testFile); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/core/service/strategy/ProcessCrudStrategyTest.java b/src/test/java/org/humancellatlas/ingest/core/service/strategy/ProcessCrudStrategyTest.java deleted file mode 100644 index fbedd745c..000000000 --- a/src/test/java/org/humancellatlas/ingest/core/service/strategy/ProcessCrudStrategyTest.java +++ /dev/null @@ -1,79 +0,0 @@ -package org.humancellatlas.ingest.core.service.strategy; - -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.core.service.strategy.impl.ProcessCrudStrategy; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.*; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = {ProcessCrudStrategy.class}) -public class ProcessCrudStrategyTest { - @Autowired private ProcessCrudStrategy processCrudStrategy; - - @MockBean private BiomaterialRepository biomaterialRepository; - @MockBean private ProcessRepository processRepository; - @MockBean private FileRepository fileRepository; - @MockBean private MessageRouter messageRouter; - - private Process testProcess; - - @BeforeEach - void setUp() { - testProcess = new Process(null); - } - - @Test - public void testRemoveLinksProcess() { - //given - File inputFile = spy(new File(null, "inputFile")); - File derivedFile = spy(new File(null, "derivedFile")); - inputFile.getInputToProcesses().add(testProcess); - derivedFile.getDerivedByProcesses().add(testProcess); - when(fileRepository.findByInputToProcessesContains(testProcess)).thenReturn(Stream.of(inputFile)); - when(fileRepository.findByDerivedByProcessesContains(testProcess)).thenReturn(Stream.of(derivedFile)); - - Biomaterial inputBio = spy(new Biomaterial(null)); - Biomaterial derivedBio = spy(new Biomaterial(null)); - inputBio.getInputToProcesses().add(testProcess); - derivedBio.getDerivedByProcesses().add(testProcess); - when(biomaterialRepository.findByInputToProcessesContains(testProcess)).thenReturn(Stream.of(inputBio)); - when(biomaterialRepository.findByDerivedByProcessesContains(testProcess)).thenReturn(Stream.of(derivedBio)); - - // when - processCrudStrategy.removeLinksToDocument(testProcess); - - // then - assertThat(inputFile.getInputToProcesses()).isEmpty(); - assertThat(derivedFile.getDerivedByProcesses()).isEmpty(); - assertThat(inputBio.getInputToProcesses()).isEmpty(); - assertThat(derivedBio.getDerivedByProcesses()).isEmpty(); - verify(fileRepository).save(inputFile); - verify(fileRepository).save(derivedFile); - verify(biomaterialRepository).save(inputBio); - verify(biomaterialRepository).save(derivedBio); - } - - @Test - public void testDeleteProcess() { - //when - processCrudStrategy.deleteDocument(testProcess); - //then - verify(processRepository).delete(testProcess); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/core/service/strategy/ProjectCrudStrategyTest.java b/src/test/java/org/humancellatlas/ingest/core/service/strategy/ProjectCrudStrategyTest.java deleted file mode 100644 index 5d2175e50..000000000 --- a/src/test/java/org/humancellatlas/ingest/core/service/strategy/ProjectCrudStrategyTest.java +++ /dev/null @@ -1,93 +0,0 @@ -package org.humancellatlas.ingest.core.service.strategy; - -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.core.service.strategy.impl.ProjectCrudStrategy; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.*; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = {ProjectCrudStrategy.class}) -public class ProjectCrudStrategyTest { - @Autowired private ProjectCrudStrategy projectCrudStrategy; - - @MockBean private ProjectRepository projectRepository; - @MockBean private ProtocolRepository protocolRepository; - @MockBean private ProcessRepository processRepository; - @MockBean private FileRepository fileRepository; - @MockBean private BiomaterialRepository biomaterialRepository; - @MockBean private MessageRouter messageRouter; - - private Project testProject; - - @BeforeEach - void setUp() { - testProject = new Project(null); - } - - @Test - public void testRemoveLinksProject() { - // Given - Biomaterial biomaterialWithProject = new Biomaterial(null); - biomaterialWithProject.setProject(testProject); - biomaterialWithProject.getProjects().add(testProject); - when(biomaterialRepository.findByProject(testProject)).thenReturn(Stream.of(biomaterialWithProject)); - when(biomaterialRepository.findByProjectsContaining(testProject)).thenReturn(Stream.of(biomaterialWithProject)); - - File fileWithProject = new File(null, "fileWithProject"); - fileWithProject.setProject(testProject); - when(fileRepository.findByProject(testProject)).thenReturn(Stream.of(fileWithProject)); - - Process processWithProject = new Process(null); - processWithProject.setProject(testProject); - processWithProject.getProjects().add(testProject); - when(processRepository.findByProject(testProject)).thenReturn(Stream.of(processWithProject)); - when(processRepository.findByProjectsContaining(testProject)).thenReturn(Stream.of(processWithProject)); - - Protocol protocolWithProject = new Protocol(null); - protocolWithProject.setProject(testProject); - when(protocolRepository.findByProject(testProject)).thenReturn(Stream.of(protocolWithProject)); - - // when - projectCrudStrategy.removeLinksToDocument(testProject); - - //then - assertThat(biomaterialWithProject.getProject()).isNull(); - assertThat(biomaterialWithProject.getProjects()).isEmpty(); - assertThat(fileWithProject.getProject()).isNull(); - assertThat(processWithProject.getProject()).isNull(); - assertThat(processWithProject.getProjects()).isEmpty(); - assertThat(protocolWithProject.getProject()).isNull(); - verify(biomaterialRepository, times(2)).save(biomaterialWithProject); - verify(fileRepository).save(fileWithProject); - verify(processRepository, times(2)).save(processWithProject); - verify(protocolRepository).save(protocolWithProject); - } - - @Test - public void testDeleteProject() { - //when - projectCrudStrategy.deleteDocument(testProject); - //then - verify(projectRepository).delete(testProject); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/core/service/strategy/ProtocolCrudStrategyTest.java b/src/test/java/org/humancellatlas/ingest/core/service/strategy/ProtocolCrudStrategyTest.java deleted file mode 100644 index 4d65d1e1d..000000000 --- a/src/test/java/org/humancellatlas/ingest/core/service/strategy/ProtocolCrudStrategyTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.humancellatlas.ingest.core.service.strategy; - -import org.humancellatlas.ingest.core.service.strategy.impl.ProtocolCrudStrategy; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.*; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = {ProtocolCrudStrategy.class}) -public class ProtocolCrudStrategyTest { - @Autowired private ProtocolCrudStrategy protocolCrudStrategy; - - @MockBean private ProtocolRepository protocolRepository; - @MockBean private ProcessRepository processRepository; - @MockBean private MessageRouter messageRouter; - - private Protocol testProtocol; - - @BeforeEach - void setUp() { - testProtocol = new Protocol(null); - } - - @Test - public void testRemoveLinksProject() { - // given - Process processWithProtocol = new Process(null); - processWithProtocol.getProtocols().add(testProtocol); - when(processRepository.findByProtocolsContains(testProtocol)).thenReturn(Stream.of(processWithProtocol)); - - // when - protocolCrudStrategy.removeLinksToDocument(testProtocol); - - //then - assertThat(processWithProtocol.getProtocols()).isEmpty(); - verify(processRepository).save(processWithProtocol); - } - - @Test - public void testDeleteProject() { - //when - protocolCrudStrategy.deleteDocument(testProtocol); - //then - verify(protocolRepository).delete(testProtocol); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/core/web/SpringLinkGeneratorTest.java b/src/test/java/org/humancellatlas/ingest/core/web/SpringLinkGeneratorTest.java deleted file mode 100644 index 90409d707..000000000 --- a/src/test/java/org/humancellatlas/ingest/core/web/SpringLinkGeneratorTest.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.humancellatlas.ingest.core.web; - -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.bundle.BundleManifest; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.protocol.Protocol; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.data.rest.core.mapping.ResourceMappings; -import org.springframework.data.rest.core.mapping.ResourceMetadata; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes={ SpringLinkGenerator.class }) -public class SpringLinkGeneratorTest { - - @MockBean - private ResourceMappings mappings; - - @Autowired - private SpringLinkGenerator linkGenerator = new SpringLinkGenerator(); - - - @Test - public void testCreateCallback() { - ResourceMetadata resourceMetadata = mock(ResourceMetadata.class); - when(resourceMetadata.getRel()).thenReturn("metadata"); - when(mappings.getMetadataFor(any())).thenReturn(resourceMetadata); - - //when: - String processCallback = linkGenerator.createCallback(Process.class, "df00e2"); - String biomaterialCallback = linkGenerator.createCallback(Biomaterial.class, "c80122"); - String fileCallback = linkGenerator.createCallback(File.class, "98dd90"); - String protocolCallback = linkGenerator.createCallback(Protocol.class, "846df1"); - String bmCallback = linkGenerator.createCallback(BundleManifest.class, "332fd9"); - - //then: - assertThat(processCallback).isEqualToIgnoringCase("/metadata/df00e2"); - assertThat(biomaterialCallback).isEqualToIgnoringCase("/metadata/c80122"); - assertThat(fileCallback).isEqualToIgnoringCase("/metadata/98dd90"); - assertThat(protocolCallback).isEqualToIgnoringCase("/metadata/846df1"); - assertThat(bmCallback).isEqualToIgnoringCase("/metadata/332fd9"); - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/errors/SubmissionErrorControllerTest.java b/src/test/java/org/humancellatlas/ingest/errors/SubmissionErrorControllerTest.java deleted file mode 100644 index db3510f5f..000000000 --- a/src/test/java/org/humancellatlas/ingest/errors/SubmissionErrorControllerTest.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.humancellatlas.ingest.errors; - -import org.humancellatlas.ingest.errors.web.SubmissionErrorController; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.data.web.PagedResourcesAssembler; -import org.springframework.http.ResponseEntity; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.mockito.Mockito.verify; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = {SubmissionErrorController.class}) -public class SubmissionErrorControllerTest { - @Autowired - private SubmissionErrorController controller; - - @MockBean - SubmissionErrorService submissionErrorService; - - @MockBean - private PagedResourcesAssembler pagedResourcesAssembler; - - @Test - public void testDeleteSubmissionErrors() { - // given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - - // when: - ResponseEntity response = controller.deleteSubmissionEnvelopeErrors(submissionEnvelope); - - // then - assertThat(response).isNotNull(); - assertThat(response.getStatusCode().value()).isEqualTo(204); - verify(submissionErrorService).deleteSubmissionEnvelopeErrors(submissionEnvelope); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/errors/SubmissionErrorServiceTest.java b/src/test/java/org/humancellatlas/ingest/errors/SubmissionErrorServiceTest.java deleted file mode 100644 index 8b0ffc1dc..000000000 --- a/src/test/java/org/humancellatlas/ingest/errors/SubmissionErrorServiceTest.java +++ /dev/null @@ -1,122 +0,0 @@ -package org.humancellatlas.ingest.errors; - -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.data.domain.PageImpl; -import org.springframework.data.domain.Pageable; -import org.springframework.test.context.junit.jupiter.SpringExtension; -import org.zalando.problem.*; - -import java.net.URI; -import java.util.Collections; -import java.util.Random; -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = {SubmissionErrorService.class}) -public class SubmissionErrorServiceTest { - @MockBean - private Pageable pageable; - @MockBean - private SubmissionErrorRepository submissionErrorRepository; - @Autowired - private SubmissionErrorService submissionErrorService; - - @Test - public void serviceCallsRepository() { - //given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - //and: - when(submissionErrorRepository.findBySubmissionEnvelope(any(SubmissionEnvelope.class), any(Pageable.class))) - .thenReturn(new PageImpl(Collections.emptyList())); - - //then: - assertThat(submissionErrorService.getErrorsFromEnvelope(submissionEnvelope, pageable)).isEmpty(); - - } - - @Test - public void errorIsGivenEnvelope() { - //given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - ArgumentCaptor insertedError = ArgumentCaptor.forClass(SubmissionError.class); - - //when: - SubmissionError submissionError = submissionErrorService.addErrorToEnvelope(submissionEnvelope, randomProblem()); - - //then: - verify(submissionErrorRepository).insert(insertedError.capture()); - assertThat(insertedError.getValue().getSubmissionEnvelope()).isEqualTo(submissionEnvelope); - assertThat(insertedError.getValue()).isEqualTo(submissionError); - } - - @Test - public void problemHasInstanceRemoved() { - //given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - ArgumentCaptor insertedError = ArgumentCaptor.forClass(SubmissionError.class); - - //when: - submissionErrorService.addErrorToEnvelope(submissionEnvelope, randomProblem()); - - //then: - verify(submissionErrorRepository).insert(insertedError.capture()); - assertThat(insertedError.getValue().getInstance()).isNull(); - } - - @Test - public void problemHasStatusRemoved() { - //given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - ArgumentCaptor insertedError = ArgumentCaptor.forClass(SubmissionError.class); - - //when: - submissionErrorService.addErrorToEnvelope(submissionEnvelope, randomProblem()); - - //then: - verify(submissionErrorRepository).insert(insertedError.capture()); - assertThat(insertedError.getValue().getStatus()).isNull(); - } - - public static Problem randomProblem() { - Random random = new Random(); - StatusType status; - URI baseType = URI.create("http://test.ingest.data.humancellatlas.org/"); - String type; - if (random.nextBoolean()) { - status = Status.valueOf(400); - type = "Error"; - } else { - status = Status.valueOf(300); - type = "Warning"; - } - - return Problem.builder() - .withStatus(status) - .withType(baseType.resolve(type)) - .withTitle("Random " + type) - .withDetail(UUID.randomUUID().toString() + UUID.randomUUID().toString()) - .withInstance(baseType.resolve(type + "/" + UUID.randomUUID())) - .build(); - } - - @Test - public void deleteSubmissionEnvelopeErrors() { - //given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - //when: - submissionErrorService.deleteSubmissionEnvelopeErrors(submissionEnvelope); - //then: - verify(submissionErrorRepository).deleteBySubmissionEnvelope(submissionEnvelope); - - } -} diff --git a/src/test/java/org/humancellatlas/ingest/exporter/DefaultExporterTest.java b/src/test/java/org/humancellatlas/ingest/exporter/DefaultExporterTest.java deleted file mode 100644 index 2f9b7f403..000000000 --- a/src/test/java/org/humancellatlas/ingest/exporter/DefaultExporterTest.java +++ /dev/null @@ -1,322 +0,0 @@ -package org.humancellatlas.ingest.exporter; - -import org.humancellatlas.ingest.bundle.BundleManifestRepository; -import org.humancellatlas.ingest.bundle.BundleManifestService; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.core.service.MetadataCrudService; -import org.humancellatlas.ingest.export.destination.ExportDestination; -import org.humancellatlas.ingest.export.entity.ExportEntityService; -import org.humancellatlas.ingest.export.job.ExportJob; -import org.humancellatlas.ingest.export.job.ExportJobRepository; -import org.humancellatlas.ingest.export.job.ExportJobService; -import org.humancellatlas.ingest.export.job.web.ExportJobRequest; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.process.ProcessService; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.json.simple.JSONObject; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; -import org.mockito.stubbing.Answer; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.time.Instant; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; - -import static java.util.stream.Collectors.toList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.humancellatlas.ingest.export.destination.ExportDestinationName.DCP; -import static org.mockito.Mockito.*; - -@ExtendWith(SpringExtension.class) -@SpringBootTest -public class DefaultExporterTest { - - @Autowired - private Exporter exporter; - - @MockBean - private ProcessService processService; - - @MockBean - private MessageRouter messageRouter; - - @MockBean - private MetadataCrudService metadataCrudService; - - @MockBean - private ExportJobService exportJobService; - - @MockBean - private ExportJobRepository exportJobRepository; - - @MockBean - private ProcessRepository processRepository; - - @MockBean - private ProjectRepository projectRepository; - - @MockBean - private ExportEntityService exportEntityService; - - @MockBean - private BundleManifestService bundleManifestService; - - @MockBean - private BundleManifestRepository bundleManifestRepository; - - SubmissionEnvelope submissionEnvelope; - - Project project; - - Set assayIds; - - @BeforeEach - void setUp() { - //given: - submissionEnvelope = new SubmissionEnvelope(); - assayIds = mockProcessIds(2); - project = new Project(null); - project.setUuid(Uuid.newUuid()); - project.getSubmissionEnvelopes().add(submissionEnvelope); - - mockProcessSvcGetProcesses(submissionEnvelope, assayIds); - - //and: - Set receivedData = mockSendingManifestThroughMessageRouter(); - } - - private String projectUuid() { - return project.getUuid().getUuid().toString(); - } - - @Test - public void testExportManifests() { - //when: - Set receivedData = mockSendingManifestThroughMessageRouter(); - - exporter.exportManifests(submissionEnvelope); - - //then: - assertAllProcessIdsProcessed(submissionEnvelope, assayIds, receivedData); - - //and: - verify(messageRouter, times(assayIds.size())) - .sendManifestForExport(any(ExperimentProcess.class)); - } - - @Test - public void testExportDataSetsContextsAndCallsMessageRouter() { - // given - mockCreateExportJob(projectUuid()); - - // when - exporter.exportData(submissionEnvelope); - - // then - var insertCaptor = ArgumentCaptor.forClass(ExportJob.class); - var sendCaptor = ArgumentCaptor.forClass(ExportJob.class); - verify(exportJobRepository).insert(insertCaptor.capture()); - verify(messageRouter).sendSubmissionForDataExport(sendCaptor.capture(), any()); - - assertThat(insertCaptor.getValue().getDestination().getContext().get("projectUuid")).isEqualTo(projectUuid()); - assertThat(insertCaptor.getValue().getContext().get("dataFileTransfer")).isEqualTo(false); - assertThat(sendCaptor.getValue().getDestination().getContext().get("projectUuid")).isEqualTo(projectUuid()); - assertThat(sendCaptor.getValue().getContext().get("dataFileTransfer")).isEqualTo(false); - } - - @Test - public void testExportMetadataFromExportJob() { - //given: - mockProcessSave(); - ExportJob newExportJob = mockCreateExportJob(projectUuid()); - Set receivedData = mockSendingProcessThroughMessageRouter(); - - //when: - exporter.exportMetadata(newExportJob); - - //then: - assertAllProcessIdsProcessed(submissionEnvelope, assayIds, receivedData); - assertDcpVersionUpdated(receivedData, newExportJob.getCreatedDate()); - verify(processRepository, times(assayIds.size())).save(any(Process.class)); - verify(messageRouter, times(assayIds.size())) - .sendExperimentForExport(any(ExperimentProcess.class), any(ExportJob.class), any()); - } - - @Test - public void testGenerateSpreadsheetFromSubmission() { - // given - mockCreateExportJob(projectUuid()); - - //when: - exporter.generateSpreadsheet(submissionEnvelope); - - // then - var insertCaptor = ArgumentCaptor.forClass(ExportJob.class); - var saveCaptor = ArgumentCaptor.forClass(ExportJob.class); - var sendCaptor = ArgumentCaptor.forClass(ExportJob.class); - verify(exportJobRepository).insert(insertCaptor.capture()); - verify(exportJobRepository).save(saveCaptor.capture()); - verify(messageRouter).sendGenerateSpreadsheet(sendCaptor.capture(), any()); - - assertThat(insertCaptor.getValue().getDestination().getContext().get("projectUuid")).isEqualTo(projectUuid()); - assertThat(saveCaptor.getValue().getContext().get("spreadsheetGeneration")).isEqualTo(false); - assertThat(sendCaptor.getValue().getDestination().getContext().get("projectUuid")).isEqualTo(projectUuid()); - assertThat(sendCaptor.getValue().getContext().get("spreadsheetGeneration")).isEqualTo(false); - } - - @Test - public void testGenerateSpreadsheetSetsContextsAndCallsMessageRouter() { - // given - ExportJob newExportJob = mockCreateExportJob(projectUuid()); - - //when: - exporter.generateSpreadsheet(newExportJob); - - // then - var saveCaptor = ArgumentCaptor.forClass(ExportJob.class); - var sendCaptor = ArgumentCaptor.forClass(ExportJob.class); - verify(exportJobRepository).save(saveCaptor.capture()); - verify(messageRouter).sendGenerateSpreadsheet(sendCaptor.capture(), any()); - - assertThat(saveCaptor.getValue().getContext().get("spreadsheetGeneration")).isEqualTo(false); - assertThat(sendCaptor.getValue().getDestination().getContext().get("projectUuid")).isEqualTo(projectUuid()); - assertThat(sendCaptor.getValue().getContext().get("spreadsheetGeneration")).isEqualTo(false); - } - - private ExportJob mockCreateExportJob(String projectUuidUuid) { - var destinationContext = new JSONObject(); - destinationContext.put("projectUuid", projectUuidUuid); - - var exportJobContext = new JSONObject(); - exportJobContext.put("totalAssayCount", assayIds.size()); - exportJobContext.put("dataFileTransfer", false); - ExportJob newExportJob = ExportJob.builder() - .submission(submissionEnvelope) - .destination(new ExportDestination(DCP, "v2", destinationContext)) - .context(exportJobContext) - .build(); - doReturn(newExportJob).when(exportJobService).createExportJob(any(SubmissionEnvelope.class), any(ExportJobRequest.class)); - doReturn(newExportJob).when(exportJobRepository).insert(any(ExportJob.class)); - doReturn(Stream.of(project)).when(projectRepository).findBySubmissionEnvelopesContains(any(SubmissionEnvelope.class)); - return newExportJob; - } - - private void assertAllProcessIdsProcessed(SubmissionEnvelope submissionEnvelope, Set assayIds, Set receivedData) { - int expectedCount = 2; - assertThat(receivedData).hasSize(expectedCount); - assertUniqueIndexes(receivedData); - assertCorrectTotalCount(receivedData, expectedCount); - assertCorrectSubmissionEnvelope(receivedData, submissionEnvelope); - assertAllProcessesExported(assayIds, receivedData); - } - - private void mockProcessSave() { - when(processRepository.save(any(Process.class))).thenAnswer( - (Answer) invocation -> { - Process process = invocation.getArgument(0); - return process; - } - ); - } - - private void mockProcessSvcGetProcesses(SubmissionEnvelope submissionEnvelope, Set assayIds) { - when(processService.getProcesses(any())).thenAnswer( - (Answer>) invocation -> { - List ids = invocation.getArgument(0); - return ids.stream().map(id -> { - Process process = spy(new Process(null)); - doReturn(id).when(process).getId(); - process.setSubmissionEnvelope(submissionEnvelope); - return process; - }); - } - ); - - doReturn(assayIds).when(processService).findAssays(any(SubmissionEnvelope.class)); - } - - private Set mockProcessIds(int max) { - return IntStream.range(0, max) - .mapToObj(count -> UUID.randomUUID().toString()) - .collect(Collectors.toSet()); - } - - private Set mockSendingManifestThroughMessageRouter() { - final Set experimentProcess = new HashSet<>(); - Answer addToSet = invocation -> { - experimentProcess.add(invocation.getArgument(0)); - return null; - }; - doAnswer(addToSet).when(messageRouter).sendManifestForExport(any(ExperimentProcess.class)); - return experimentProcess; - } - - private Set mockSendingProcessThroughMessageRouter() { - final Set experimentProcess = new HashSet<>(); - Answer addToSet = invocation -> { - experimentProcess.add(invocation.getArgument(0)); - return null; - }; - doAnswer(addToSet).when(messageRouter).sendExperimentForExport(any(ExperimentProcess.class), any(ExportJob.class), any()); - return experimentProcess; - } - - private void assertUniqueIndexes(Set receivedData) { - List indexes = receivedData.stream() - .map(ExperimentProcess::getIndex) - .collect(toList()); - assertThat(indexes).containsOnlyOnce(0, 1); - } - - private void assertDcpVersionUpdated(Set receivedData, Instant dcpVersion) { - receivedData.stream() - .map(ExperimentProcess::getProcess) - .forEach(process -> assertThat(process.getDcpVersion()).isEqualTo(dcpVersion)); - } - - private void assertCorrectTotalCount(Set receivedData, int expectedCount) { - receivedData.stream().forEach(exporterData -> { - assertThat(exporterData.getTotalCount()).isEqualTo(expectedCount); - }); - } - - private void assertCorrectSubmissionEnvelope(Set receivedData, - SubmissionEnvelope submissionEnvelope) { - receivedData.forEach(exporterData -> assertThat(exporterData.getSubmissionEnvelope()).isEqualTo(submissionEnvelope)); - } - - private void assertAllProcessesExported(Set assayIds, - Set exporterData) { - - List sentProcesses = exporterData.stream() - .map(ExperimentProcess::getProcess) - .collect(toList()); - - assertThat(sentProcesses.stream().map(Process::getId)).containsAll(assayIds); - } - - @Configuration - static class TestConfiguration { - - @Bean - Exporter defaultExporter() { - return new DefaultExporter(); - } - - } - -} \ No newline at end of file diff --git a/src/test/java/org/humancellatlas/ingest/file/FileTest.java b/src/test/java/org/humancellatlas/ingest/file/FileTest.java deleted file mode 100644 index 8400f31ef..000000000 --- a/src/test/java/org/humancellatlas/ingest/file/FileTest.java +++ /dev/null @@ -1,91 +0,0 @@ -package org.humancellatlas.ingest.file; - -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; - -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; - -public class FileTest { - Process process; - File file; - - @BeforeEach - void setUp() { - //given: - file = new File(null, "fileName"); - - process = spy(new Process(null)); - doReturn("fe89a0").when(process).getId(); - } - - @Test - public void testAddAsDerivedByProcess() { - //when: - file.addAsDerivedByProcess(process); - - //then: - assertThat(file.getDerivedByProcesses()).containsExactly(process); - } - - @Test - public void testAddDerivedByProcessdNoDuplication() { - // when: - file.addAsDerivedByProcess(process); - file.addAsDerivedByProcess(process); - - //then: - assertThat(file.getDerivedByProcesses()).hasSize(1); - } - - @Test - public void testAddToAnalysis() { - //given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - process.setSubmissionEnvelope(submissionEnvelope); - - //when: - file.addToAnalysis(process); - - //then: - assertThat(file.getDerivedByProcesses()).contains(process); - assertThat(file.getSubmissionEnvelope()).isEqualTo(submissionEnvelope); - } - - @Test - public void testAddToAnalysisWhenFileAlreadyLinkedToSubmissionEnvelope() { - //given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - process.setSubmissionEnvelope(submissionEnvelope); - file.setSubmissionEnvelope(submissionEnvelope); - - //when: - file.addToAnalysis(process); - - //then: - assertThat(file.getDerivedByProcesses()).contains(process); - assertThat(file.getSubmissionEnvelope()).isEqualTo(submissionEnvelope); - } - - @ParameterizedTest - @MethodSource("testFiles") - public void newFileHasDataFileUuidNotNull(File file) { - assertThat(file) - .extracting("dataFileUuid") - .doesNotContainNull(); - } - - private static Stream testFiles() { - return Stream.of( - new File(), - new File(null, "test-File") - ); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/messaging/MessageRouterTest.java b/src/test/java/org/humancellatlas/ingest/messaging/MessageRouterTest.java deleted file mode 100644 index c69e54657..000000000 --- a/src/test/java/org/humancellatlas/ingest/messaging/MessageRouterTest.java +++ /dev/null @@ -1,215 +0,0 @@ -package org.humancellatlas.ingest.messaging; - -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.config.ConfigurationService; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.core.web.LinkGenerator; -import org.humancellatlas.ingest.export.ExportState; -import org.humancellatlas.ingest.export.destination.ExportDestination; -import org.humancellatlas.ingest.export.job.ExportJob; -import org.humancellatlas.ingest.exporter.ExperimentProcess; -import org.humancellatlas.ingest.messaging.model.ExportSubmissionMessage; -import org.humancellatlas.ingest.messaging.model.ManifestMessage; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.json.simple.JSONObject; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.rest.core.config.RepositoryRestConfiguration; -import org.springframework.data.rest.core.mapping.ResourceMappings; - -import java.net.URI; -import java.time.Instant; -import java.util.ArrayList; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.humancellatlas.ingest.export.destination.ExportDestinationName.DCP; -import static org.humancellatlas.ingest.messaging.Constants.Exchanges.EXPORTER_EXCHANGE; -import static org.humancellatlas.ingest.messaging.Constants.Routing.MANIFEST_SUBMITTED; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.*; - - -@SpringBootTest -public class MessageRouterTest { - - @Autowired - private MessageRouter messageRouter; - - @MockBean - private MessageSender messageSender; - - @MockBean - private ResourceMappings resourceMappings; - - @MockBean - private RepositoryRestConfiguration config; - - @MockBean - private LinkGenerator linkGenerator; - - @MockBean - private ConfigurationService configurationService; - - @Test - public void testSendManifestForExport() { - //expect: - doTestSendForExport(MANIFEST_SUBMITTED); - } - - @Test - public void testSendSubmissionForDataExport() { - // given - var submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.setUuid(Uuid.newUuid()); - var project = new Project(null); - project.setUuid(Uuid.newUuid()); - project.getSubmissionEnvelopes().add(submissionEnvelope); - var exportJob = exportJob(submissionEnvelope, project); - var context = new JSONObject(); - - // when - messageRouter.sendSubmissionForDataExport(exportJob, context); - - // then - var argumentCaptor = ArgumentCaptor.forClass(ExportSubmissionMessage.class); - verify(messageSender).queueNewExportMessage(anyString(), anyString(), argumentCaptor.capture(), anyLong()); - verify(linkGenerator).createCallback(any(), anyString()); - - var capturedArgument = argumentCaptor.getValue(); - assertThat(capturedArgument.getExportJobId()).isEqualTo(exportJob.getId()); - assertThat(capturedArgument.getSubmissionUuid()).isEqualTo(submissionEnvelope.getUuid().getUuid().toString()); - assertThat(capturedArgument.getProjectUuid()).isEqualTo(project.getUuid().getUuid().toString()); - assertThat(capturedArgument.getContext()).isEqualTo(context); - } - - private ExportJob exportJob(SubmissionEnvelope submissionEnvelope, Project project) { - var destinationContext = new JSONObject(); - destinationContext.put("projectUuid", project.getUuid().getUuid().toString()); - - var exportJobContext = new JSONObject(); - exportJobContext.put("dataFileTransfer", false); - return ExportJob.builder() - .id("testExportJobId") - .submission(submissionEnvelope) - .destination(new ExportDestination(DCP, "v2", destinationContext)) - .context(exportJobContext) - .build(); - } - - @Test - public void testRouteStateTrackingUpdateMessageFor() { - // given: - Project project = mock(Project.class); - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - doReturn(Instant.now()).when(project).getUpdateDate(); - doReturn(submissionEnvelope).when(project).getSubmissionEnvelope(); - - // when: - messageRouter.routeStateTrackingUpdateMessageFor(project); - - verify(messageSender, times(1)).queueDocumentStateUpdateMessage(any(URI.class), any(), anyLong()); - } - - @Test - public void testRouteStateTrackingUpdateMessageForBiomaterial() { - // given: - Biomaterial project = mock(Biomaterial.class); - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - doReturn(Instant.now()).when(project).getUpdateDate(); - doReturn(submissionEnvelope).when(project).getSubmissionEnvelope(); - - // when: - messageRouter.routeStateTrackingUpdateMessageFor(project); - - verify(messageSender, times(1)).queueDocumentStateUpdateMessage(any(URI.class), any(), anyLong()); - } - - @Test - public void testRouteStateTrackingUpdateMessageForProject() { - // given: - Project project = new Project(null); - // when: - messageRouter.routeStateTrackingUpdateMessageFor(project); - - verify(messageSender, never()).queueDocumentStateUpdateMessage(any(URI.class), any(), anyLong()); - } - - @Test - public void testRouteStateTrackingUpdateMessageForBiomaterialWithoutSubmission() { - // given: - Biomaterial project = mock(Biomaterial.class); - - // when: - Exception exception = assertThrows(RuntimeException.class, () -> { - messageRouter.routeStateTrackingUpdateMessageFor(project); - }); - - verify(messageSender, never()).queueDocumentStateUpdateMessage(any(URI.class), any(), anyLong()); - } - - private void doTestSendForExport(String routingKey) { - //given: - String processId = "78bbd9"; - Process process = spy(new Process(null)); - doReturn(processId).when(process).getId(); - - Uuid processUuid = Uuid.newUuid(); - process.setUuid(processUuid); - Instant version = Instant.now(); - process.setDcpVersion(version); - - //and: - String envelopeId = "87bcf3"; - SubmissionEnvelope submissionEnvelope = spy(new SubmissionEnvelope()); - doReturn(envelopeId).when(submissionEnvelope).getId(); - Uuid envelopeUuid = Uuid.newUuid(); - submissionEnvelope.setUuid(envelopeUuid); - - process.setSubmissionEnvelope(submissionEnvelope); - - //and: - ExperimentProcess exporterData = new ExperimentProcess(2, 4, process, submissionEnvelope, null); - - //and: - String callbackLink = "/processes/78bbd9"; - doReturn(callbackLink).when(linkGenerator).createCallback(any(Class.class), anyString()); - - //when: - messageRouter.sendManifestForExport(exporterData); - - //then: - ArgumentCaptor messageCaptor = ArgumentCaptor.forClass(ManifestMessage.class); - verify(messageSender).queueNewExportMessage(eq(EXPORTER_EXCHANGE), eq(routingKey), - messageCaptor.capture(), anyLong()); - - //and: - ManifestMessage submittedMessage = messageCaptor.getValue(); - assertThat(submittedMessage) - .extracting("documentId", "documentUuid", "callbackLink", "documentType", - "envelopeId", "envelopeUuid", "index", "total") - .containsExactly(processId, processUuid.getUuid().toString(), callbackLink, - process.getClass().getSimpleName(), envelopeId, envelopeUuid.getUuid().toString(), 2, 4); - } - - @Configuration - static class TestConfiguration { - - @Bean - MessageRouter messageRouter() { - return new MessageRouter(); - } - - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/messaging/MessageServiceTest.java b/src/test/java/org/humancellatlas/ingest/messaging/MessageServiceTest.java deleted file mode 100644 index 9d8ac82c7..000000000 --- a/src/test/java/org/humancellatlas/ingest/messaging/MessageServiceTest.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.humancellatlas.ingest.messaging; - -import org.junit.jupiter.api.Test; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.springframework.amqp.rabbit.core.RabbitMessagingTemplate; -import org.springframework.context.annotation.Configuration; -import org.springframework.messaging.MessagingException; -import org.springframework.messaging.converter.MessageConversionException; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.verify; - - -public class MessageServiceTest { - - private RabbitMessagingTemplate rabbitMessagingTemplate = Mockito.mock(RabbitMessagingTemplate.class); - - private MessageService messageService = new MessageService(rabbitMessagingTemplate); - - - @Test - public void testPublish() { - //given: - Message message = new Message("exchange", "routingKey", "payload"); - - //when: - messageService.publish(message); - - //then: - verify(rabbitMessagingTemplate).convertAndSend(message.getExchange(), message.getRoutingKey(), message.getPayload()); - } - - @Test - public void testPublishMessageConversionException() { - //given: - Message message = new Message("", "", ""); - - //when: - doThrow(MessageConversionException.class).when(rabbitMessagingTemplate).convertAndSend("","",""); - - //then: - assertThatThrownBy(() -> { messageService.publish(message); }).isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Unable to convert payload"); - } - - @Test - public void testPublishMessagingException() { - //given: - Message message = new Message("", "", ""); - - //when: - doThrow(MessagingException.class).when(rabbitMessagingTemplate).convertAndSend("","",""); - - //then: - assertThatThrownBy(() -> { messageService.publish(message); }).isInstanceOf(RuntimeException.class) - .hasMessageContaining("There was a problem sending message"); - } - - @Configuration - static class TestConfiguration {} - -} diff --git a/src/test/java/org/humancellatlas/ingest/notifications/NotificationCoordinatorTest.java b/src/test/java/org/humancellatlas/ingest/notifications/NotificationCoordinatorTest.java deleted file mode 100644 index 565b2568c..000000000 --- a/src/test/java/org/humancellatlas/ingest/notifications/NotificationCoordinatorTest.java +++ /dev/null @@ -1,173 +0,0 @@ -package org.humancellatlas.ingest.notifications; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.assertj.core.api.Assertions; -import org.humancellatlas.ingest.notifications.exception.ProcessingException; -import org.humancellatlas.ingest.notifications.model.Checksum; -import org.humancellatlas.ingest.notifications.model.Notification; -import org.humancellatlas.ingest.notifications.model.NotificationState; -import org.humancellatlas.ingest.notifications.processors.NotificationProcessor; -import org.humancellatlas.ingest.notifications.sources.NotificationSource; -import org.humancellatlas.ingest.notifications.sources.impl.inmemory.InmemoryNotificationSource; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; - -import org.mockito.stubbing.Answer; - -public class NotificationCoordinatorTest { - - private Notification generateTestNotification(String checksumValue) { - return Notification.buildNew() - .metadata(new HashMap<>()) - .content("testcontent") - .checksum(new Checksum("testtype", checksumValue)) - .build(); - } - - private NotificationService mockNotificationService() { - NotificationService notificationService = mock(NotificationService.class); - - Answer mockChangeStateFn = invocation -> { - Notification notification = invocation.getArgument(0); - NotificationState toState = invocation.getArgument(1); - return Notification.builder() - .checksum(notification.getChecksum()) - .content(notification.getContent()) - .metadata(notification.getMetadata()) - .notifyAt(notification.getNotifyAt()) - .state(toState).build(); - }; - - Mockito.doAnswer(mockChangeStateFn) - .when(notificationService).changeState(any(Notification.class), any(NotificationState.class)); - - return notificationService; - } - - @Test - public void testQueue() { - List testNotifications = List.of(generateTestNotification("testvalue1"), - generateTestNotification("testvalue2")); - NotificationSource testInmemorySource = new InmemoryNotificationSource(); - - NotificationService notificationService = mockNotificationService(); - Mockito.doReturn(testNotifications.stream()) - .when(notificationService).getUnhandledNotifications(); - - NotificationCoordinator notificationCoordinator = new NotificationCoordinator(Collections.emptyList(), - testInmemorySource, - notificationService); - - notificationCoordinator.queue(); - - Assertions.assertThat(testInmemorySource.stream() - .map(n -> n.getChecksum().getValue())) - .containsSequence(testNotifications.stream() - .map(n -> n.getChecksum().getValue()) - .collect(Collectors.toList())); - } - - @Test - public void testProcess() { - NotificationSource mockInmemorySource = mock(NotificationSource.class); - Mockito.doReturn(Stream.of(generateTestNotification("testvalue1"), - generateTestNotification("testvalue2"))) - .when(mockInmemorySource).stream(); - - NotificationService notificationService = mockNotificationService(); - - NotificationProcessor mockHappyNotificationProcessor = mock(NotificationProcessor.class); - Mockito.doNothing() - .when(mockHappyNotificationProcessor).handle(any(Notification.class)); - - NotificationCoordinator notificationCoordinator = new NotificationCoordinator(List.of(mockHappyNotificationProcessor), - mockInmemorySource, - notificationService); - - notificationCoordinator.process(); - - Mockito.verify(notificationService, times(2)) - .changeState(any(Notification.class), eq(NotificationState.PROCESSING)); - - Mockito.verify(notificationService, times(2)) - .changeState(any(Notification.class), eq(NotificationState.PROCESSED)); - } - - @Test - public void testProcessingFailure() { - NotificationSource mockInmemorySource = mock(NotificationSource.class); - Mockito.doReturn(Stream.of(generateTestNotification("testvalue1"), - generateTestNotification("testvalue2"))) - .when(mockInmemorySource).stream(); - - NotificationService notificationService = mockNotificationService(); - - NotificationProcessor mockUnhappyNotificationProcessor = mock(NotificationProcessor.class); - Mockito.doThrow(new ProcessingException("")) - .when(mockUnhappyNotificationProcessor).handle(any(Notification.class)); - Mockito.doReturn(true) - .when(mockUnhappyNotificationProcessor).isEligible(any(Notification.class)); - - NotificationCoordinator notificationCoordinator = new NotificationCoordinator(List.of(mockUnhappyNotificationProcessor), - mockInmemorySource, - notificationService); - - notificationCoordinator.process(); - - Mockito.verify(notificationService, times(2)) - .changeState(any(Notification.class), eq(NotificationState.PROCESSING)); - - Mockito.verify(notificationService, times(2)) - .changeState(any(Notification.class), eq(NotificationState.FAILED)); - } - - /** - * Test behaviour when one processor succeeds, but another fails. - * Expect an overall failure outcome. - */ - @Test - public void testMultipleProcessors_OneFailing() { - NotificationSource fakeSource = new InmemoryNotificationSource(); - NotificationService notificationService = mockNotificationService(); - - NotificationProcessor mockHappyNotificationProcessor = mock(NotificationProcessor.class); - Mockito.doNothing() - .when(mockHappyNotificationProcessor).handle(any(Notification.class)); - Mockito.doReturn(true) - .when(mockHappyNotificationProcessor).isEligible(any(Notification.class)); - - NotificationProcessor mockUnhappyNotificationProcessor = mock(NotificationProcessor.class); - Mockito.doThrow(new ProcessingException("")) - .when(mockUnhappyNotificationProcessor).handle(any(Notification.class)); - Mockito.doReturn(true) - .when(mockUnhappyNotificationProcessor).isEligible(any(Notification.class)); - - NotificationCoordinator notificationCoordinator = new NotificationCoordinator(List.of(mockUnhappyNotificationProcessor, - mockHappyNotificationProcessor), - fakeSource, - notificationService); - - fakeSource.supply(List.of(generateTestNotification("testvalue1"), - generateTestNotification("testvalue2"))); - notificationCoordinator.process(); - - Mockito.verify(notificationService, times(2)) - .changeState(any(Notification.class), eq(NotificationState.PROCESSING)); - - Mockito.verify(notificationService, times(2)) - .changeState(any(Notification.class), eq(NotificationState.FAILED)); - - Mockito.verify(notificationService, never()) - .changeState(any(Notification.class), eq(NotificationState.PROCESSED)); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/notifications/NotificationServicesTest.java b/src/test/java/org/humancellatlas/ingest/notifications/NotificationServicesTest.java deleted file mode 100644 index ce832745a..000000000 --- a/src/test/java/org/humancellatlas/ingest/notifications/NotificationServicesTest.java +++ /dev/null @@ -1,118 +0,0 @@ -package org.humancellatlas.ingest.notifications; - -import java.util.HashMap; -import java.util.Optional; -import org.assertj.core.api.Assertions; -import org.humancellatlas.ingest.notifications.exception.DuplicateNotification; -import org.humancellatlas.ingest.notifications.model.Checksum; -import org.humancellatlas.ingest.notifications.model.Notification; -import org.humancellatlas.ingest.notifications.model.NotificationRequest; -import org.humancellatlas.ingest.notifications.model.NotificationState; -import org.junit.jupiter.api.Test; -import static org.mockito.AdditionalAnswers.returnsFirstArg; -import org.mockito.Mockito; -import org.springframework.dao.DuplicateKeyException; - -import static org.mockito.Mockito.*; -import static org.assertj.core.api.Assertions.*; - -public class NotificationServicesTest { - private NotificationRepository notificationRepository = mock(NotificationRepository.class); - private NotificationService notificationService = new NotificationService(notificationRepository); - - @Test - public void testCreateNotification() { - Checksum testChecksum = new Checksum("testtype", "testvalue"); - NotificationRequest notificationRequest = new NotificationRequest("testcontent", - new HashMap<>(), - testChecksum); - - Mockito.doReturn(Notification.buildNew() - .metadata(new HashMap<>()) - .content("testcontent") - .checksum(testChecksum) - .build()) - .when(notificationRepository).save(any(Notification.class)); - - Notification createdNotification = notificationService.createNotification(notificationRequest); - - assertThat(createdNotification.getChecksum()).isEqualTo(testChecksum); - assertThat(createdNotification.getMetadata()).isEqualTo(new HashMap<>()); - assertThat(createdNotification.getState()).isEqualTo(NotificationState.PENDING); - assertThat(createdNotification.getContent()).isEqualTo("testcontent"); - } - - @Test - public void testCreateDuplicateNotification() { - Checksum testChecksum = new Checksum("testtype", "testvalue"); - NotificationRequest notificationRequest = new NotificationRequest("testcontent", - new HashMap<>(), - testChecksum); - - Notification testExistingNotification = Notification.buildNew() - .metadata(new HashMap<>()) - .content("testcontent") - .checksum(testChecksum) - .build(); - - - Mockito.doThrow(new DuplicateKeyException("")) - .when(notificationRepository).save(any(Notification.class)); - - Mockito.doReturn(Optional.of(testExistingNotification)) - .when(notificationRepository).findByChecksum_Value("testvalue"); - - - Assertions.assertThatExceptionOfType(DuplicateNotification.class) - .isThrownBy(() -> notificationService.createNotification(notificationRequest)); - } - - @Test - public void testRetrieveByChecksum() { - Checksum testChecksum = new Checksum("testtype", "testvalue"); - - Mockito.doReturn(Optional.of(Notification.buildNew() - .metadata(new HashMap<>()) - .content("testcontent") - .checksum(testChecksum) - .build())) - .when(notificationRepository).findByChecksum(testChecksum); - - Assertions.assertThat(notificationService.retrieveForChecksum(testChecksum).orElseThrow().getChecksum()) - .isEqualTo(testChecksum); - } - - @Test - public void testChangeState() { - Checksum testChecksum = new Checksum("testtype", "testvalue"); - Notification testNotification = Notification.buildNew() - .metadata(new HashMap<>()) - .content("testcontent") - .checksum(testChecksum) - .build(); - - - Mockito.doAnswer(returnsFirstArg()) - .when(notificationRepository).save(any(Notification.class)); - - - Notification changedState = notificationService.changeState(testNotification, NotificationState.QUEUED); - assertThat(changedState.getState()).isEqualTo(NotificationState.QUEUED); - } - - @Test - public void testIllegalStateChange() { - Checksum testChecksum = new Checksum("testtype", "testvalue"); - Notification testNotification = Notification.buildNew() - .metadata(new HashMap<>()) - .content("testcontent") - .checksum(testChecksum) - .build(); - - Mockito.doAnswer(returnsFirstArg()) - .when(notificationRepository).save(any(Notification.class)); - - Assertions.assertThatExceptionOfType(IllegalStateException.class) - .isThrownBy(() -> notificationService.changeState(testNotification, NotificationState.PROCESSED)); - } -} \ No newline at end of file diff --git a/src/test/java/org/humancellatlas/ingest/notifications/processors/EmailNotificationsProcessorTest.java b/src/test/java/org/humancellatlas/ingest/notifications/processors/EmailNotificationsProcessorTest.java deleted file mode 100644 index c9a34fe2a..000000000 --- a/src/test/java/org/humancellatlas/ingest/notifications/processors/EmailNotificationsProcessorTest.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.humancellatlas.ingest.notifications.processors; - -public class EmailNotificationsProcessorTest { - -} diff --git a/src/test/java/org/humancellatlas/ingest/process/ProcessServiceTest.java b/src/test/java/org/humancellatlas/ingest/process/ProcessServiceTest.java deleted file mode 100644 index 598a57956..000000000 --- a/src/test/java/org/humancellatlas/ingest/process/ProcessServiceTest.java +++ /dev/null @@ -1,114 +0,0 @@ -package org.humancellatlas.ingest.process; - -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.bundle.BundleManifestRepository; -import org.humancellatlas.ingest.core.service.MetadataCrudService; -import org.humancellatlas.ingest.core.service.MetadataUpdateService; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.state.MetadataDocumentEventHandler; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.*; - -@ExtendWith(SpringExtension.class) -public class ProcessServiceTest { - - @Autowired - private ProcessService service; - - @MockBean - private SubmissionEnvelopeRepository submissionEnvelopeRepository; - @MockBean - private ProcessRepository processRepository; - @MockBean - private FileRepository fileRepository; - @MockBean - private BiomaterialRepository biomaterialRepository; - @MockBean - private BundleManifestRepository bundleManifestRepository; - @MockBean - private ProjectRepository projectRepository; - @MockBean - private MetadataDocumentEventHandler metadataDocumentEventHandler; - @MockBean - private MetadataCrudService metadataCrudService; - @MockBean - private MetadataUpdateService metadataUpdateService; - - String fileName = "ERR1630013.fastq.gz"; - File file; - Process analysis; - SubmissionEnvelope submissionEnvelope; - - @BeforeEach - void setUp() { - // Given: - file = spy(new File(null, fileName)); - analysis = new Process(null); - submissionEnvelope = new SubmissionEnvelope(); - analysis.setSubmissionEnvelope(submissionEnvelope); - } - - @Test - public void testAddFileToAnalysisProcess() { - //given: - doReturn(Collections.emptyList()) - .when(fileRepository) - .findBySubmissionEnvelopeAndFileName(any(SubmissionEnvelope.class), anyString()); - - //when: - Process result = service.addOutputFileToAnalysisProcess(analysis, file); - - //then: - assertThat(result).isEqualTo(analysis); - verify(file).addToAnalysis(analysis); - verify(fileRepository).save(file); - } - - @Test - public void testAddFileToAnalysisProcessWhenFileAlreadyExists() { - //given: - List persistentFiles = Arrays.asList(file); - doReturn(persistentFiles).when(fileRepository) - .findBySubmissionEnvelopeAndFileName(submissionEnvelope, fileName); - - //when: - Process result = service.addOutputFileToAnalysisProcess(analysis, file); - - //then: - assertThat(result).isEqualTo(analysis); - - //and: - verify(file).addToAnalysis(analysis); - verify(fileRepository).save(file); - } - - @Configuration - static class TestConfiguration { - - @Bean - ProcessService processService() { - return new ProcessService(); - } - - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/process/ProcessTest.java b/src/test/java/org/humancellatlas/ingest/process/ProcessTest.java deleted file mode 100644 index d0f554000..000000000 --- a/src/test/java/org/humancellatlas/ingest/process/ProcessTest.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.humancellatlas.ingest.process; - -public class ProcessTest { - -// @Test -// public void testIsAssaying() { -// //given: -// Process nonAssayingProcess = new Process(); -// -// //and: -// Process hasInputBiomaterial = new Process(); -// hasInputBiomaterial.addInput(new Biomaterial(null)); -// -// //and: -// Process hasDerivedFile = new Process(); -// hasDerivedFile.addDerivative(new File(null)); -// -// //and: -// Process assayingProcess = new Process(); -// assayingProcess.addInput(new Biomaterial(null)); -// assayingProcess.addDerivative(new File(null)); -// -// //expect: -// String notAssaying = "Expected Process to be NON assaying."; -// assertThat(nonAssayingProcess.isAssaying()).as(notAssaying).isFalse(); -// assertThat(hasInputBiomaterial.isAssaying()).as(notAssaying).isFalse(); -// assertThat(hasDerivedFile.isAssaying()).as(notAssaying).isFalse(); -// -// //and: -// assertThat(assayingProcess.isAssaying()).as("Expected Process to be assaying.").isTrue(); -// } -// -// @Test -// public void testIsAnalysis() { -// //given: -// Process nonAnalysis = new Process(); -// -// //and: -// Process hasInputFile = new Process(); -// hasInputFile.addInput(new File("input")); -// -// //and: -// Process hasDerivedFile = new Process(); -// hasDerivedFile.addDerivative(new File("output")); -// -// //and: -// Process analysis = new Process(); -// analysis.addInput(new File("input")); -// analysis.addDerivative(new File("output")); -// -// //then: -// String notAnalysis = "Expected Process to be non Analysis"; -// assertThat(nonAnalysis.isAnalysis()).as(notAnalysis).isFalse(); -// assertThat(hasInputFile.isAnalysis()).as(notAnalysis).isFalse(); -// assertThat(hasDerivedFile.isAnalysis()).as(notAnalysis).isFalse(); -// -// //then: -// assertThat(analysis.isAnalysis()).as("Expected Process to be Analysis.").isTrue(); -// } - -} diff --git a/src/test/java/org/humancellatlas/ingest/project/ProjectFilterQueryBuilderTest.java b/src/test/java/org/humancellatlas/ingest/project/ProjectFilterQueryBuilderTest.java deleted file mode 100644 index 3ddbcaab6..000000000 --- a/src/test/java/org/humancellatlas/ingest/project/ProjectFilterQueryBuilderTest.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.humancellatlas.ingest.project; - -import org.assertj.core.api.Assertions; -import org.humancellatlas.ingest.project.web.SearchFilter; -import org.humancellatlas.ingest.project.web.SearchType; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; - -import java.util.Optional; - -public class ProjectFilterQueryBuilderTest { - @Test - void null_search_type_with_non_null_text() { - SearchFilter searchFilter = SearchFilter.builder() - .search("project keyword") - .wranglingState(null) - .primaryWrangler(null) - .wranglingPriority(null) - .hasOfficialHcaPublication(null) - .identifyingOrganism(null) - .organOntology(null) - .minCellCount(null) - .maxCellCount(null) - .projectLabels(null) - .dcpReleaseNumber(null) - .dataAccess(null) - .searchType(null) - .build(); - - - ProjectQueryBuilder.buildProjectsQuery(searchFilter); - // no exception thrown when searchType is null - } - - @ParameterizedTest - @CsvSource({ - "AllKeywords,k1 k2,\"k1\" \"k2\"", - "AnyKeyword,k1 k2,k1 k2", - "ExactMatch,k1 k2,\"k1 k2\"", - - "null,k1 k2,k1 k2", - - "AllKeywords,\"k1 k2\",\"k1 k2\"", - "AnyKeyword,k1 \"k2\" k3,k1 \"k2\" k3", - "ExactMatch,\"k1\" k2,\"k1\" k2", - }) - public void quoting_in_mongo_syntax_by_search_type(String searchTypeStr, String input, String expected) { - SearchType searchType = searchTypeStr.equals("null") ? null : SearchType.valueOf(searchTypeStr); - SearchFilter searchFilter = SearchFilter.builder() - .search(input) - .searchType(searchType) - .build(); - Assertions.assertThat(ProjectQueryBuilder.formatSearchString(searchFilter)) - .isEqualTo(expected); - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/project/ProjectLinkChangeListenerTest.java b/src/test/java/org/humancellatlas/ingest/project/ProjectLinkChangeListenerTest.java deleted file mode 100644 index 68507e36b..000000000 --- a/src/test/java/org/humancellatlas/ingest/project/ProjectLinkChangeListenerTest.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.humancellatlas.ingest.project; - -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.verify; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = ProjectLinkChangeListener.class) -class ProjectLinkChangeListenerTest { - @MockBean - ProjectService projectService; - @Autowired - ProjectLinkChangeListener projectLinkChangeListener; - - - @Test - void test_whenUpdatingStatus_usingProjectService() { - // given - Project project = new Project(null); - project.setUuid(Uuid.newUuid()); - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - project.addToSubmissionEnvelopes(submissionEnvelope); - List submissions = List.of(submissionEnvelope); - - // then - projectLinkChangeListener.beforeLinkSave(project, submissions); - - // then - verify(projectService).updateWranglingState(eq(project), any()); - } -} \ No newline at end of file diff --git a/src/test/java/org/humancellatlas/ingest/project/ProjectServiceTest.java b/src/test/java/org/humancellatlas/ingest/project/ProjectServiceTest.java deleted file mode 100644 index 34fb2246c..000000000 --- a/src/test/java/org/humancellatlas/ingest/project/ProjectServiceTest.java +++ /dev/null @@ -1,321 +0,0 @@ -package org.humancellatlas.ingest.project; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.assertj.core.api.Assertions; -import org.humancellatlas.ingest.audit.AuditEntry; -import org.humancellatlas.ingest.audit.AuditEntryService; -import org.humancellatlas.ingest.bundle.BundleManifestRepository; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.core.service.MetadataCrudService; -import org.humancellatlas.ingest.core.service.MetadataUpdateService; -import org.humancellatlas.ingest.project.exception.NonEmptyProject; -import org.humancellatlas.ingest.schemas.Schema; -import org.humancellatlas.ingest.schemas.SchemaService; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mockito; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.ApplicationContext; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.util.*; -import java.util.stream.IntStream; -import java.util.stream.Stream; - -import static java.util.stream.Collectors.toList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.humancellatlas.ingest.audit.AuditType.STATUS_UPDATED; -import static org.humancellatlas.ingest.project.WranglingState.ELIGIBLE; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = { - ProjectService.class, - ProjectRepository.class, - SubmissionEnvelopeRepository.class, -}) -public class ProjectServiceTest { - - @Autowired - private ApplicationContext applicationContext; - - @Autowired - private ProjectService projectService; - - @MockBean - private MongoTemplate mongoTemplate; - - @MockBean - private SubmissionEnvelopeRepository submissionEnvelopeRepository; - - @MockBean - private ProjectRepository projectRepository; - - @MockBean - private MetadataCrudService metadataCrudService; - - @MockBean - private MetadataUpdateService metadataUpdateService; - - @MockBean - private SchemaService schemaService; - - @MockBean - private BundleManifestRepository bundleManifestRepository; - - @MockBean - private ProjectEventHandler projectEventHandler; - - @MockBean - private AuditEntryService auditEntryService; - - @BeforeEach - void setUp() { - applicationContext.getBeansWithAnnotation(MockBean.class).forEach(Mockito::reset); - } - - @Nested - class SubmissionEnvelopes { - Project project1; - Project project2; - Set submissionSet1; - Set submissionSet2; - - @BeforeEach - void setup(){ - // given - project1 = spy(new Project(null)); - doReturn("project1").when(project1).getId(); - project1.setUuid(Uuid.newUuid()); - - submissionSet1 = new HashSet<>(); - IntStream.range(0, 3).mapToObj(Integer::toString).forEach(id -> { - var sub = spy(new SubmissionEnvelope()); - doReturn(id).when(sub).getId(); - submissionSet1.add(sub); - }); - submissionSet1.forEach(project1::addToSubmissionEnvelopes); - - //and: - project2 = spy(new Project(null)); - doReturn("project2").when(project2).getId(); - project2.setUuid(project1.getUuid()); - - submissionSet2 = new HashSet<>(); - IntStream.range(10, 15).mapToObj(Integer::toString).forEach(id -> { - var sub = spy(new SubmissionEnvelope()); - doReturn(id).when(sub).getId(); - submissionSet2.add(sub); - }); - submissionSet2.forEach(project2::addToSubmissionEnvelopes); - } - - @Test - @DisplayName("get all submissions") - void getFromAllCopiesOfProjects() { - // given - when(projectRepository.findByUuid(project1.getUuid())).thenReturn(Stream.of(project1, project2)); - - // when: - var submissionEnvelopes = projectService.getSubmissionEnvelopes(project1); - - //then: - assertThat(submissionEnvelopes) - .containsAll(submissionSet1) - .containsAll(submissionSet2); - } - - @Test - @DisplayName("no duplicate submissions") - void getFromAllCopiesOfProjectsNoDuplicates() { - // given - var project3 = spy(new Project(null)); - doReturn("project3").when(project3).getId(); - project3.setUuid(project1.getUuid()); - - submissionSet1.forEach(project3::addToSubmissionEnvelopes); - - var documentIds = new ArrayList(); - submissionSet1.forEach(submission -> documentIds.add(submission.getId())); - submissionSet2.forEach(submission -> documentIds.add(submission.getId())); - - //and: - when(projectRepository.findByUuid(project1.getUuid())).thenReturn(Stream.of(project1, project2, project3)); - - //when: - var submissionEnvelopes = projectService.getSubmissionEnvelopes(project1); - - //then: - var returnDocumentIds = new ArrayList(); - submissionEnvelopes.forEach(submission -> returnDocumentIds.add(submission.getId())); - - assertThat(returnDocumentIds).containsExactlyInAnyOrderElementsOf(documentIds); - } - - } - - @Nested - class Registration { - - @Test - @DisplayName("success") - void succeed() { - //given: - String content = "{\"name\": \"project\"}"; - Project project = new Project(content); - - //and: - Project persistentProject = new Project(content); - doReturn(persistentProject).when(projectRepository).save(project); - - //when: - Project result = projectService.register(project); - - //then: - verify(projectRepository).save(project); - assertThat(result).isEqualTo(persistentProject); - verify(projectEventHandler).registeredProject(persistentProject); - } - - } - - @Nested - class Deletion { - - @Test - @DisplayName("success") - void succeedForEmptyProject() throws Exception { - //given: - var project = new Project("{\"name\": \"test\"}"); - - //and: - var persistentProjects = IntStream.range(0, 3) - .mapToObj(number -> new Project(null)) - .collect(toList()); - persistentProjects.forEach(persistentProject -> - BeanUtils.copyProperties(project, persistentProject) - ); - doReturn(persistentProjects.stream()) - .when(projectRepository).findByUuid(project.getUuid()); - - //when: - projectService.delete(project); - - //then: - persistentProjects.forEach(persistentProject -> { - verify(metadataCrudService).deleteDocument(persistentProject); - verify(projectEventHandler).deletedProject(persistentProject); - }); - } - - @Test - @DisplayName("fails for non-empty Project") - void failForProjectWithSubmissions() { - //given: - var project = new Project(null); - - //and: copy of project with no submissions - var persistentEmptyProject = new Project(null); - BeanUtils.copyProperties(project, persistentEmptyProject); - - //and: copy of project with submissions - var persistentNonEmptyProject = new Project(null); - BeanUtils.copyProperties(project, persistentNonEmptyProject); - IntStream.range(0, 3) - .mapToObj(Integer::toString) - .forEach(id -> { - SubmissionEnvelope sub = spy(new SubmissionEnvelope()); - doReturn(id).when(sub).getId(); - persistentNonEmptyProject.addToSubmissionEnvelopes(sub); - }); - - //and: - doReturn(Stream.of(persistentEmptyProject, persistentNonEmptyProject)) - .when(projectRepository).findByUuid(project.getUuid()); - - //expect: - Assertions.assertThatThrownBy(() -> - projectService.delete(project) - ).isInstanceOf(NonEmptyProject.class); - } - - } - - @Nested - class SuggestedProject { - - @Test - @DisplayName("Register a project") - void givenSuggestionCreatesProjectSuccessfully() { - //given: - ObjectMapper mapper = new ObjectMapper(); - ObjectNode suggestion = mapper.createObjectNode(); - suggestion.put("doi", "doi123"); - suggestion.put("name", "Test User"); - suggestion.put("email", "test@example.com"); - suggestion.put("comments", "This is a comment"); - - //and: - final String highLevelEntity = "type"; - Schema projectSchema = new Schema(highLevelEntity, "2.0","project", "project", "project", "mock.io/mock-schema-project"); - doReturn(projectSchema).when(schemaService).getLatestSchemaByEntityType(highLevelEntity, "project"); - - Map content = new HashMap<>(); - final String entityType = "project"; - content.put("describedBy", schemaService.getLatestSchemaByEntityType(highLevelEntity, entityType).getSchemaUri()); - content.put("schema_type", entityType); - - Project persistentProject = new Project(content); - persistentProject.setWranglingState(WranglingState.NEW_SUGGESTION); - persistentProject.setWranglingNotes( - String.format( - "DOI: %s \nName: %s \nEmail: %s \nComments: %s", - suggestion.get("doi"), - suggestion.get("name"), - suggestion.get("email"), - suggestion.get("comments") - ) - ); - doReturn(persistentProject).when(projectRepository).save(any(Project.class)); - - - //when: - Project result = projectService.createSuggestedProject(suggestion); - - //then: - assertThat(result).isEqualTo(persistentProject); - } - } - - @Nested - class ProjectUpdate { - - @Test - @DisplayName("state update adds a history entry") - void statusUpdatesAddsHistoryRecord() { - // given - Project project = new Project(null); - - // when - projectService.updateWranglingState(project, ELIGIBLE); - - // then - verify(auditEntryService) - .addAuditEntry(new AuditEntry(STATUS_UPDATED, - any(), - ELIGIBLE, - project)); - } - } -} diff --git a/src/test/java/org/humancellatlas/ingest/project/ProjectTest.java b/src/test/java/org/humancellatlas/ingest/project/ProjectTest.java deleted file mode 100644 index cfb86a063..000000000 --- a/src/test/java/org/humancellatlas/ingest/project/ProjectTest.java +++ /dev/null @@ -1,84 +0,0 @@ -package org.humancellatlas.ingest.project; - -import org.humancellatlas.ingest.state.SubmissionState; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; - -public class ProjectTest { - @Test - public void testGetOpenSubmissionEnvelopes() { - //given: - SubmissionEnvelope openSubmissionEnvelope = new SubmissionEnvelope(); - openSubmissionEnvelope.enactStateTransition(SubmissionState.DRAFT); - - SubmissionEnvelope openSubmissionEnvelope2 = new SubmissionEnvelope(); - openSubmissionEnvelope2.enactStateTransition(SubmissionState.DRAFT); - openSubmissionEnvelope2.enactStateTransition(SubmissionState.METADATA_VALID); - openSubmissionEnvelope2.enactStateTransition(SubmissionState.SUBMITTED); - - - Project project = new Project(null); - project.addToSubmissionEnvelopes(openSubmissionEnvelope); - project.addToSubmissionEnvelopes(openSubmissionEnvelope2); - - //when: - List openSubmissionEnvelopes = project.getOpenSubmissionEnvelopes(); - - //then: - assertThat(openSubmissionEnvelopes).hasSize(1); - } - - @Test - public void testGetOpenSubmissionEnvelopesNone() { - //given: - SubmissionEnvelope completeSubmission = new SubmissionEnvelope(); - completeSubmission.enactStateTransition(SubmissionState.DRAFT); - completeSubmission.enactStateTransition(SubmissionState.METADATA_VALID); - completeSubmission.enactStateTransition(SubmissionState.SUBMITTED); - completeSubmission.enactStateTransition(SubmissionState.PROCESSING); - completeSubmission.enactStateTransition(SubmissionState.COMPLETE); - - SubmissionEnvelope submittedSubmission = new SubmissionEnvelope(); - submittedSubmission.enactStateTransition(SubmissionState.DRAFT); - submittedSubmission.enactStateTransition(SubmissionState.METADATA_VALID); - submittedSubmission.enactStateTransition(SubmissionState.SUBMITTED); - - Project project = new Project(null); - project.addToSubmissionEnvelopes(submittedSubmission); - project.addToSubmissionEnvelopes(completeSubmission); - - //when: - List openSubmissionEnvelopes = project.getOpenSubmissionEnvelopes(); - - //then: - assertThat(openSubmissionEnvelopes).hasSize(0); - } - - @Test - public void testIsEditable() { - Project project = new Project(null); - assertThat(project.isEditable()).isTrue(); - - SubmissionEnvelope submissionOne = new SubmissionEnvelope(); - submissionOne.enactStateTransition(SubmissionState.METADATA_VALID); - SubmissionEnvelope submissionTwo = new SubmissionEnvelope(); - submissionTwo.enactStateTransition(SubmissionState.METADATA_INVALID); - project.addToSubmissionEnvelopes(submissionOne); - project.addToSubmissionEnvelopes(submissionTwo); - - assertThat(project.isEditable()).isTrue(); - - submissionOne.enactStateTransition(SubmissionState.GRAPH_VALIDATION_REQUESTED); - assertThat(project.isEditable()).isFalse(); - - submissionOne.enactStateTransition(SubmissionState.GRAPH_VALID); - assertThat(project.isEditable()).isTrue(); - - submissionTwo.enactStateTransition(SubmissionState.GRAPH_VALIDATION_REQUESTED); - assertThat(project.isEditable()).isFalse(); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/project/WranglingStateTest.java b/src/test/java/org/humancellatlas/ingest/project/WranglingStateTest.java deleted file mode 100644 index b4add5f34..000000000 --- a/src/test/java/org/humancellatlas/ingest/project/WranglingStateTest.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.humancellatlas.ingest.project; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.humancellatlas.ingest.project.WranglingState; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.json.JsonTest; - -import java.io.IOException; - -import static junit.framework.TestCase.assertEquals; - -@JsonTest -public class WranglingStateTest { - @Autowired - private ObjectMapper objectMapper; - - @Test - public void testDeserialize() throws IOException { - assertEquals(objectMapper.readValue("\"New\"", WranglingState.class), WranglingState.NEW); - assertEquals(objectMapper.readValue("\"Eligible\"", WranglingState.class), WranglingState.ELIGIBLE); - assertEquals(objectMapper.readValue("\"Not eligible\"", WranglingState.class), WranglingState.NOT_ELIGIBLE); - assertEquals(objectMapper.readValue("\"In progress\"", WranglingState.class), WranglingState.IN_PROGRESS); - assertEquals(objectMapper.readValue("\"Stalled\"", WranglingState.class), WranglingState.STALLED); - assertEquals(objectMapper.readValue("\"Submitted\"", WranglingState.class), WranglingState.SUBMITTED); - assertEquals(objectMapper.readValue("\"Published in DCP\"", WranglingState.class), WranglingState.PUBLISHED_IN_DCP); - assertEquals(objectMapper.readValue("\"Deleted\"", WranglingState.class), WranglingState.DELETED); - } - - @Test - public void testSerialize() throws JsonProcessingException { - assertEquals(objectMapper.writeValueAsString(WranglingState.NEW),"\"New\""); - assertEquals(objectMapper.writeValueAsString(WranglingState.ELIGIBLE),"\"Eligible\""); - assertEquals(objectMapper.writeValueAsString(WranglingState.NOT_ELIGIBLE),"\"Not eligible\""); - assertEquals(objectMapper.writeValueAsString(WranglingState.IN_PROGRESS),"\"In progress\""); - assertEquals(objectMapper.writeValueAsString(WranglingState.STALLED),"\"Stalled\""); - assertEquals(objectMapper.writeValueAsString(WranglingState.PUBLISHED_IN_DCP),"\"Published in DCP\""); - assertEquals(objectMapper.writeValueAsString(WranglingState.DELETED),"\"Deleted\""); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/protocol/ProtocolServiceTest.java b/src/test/java/org/humancellatlas/ingest/protocol/ProtocolServiceTest.java deleted file mode 100644 index 4f6ee7344..000000000 --- a/src/test/java/org/humancellatlas/ingest/protocol/ProtocolServiceTest.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.humancellatlas.ingest.protocol; - -import org.humancellatlas.ingest.core.service.MetadataCrudService; -import org.humancellatlas.ingest.core.service.MetadataUpdateService; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageImpl; -import org.springframework.data.domain.Pageable; - -import java.util.Optional; - -import static java.util.Arrays.asList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; - -@SpringBootTest(classes=ProtocolService.class) -public class ProtocolServiceTest { - - @Autowired - private ProtocolService protocolService; - - @MockBean - private MetadataCrudService metadataCrudService; - - @MockBean - private MetadataUpdateService metadataUpdateService; - - @MockBean - private ProtocolRepository protocolRepository; - - @MockBean - private ProcessRepository processRepository; - - @MockBean - private ProjectRepository projectRepository; - - @MockBean - private SubmissionEnvelopeRepository submissionEnvelopeRepository; - - @Nested - class Submission { - - @Test - void determineLinking() { - //given: - SubmissionEnvelope submission = new SubmissionEnvelope(); - - //and: - Protocol linked = new Protocol("linked"); - Protocol notLinked = new Protocol("not linked"); - Pageable pageable = mock(Pageable.class); - doReturn(new PageImpl(asList(linked, notLinked))).when(protocolRepository) - .findBySubmissionEnvelope(submission, pageable); - - //and: - doReturn(Optional.of(new Process(null))).when(processRepository).findFirstByProtocolsContains(linked); - doReturn(Optional.empty()).when(processRepository).findFirstByProtocolsContains(notLinked); - - //when: - Page results = protocolService.retrieve(submission, pageable); - - //then: - assertThat(results).isNotNull(); - assertThat(results.getTotalElements()).isEqualTo(2); - - //and: - assertThat(linked.isLinked()).isTrue(); - assertThat(notLinked.isLinked()).isFalse(); - } - - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/query/QueryBuilderTest.java b/src/test/java/org/humancellatlas/ingest/query/QueryBuilderTest.java deleted file mode 100644 index b95ee21d6..000000000 --- a/src/test/java/org/humancellatlas/ingest/query/QueryBuilderTest.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.humancellatlas.ingest.query; - -import org.junit.jupiter.api.Test; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; - -import java.util.ArrayList; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; - -public class QueryBuilderTest { - @Test - public void testBuildOr() { - //given: - QueryBuilder queryBuilder = new QueryBuilder(); - - List metadataCriteriaList = new ArrayList<>(); - metadataCriteriaList.add(new MetadataCriteria("field", Operator.IS, "value")); - Query expectedQuery = new Query(); - Criteria criteria = Criteria.where("field").is("value"); - expectedQuery.addCriteria(new Criteria().orOperator(new Criteria[]{criteria})); - - //when: - Query actualQuery = queryBuilder.build(metadataCriteriaList, false); - - //then: - assertThat(actualQuery).isEqualTo(expectedQuery); - - } - - @Test - public void testBuildAnd() { - //given: - QueryBuilder queryBuilder = new QueryBuilder(); - - List metadataCriteriaList = new ArrayList<>(); - metadataCriteriaList.add(new MetadataCriteria("field", Operator.IS, "value")); - Query expectedQuery = new Query(); - Criteria criteria = Criteria.where("field").is("value"); - expectedQuery.addCriteria(new Criteria().andOperator(new Criteria[]{criteria})); - - //when: - Query actualQuery = queryBuilder.build(metadataCriteriaList, true); - - //then: - assertThat(actualQuery).isEqualTo(expectedQuery); - - } -} \ No newline at end of file diff --git a/src/test/java/org/humancellatlas/ingest/repository/ProcessRepositoryTest.java b/src/test/java/org/humancellatlas/ingest/repository/ProcessRepositoryTest.java deleted file mode 100644 index 2e7e1cae8..000000000 --- a/src/test/java/org/humancellatlas/ingest/repository/ProcessRepositoryTest.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.humancellatlas.ingest.repository; - -/** - * Created by rolando on 16/02/2018. - */ -public class ProcessRepositoryTest { - -} diff --git a/src/test/java/org/humancellatlas/ingest/schemas/SchemaServiceTest.java b/src/test/java/org/humancellatlas/ingest/schemas/SchemaServiceTest.java deleted file mode 100644 index 5f036c10e..000000000 --- a/src/test/java/org/humancellatlas/ingest/schemas/SchemaServiceTest.java +++ /dev/null @@ -1,136 +0,0 @@ -package org.humancellatlas.ingest.schemas; - -import org.humancellatlas.ingest.schemas.schemascraper.SchemaScraper; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.util.List; - -import static java.util.Arrays.asList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.doReturn; - -@ExtendWith(SpringExtension.class) -public class SchemaServiceTest { - - @Autowired - private SchemaService schemaService; - - @MockBean - private SchemaRepository schemaRepository; - - @MockBean - private SchemaScraper schemaScraper; - - @Test - public void testGetLatestSchemasVersions() { - //given: - Schema version1_2_3 = createTestSchema("1.2.3", "process_core"); - Schema version1_2_4 = createTestSchema("1.2.4", "process_core"); - Schema version2_0 = createTestSchema("2.0", "process_core"); - Schema version11_1_1 = createTestSchema("11.1.1", "process_core"); - - //and: - List schemas = asList(version2_0, version1_2_3, version11_1_1, version1_2_4); - doReturn(schemas).when(schemaRepository).findAll(); - - //when: - List latestSchemas = schemaService.getLatestSchemas(); - - //then: - assertThat(latestSchemas).hasSize(1); - Schema latestSchema = latestSchemas.get(0); - assertThat(latestSchema.getSchemaVersion()).isEqualTo(version11_1_1.getSchemaVersion()); - } - - @Test - public void testGetLatestSchemasCount() { - //given: - Schema version1_2_3 = createTestSchema("1.2.3", "process_core"); - Schema version1_2_4 = createTestSchema("1.2.4", "process_core"); - Schema version2_0 = createTestSchema("2.0", "process_core"); - Schema version11_1_1 = createTestSchema("11.1.1", "process_core"); - Schema protocol_version1_2_3 = createTestSchema("1.2.3", "protocol"); - Schema biomaterial_version1_2_4 = createTestSchema("1.2.4", "biomaterial"); - Schema project_version2_0 = createTestSchema("2.0", "project"); - Schema project_version11_1_1 = createTestSchema("11.1.1", "project"); - - //and: - List schemas = asList(version2_0, version1_2_3, version11_1_1, version1_2_4, - protocol_version1_2_3, biomaterial_version1_2_4, project_version2_0, project_version11_1_1); - doReturn(schemas).when(schemaRepository).findAll(); - - //when: - List latestSchemas = schemaService.getLatestSchemas(); - - //then: - assertThat(latestSchemas).hasSize(4); - } - - - @Test - public void testGetLatestSchemaByEntityTypeWithExistingType() { - //given: - final String projectEntityType = "project"; - final String oldSchemaVersion = "2.0"; - final String newSchemaVersion = "11.1.1"; - Schema protocol_version1_2_3 = createTestSchema(oldSchemaVersion, "protocol"); - Schema biomaterial_version1_2_4 = createTestSchema(newSchemaVersion, "biomaterial"); - Schema project_version2_0 = createTestSchema(oldSchemaVersion, projectEntityType); - Schema project_version11_1_1 = createTestSchema(newSchemaVersion, projectEntityType); - - //and: - List schemas = asList(project_version2_0, protocol_version1_2_3, project_version11_1_1, biomaterial_version1_2_4); - doReturn(schemas).when(schemaRepository).findAll(); - - //when: - Schema latestSchema = schemaService.getLatestSchemaByEntityType("type", projectEntityType); - - //then: - assertThat(latestSchema.getConcreteEntity()).isEqualTo(projectEntityType); - assertThat(latestSchema.getSchemaVersion()).isEqualTo(newSchemaVersion); - } - - @Test - public void testGetLatestSchemaByEntityTypeWithNonExistingType() { - //given: - final String projectEntityType = "project"; - final String oldSchemaVersion = "2.0"; - final String newSchemaVersion = "11.1.1"; - Schema protocol_version1_2_3 = createTestSchema(oldSchemaVersion, "protocol"); - Schema biomaterial_version1_2_4 = createTestSchema(newSchemaVersion, "biomaterial"); - Schema project_version2_0 = createTestSchema(oldSchemaVersion, projectEntityType); - Schema project_version11_1_1 = createTestSchema(newSchemaVersion, projectEntityType); - - //and: - List schemas = asList(project_version2_0, protocol_version1_2_3, project_version11_1_1, biomaterial_version1_2_4); - doReturn(schemas).when(schemaRepository).findAll(); - - //when: - Schema latestSchema = schemaService.getLatestSchemaByEntityType("type", "non_exists"); - - //then: - assertThat(latestSchema).isNull(); - } - - private Schema createTestSchema(String schemaVersion, String entityType) { - return new Schema("type", schemaVersion, entityType, entityType, entityType, - "http://schema.humancellatlas.org"); - } - - @Configuration - static class TestConfiguration { - - @Bean - SchemaService schemaService() { - return new SchemaService(); - } - - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/schemas/SchemaTest.java b/src/test/java/org/humancellatlas/ingest/schemas/SchemaTest.java deleted file mode 100644 index c69f11a2b..000000000 --- a/src/test/java/org/humancellatlas/ingest/schemas/SchemaTest.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.humancellatlas.ingest.schemas; - - -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assumptions.assumeThat; - -public class SchemaTest { - - @Test - public void testCompareToSameVersion() { - //given: - Schema schema = createTestSchema("7.3.1"); - - //expect: - assertThat(schema.compareTo(schema)).isEqualTo(0); - } - - @Test - public void testCompareToOlderVersion() { - //given: - Schema schemaVersion10 = createTestSchema("10.9"); - Schema schemaVersion7 = createTestSchema("7.3.1"); - Schema schemaVersion6_7 = createTestSchema("6.7.3"); - Schema schemaVersion6_3 = createTestSchema("6.3.11"); - - //expect: - assertThat(schemaVersion10.compareTo(schemaVersion7)).isGreaterThan(0); - assertThat(schemaVersion7.compareTo(schemaVersion6_7)).isGreaterThan(0); - assertThat(schemaVersion6_7.compareTo(schemaVersion6_3)).isGreaterThan(0); - } - - @Test - public void testCompareToNewerVersion() { - //given: - Schema schemaVersion5 = createTestSchema("5"); - Schema schemaVersion5_1 = createTestSchema("5.1"); - Schema schemaVersion5_1_3 = createTestSchema("5.1.3"); - - //expect: - assertThat(schemaVersion5.compareTo(schemaVersion5_1)).isLessThan(0); - assertThat(schemaVersion5_1.compareTo(schemaVersion5_1_3)).isLessThan(0); - assertThat(schemaVersion5.compareTo(schemaVersion5_1_3)).isLessThan(0); - } - - @Test - public void testCompareDifferentSchemas() { - //given: - Schema biomaterialCore = new Schema("core", "5.9.10", "biomaterial", "", - "biomaterial_core", "http://schema.humancellatlas.org"); - Schema processCore = createTestSchema("7.4.3"); - assumeThat(biomaterialCore.getConcreteEntity()) - .isNotEqualToIgnoringCase(processCore.getConcreteEntity()); - - //expect: - assertThat(biomaterialCore.compareTo(processCore)).isLessThan(0); - assertThat(processCore.compareTo(biomaterialCore)).isGreaterThan(0); - } - - private Schema createTestSchema(String schemaVersion) { - return new Schema("core", schemaVersion, "process", "", "process_core", - "http://schema.humancellatlas.org"); - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/security/AccountServiceTest.java b/src/test/java/org/humancellatlas/ingest/security/AccountServiceTest.java deleted file mode 100644 index cc4471299..000000000 --- a/src/test/java/org/humancellatlas/ingest/security/AccountServiceTest.java +++ /dev/null @@ -1,85 +0,0 @@ -package org.humancellatlas.ingest.security; - - -import org.humancellatlas.ingest.security.exception.DuplicateAccount; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.assertj.core.api.Assumptions.assumeThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.verify; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = {DefaultAccountService.class}) -public class AccountServiceTest { - - @Autowired - private AccountService accountService; - - @MockBean - private AccountRepository accountRepository; - - @Nested - @DisplayName("Registration") - class Registration { - - @Test - void success() { - //given: - String providerReference = "67fe90"; - Account account = new Account(providerReference); - assumeThat(account.getRoles()).isEmpty(); - - //and: - String name = "Juan dela Cruz"; - account.setName(name); - - //and: - Account persistentAccount = new Account("773b471", providerReference); - persistentAccount.setName(name); - doReturn(persistentAccount).when(accountRepository).save(any(Account.class)); - - //when: - Account result = accountService.register(account); - - //then: - assertThat(result).isEqualTo(persistentAccount); - var accountCaptor = ArgumentCaptor.forClass(Account.class); - verify(accountRepository).save(accountCaptor.capture()); - - //and: - var savedAccount = accountCaptor.getValue(); - assertThat(savedAccount) - .extracting(Account::getProviderReference, Account::getName) - .containsExactly(providerReference, name); - assertThat(savedAccount.getRoles()).containsOnly(Role.CONTRIBUTOR); - } - - @Test - void duplicateAccount() { - //given: - String providerReference = "84cd01b"; - Account account = new Account(providerReference); - - //and: - Account persistentAccount = new Account("72b1c9e", providerReference); - doReturn(persistentAccount).when(accountRepository).findByProviderReference(providerReference); - - //expect: - assertThatThrownBy(() -> { - accountService.register(account); - }).isInstanceOf(DuplicateAccount.class); - } - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/security/AccountTest.java b/src/test/java/org/humancellatlas/ingest/security/AccountTest.java deleted file mode 100644 index ef6a89cd0..000000000 --- a/src/test/java/org/humancellatlas/ingest/security/AccountTest.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.humancellatlas.ingest.security; - -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -public class AccountTest { - - @Nested - class Guest { - - @Test - void ensureNoNullFields() { - //expect: - assertThat(Account.GUEST).hasNoNullFieldsOrProperties(); - } - - @Test - void ensureGuestRole() { - //expect: - assertThat(Account.GUEST.getRoles()).containsOnly(Role.GUEST); - } - - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/security/JwtGenerator.java b/src/test/java/org/humancellatlas/ingest/security/JwtGenerator.java deleted file mode 100644 index 0c8dca621..000000000 --- a/src/test/java/org/humancellatlas/ingest/security/JwtGenerator.java +++ /dev/null @@ -1,105 +0,0 @@ -package org.humancellatlas.ingest.security; - -import com.auth0.jwt.JWT; -import com.auth0.jwt.JWTCreator; -import com.auth0.jwt.algorithms.Algorithm; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.humancellatlas.ingest.security.authn.oidc.UserInfo; - -import javax.annotation.Nullable; -import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.security.NoSuchAlgorithmException; -import java.security.interfaces.RSAPrivateKey; -import java.security.interfaces.RSAPublicKey; -import java.util.*; - -import static java.util.Map.entry; - -public class JwtGenerator { - - public static final String DEFAULT_ISSUER = "https://humancellatlas.auth0.com"; - public static final String DEFAULT_KEY_ID = "MDc2OTM3ODI4ODY2NUU5REVGRDVEM0MyOEYwQTkzNDZDRDlEQzNBRQ"; - - public static final String OIDC_ISS = "iss"; - public static final String OIDC_SUB = "sub"; - - private final ObjectMapper objectMapper = new ObjectMapper(); - - private final KeyPair keyPair; - - private final String issuer; - - public JwtGenerator() { - this(DEFAULT_ISSUER); - } - - /** - * Creates an instance with a pre-defined default issuer. This issuer >will be overridden by "iss" claim - * during when generating the JWT if it is set. - * - * @param issuer - */ - //The decision to allow "iss" claim to override the issuer field is so that UserInfo can be encoded with little - //modifications to this utility class. - public JwtGenerator(@Nullable String issuer) { - KeyPairGenerator keyGenerator = getKeyPairGenerator(); - this.keyPair = keyGenerator.generateKeyPair(); - this.issuer = Optional.ofNullable(issuer).orElse(DEFAULT_ISSUER); - } - - private KeyPairGenerator getKeyPairGenerator() { - try { - KeyPairGenerator keyGenerator = KeyPairGenerator.getInstance("RSA"); - keyGenerator.initialize(2048); - return keyGenerator; - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - } - - - public RSAPublicKey getPublicKey() { - return (RSAPublicKey) keyPair.getPublic(); - } - - public String generate() { - return generate(null, null, null); - } - - public String generate(Map claims) { - return generate(null, null, claims); - } - - public String generate(@Nullable String keyId, @Nullable String subject, @Nullable Map claims) { - var kid = Optional.ofNullable(keyId); - Map header = Map.ofEntries(entry("kid", kid.orElse(DEFAULT_KEY_ID))); - - Map allClaims = new HashMap<>(); - Optional.ofNullable(claims).ifPresent(allClaims::putAll); - JWTCreator.Builder builder = JWT.create() - .withHeader(header) - .withIssuer(Optional.ofNullable(allClaims.get(OIDC_ISS)) - .orElse(issuer)) - .withSubject(Optional.ofNullable(subject) - .or(() -> Optional.ofNullable(allClaims.get(OIDC_SUB))) - .orElse(UUID.randomUUID().toString())); - - Arrays.asList(OIDC_ISS, OIDC_SUB).forEach(allClaims::remove); - allClaims.forEach(builder::withClaim); - - var rsa256 = Algorithm.RSA256(null, (RSAPrivateKey) keyPair.getPrivate()); - return builder.sign(rsa256); - } - - public String generateWithSubject(String subject) { - return generate(null, subject, null); - } - - public String encode(UserInfo userInfo) { - var claims = objectMapper.convertValue(userInfo, new TypeReference>() {}); - return generate(null, null, claims); - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/security/JwtVerifierResolverTest.java b/src/test/java/org/humancellatlas/ingest/security/JwtVerifierResolverTest.java deleted file mode 100644 index bbc225c22..000000000 --- a/src/test/java/org/humancellatlas/ingest/security/JwtVerifierResolverTest.java +++ /dev/null @@ -1,182 +0,0 @@ -package org.humancellatlas.ingest.security; - -import com.auth0.jwt.JWT; -import com.auth0.jwt.interfaces.DecodedJWT; -import com.auth0.jwt.interfaces.JWTVerifier; -import org.humancellatlas.ingest.security.authn.provider.gcp.GcpJwkVault; -import org.humancellatlas.ingest.security.common.jwk.DelegatingJwtVerifier; -import org.humancellatlas.ingest.security.common.jwk.JwtVerifierResolver; -import org.junit.jupiter.api.Test; - -import java.security.interfaces.RSAPublicKey; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; - -public class JwtVerifierResolverTest { - - @Test - public void testResolveForJwt() { - //given: - JwtGenerator jwtGenerator = new JwtGenerator("issuerFromToken"); - RSAPublicKey publicKey = jwtGenerator.getPublicKey(); - - //and: - String audience = "https://dev.data.humancellatlas.org/"; - GcpJwkVault jwkVault = mock(GcpJwkVault.class); - doReturn(publicKey).when(jwkVault).getPublicKey(any(DecodedJWT.class)); - - //and: - JwtVerifierResolver jwtVerifierResolver = new JwtVerifierResolver(jwkVault, audience, null); - - //and: given the token - String jwt = jwtGenerator.generate(); - DecodedJWT token = JWT.decode(jwt); - - //when: - JWTVerifier verifier = jwtVerifierResolver.resolve(jwt); - - //then: - assertThat(verifier).isNotNull(); - - //and: inspect using verifier with extended interface as a work around - assertThat(verifier).isInstanceOf(DelegatingJwtVerifier.class); - DelegatingJwtVerifier delegatingVerifier = (DelegatingJwtVerifier) verifier; - assertThat(delegatingVerifier) - .extracting("audience", "issuer") - .containsExactly(audience, token.getIssuer()); - } - - @Test - public void testResolveForJwtWithIssuer() { - //given: - String issuerFromToken = "issuerFromToken"; - JwtGenerator jwtGenerator = new JwtGenerator(issuerFromToken); - RSAPublicKey publicKey = jwtGenerator.getPublicKey(); - - //and: - String audience = "https://dev.data.humancellatlas.org/"; - String issuer = "auth0"; - GcpJwkVault jwkVault = mock(GcpJwkVault.class); - doReturn(publicKey).when(jwkVault).getPublicKey(any(DecodedJWT.class)); - - //and: - JwtVerifierResolver jwtVerifierResolver = new JwtVerifierResolver(jwkVault, audience, issuer); - - //and: given the token - String jwt = jwtGenerator.generate(); - - //when: - JWTVerifier verifier = jwtVerifierResolver.resolve(jwt); - - //then: - assertThat(verifier).isNotNull(); - - //and: inspect using verifier with extended interface as a work around - assertThat(verifier).isInstanceOf(DelegatingJwtVerifier.class); - DelegatingJwtVerifier delegatingVerifier = (DelegatingJwtVerifier) verifier; - assertThat(delegatingVerifier) - .extracting("audience", "issuer") - .containsExactly(audience, issuer); - } - - @Test - public void testResolveForJwtWithNoAudience() { - //given: - JwtGenerator jwtGenerator = new JwtGenerator(); - RSAPublicKey publicKey = jwtGenerator.getPublicKey(); - - //and: - String issuer = "auth0"; - GcpJwkVault jwkVault = mock(GcpJwkVault.class); - doReturn(publicKey).when(jwkVault).getPublicKey(any(DecodedJWT.class)); - - //and: - JwtVerifierResolver jwtVerifierResolver = new JwtVerifierResolver(jwkVault, null, issuer); - - //and: given the token - String jwt = jwtGenerator.generate(); - - //when: - JWTVerifier verifier = jwtVerifierResolver.resolve(jwt); - - //then: - assertThat(verifier).isNotNull(); - - //and: inspect using verifier with extended interface as a work around - assertThat(verifier).isInstanceOf(DelegatingJwtVerifier.class); - DelegatingJwtVerifier delegatingVerifier = (DelegatingJwtVerifier) verifier; - assertThat(delegatingVerifier) - .extracting("audience", "issuer") - .containsExactly(null, issuer); - } - - - @Test - public void testResolveForJwtWithNoAudienceAndNoIssuer() { - //given: - String issuerFromToken = "issuerFromToken"; - JwtGenerator jwtGenerator = new JwtGenerator(issuerFromToken); - RSAPublicKey publicKey = jwtGenerator.getPublicKey(); - - //and: - String issuer = "auth0"; - GcpJwkVault jwkVault = mock(GcpJwkVault.class); - doReturn(publicKey).when(jwkVault).getPublicKey(any(DecodedJWT.class)); - - //and: - JwtVerifierResolver jwtVerifierResolver = new JwtVerifierResolver(jwkVault, null, null); - - //and: given the token - String jwt = jwtGenerator.generate(); - DecodedJWT token = JWT.decode(jwt); - - //when: - JWTVerifier verifier = jwtVerifierResolver.resolve(jwt); - - //then: - assertThat(verifier).isNotNull(); - - //and: inspect using verifier with extended interface as a work around - assertThat(verifier).isInstanceOf(DelegatingJwtVerifier.class); - DelegatingJwtVerifier delegatingVerifier = (DelegatingJwtVerifier) verifier; - assertThat(delegatingVerifier) - .extracting("audience", "issuer") - .containsExactly(null, token.getIssuer()); - } - - @Test - public void testResolveForJwtWithAudienceAndNoIssuer() { - //given: - JwtGenerator jwtGenerator = new JwtGenerator("issuerFromToken"); - RSAPublicKey publicKey = jwtGenerator.getPublicKey(); - - //and: - String audience = "https://dev.data.humancellatlas.org/"; - GcpJwkVault jwkVault = mock(GcpJwkVault.class); - doReturn(publicKey).when(jwkVault).getPublicKey(any(DecodedJWT.class)); - - //and: - JwtVerifierResolver jwtVerifierResolver = new JwtVerifierResolver(jwkVault, audience, null); - - //and: given the token - String jwt = jwtGenerator.generate(); - DecodedJWT token = JWT.decode(jwt); - - //when: - JWTVerifier verifier = jwtVerifierResolver.resolve(jwt); - - //then: - assertThat(verifier).isNotNull(); - - //and: inspect using verifier with extended interface as a work around - assertThat(verifier).isInstanceOf(DelegatingJwtVerifier.class); - DelegatingJwtVerifier delegatingVerifier = (DelegatingJwtVerifier) verifier; - assertThat(delegatingVerifier) - .extracting("audience", "issuer") - .containsExactly(audience, token.getIssuer()); - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/security/UserAuditingTest.java b/src/test/java/org/humancellatlas/ingest/security/UserAuditingTest.java deleted file mode 100644 index d456af7fc..000000000 --- a/src/test/java/org/humancellatlas/ingest/security/UserAuditingTest.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.humancellatlas.ingest.security; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.context.SecurityContext; -import org.springframework.security.core.context.SecurityContextHolder; -import org.springframework.security.core.context.SecurityContextImpl; - -import static java.util.Arrays.asList; -import static org.assertj.core.api.Java6Assertions.assertThat; -import static org.mockito.Mockito.*; - -public class UserAuditingTest { - - private UserAuditing userAuditing; - - @BeforeEach - void setUp() { - userAuditing = new UserAuditing(); - } - - @Nested - @DisplayName("getCurrentAuditor") - class GetCurrentAuditor { - - @Test - void accountTypePrincipal() { - //given: - String providerReference = "6700ed52"; - Account userAccount = new Account(providerReference, "elixir-1"); - - //and: - Authentication authentication = mock(Authentication.class); - doReturn(userAccount).when(authentication).getPrincipal(); - when(authentication.isAuthenticated()).thenReturn(true); - - //and: - SecurityContext securityContext = new SecurityContextImpl(authentication); - SecurityContextHolder.setContext(securityContext); - - //when: - String auditor = userAuditing.getCurrentAuditor().orElseThrow(); - - //then: - assertThat(auditor).isEqualTo(providerReference); - } - - @Test - void nonAccountTypePrincipal() { - //given: - String principal = "jdelacruz"; - Authentication authentication = new UsernamePasswordAuthenticationToken(principal, "pas$w0rd", - asList(Role.CONTRIBUTOR)); - SecurityContextImpl securityContext = new SecurityContextImpl(authentication); - SecurityContextHolder.setContext(securityContext); - - //when: - String auditor = userAuditing.getCurrentAuditor().orElseThrow(); - - //then: - assertThat(auditor).isEqualTo(principal); - } - - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/security/authn/oidc/OpenIdAuthenticationTest.java b/src/test/java/org/humancellatlas/ingest/security/authn/oidc/OpenIdAuthenticationTest.java deleted file mode 100644 index c221fccb9..000000000 --- a/src/test/java/org/humancellatlas/ingest/security/authn/oidc/OpenIdAuthenticationTest.java +++ /dev/null @@ -1,139 +0,0 @@ -package org.humancellatlas.ingest.security.authn.oidc; - -import org.humancellatlas.ingest.security.Account; -import org.humancellatlas.ingest.security.Role; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.GrantedAuthority; - -import java.util.Collection; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assumptions.assumeThat; - -public class OpenIdAuthenticationTest { - - private final String subjectId = "73985cc"; - private final UserInfo userInfo = new UserInfo(subjectId, ""); - - private Account account; - private Authentication authentication; - - @BeforeEach - void setUp() { - account = new Account(subjectId); - authentication = new OpenIdAuthentication(account); - ((OpenIdAuthentication) authentication).authenticateWith(userInfo); - } - - @Nested - @DisplayName("Authentication") - class AuthenticationTest { - - private OpenIdAuthentication authentication; - - @BeforeEach - void setUp() { - authentication = new OpenIdAuthentication(account); - } - - @Test - public void successful() { - //when: - authentication.authenticateWith(userInfo); - - //expect: - assertThat(authentication.isAuthenticated()).isTrue(); - assertThat(authentication.getCredentials()).isEqualTo(userInfo); - } - - @Test - public void noPrincipalAsGuest() { - //given: - authentication = new OpenIdAuthentication((Account) null); - - //when: - authentication.authenticateWith(userInfo); - - //expect: - assertThat(authentication.isAuthenticated()).isTrue(); - assertThat(authentication.getPrincipal()).isEqualTo(Account.GUEST); - assertThat(authentication.getCredentials()).isEqualTo(userInfo); - } - - @Test - public void nonMatchingSubjectId() { - //given: - String anotherSubjectId = "82909a1"; - UserInfo anotherUserInfo = new UserInfo(anotherSubjectId, ""); - assumeThat(anotherSubjectId).isNotEqualTo(subjectId); - - //when: - authentication.authenticateWith(anotherUserInfo); - - //then: - assertThat(authentication.isAuthenticated()).isFalse(); - assertThat(authentication.getCredentials()).isEqualTo(anotherUserInfo); - } - - @Test - public void authenticatedGuest() { - //given: - var authentication = new OpenIdAuthentication((Account) null); - - //when: - authentication.authenticateWith(userInfo); - - //then: - assertThat(authentication.isAuthenticated()).isTrue(); - assertThat(authentication.getCredentials()).isEqualTo(userInfo); - } - - } - - @Test - public void testGetPrincipal() { - //expect: - assertThat(authentication.getPrincipal()).isEqualTo(account); - } - - @Test - public void testGetCredentials() { - //expect: - assertThat(authentication.getCredentials()).isEqualTo(userInfo); - } - - @Test - public void testGetAuthorities() { - //given: - account.addRole(Role.CONTRIBUTOR); - - //expect: - //this assignment is to work around the weirdness with generic type that I couldn't figure out - Collection authorities = (Collection) authentication.getAuthorities(); - assertThat(authorities).containsExactly(Role.CONTRIBUTOR); - } - - @Test - public void testGetName() { - //expect: - assertThat(authentication.getName()).isEqualTo(subjectId); - } - - @Test - public void testGetDetails() { - //expect: - assertThat(authentication.getDetails()).isEqualTo(userInfo); - } - - @Test - public void ensureNonNullPrincipal() { - //expect: - Authentication authentication = new OpenIdAuthentication((Account) null); - assertThat(authentication.getPrincipal()).isSameAs(Account.GUEST); - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/security/authn/oidc/UserInfoTest.java b/src/test/java/org/humancellatlas/ingest/security/authn/oidc/UserInfoTest.java deleted file mode 100644 index 196125759..000000000 --- a/src/test/java/org/humancellatlas/ingest/security/authn/oidc/UserInfoTest.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.humancellatlas.ingest.security.authn.oidc; - -import org.humancellatlas.ingest.security.Account; -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -public class UserInfoTest { - - @Test - public void convertToAccount() { - //given: - String subjectId = "723b4001"; - String name = "Jean Valjean"; - UserInfo userInfo = new UserInfo(subjectId, name); - - //when: - Account account = userInfo.toAccount(); - - //then: - assertThat(account) - .extracting("providerReference", "name") - .containsExactly(subjectId, name); - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/security/authn/provider/auth0/TestUserWhitelist.java b/src/test/java/org/humancellatlas/ingest/security/authn/provider/auth0/TestUserWhitelist.java deleted file mode 100644 index ba37822ae..000000000 --- a/src/test/java/org/humancellatlas/ingest/security/authn/provider/auth0/TestUserWhitelist.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.humancellatlas.ingest.security.authn.provider.auth0; - -import org.humancellatlas.ingest.security.authn.provider.gcp.GcpDomainWhiteList; -import org.junit.jupiter.api.Test; - -import static java.util.Arrays.asList; -import static org.assertj.core.api.Assertions.assertThat; - -public class TestUserWhitelist { - - @Test - public void testLists() { - //given: - GcpDomainWhiteList userWhiteList = new GcpDomainWhiteList("trusteddomain.com", "friendlypeople.net"); - - //expect: - asList("goodguy@trusteddomain.com", "upstandinglass@friendlypeople.net", "cooldude@friendlypeople.net") - .forEach(email -> assertThat(userWhiteList.lists(email)).isTrue()); - - //and: - asList("maninavan@shadycharacters.tv", "suspicious@darkcorner.xyz") - .forEach(email -> assertThat(userWhiteList.lists(email)).isFalse()); - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/security/authn/provider/auth0/UserJwtAuthenticationProviderTest.java b/src/test/java/org/humancellatlas/ingest/security/authn/provider/auth0/UserJwtAuthenticationProviderTest.java deleted file mode 100644 index b284600ef..000000000 --- a/src/test/java/org/humancellatlas/ingest/security/authn/provider/auth0/UserJwtAuthenticationProviderTest.java +++ /dev/null @@ -1,117 +0,0 @@ -package org.humancellatlas.ingest.security.authn.provider.auth0; - -import com.auth0.spring.security.api.JwtAuthenticationProvider; -import com.auth0.spring.security.api.authentication.PreAuthenticatedAuthenticationJsonWebToken; -import org.humancellatlas.ingest.security.JwtGenerator; -import org.humancellatlas.ingest.security.authn.provider.auth0.UserJwtAuthenticationProvider; -import org.humancellatlas.ingest.security.authn.provider.gcp.GcpDomainWhiteList; -import org.humancellatlas.ingest.security.exception.InvalidUserGroup; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.springframework.security.authentication.AuthenticationProvider; -import org.springframework.security.core.Authentication; - -import java.util.Map; - -import static java.util.Map.entry; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; - -public class UserJwtAuthenticationProviderTest { - - @Nested - @DisplayName("authentication") - class AuthenticationTest { - private JwtGenerator jwtGenerator = new JwtGenerator(); - - @Test - @DisplayName("authentication succeeds") - public void testAuthenticate() { - //given: JWT authentication - String userEmail = "trustedfellow@friendlysite.com"; - Map claims = Map.ofEntries( - entry("https://auth.data.humancellatlas.org/group", "hca") - ); - - String jwt = jwtGenerator.generate(null, userEmail, claims); - Authentication jwtAuthentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); - - //and: - JwtAuthenticationProvider delegate = mock(JwtAuthenticationProvider.class); - doReturn(jwtAuthentication).when(delegate).authenticate(any(Authentication.class)); - - //and: - GcpDomainWhiteList userWhitelist = mock(GcpDomainWhiteList.class); - doReturn(true).when(userWhitelist).lists(anyString()); - - //and: - AuthenticationProvider authenticationProvider = new UserJwtAuthenticationProvider(delegate); - - //when: - Authentication authentication = authenticationProvider.authenticate(jwtAuthentication); - - //then: - assertThat(authentication).extracting("principal").containsExactly(userEmail); - } - - @Test - @DisplayName("no user group") - public void testForNoUserGroup() { - //given: JWT authentication - String userGroup = "null"; - String jwt = jwtGenerator.generateWithSubject(userGroup); - Authentication authentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); - - //and: - JwtAuthenticationProvider delegate = mock(JwtAuthenticationProvider.class); - doReturn(authentication).when(delegate).authenticate(any(Authentication.class)); - - //and: - GcpDomainWhiteList userWhitelist = mock(GcpDomainWhiteList.class); - doReturn(false).when(userWhitelist).lists(anyString()); - - //and: - AuthenticationProvider authenticationProvider = new UserJwtAuthenticationProvider(delegate); - - //expect: - assertThatThrownBy(() -> { - authenticationProvider.authenticate(authentication); - }).isExactlyInstanceOf(InvalidUserGroup.class).hasMessageContaining(userGroup); - } - - @Test - @DisplayName("invalid user group") - public void testForInvalidUserGroup() { - //given: JWT authentication - String userGroup = "public"; - Map claims = Map.ofEntries( - entry("https://auth.data.humancellatlas.org/group", userGroup) - ); - String jwt = jwtGenerator.generate(null, userGroup, claims); - Authentication authentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); - - //and: - JwtAuthenticationProvider delegate = mock(JwtAuthenticationProvider.class); - doReturn(authentication).when(delegate).authenticate(any(Authentication.class)); - - //and: - GcpDomainWhiteList userWhitelist = mock(GcpDomainWhiteList.class); - doReturn(false).when(userWhitelist).lists(anyString()); - - //and: - AuthenticationProvider authenticationProvider = new UserJwtAuthenticationProvider(delegate); - - //expect: - assertThatThrownBy(() -> { - authenticationProvider.authenticate(authentication); - }).isExactlyInstanceOf(InvalidUserGroup.class).hasMessageContaining(userGroup); - } - - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/security/authn/provider/elixir/ElixirAaiAuthenticationProviderTest.java b/src/test/java/org/humancellatlas/ingest/security/authn/provider/elixir/ElixirAaiAuthenticationProviderTest.java deleted file mode 100644 index 4c4f92567..000000000 --- a/src/test/java/org/humancellatlas/ingest/security/authn/provider/elixir/ElixirAaiAuthenticationProviderTest.java +++ /dev/null @@ -1,217 +0,0 @@ -package org.humancellatlas.ingest.security.authn.provider.elixir; - -import com.auth0.jwt.exceptions.JWTVerificationException; -import com.auth0.jwt.interfaces.DecodedJWT; -import com.auth0.jwt.interfaces.JWTVerifier; -import com.auth0.spring.security.api.authentication.PreAuthenticatedAuthenticationJsonWebToken; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import okhttp3.mockwebserver.MockResponse; -import okhttp3.mockwebserver.MockWebServer; -import okhttp3.mockwebserver.RecordedRequest; -import org.humancellatlas.ingest.security.Account; -import org.humancellatlas.ingest.security.AccountRepository; -import org.humancellatlas.ingest.security.JwtGenerator; -import org.humancellatlas.ingest.security.authn.oidc.UserInfo; -import org.humancellatlas.ingest.security.common.jwk.JwtVerifierResolver; -import org.humancellatlas.ingest.security.exception.JwtVerificationFailed; -import org.junit.jupiter.api.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.test.autoconfigure.web.client.AutoConfigureWebClient; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.security.authentication.AuthenticationProvider; -import org.springframework.security.core.Authentication; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.assertj.core.api.Assumptions.assumeThat; -import static org.humancellatlas.ingest.security.ElixirConfig.ELIXIR; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.*; - -@SpringBootTest(classes={ElixirAaiAuthenticationProviderTest.Config.class}) -@AutoConfigureWebClient -public class ElixirAaiAuthenticationProviderTest { - - @Configuration - @Import(ElixirAaiAuthenticationProvider.class) - static class Config {} - - private MockWebServer mockBackEnd; - - @MockBean - private JWTVerifier jwtVerifier; - - @MockBean - @Qualifier(ELIXIR) - private JwtVerifierResolver jwtVerifierResolver; - - @MockBean - private AccountRepository accountRepository; - - @Autowired - private AuthenticationProvider authenticationProvider; - - @Nested - @DisplayName("Authenticate") - class AuthenticationTests { - - private ObjectMapper objectMapper = new ObjectMapper(); - - @BeforeEach - public void setUp() throws Exception { - mockBackEnd = new MockWebServer(); - mockBackEnd.start(); - - doReturn(jwtVerifier).when(jwtVerifierResolver).resolve(anyString()); - String baseUrl = String.format("http://localhost:%s", mockBackEnd.getPort()); - doReturn(baseUrl).when(jwtVerifierResolver).getIssuer(); - } - - @AfterEach - public void tearDown() throws Exception { - mockBackEnd.shutdown(); - } - - @Test - @DisplayName("success") - public void testAuthenticate() throws Exception { - //given: JWT - String subject = "johndoe@elixirdomain.tld"; - UserInfo userInfo = new UserInfo(subject, "name", "pref", "giv", "fam", "email@ebi.ac.uk"); - JwtGenerator jwtGenerator = new JwtGenerator("elixir"); - String jwt = jwtGenerator.encode(userInfo); - - //and: given a JWT Authentication - var jwtAuthentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); - assumeThat(jwtAuthentication).isNotNull(); - - //and: given JWT Verifier will verify token successfully - DecodedJWT token = mock(DecodedJWT.class); - doReturn(jwt).when(token).getToken(); - doReturn(token).when(jwtVerifier).verify(jwtAuthentication.getToken()); - - //and: given account with same provider reference will be found - Account account = new Account("73bbc45", subject); - doReturn(account).when(accountRepository).findByProviderReference(subject); - - //and: Elixir user info will be returned - mockBackEnd.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(userInfo)) - .addHeader("Content-Type", "application/json")); - - //when: - Authentication authentication = authenticationProvider.authenticate(jwtAuthentication); - - //then: - assertThat(authentication).extracting("authenticated", "principal"); - assertThat(authentication.isAuthenticated()).isTrue(); - assertThat(authentication.getPrincipal()).isEqualTo(account); - assertCorrectRequest(jwtAuthentication.getToken()); - } - - @Test - @DisplayName("no account") - public void testForNoAccount() throws Exception { - //given: JWT - String subject = "johndoe@elixirdomain.tld"; - UserInfo userInfo = new UserInfo(subject, "name", "pref", "giv", "fam", "email@ebi.ac.uk"); - String jwt = new JwtGenerator("elixir").encode(userInfo); - - //and: given a JWT Authentication - var jwtAuthentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); - assumeThat(jwtAuthentication).isNotNull(); - - //and: given JWT Verifier will verify token successfully - DecodedJWT token = mock(DecodedJWT.class); - doReturn(jwt).when(token).getToken(); - doReturn(token).when(jwtVerifier).verify(jwtAuthentication.getToken()); - - //and: Elixir user info will be returned - mockBackEnd.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(userInfo)) - .addHeader("Content-Type", "application/json")); - - //and: no matching records in the database - doReturn(null).when(accountRepository).findByProviderReference(anyString()); - - //when: - Authentication authentication = authenticationProvider.authenticate(jwtAuthentication); - - //then: - assertThat(authentication).isNotNull(); - assertThat(authentication.getPrincipal()).isEqualTo(Account.GUEST); - assertCorrectRequest(jwtAuthentication.getToken()); - - //and: - assertThat(authentication.getCredentials()).isInstanceOf(UserInfo.class); - UserInfo credentials = (UserInfo) authentication.getCredentials(); - assertThat(credentials).isEqualToComparingFieldByField(userInfo); - } - - private void assertCorrectRequest(String token) throws InterruptedException { - RecordedRequest request = mockBackEnd.takeRequest(); - assertThat(request.getMethod()).isEqualToIgnoringCase("GET"); - String bearerToken = String.format("Bearer %s", token); - assertThat(request.getHeaders().get("Authorization")).isEqualTo(bearerToken); - } - - @Test - @DisplayName("valid user email") - public void testForValidUserEmail() throws JsonProcessingException { - //given: - UserInfo userInfo = new UserInfo("subject", "name", "pref", "giv", "fam", "email@embl.ac.uk"); - mockBackEnd.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(userInfo)) - .addHeader("Content-Type", "application/json")); - - //and: - JwtGenerator jwtGenerator = new JwtGenerator("elixir"); - String jwt = jwtGenerator.generate(); - var jwtAuthentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); - - DecodedJWT token = mock(DecodedJWT.class); - doReturn(jwt).when(token).getToken(); - doReturn(token).when(jwtVerifier).verify(jwtAuthentication.getToken()); - Account account = mock(Account.class); - doReturn(account).when(accountRepository).findByProviderReference("sub"); - - //when: - Authentication auth = authenticationProvider.authenticate(jwtAuthentication); - - //then: - assertThat(auth).isNotNull(); - } - - @Test - @DisplayName("verification failed") - public void testForFailedVerification() throws JsonProcessingException { - //given: - UserInfo userInfo = new UserInfo("subject", "name", "pref", "giv", "fam", "email@ebi.ac.uk"); - mockBackEnd.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(userInfo)) - .addHeader("Content-Type", "application/json")); - - //and: given a JWT Authentication - JwtGenerator jwtGenerator = new JwtGenerator("elixir"); - String jwt = jwtGenerator.generateWithSubject("sub"); - var jwtAuthentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); - - //and: JWT verifier will fail - Exception verificationFailed = new JWTVerificationException("verification failed"); - doThrow(verificationFailed).when(jwtVerifier).verify(jwtAuthentication.getToken()); - - - //expect: - assertThatThrownBy(() -> { - authenticationProvider.authenticate(jwtAuthentication); - }).isInstanceOf(JwtVerificationFailed.class); - } - - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/security/authn/provider/elixir/ElixirJwkVaultTest.java b/src/test/java/org/humancellatlas/ingest/security/authn/provider/elixir/ElixirJwkVaultTest.java deleted file mode 100644 index acd48855f..000000000 --- a/src/test/java/org/humancellatlas/ingest/security/authn/provider/elixir/ElixirJwkVaultTest.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.humancellatlas.ingest.security.authn.provider.elixir; - -import com.auth0.jwk.Jwk; -import com.auth0.jwk.UrlJwkProvider; -import com.auth0.jwt.JWT; -import org.humancellatlas.ingest.security.JwtGenerator; -import org.humancellatlas.ingest.security.common.jwk.JwkVault; -import org.humancellatlas.ingest.security.common.jwk.UrlJwkProviderResolver; -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; - -public class ElixirJwkVaultTest { - - @Test - public void testGetPublicKey() throws Exception { - //given: JWT - String issuer = "https://login.elixir-czech.org/oidc"; - JwtGenerator generator = new JwtGenerator(issuer); - var jwt = generator.generate(); - - Jwk jwk = mock(Jwk.class); - doReturn(generator.getPublicKey()).when(jwk).getPublicKey(); - - //and: - UrlJwkProvider urlJwkProvider = mock(UrlJwkProvider.class); - doReturn(jwk).when(urlJwkProvider).get(JwtGenerator.DEFAULT_KEY_ID); - - //and: - UrlJwkProviderResolver urlJwkProviderResolver = mock(UrlJwkProviderResolver.class); - doReturn(urlJwkProvider).when(urlJwkProviderResolver).resolve(); - - //and: - JwkVault jwkVault = new ElixirJwkVault(urlJwkProviderResolver); - - //when: - var token = JWT.decode(jwt); - var publicKey = jwkVault.getPublicKey(token); - - //then: - assertThat(publicKey).isNotNull(); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/security/authn/provider/gcp/GcpJwkVaultTest.java b/src/test/java/org/humancellatlas/ingest/security/authn/provider/gcp/GcpJwkVaultTest.java deleted file mode 100644 index 9fd61b938..000000000 --- a/src/test/java/org/humancellatlas/ingest/security/authn/provider/gcp/GcpJwkVaultTest.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.humancellatlas.ingest.security.authn.provider.gcp; - -import com.auth0.jwk.Jwk; -import com.auth0.jwk.UrlJwkProvider; -import com.auth0.jwt.JWT; -import org.humancellatlas.ingest.security.JwtGenerator; -import org.humancellatlas.ingest.security.authn.provider.elixir.ElixirJwkVault; -import org.humancellatlas.ingest.security.authn.provider.gcp.GcpJwkVault; -import org.humancellatlas.ingest.security.common.jwk.JwkVault; -import org.humancellatlas.ingest.security.common.jwk.UrlJwkProviderResolver; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static java.util.Map.entry; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; - -public class GcpJwkVaultTest { - - @Test - public void testGetPublicKeyForJwt() throws Exception { - //given: JWT - String issuer = "https://humancellatlas.auth0.com"; - JwtGenerator generator = new JwtGenerator(issuer); - var customClaims = Map.ofEntries( - entry("https://auth.data.humancellatlas.org/email", "sample@domain.tld") - ); - var jwt = generator.generate(customClaims); - - //and: JWK from remote service - Jwk jwk = mock(Jwk.class); - doReturn(generator.getPublicKey()).when(jwk).getPublicKey(); - - //and: - UrlJwkProvider urlJwkProvider = mock(UrlJwkProvider.class); - doReturn(jwk).when(urlJwkProvider).get(JwtGenerator.DEFAULT_KEY_ID); - - //and: - UrlJwkProviderResolver urlJwkProviderResolver = mock(UrlJwkProviderResolver.class); - doReturn(urlJwkProvider).when(urlJwkProviderResolver).resolve(issuer); - - //and: GoogleServiceJwkVault - JwkVault jwkVault = new GcpJwkVault(urlJwkProviderResolver); - - //when: - var token = JWT.decode(jwt); - var publicKey = jwkVault.getPublicKey(token); - - //then: - assertThat(publicKey).isNotNull(); - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/security/authn/provider/gcp/GoogleServiceJwtAuthenticationProviderTest.java b/src/test/java/org/humancellatlas/ingest/security/authn/provider/gcp/GoogleServiceJwtAuthenticationProviderTest.java deleted file mode 100644 index ba9a9a4e5..000000000 --- a/src/test/java/org/humancellatlas/ingest/security/authn/provider/gcp/GoogleServiceJwtAuthenticationProviderTest.java +++ /dev/null @@ -1,117 +0,0 @@ -package org.humancellatlas.ingest.security.authn.provider.gcp; - -import com.auth0.jwt.exceptions.JWTVerificationException; -import com.auth0.jwt.interfaces.JWTVerifier; -import com.auth0.spring.security.api.authentication.PreAuthenticatedAuthenticationJsonWebToken; -import org.humancellatlas.ingest.security.JwtGenerator; -import org.humancellatlas.ingest.security.common.jwk.JwtVerifierResolver; -import org.humancellatlas.ingest.security.exception.JwtVerificationFailed; -import org.humancellatlas.ingest.security.exception.UnlistedJwtIssuer; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.springframework.security.authentication.AuthenticationProvider; -import org.springframework.security.core.Authentication; - -import java.util.Map; - -import static java.util.Map.entry; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.assertj.core.api.Assumptions.assumeThat; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.*; - -public class GoogleServiceJwtAuthenticationProviderTest { - - @Nested - @DisplayName("Authenticate") - class AuthenticationTests { - private JWTVerifier jwtVerifier; - - private JwtVerifierResolver jwtVerifierResolver; - - private GcpDomainWhiteList projectWhitelist; - - @BeforeEach - public void setUp() { - jwtVerifier = mock(JWTVerifier.class); - jwtVerifierResolver = mock(JwtVerifierResolver.class); - doReturn(jwtVerifier).when(jwtVerifierResolver).resolve(anyString()); - - projectWhitelist = mock(GcpDomainWhiteList.class); - doReturn(true).when(projectWhitelist).lists("sample@domain.tld"); - } - - @Test - @DisplayName("success") - public void testAuthenticate() { - //given: JWT - Map claims = Map.ofEntries( - entry("https://auth.data.humancellatlas.org/group", "public") - ); - String keyId = "MDc2OTM3ODI4ODY2NUU5REVGRDVEM0MyOEYwQTkzNDZDRDlEQzNBRQ"; - String subject = "johndoe@somedomain.tld"; - - JwtGenerator jwtGenerator = new JwtGenerator("sample@domain.tld"); - String jwt = jwtGenerator.generate(keyId, subject, claims); - - //and: given a JWT Authentication - Authentication jwtAuthentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); - assumeThat(jwtAuthentication).isNotNull(); - - //and: - AuthenticationProvider authenticationProvider = new GoogleServiceJwtAuthenticationProvider( - projectWhitelist, jwtVerifierResolver); - - //when: - Authentication authentication = authenticationProvider.authenticate(jwtAuthentication); - - //then: - assertThat(authentication).isNotNull(); - } - - @Test - @DisplayName("unlisted issuer") - public void testForUnlistedIssuer() { - //given: - AuthenticationProvider authenticationProvider = new GoogleServiceJwtAuthenticationProvider( - projectWhitelist, jwtVerifierResolver); - - //and: - JwtGenerator jwtGenerator = new JwtGenerator("sample@otherdomain.tld"); - String jwt = jwtGenerator.generate(); - Authentication jwtAuthentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); - - //expect: - assertThatThrownBy(() -> { - authenticationProvider.authenticate(jwtAuthentication); - }).isInstanceOf(UnlistedJwtIssuer.class).hasMessageContaining("sample@otherdomain.tld"); - } - - @Test - @DisplayName("verification failed") - public void testForFailedVerification() { - //given: - AuthenticationProvider authenticationProvider = new GoogleServiceJwtAuthenticationProvider( - projectWhitelist, jwtVerifierResolver); - - //and: - Exception verificationFailed = new JWTVerificationException("verification failed"); - doThrow(verificationFailed).when(jwtVerifier).verify(anyString()); - - //and: - JwtGenerator jwtGenerator = new JwtGenerator("sample@domain.tld"); - String jwt = jwtGenerator.generate(); - Authentication jwtAuthentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); - - //expect: - assertThatThrownBy(() -> { - authenticationProvider.authenticate(jwtAuthentication); - }).isInstanceOf(JwtVerificationFailed.class); - } - - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/security/common/jwk/UrlJwkProviderResolverTest.java b/src/test/java/org/humancellatlas/ingest/security/common/jwk/UrlJwkProviderResolverTest.java deleted file mode 100644 index 74a87566b..000000000 --- a/src/test/java/org/humancellatlas/ingest/security/common/jwk/UrlJwkProviderResolverTest.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.humancellatlas.ingest.security.common.jwk; - -import com.auth0.jwk.UrlJwkProvider; -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -public class UrlJwkProviderResolverTest { - - @Test - public void testResolve() { - //given: - String baseUrl = "https://sample.service.tld"; - UrlJwkProviderResolver resolver = new UrlJwkProviderResolver(baseUrl); - - //when: - String relativePath = "issuer.service.tld"; - UrlJwkProvider provider = resolver.resolve(relativePath); - - //then: - assertThat(provider).isNotNull(); - - //and: inspect assigned URL through sub-class interface as a work-around - var url = ((RemoteJwkProvider) provider).getUrl(); - assertThat(url.toString()).isEqualTo("https://sample.service.tld/issuer.service.tld"); - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/security/web/AuthenticationControllerTest.java b/src/test/java/org/humancellatlas/ingest/security/web/AuthenticationControllerTest.java deleted file mode 100644 index 7f0ff7983..000000000 --- a/src/test/java/org/humancellatlas/ingest/security/web/AuthenticationControllerTest.java +++ /dev/null @@ -1,219 +0,0 @@ -package org.humancellatlas.ingest.security.web; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.humancellatlas.ingest.security.Account; -import org.humancellatlas.ingest.security.AccountService; -import org.humancellatlas.ingest.security.Role; -import org.humancellatlas.ingest.security.SecurityConfig; -import org.humancellatlas.ingest.security.authn.oidc.OpenIdAuthentication; -import org.humancellatlas.ingest.security.authn.oidc.UserInfo; -import org.humancellatlas.ingest.security.exception.DuplicateAccount; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.http.HttpStatus; -import org.springframework.mock.web.MockHttpServletResponse; -import org.springframework.security.authentication.AuthenticationProvider; -import org.springframework.security.core.Authentication; -import org.springframework.security.test.context.support.WithAnonymousUser; -import org.springframework.security.test.context.support.WithMockUser; -import org.springframework.test.context.junit.jupiter.web.SpringJUnitWebConfig; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.MvcResult; -import org.springframework.web.context.WebApplicationContext; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.humancellatlas.ingest.security.ElixirConfig.ELIXIR; -import static org.humancellatlas.ingest.security.GcpConfig.GCP; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.authentication; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -@WebMvcTest(AuthenticationController.class) -@AutoConfigureMockMvc(printOnlyOnFailure = false) -public class AuthenticationControllerTest { - - private static final String BASE_PATH = "/auth"; - - @Autowired - private WebApplicationContext applicationContext; - - @Autowired - private MockMvc webApp; - - @MockBean(name = GCP) - private AuthenticationProvider gcp; - - @MockBean(name = ELIXIR) - private AuthenticationProvider elixir; - - @MockBean - private AccountService accountService; - - @Nested - @DisplayName("Registration") - class Registration { - - private static final String PATH = "/auth/registration"; - - @Test - void byAuthenticatedGuest() throws Exception { - //given: - String subjectId = "cf12881b"; - UserInfo userInfo = new UserInfo(subjectId, "Jane Doe"); - Authentication authentication = new OpenIdAuthentication(null, userInfo); - - //and: - String accountId = "b4912b3"; - Account persistentAccount = new Account(accountId, subjectId); - doReturn(persistentAccount) - .when(accountService) - .register(any(Account.class)); - - //when: - MvcResult result = webApp - .perform(post(PATH) - .with(authentication(authentication)) - .with(csrf())) - .andReturn(); - - //then: - MockHttpServletResponse response = result.getResponse(); - assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); - - //and: - ObjectMapper objectMapper = new ObjectMapper(); - var resultingAccount = objectMapper.readValue(response.getContentAsString(), Account.class); - assertThat(resultingAccount.getId()).isEqualTo(accountId); - assertCorrectRegisteredAccount(userInfo); - } - - private void assertCorrectRegisteredAccount(UserInfo userInfo) { - var accountCaptor = ArgumentCaptor.forClass(Account.class); - verify(accountService).register(accountCaptor.capture()); - - var registeredAccount = accountCaptor.getValue(); - assertThat(registeredAccount) - .extracting("providerReference", "name") - .containsExactly(userInfo.getSubjectId(), userInfo.getName()); - assertThat(registeredAccount.getRoles()).isEmpty(); - } - - @Test - @WithMockUser(roles = {"CONTRIBUTOR"}) - void byRegisteredUser() throws Exception { - // expect: - webApp - .perform(post(PATH)) - .andExpect(status().isForbidden()); - } - - /* - Similar scenario to byRegisteredUser but somehow the Account was either, - 1) unrecognised and so was treated as an authenticated Guest, or - 2) Account was erroneously assigned the Guest role. - Essentially, we want to handle duplicated subject id in our system. - */ - @Test - void byUnrecognisedRegisteredUser() throws Exception { - //given: - UserInfo userInfo = new UserInfo("cc9a9a1", ""); - Authentication authentication = new OpenIdAuthentication(userInfo); - - //and: - doThrow(new DuplicateAccount()) - .when(accountService) - .register(any(Account.class)); - - //expect: - webApp - .perform(post(PATH) - .with(authentication(authentication)) - .with(csrf())) - .andExpect(status().isConflict()); - } - - @Test - void byAnonymousUser() throws Exception { - // expect: - webApp - .perform(post(PATH)) - .andExpect(status().isUnauthorized()); - } - - } - - @Nested - @DisplayName("Account Retrieval") - class AccountRetrieval { - - private final String PATH = String.format("%s/account", BASE_PATH); - - @Test - void registeredUser() throws Exception { - //given: - String accountId = "bcdde10"; - String subjectId = "67135cc"; - - //and: - Account account = new Account(accountId, subjectId); - account.addRole(Role.CONTRIBUTOR); - - //and: - UserInfo credentials = new UserInfo(subjectId, ""); - Authentication authentication = new OpenIdAuthentication(account, credentials); - - //when: - MvcResult result = webApp - .perform(get(PATH).with(authentication(authentication))) - .andReturn(); - - //then: - MockHttpServletResponse response = result.getResponse(); - assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); - assertCorrectAccountDetails(response, accountId, subjectId); - } - - private void assertCorrectAccountDetails(MockHttpServletResponse response, String accountId, - String subjectId) throws Exception { - ObjectMapper objectMapper = new ObjectMapper(); - Account retrievedAccount = objectMapper.readValue(response.getContentAsString(), Account.class); - assertThat(retrievedAccount) - .extracting("id", "providerReference") - .containsExactly(accountId, subjectId); - assertThat(retrievedAccount.getRoles()).containsExactly(Role.CONTRIBUTOR); - } - - @Test - void authenticatedGuest() throws Exception { - //given: - UserInfo userInfo = new UserInfo("82ffab9", ""); - Authentication authentication = new OpenIdAuthentication(userInfo); - - //expect: - webApp - .perform(get(PATH).with(authentication(authentication))) - .andExpect(status().isNotFound()); - } - - @Test - void unknownGuest() throws Exception { - //expect: - webApp - .perform(get(PATH)) - .andExpect(status().isUnauthorized()); - } - - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/stagingjob/StagingJobServiceTest.java b/src/test/java/org/humancellatlas/ingest/stagingjob/StagingJobServiceTest.java deleted file mode 100644 index bd77aca5e..000000000 --- a/src/test/java/org/humancellatlas/ingest/stagingjob/StagingJobServiceTest.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.humancellatlas.ingest.stagingjob; - -import org.humancellatlas.ingest.stagingjob.StagingJobService.JobAlreadyRegisteredException; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.springframework.dao.DuplicateKeyException; - -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatExceptionOfType; -import static org.mockito.Mockito.*; - -public class StagingJobServiceTest { - - private StagingJobRepository stagingJobRepository = mock(StagingJobRepository.class); - private StagingJobService stagingJobService = new StagingJobService(stagingJobRepository); - - @BeforeEach - public void setUp() { - reset(stagingJobRepository); - } - - @Nested - class Registration { - - @Test - public void validJob() { - // given: - UUID stagingAreaUUid = UUID.randomUUID(); - String fileName = "test_1.fastq.gz"; - String metadataUuid = UUID.randomUUID().toString(); - StagingJob stagingJob = new StagingJob(stagingAreaUUid, metadataUuid, fileName); - - // and: - StagingJob persistentJob = spy(stagingJob); - doReturn("_generated_id_1").when(persistentJob).getId(); - doReturn(persistentJob).when(stagingJobRepository).save(any()); - - //when: - StagingJob resultingJob = stagingJobService.register(stagingJob); - - //then: - verify(stagingJobRepository).save(stagingJob); - assertThat(resultingJob).isEqualTo(persistentJob); - } - - @Test - public void duplicateJob() { - // given: - UUID stagingAreaUuid = UUID.randomUUID(); - String metadataUuid = UUID.randomUUID().toString(); - String fileName = "test.fastq.gz"; - StagingJob stagingJob = new StagingJob(stagingAreaUuid, metadataUuid, fileName); - - // and: - doThrow(new DuplicateKeyException("duplicate key")).when(stagingJobRepository).save(any()); - - // expect : - assertThatExceptionOfType(JobAlreadyRegisteredException.class) - .isThrownBy(() -> stagingJobService.register(stagingJob)); - } - } -} diff --git a/src/test/java/org/humancellatlas/ingest/stagingjob/web/StagingJobControllerTest.java b/src/test/java/org/humancellatlas/ingest/stagingjob/web/StagingJobControllerTest.java deleted file mode 100644 index 4b5d0c10b..000000000 --- a/src/test/java/org/humancellatlas/ingest/stagingjob/web/StagingJobControllerTest.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.humancellatlas.ingest.stagingjob.web; - -import org.humancellatlas.ingest.stagingjob.StagingJob; -import org.humancellatlas.ingest.stagingjob.StagingJobService; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.data.mapping.PersistentEntity; -import org.springframework.data.rest.webmvc.PersistentEntityResource; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; - -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.BDDMockito.given; -import static org.mockito.Mockito.*; - -@ExtendWith(MockitoExtension.class) -public class StagingJobControllerTest { - - private StagingJobService stagingJobService; - - private StagingJobController controller; - - @BeforeEach - public void setUp(@Mock StagingJobService stagingJobService) { - this.stagingJobService = stagingJobService; - controller = new StagingJobController(stagingJobService); - } - - @Test - public void createStagingJob() { - //given: - StagingJob stagingJob = new StagingJob(UUID.randomUUID(), "file_1.json"); - StagingJob persistentJob = spy(stagingJob); - given(stagingJobService.register(any(StagingJob.class))).willReturn(persistentJob); - - //and: - PersistentEntityResourceAssembler resourceAssembler = mock(PersistentEntityResourceAssembler.class); - given(resourceAssembler.toFullResource(any())).willAnswer(invocation -> { - Object entity = invocation.getArgument(0); - return PersistentEntityResource.build(entity, mock(PersistentEntity.class)).build(); - }); - - //when: - ResponseEntity response = controller.createStagingJob(stagingJob, resourceAssembler); - - //then: - verify(stagingJobService).register(any(StagingJob.class)); - - //and: - assertThat(response).isNotNull() - .extracting("status").containsExactly(HttpStatus.OK); - assertThat(response.getBody()).isInstanceOf(PersistentEntityResource.class); - PersistentEntityResource responseBody = (PersistentEntityResource) response.getBody(); - assertThat(responseBody.getContent()).isEqualTo(persistentJob); - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/state/MetadataDocumentEventHandlerTest.java b/src/test/java/org/humancellatlas/ingest/state/MetadataDocumentEventHandlerTest.java deleted file mode 100644 index 438366710..000000000 --- a/src/test/java/org/humancellatlas/ingest/state/MetadataDocumentEventHandlerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.humancellatlas.ingest.state; - -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import static org.mockito.Mockito.*; - - -public class MetadataDocumentEventHandlerTest { - - private MessageRouter messageRouter = mock(MessageRouter.class); - MetadataDocumentEventHandler handler = new MetadataDocumentEventHandler(messageRouter); - - @Test - public void testHandleCreateDocumentsWithoutSubmissionEnvelope() { - Biomaterial biomaterial = new Biomaterial(null); - handler.handleMetadataDocumentCreate(biomaterial); - Mockito.verify(messageRouter, times(1)).routeValidationMessageFor(biomaterial); - Mockito.verify(messageRouter, times(1)).routeStateTrackingUpdateMessageFor(biomaterial); - } - - @Test - public void testHandleCreateDocumentsWithSubmissionEnvelope() { - Project project = new Project(null); - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - project.getSubmissionEnvelopes().add(submissionEnvelope); - handler.handleMetadataDocumentCreate(project); - Mockito.verify(messageRouter, times(1)).routeValidationMessageFor(project); - Mockito.verify(messageRouter, times(1)).routeStateTrackingUpdateMessageFor(project); - } - -} diff --git a/src/test/java/org/humancellatlas/ingest/state/MetadataStateChangeListenerTest.java b/src/test/java/org/humancellatlas/ingest/state/MetadataStateChangeListenerTest.java deleted file mode 100644 index 620da7657..000000000 --- a/src/test/java/org/humancellatlas/ingest/state/MetadataStateChangeListenerTest.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.humancellatlas.ingest.state; - -import org.humancellatlas.ingest.core.MetadataDocument; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.notifications.NotificationService; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectService; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.springframework.data.mongodb.core.mapping.event.AfterSaveEvent; -import org.springframework.data.mongodb.core.mapping.event.BeforeConvertEvent; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.*; - -public class MetadataStateChangeListenerTest { - private MessageRouter messageRouter = mock(MessageRouter.class); - - MetadataStateChangeListener metadataDocumentMongoEventListener = new MetadataStateChangeListener(messageRouter); - - @Test - public void testOnBeforeConvert() { - Project project = new Project(null); - metadataDocumentMongoEventListener.onBeforeConvert(new BeforeConvertEvent(project, "project")); - assertThat(project.getUuid()).isNotNull(); - assertThat(project.getDcpVersion()).isEqualTo(project.getSubmissionDate()); - } - - @Test - public void testOnAfterSave() { - Project project = new Project(null); - AfterSaveEvent afterSaveEvent = mock(AfterSaveEvent.class); - doReturn(project).when(afterSaveEvent).getSource(); - metadataDocumentMongoEventListener.onAfterSave(afterSaveEvent); - Mockito.verify(messageRouter, times(1)).routeValidationMessageFor(project); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/state/ValidationStateTest.java b/src/test/java/org/humancellatlas/ingest/state/ValidationStateTest.java deleted file mode 100644 index 21f59c183..000000000 --- a/src/test/java/org/humancellatlas/ingest/state/ValidationStateTest.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.humancellatlas.ingest.state; - -import org.humancellatlas.ingest.file.ValidationReport; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.json.JsonTest; -import org.springframework.boot.test.json.JacksonTester; - -import java.util.stream.Stream; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; - - -@JsonTest -public class ValidationStateTest { - @Autowired - private JacksonTester json; - - private static Stream provideStatesForTestFromJSON() { - return Stream.of( - Arguments.of(ValidationState.INVALID, "invalid"), - Arguments.of(ValidationState.INVALID, "Invalid"), - Arguments.of(ValidationState.INVALID, "INVALID"), - Arguments.of(ValidationState.VALID, "valid"), - Arguments.of(ValidationState.VALID, "Valid"), - Arguments.of(ValidationState.VALID, "VALID"), - Arguments.of(ValidationState.VALIDATING, "validating"), - Arguments.of(ValidationState.VALIDATING, "Validating"), - Arguments.of(ValidationState.VALIDATING, "VALIDATING") - ); - } - - @ParameterizedTest - @MethodSource("provideStatesForTestFromJSON") - public void testFromJSON(ValidationState expected, String given) throws Exception{ - var jsonValue = String.format("{ \"validationState\": \"%s\" }", given); - assertThat(json.parse(jsonValue).getObject().getValidationState()).isEqualTo(expected); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeServiceTest.java b/src/test/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeServiceTest.java deleted file mode 100644 index 5ce5f90cb..000000000 --- a/src/test/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeServiceTest.java +++ /dev/null @@ -1,478 +0,0 @@ -package org.humancellatlas.ingest.submission; - -import org.humancellatlas.ingest.biomaterial.Biomaterial; -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.bundle.BundleManifestRepository; -import org.humancellatlas.ingest.core.Uuid; -import org.humancellatlas.ingest.core.service.MetadataCrudService; -import org.humancellatlas.ingest.core.service.MetadataUpdateService; -import org.humancellatlas.ingest.errors.SubmissionErrorRepository; -import org.humancellatlas.ingest.exporter.Exporter; -import org.humancellatlas.ingest.file.File; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.patch.PatchRepository; -import org.humancellatlas.ingest.process.Process; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.project.Project; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.project.ProjectService; -import org.humancellatlas.ingest.project.WranglingState; -import org.humancellatlas.ingest.protocol.Protocol; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.humancellatlas.ingest.state.SubmissionState; -import org.humancellatlas.ingest.state.SubmitAction; -import org.humancellatlas.ingest.state.ValidationState; -import org.humancellatlas.ingest.submissionmanifest.SubmissionManifestRepository; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.domain.*; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = {SubmissionEnvelopeService.class}) -@AutoConfigureMockMvc(printOnlyOnFailure = false) -public class SubmissionEnvelopeServiceTest { - @Autowired - private SubmissionEnvelopeService service; - - @MockBean - private MessageRouter messageRouter; - - @MockBean - private Exporter exporter; - - @MockBean - private MetadataCrudService metadataCrudService; - - @MockBean - private MetadataUpdateService metadataUpdateService; - - @MockBean - private ExecutorService executorService; - - @MockBean - private SubmissionEnvelopeRepository submissionEnvelopeRepository; - - @MockBean - private SubmissionEnvelopeCreateHandler submissionEnvelopeCreateHandler; - - @MockBean - private SubmissionManifestRepository submissionManifestRepository; - - @MockBean - private BundleManifestRepository bundleManifestRepository; - - @MockBean - private ProjectRepository projectRepository; - - @MockBean - private ProcessRepository processRepository; - - @MockBean - private ProtocolRepository protocolRepository; - - @MockBean - private FileRepository fileRepository; - - @MockBean - private BiomaterialRepository biomaterialRepository; - - @MockBean - private PatchRepository patchRepository; - - @MockBean - private SubmissionErrorRepository submissionErrorRepository; - - @MockBean - ProjectService projectService; - - @Configuration - static class TestConfiguration { - } - - @Test - public void testContentLastUpdated() { - // given - SubmissionEnvelope submission = mock(SubmissionEnvelope.class); - Project project = mock(Project.class); - Biomaterial biomaterial = mock(Biomaterial.class); - Process process = mock(Process.class); - File file = mock(File.class); - - PageRequest request = PageRequest.of(0, 1, new Sort(Sort.Direction.DESC, "updateDate")); - when(projectRepository.findBySubmissionEnvelopesContaining(submission, request)) - .thenReturn(new PageImpl<>(List.of(project), request, 1)); - when(protocolRepository.findBySubmissionEnvelope(submission, request)) - .thenReturn(Page.empty()); - when(biomaterialRepository.findBySubmissionEnvelope(submission, request)) - .thenReturn(new PageImpl<>(List.of(biomaterial), request, 1)); - when(processRepository.findBySubmissionEnvelope(submission, request)) - .thenReturn(new PageImpl<>(List.of(process), request, 1)); - when(fileRepository.findBySubmissionEnvelope(submission, request)) - .thenReturn(new PageImpl<>(List.of(file), request, 1)); - - Instant now = Instant.now(); - Instant yesterday = now.minus(1, ChronoUnit.DAYS); - when(project.getUpdateDate()).thenReturn(yesterday); - when(biomaterial.getUpdateDate()).thenReturn(yesterday); - when(process.getUpdateDate()).thenReturn(yesterday); - when(file.getUpdateDate()).thenReturn(now); - - // when - Optional lastUpdateDate = service.getSubmissionContentLastUpdated(submission); - - // then - assertThat(lastUpdateDate.isPresent()).isTrue(); - assertThat(lastUpdateDate.get().toString()).isEqualTo(now.toString()); - } - - @Test - public void testContentLastUpdatedEmptySubmission() { - // given - SubmissionEnvelope submission = mock(SubmissionEnvelope.class); - - PageRequest request = PageRequest.of(0, 1, new Sort(Sort.Direction.DESC, "updateDate")); - when(projectRepository.findBySubmissionEnvelopesContaining(submission, request)) - .thenReturn(Page.empty()); - when(protocolRepository.findBySubmissionEnvelope(submission, request)) - .thenReturn(Page.empty()); - when(biomaterialRepository.findBySubmissionEnvelope(submission, request)) - .thenReturn(Page.empty()); - when(processRepository.findBySubmissionEnvelope(submission, request)) - .thenReturn(Page.empty()); - when(fileRepository.findBySubmissionEnvelope(submission, request)) - .thenReturn(Page.empty()); - - // when - Optional lastUpdateDate = service.getSubmissionContentLastUpdated(submission); - - // then - assertThat(lastUpdateDate.isPresent()).isFalse(); - } - - @Test - public void testDeleteSubmission() { - //given SubmissionEnvelope - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.setUuid(Uuid.newUuid()); - - //given metadata within the SubmissionEnvelope - Biomaterial testBiomaterial = new Biomaterial(Map.ofEntries(Map.entry("key", UUID.randomUUID()))); - Protocol testProtocol = new Protocol(Map.ofEntries(Map.entry("key", UUID.randomUUID()))); - Process testProcess = new Process(Map.ofEntries(Map.entry("key", UUID.randomUUID()))); - - testProcess.setSubmissionEnvelope(submissionEnvelope); - testProtocol.setSubmissionEnvelope(submissionEnvelope); - testBiomaterial.setSubmissionEnvelope(submissionEnvelope); - - //given metadata outside the SubmissionEnvelope - Biomaterial testOutsideBiomaterial = new Biomaterial(Map.ofEntries(Map.entry("key", UUID.randomUUID()))); - File testOutsideFile = new File(Map.ofEntries(Map.entry("key", UUID.randomUUID())), ""); - Process testOutsideProcess = new Process(Map.ofEntries(Map.entry("key", UUID.randomUUID()))); - - //given links to metadata outside of the SubmissionEnvelope - testOutsideBiomaterial.getInputToProcesses().add(testProcess); - testOutsideFile.getDerivedByProcesses().add(testProcess); - testOutsideProcess.getProtocols().add(testProtocol); - - //given File - File file = new File(null, "testFile.txt"); - file.setSubmissionEnvelope(submissionEnvelope); - - //given Project - Project project = new Project(new Object()); - project.setUuid(Uuid.newUuid()); - project.addToSubmissionEnvelopes(submissionEnvelope); - assertThat(project.getSubmissionEnvelopes()).contains(submissionEnvelope); - - //given SupplementaryFile - project.getSupplementaryFiles().add(file); - assertThat(project.getSupplementaryFiles()).contains(file); - - //given ProjectRepository - List projectList = new ArrayList<>(); - projectList.add(project); - when(projectRepository.findBySubmissionEnvelopesContaining(any(), any())) - .thenReturn(new PageImpl<>(projectList, Pageable.unpaged(), 1)); - - when(projectRepository.findBySubmissionEnvelopesContains(any())) - .thenReturn(Stream.of(project)); - - when(projectRepository.findBySupplementaryFilesContains(any())) - .thenReturn(Stream.of(project)); - - //when - when(processRepository.findBySubmissionEnvelope(submissionEnvelope)).thenReturn(Stream.of(testProcess)); - when(biomaterialRepository.findBySubmissionEnvelope(submissionEnvelope)).thenReturn(Stream.of(testBiomaterial)); - when(protocolRepository.findBySubmissionEnvelope(submissionEnvelope)).thenReturn(Stream.of(testProtocol)); - when(fileRepository.findBySubmissionEnvelope(submissionEnvelope)).thenReturn(Stream.of(file)); - - when(biomaterialRepository.findByInputToProcessesContains(testProcess)).thenReturn(Stream.of(testOutsideBiomaterial)); - when(fileRepository.findByDerivedByProcessesContains(testProcess)).thenReturn(Stream.of(testOutsideFile)); - when(processRepository.findByProtocolsContains(testProtocol)).thenReturn(Stream.of(testOutsideProcess)); - - service.deleteSubmission(submissionEnvelope, false); - - //then: - verify(metadataCrudService).removeLinksToDocument(testProcess); - verify(metadataCrudService).removeLinksToDocument(testProtocol); - verify(metadataCrudService).removeLinksToDocument(file); - - verify(biomaterialRepository).deleteBySubmissionEnvelope(submissionEnvelope); - verify(processRepository).deleteBySubmissionEnvelope(submissionEnvelope); - verify(protocolRepository).deleteBySubmissionEnvelope(submissionEnvelope); - verify(fileRepository).deleteBySubmissionEnvelope(submissionEnvelope); - verify(bundleManifestRepository).deleteByEnvelopeUuid(submissionEnvelope.getUuid().getUuid().toString()); - verify(patchRepository).deleteBySubmissionEnvelope(submissionEnvelope); - verify(submissionManifestRepository).deleteBySubmissionEnvelope(submissionEnvelope); - verify(submissionErrorRepository).deleteBySubmissionEnvelope(submissionEnvelope); - - verify(projectRepository).findBySubmissionEnvelopesContains(submissionEnvelope); - assertThat(project.getSubmissionEnvelopes()).isEmpty(); - verify(projectRepository, atLeastOnce()).save(project); - verify(submissionEnvelopeRepository).delete(submissionEnvelope); - } - - @ParameterizedTest - @EnumSource(value = SubmissionState.class, names = { - "PENDING", - "DRAFT", - "METADATA_VALIDATING", - "METADATA_VALID", - "METADATA_INVALID", - "GRAPH_VALIDATION_REQUESTED", - "GRAPH_VALIDATING", - "GRAPH_VALID", - "GRAPH_INVALID", - "SUBMITTED", - "PROCESSING", - "ARCHIVING", - "ARCHIVED", - "EXPORTING", - "EXPORTED", - "CLEANUP", - "COMPLETE" - }) - public void testRedundantHandleEnvelopeStateUpdateRequest(SubmissionState state) { - // Given - var submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.enactStateTransition(state); - - // When - service.handleEnvelopeStateUpdateRequest(submissionEnvelope, state); - - // Then - // no errors - } - - @Nested - @DisplayName("SubmitRequestTests") - class SubmitRequestTests { - SubmissionEnvelope submissionEnvelope; - Project project; - - @BeforeEach - public void setup() { - submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALID); - project = new Project(null); - project.setValidationState(ValidationState.VALID); - when(projectRepository.findBySubmissionEnvelopesContains(any())) - .thenReturn(Stream.of(project)); - } - - @Test - public void testSubmissionBlocked() { - //given: - submissionEnvelope.enactStateTransition(SubmissionState.METADATA_VALID); - - //when - Throwable exception = assertThrows(RuntimeException.class, - () -> service.handleSubmitRequest(submissionEnvelope, List.of(SubmitAction.EXPORT)) - ); - - // then: - assertErrorMessageContains(exception, "without a graph valid state"); - } - - @Test - public void testSubmissionUnblocked() { - //when - service.handleSubmitRequest(submissionEnvelope, List.of(SubmitAction.EXPORT)); - - // then: - verify(submissionEnvelopeRepository).save(submissionEnvelope); - } - - @Test - public void testGraphValidationErrorsCleared() { - //given envelope: - submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_INVALID); - - //given metadata within the SubmissionEnvelope - Biomaterial testBiomaterial = new Biomaterial(Map.ofEntries(Map.entry("key", UUID.randomUUID()))); - Protocol testProtocol = new Protocol(Map.ofEntries(Map.entry("key", UUID.randomUUID()))); - Process testProcess = new Process(Map.ofEntries(Map.entry("key", UUID.randomUUID()))); - File testFile = new File(null, "testFile.txt"); - - testProcess.setSubmissionEnvelope(submissionEnvelope); - testProtocol.setSubmissionEnvelope(submissionEnvelope); - testBiomaterial.setSubmissionEnvelope(submissionEnvelope); - testFile.setSubmissionEnvelope(submissionEnvelope); - - // given graph validation errors on the metadata - testBiomaterial.setGraphValidationErrors(Arrays.asList("test1", "test2")); - testProcess.setGraphValidationErrors(Arrays.asList("test1", "test2")); - testProtocol.setGraphValidationErrors(Arrays.asList("test1", "test2")); - testFile.setGraphValidationErrors(Arrays.asList("test1", "test2")); - - // when - when(biomaterialRepository.findBySubmissionEnvelope(any())) - .thenReturn(Stream.of(testBiomaterial)); - when(processRepository.findBySubmissionEnvelope(any())) - .thenReturn(Stream.of(testProcess)); - when(protocolRepository.findBySubmissionEnvelope(any())) - .thenReturn(Stream.of(testProtocol)); - when(fileRepository.findBySubmissionEnvelope(any())) - .thenReturn(Stream.of(testFile)); - - service.handleEnvelopeStateUpdateRequest(submissionEnvelope, SubmissionState.GRAPH_VALIDATION_REQUESTED); - submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALIDATION_REQUESTED); - //then: - assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(SubmissionState.GRAPH_VALIDATION_REQUESTED); - assertThat(testBiomaterial.getGraphValidationErrors()).isEqualTo(new ArrayList<>()); - assertThat(testProcess.getGraphValidationErrors()).isEqualTo(new ArrayList<>()); - assertThat(testProtocol.getGraphValidationErrors()).isEqualTo(new ArrayList<>()); - assertThat(testFile.getGraphValidationErrors()).isEqualTo(new ArrayList<>()); - } - - @Test - public void testSubmissionInvalidProject() { - //given: - project.setValidationState(ValidationState.INVALID); - - //when - Throwable exception = assertThrows(RuntimeException.class, - () -> service.handleSubmitRequest(submissionEnvelope, List.of(SubmitAction.EXPORT)) - ); - - // then: - assertErrorMessageContains(exception, "cannot be submitted when the project is invalid"); - } - - @Test - public void testSubmissionNoProject() { - //given: - when(projectRepository.findBySubmissionEnvelopesContains(any())) - .thenReturn(Stream.empty()); - - //when - Throwable exception = assertThrows(RuntimeException.class, - () -> service.handleSubmitRequest(submissionEnvelope, List.of(SubmitAction.EXPORT)) - ); - - // then: - assertErrorMessageContains(exception, "cannot be submitted without a project"); - } - - @Test void testExportedEventUpdatesHistory() { - // given - // submission from setUp() - // when - service.handleCommitExported(submissionEnvelope); - - // then - verify(projectService).updateWranglingState(project, WranglingState.SUBMITTED); - } - private void assertErrorMessageContains(Throwable exception, String s) { - assertThat(exception.getMessage()).contains(s); - verify(submissionEnvelopeRepository, never()).save(submissionEnvelope); - } - } - - @Nested - @DisplayName("StateUpdateRequestTests") - class StateUpdateRequestTests { - SubmissionEnvelope submissionEnvelope; - HashSet submitActions; - - @BeforeEach - public void setup() { - submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.enactStateTransition(SubmissionState.SUBMITTED); - submitActions = new HashSet<>(); - submissionEnvelope.setSubmitActions(submitActions); - } - - @Test - public void testHandleEnvelopeArchivingRequest(){ - // given - submitActions.add(SubmitAction.ARCHIVE); - - // when - service.handleCommitSubmit(submissionEnvelope); - - // then - verify(messageRouter).routeStateTrackingUpdateMessageForEnvelopeEvent(submissionEnvelope, SubmissionState.PROCESSING); - } - - @Test - public void testHandleEnvelopeExportingDataRequest(){ - // given - submitActions.add(SubmitAction.EXPORT); - - // when - service.handleCommitSubmit(submissionEnvelope); - - // then - verify(messageRouter).routeStateTrackingUpdateMessageForEnvelopeEvent(submissionEnvelope, SubmissionState.EXPORTING); - } - - @Test - public void testHandleEnvelopeExportingMetadataRequest(){ - // given - submitActions.add(SubmitAction.EXPORT_METADATA); - - // when - service.handleCommitSubmit(submissionEnvelope); - - // then - verify(messageRouter).routeStateTrackingUpdateMessageForEnvelopeEvent(submissionEnvelope, SubmissionState.EXPORTING); - } - - @Test - public void testHandleEnvelopeCleanupRequest(){ - // given - submissionEnvelope.enactStateTransition(SubmissionState.EXPORTED); - submitActions.add(SubmitAction.CLEANUP); - - // when - service.handleCommitSubmit(submissionEnvelope); - - // then - verify(messageRouter).routeStateTrackingUpdateMessageForEnvelopeEvent(submissionEnvelope, SubmissionState.CLEANUP); - } - } -} diff --git a/src/test/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeTest.java b/src/test/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeTest.java deleted file mode 100644 index 0132e0603..000000000 --- a/src/test/java/org/humancellatlas/ingest/submission/SubmissionEnvelopeTest.java +++ /dev/null @@ -1,204 +0,0 @@ -package org.humancellatlas.ingest.submission; - - -import org.humancellatlas.ingest.state.SubmissionState; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; - -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; - -public class SubmissionEnvelopeTest { - - @Test - public void testAllowedSubmissionStateTransitions() { - List states = getAllowedStates(SubmissionState.PENDING); - assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.DRAFT)); - } - - @Test - public void testAllowedSubmissionStateTransitionsForDraft() { - List states = getAllowedStates(SubmissionState.DRAFT); - assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.METADATA_VALIDATING)); - } - - @Test - public void testAllowedSubmissionStateTransitionsForMetadataValidating() { - List states = getAllowedStates(SubmissionState.METADATA_VALIDATING); - assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.DRAFT, SubmissionState.METADATA_INVALID, SubmissionState.METADATA_VALID)); - } - - @Test - public void testAllowedSubmissionStateTransitionsForMetadataValid() { - List states = getAllowedStates(SubmissionState.METADATA_VALID); - assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.DRAFT, SubmissionState.GRAPH_VALIDATION_REQUESTED)); - } - - @Test - public void testAllowedSubmissionStateTransitionsForMetadataInvalid() { - List states = getAllowedStates(SubmissionState.METADATA_INVALID); - assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.DRAFT, SubmissionState.METADATA_VALIDATING, SubmissionState.GRAPH_VALIDATION_REQUESTED)); - } - - @Test - public void testAllowedSubmissionStateTransitionsForGraphValidationRequested() { - List states = getAllowedStates(SubmissionState.GRAPH_VALIDATION_REQUESTED); - assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.GRAPH_VALIDATING, SubmissionState.DRAFT)); - } - - @Test - public void testAllowedSubmissionStateTransitionsForGraphValidating() { - List states = getAllowedStates(SubmissionState.GRAPH_VALIDATING); - assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.GRAPH_INVALID, SubmissionState.GRAPH_VALID, SubmissionState.DRAFT)); - } - - @Test - public void testAllowedSubmissionStateTransitionsForGraphValid() { - List states = getAllowedStates(SubmissionState.GRAPH_VALID); - assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.DRAFT, SubmissionState.SUBMITTED)); - } - - @Test - public void testAllowedSubmissionStateTransitionsForGraphInvalid() { - List states = getAllowedStates(SubmissionState.GRAPH_INVALID); - assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.GRAPH_VALIDATION_REQUESTED, SubmissionState.DRAFT)); - } - - @Test - public void testAllowedSubmissionStateTransitionsForSubmitted() { - List states = getAllowedStates(SubmissionState.SUBMITTED); - assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.PROCESSING, SubmissionState.EXPORTING)); - - } - - @Test - public void testAllowedSubmissionStateTransitionsForProcessing() { - List states = getAllowedStates(SubmissionState.PROCESSING); - assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.ARCHIVING)); - } - - @Test - public void testAllowedSubmissionStateTransitionsForArchiving() { - List states = getAllowedStates(SubmissionState.ARCHIVING); - assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.ARCHIVED)); - } - - @Test - public void testAllowedSubmissionStateTransitionsForArchived() { - List states = getAllowedStates(SubmissionState.ARCHIVED); - assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.EXPORTING)); - } - - @Test - public void testAllowedSubmissionStateTransitionsForExported() { - List states = getAllowedStates(SubmissionState.EXPORTED); - assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.CLEANUP)); - } - - @Test - public void testAllowedSubmissionStateTransitionsForCleanup() { - List states = getAllowedStates(SubmissionState.CLEANUP); - assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.COMPLETE)); - } - - @Test - public void testAllowedSubmissionStateTransitionsForComplete() { - List states = getAllowedStates(SubmissionState.COMPLETE); - assertThat(states).isEmpty(); - } - - - @ParameterizedTest - @EnumSource(value = SubmissionState.class, names = { - "PENDING", - "METADATA_VALIDATING", - "GRAPH_VALIDATION_REQUESTED", - "GRAPH_VALIDATING", - "EXPORTING", - "PROCESSING", - "CLEANUP", - "ARCHIVED", - "SUBMITTED" - }) - public void testIsNotEditable(SubmissionState state) { - //given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.enactStateTransition(state); - assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(state); - - //then: - assertThat(submissionEnvelope.isEditable()).isFalse(); - } - - @ParameterizedTest - @EnumSource(value = SubmissionState.class, names = { - "METADATA_VALID", - "METADATA_INVALID", - "EXPORTED", - "GRAPH_VALID", - "GRAPH_INVALID", - "COMPLETE", - "DRAFT", - "ARCHIVING" - }) - public void testIsEditable(SubmissionState state) { - //given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.enactStateTransition(state); - assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(state); - - //then: - assertThat(submissionEnvelope.isEditable()).isTrue(); - } - - @ParameterizedTest - @EnumSource(value = SubmissionState.class, names = { - "GRAPH_VALIDATION_REQUESTED", - "GRAPH_VALIDATING", - "EXPORTING", - "PROCESSING", - "ARCHIVED", - "SUBMITTED" - }) - public void testCannotAddTo(SubmissionState state) { - //given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.enactStateTransition(state); - assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(state); - - //then: - assertThat(submissionEnvelope.isSystemEditable()).isFalse(); - } - - @ParameterizedTest - @EnumSource(value = SubmissionState.class, names = { - "METADATA_VALIDATING", - "PENDING", - "METADATA_VALID", - "METADATA_INVALID", - "EXPORTED", - "GRAPH_VALID", - "GRAPH_INVALID", - "COMPLETE", - "DRAFT", - "ARCHIVING", - "CLEANUP" - }) - public void testCanAddTo(SubmissionState state) { - //given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.enactStateTransition(state); - assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(state); - - //then: - assertThat(submissionEnvelope.isSystemEditable()).isTrue(); - } - - private List getAllowedStates(SubmissionState state) { - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.enactStateTransition(state); - return submissionEnvelope.allowedSubmissionStateTransitions(); - } -} diff --git a/src/test/java/org/humancellatlas/ingest/submission/web/SubmissionControllerTest.java b/src/test/java/org/humancellatlas/ingest/submission/web/SubmissionControllerTest.java deleted file mode 100644 index 550a07f79..000000000 --- a/src/test/java/org/humancellatlas/ingest/submission/web/SubmissionControllerTest.java +++ /dev/null @@ -1,198 +0,0 @@ -package org.humancellatlas.ingest.submission.web; - -import org.humancellatlas.ingest.biomaterial.BiomaterialRepository; -import org.humancellatlas.ingest.bundle.BundleManifestRepository; -import org.humancellatlas.ingest.exporter.Exporter; -import org.humancellatlas.ingest.file.FileRepository; -import org.humancellatlas.ingest.messaging.MessageRouter; -import org.humancellatlas.ingest.process.ProcessRepository; -import org.humancellatlas.ingest.process.ProcessService; -import org.humancellatlas.ingest.project.ProjectRepository; -import org.humancellatlas.ingest.protocol.ProtocolRepository; -import org.humancellatlas.ingest.protocol.ProtocolService; -import org.humancellatlas.ingest.submission.SubmissionEnvelope; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeRepository; -import org.humancellatlas.ingest.submission.SubmissionEnvelopeService; -import org.humancellatlas.ingest.submission.SubmissionStateMachineService; -import org.humancellatlas.ingest.submissionmanifest.SubmissionManifestRepository; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; -import org.springframework.data.web.PagedResourcesAssembler; -import org.springframework.http.HttpEntity; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.humancellatlas.ingest.state.SubmissionState.*; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes={ SubmissionController.class }) -public class SubmissionControllerTest { - - @Autowired - private SubmissionController controller; - - @MockBean - private Exporter exporter; - - @MockBean - private SubmissionEnvelopeService submissionEnvelopeService; - @MockBean - private ProcessService processService; - @MockBean - private ProtocolService protocolService; - - @MockBean - private SubmissionEnvelopeRepository submissionEnvelopeRepository; - @MockBean - private FileRepository fileRepository; - @MockBean - private ProjectRepository projectRepository; - @MockBean - private ProtocolRepository protocolRepository; - @MockBean - private BiomaterialRepository biomaterialRepository; - @MockBean - private ProcessRepository processRepository; - @MockBean - private BundleManifestRepository bundleManifestRepository; - @MockBean - private SubmissionManifestRepository submissionManifestRepository; - @MockBean - private PagedResourcesAssembler pagedResourcesAssembler; - @MockBean - private SubmissionStateMachineService submissionStateMachineService; - @MockBean - private MessageRouter messageRouter; - - @Test - public void testEnactSubmitEnvelope() { - //given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - assertThat(submissionEnvelope.getSubmissionState()).isNotEqualTo(SUBMITTED); - - //and: - PersistentEntityResourceAssembler resourceAssembler = - mock(PersistentEntityResourceAssembler.class); - - //when: - HttpEntity response = controller.enactSubmitEnvelope(submissionEnvelope, - resourceAssembler); - - //then: - assertThat(response).isNotNull(); - assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(SUBMITTED); - verify(submissionEnvelopeRepository).save(submissionEnvelope); - verify(submissionEnvelopeService).handleCommitSubmit(submissionEnvelope); - } - - @Test - public void testDeleteSubmissionEnvelopeWithoutForce() { - //given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - - //when: - HttpEntity response = controller.forceDeleteSubmission(submissionEnvelope, false); - - //then: - assertThat(response).isNotNull(); - verify(submissionEnvelopeService).deleteSubmission(submissionEnvelope, false); - } - - @Test - public void testDeleteSubmissionEnvelopeWithForce() { - //given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - - //when: - HttpEntity response = controller.forceDeleteSubmission(submissionEnvelope, true); - - //then: - assertThat(response).isNotNull(); - verify(submissionEnvelopeService).deleteSubmission(submissionEnvelope, true); - } - - @Test - public void testDraftStateTransition() { - //given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.enactStateTransition(GRAPH_VALID); - assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(GRAPH_VALID); - - //and: - PersistentEntityResourceAssembler resourceAssembler = - mock(PersistentEntityResourceAssembler.class); - - // When: - HttpEntity response = controller.enactDraftEnvelope(submissionEnvelope, - resourceAssembler); - - //then: - assertThat(response).isNotNull(); - assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(DRAFT); - } - - @Test - public void testHappyValidationPath() { - //given: - SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); - submissionEnvelope.enactStateTransition(DRAFT); - assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(DRAFT); - - //and: - PersistentEntityResourceAssembler resourceAssembler = - mock(PersistentEntityResourceAssembler.class); - - // Test metadata validation happy path - // Metadata validation is triggered when documents are added to the submission - // so no endpoints fro requesting the state tracker to change the submission state - - // draft -> metadata validating - HttpEntity response = controller.enactValidatingEnvelope(submissionEnvelope, resourceAssembler); - assertThat(response).isNotNull(); - assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(METADATA_VALIDATING); - - // metadata validating -> metadata valid - response = controller.enactValidEnvelope(submissionEnvelope, resourceAssembler); - assertThat(response).isNotNull(); - assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(METADATA_VALID); - - - // Test graph validation happy path - // endpoints for requesting the state tracker to change the submission state are used here - - // metadata valid -> graph validation requested - HttpEntity requestResponse = controller.requestGraphValidation(submissionEnvelope, resourceAssembler); - assertThat(requestResponse).isNotNull(); - verify(submissionEnvelopeService).handleEnvelopeStateUpdateRequest(submissionEnvelope, GRAPH_VALIDATION_REQUESTED); - HttpEntity enactResponse = controller.enactGraphValidationRequested(submissionEnvelope, resourceAssembler); - assertThat(enactResponse).isNotNull(); - assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(GRAPH_VALIDATION_REQUESTED); - - // graph validation requested -> graph validating - requestResponse = controller.requestGraphValidating(submissionEnvelope, resourceAssembler); - assertThat(requestResponse).isNotNull(); - verify(submissionEnvelopeService).handleEnvelopeStateUpdateRequest(submissionEnvelope, GRAPH_VALIDATING); - enactResponse = controller.enactGraphValidating(submissionEnvelope, resourceAssembler); - assertThat(enactResponse).isNotNull(); - assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(GRAPH_VALIDATING); - - // graph validating -> graph valid - requestResponse = controller.requestGraphValid(submissionEnvelope, resourceAssembler); - assertThat(requestResponse).isNotNull(); - verify(submissionEnvelopeService).handleEnvelopeStateUpdateRequest(submissionEnvelope, GRAPH_VALID); - enactResponse = controller.enactGraphValid(submissionEnvelope, resourceAssembler); - assertThat(enactResponse).isNotNull(); - assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(GRAPH_VALID); - } - - @Configuration - static class TestConfiguration {} - -} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/ProjectJson.java b/src/test/java/uk/ac/ebi/subs/ingest/ProjectJson.java new file mode 100644 index 000000000..274245464 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/ProjectJson.java @@ -0,0 +1,35 @@ +package uk.ac.ebi.subs.ingest; + +import java.util.Map; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class ProjectJson { + String title; + + public static ProjectJson fromTitle(String title) { + ProjectJson project = new ProjectJson(); + project.title = title; + return project; + } + + public ObjectNode toObjectNode() { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode content = mapper.createObjectNode(); + ObjectNode projectCore0 = content.putObject("project_core"); + projectCore0.put("project_title", this.title); + + ObjectNode metadata = mapper.createObjectNode(); + metadata.set("content", content); + + return metadata; + } + + public Map toMap() { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode project = this.toObjectNode(); + return mapper.convertValue(project, new TypeReference>() {}); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/audit/AuditTypeTest.java b/src/test/java/uk/ac/ebi/subs/ingest/audit/AuditTypeTest.java new file mode 100644 index 000000000..c5634fcf7 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/audit/AuditTypeTest.java @@ -0,0 +1,28 @@ +package uk.ac.ebi.subs.ingest.audit; + +import static junit.framework.TestCase.assertEquals; + +import java.io.IOException; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.json.JsonTest; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +@JsonTest +public class AuditTypeTest { + @Autowired private ObjectMapper objectMapper; + + @Test + public void testDeserialize() throws IOException { + assertEquals( + objectMapper.readValue("\"Status updated\"", AuditType.class), AuditType.STATUS_UPDATED); + } + + @Test + public void testSerialize() throws JsonProcessingException { + assertEquals(objectMapper.writeValueAsString(AuditType.STATUS_UPDATED), "\"Status updated\""); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/core/MetadataDocumentTest.java b/src/test/java/uk/ac/ebi/subs/ingest/core/MetadataDocumentTest.java new file mode 100644 index 000000000..7b0b3d5f0 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/core/MetadataDocumentTest.java @@ -0,0 +1,111 @@ +package uk.ac.ebi.subs.ingest.core; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import lombok.*; + +@ExtendWith(SpringExtension.class) +@SpringBootTest( + classes = { + MetadataDocument.class, + }) +public class MetadataDocumentTest { + @Test + @DisplayName("Is Equal with matching Static Members") + public void testSameStatics() { + // given + var map1 = Map.of("Key1", "Value1", "Key2", "Value2"); + var map2 = Map.of("Key1", "Value1", "Key2", "Value2"); + + MetadataDocument doc1 = new DocumentTest(EntityType.PROJECT, "Identifier", map1); + doc1.setUuid(Uuid.newUuid()); + MetadataDocument doc2 = new DocumentTest(EntityType.PROJECT, "Identifier", map2); + doc2.setUuid(doc1.getUuid()); + + assertThat(doc1).isEqualTo(doc2); + } + + @Test + @DisplayName("Is Equal with similar content") + public void testSimilarContent() { + // given + var map1 = Map.of("Key1", "Value1", "Key2", "Value2"); + var map2 = Map.of("Key2", "Value2", "Key1", "Value1"); + MetadataDocument doc1 = new DocumentTest(EntityType.PROJECT, "Identifier", map1); + MetadataDocument doc2 = new DocumentTest(EntityType.PROJECT, "Identifier", map2); + + assertThat(doc1).isEqualTo(doc2); + } + + @Test + @DisplayName("Is Not Equal with different id") + public void testDifferentID() { + // given + var map1 = Map.of("Key1", "Value1", "Key2", "Value2"); + var map2 = Map.of("Key1", "Value1", "Key2", "Value2"); + MetadataDocument doc1 = new DocumentTest(EntityType.PROJECT, "Identifier-One", map1); + MetadataDocument doc2 = new DocumentTest(EntityType.PROJECT, "Identifier-Two", map2); + + assertThat(doc1).isNotEqualTo(doc2); + } + + @Test + @DisplayName("Is Not Equal with different content") + public void testDifferentContent() { + // given + var map1 = Map.of("Key1", "Value1", "Key2", "Value2"); + var map2 = Map.of("Key1", "Value1", "Key3", "Value3"); + MetadataDocument doc1 = new DocumentTest(EntityType.PROJECT, "Identifier", map1); + MetadataDocument doc2 = new DocumentTest(EntityType.PROJECT, "Identifier", map2); + + assertThat(doc1).isNotEqualTo(doc2); + } + + @Test + @DisplayName("Is Not Equal with different content") + public void testDifferentUUIDs() { + // given + var map1 = Map.of("Key1", "Value1", "Key2", "Value2"); + var map2 = Map.of("Key1", "Value1", "Key2", "Value2"); + MetadataDocument doc1 = new DocumentTest(EntityType.PROJECT, "Identifier", map1); + doc1.setUuid(Uuid.newUuid()); + MetadataDocument doc2 = new DocumentTest(EntityType.PROJECT, "Identifier", map2); + doc2.setUuid(Uuid.newUuid()); + + assertThat(doc1).isNotEqualTo(doc2); + } + + @Getter + @EqualsAndHashCode(callSuper = true) + static class DocumentTest extends MetadataDocument { + DocumentTest(EntityType type, String id, Object content) { + super(type, content); + this.id = id; + } + } + + @Test + @DisplayName("Is Equal with matching accessions") + public void testAccessionsEquality() { + // given + var accessions1 = List.of("ENA:12345", "ENA:67890"); + var accessions2 = List.of("ENA:12345", "ENA:67890"); + + MetadataDocument doc1 = new DocumentTest(EntityType.PROJECT, "Identifier", Map.of()); + doc1.setAccessions(accessions1); + MetadataDocument doc2 = new DocumentTest(EntityType.PROJECT, "Identifier", Map.of()); + doc2.setAccessions(accessions2); + + // then + assertThat(doc1).isEqualTo(doc2); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/core/service/MetadataCrudServiceTest.java b/src/test/java/uk/ac/ebi/subs/ingest/core/service/MetadataCrudServiceTest.java new file mode 100644 index 000000000..d2234c364 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/core/service/MetadataCrudServiceTest.java @@ -0,0 +1,66 @@ +package uk.ac.ebi.subs.ingest.core.service; + +import java.util.stream.Stream; + +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.provider.Arguments; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.core.service.strategy.impl.*; +import uk.ac.ebi.subs.ingest.dataset.DatasetRepository; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; +import uk.ac.ebi.subs.ingest.study.StudyRepository; + +@ExtendWith(SpringExtension.class) +@SpringBootTest( + classes = { + MetadataCrudService.class, + BiomaterialCrudStrategy.class, + FileCrudStrategy.class, + ProcessCrudStrategy.class, + ProjectCrudStrategy.class, + ProtocolCrudStrategy.class, + StudyCrudStrategy.class, + DatasetCrudStrategy.class + }) +public class MetadataCrudServiceTest { + @Autowired private MetadataCrudService crudService; + @Autowired private BiomaterialCrudStrategy biomaterialCrudStrategy; + @Autowired private FileCrudStrategy fileCrudStrategy; + @Autowired private ProcessCrudStrategy processCrudStrategy; + @Autowired private ProjectCrudStrategy projectCrudStrategy; + @Autowired private ProtocolCrudStrategy protocolCrudStrategy; + @Autowired private StudyCrudStrategy studyCrudStrategy; + @Autowired private DatasetCrudStrategy datasetCrudStrategy; + + @MockBean private MessageRouter messageRouter; + @MockBean private BiomaterialRepository biomaterialRepository; + @MockBean private FileRepository fileRepository; + @MockBean private ProcessRepository processRepository; + @MockBean private ProjectRepository projectRepository; + @MockBean private ProtocolRepository protocolRepository; + @MockBean private StudyRepository studyRepository; + @MockBean private DatasetRepository datasetRepository; + + private static Stream providedTestDocuments() { + return Stream.of( + Arguments.of(new Biomaterial(null)), + Arguments.of(new File(null, "fileName")), + Arguments.of(new Process(null)), + Arguments.of(new Project(null)), + Arguments.of(new Protocol(null))); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/core/service/MetadataLinkingServiceTest.java b/src/test/java/uk/ac/ebi/subs/ingest/core/service/MetadataLinkingServiceTest.java new file mode 100644 index 000000000..1ec01b7f4 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/core/service/MetadataLinkingServiceTest.java @@ -0,0 +1,153 @@ +package uk.ac.ebi.subs.ingest.core.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; + +import java.lang.reflect.InvocationTargetException; +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.dao.OptimisticLockingFailureException; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = {MetadataLinkingService.class}) +public class MetadataLinkingServiceTest { + @Autowired private MetadataLinkingService service; + + @MockBean private ValidationStateChangeService validationStateChangeService; + + @MockBean private MongoTemplate mongoTemplate; + + Protocol protocol; + Protocol protocol2; + Process process; + SubmissionEnvelope submission; + + @BeforeEach + void setUp() { + submission = new SubmissionEnvelope(); + submission.enactStateTransition(SubmissionState.GRAPH_VALID); + protocol = spy(new Protocol(null)); + doReturn("protocol1").when(protocol).getId(); + protocol2 = spy(new Protocol(null)); + doReturn("protocol2").when(protocol2).getId(); + process = spy(new Process(null)); + doReturn("process").when(process).getId(); + process.setSubmissionEnvelope(submission); + } + + @Test + public void testReplaceLink() + throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + process.addProtocol(protocol); + + // when + service.replaceLinks(process, List.of(protocol2), "protocols"); + + // then + assertThat(process.getProtocols().size()).isEqualTo(1); + assertThat(process.getProtocols().contains(protocol2)).isTrue(); + + verify(validationStateChangeService, times(0)) + .changeValidationState(protocol.getType(), protocol.getId(), ValidationState.DRAFT); + verify(validationStateChangeService, times(0)) + .changeValidationState(protocol2.getType(), protocol2.getId(), ValidationState.DRAFT); + verify(validationStateChangeService) + .changeValidationState(process.getType(), process.getId(), ValidationState.DRAFT); + verify(mongoTemplate).save(process); + } + + @Test + public void testReplaceLinkNotGraphValid() + throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + process.addProtocol(protocol); + submission.enactStateTransition(SubmissionState.METADATA_VALID); + + // when + service.replaceLinks(process, List.of(protocol2), "protocols"); + + // then + assertThat(process.getProtocols().size()).isEqualTo(1); + assertThat(process.getProtocols().contains(protocol2)).isTrue(); + + verify(validationStateChangeService, times(0)) + .changeValidationState(protocol.getType(), protocol.getId(), ValidationState.DRAFT); + verify(validationStateChangeService, times(0)) + .changeValidationState(protocol2.getType(), protocol2.getId(), ValidationState.DRAFT); + verify(validationStateChangeService, times(0)) + .changeValidationState(process.getType(), process.getId(), ValidationState.DRAFT); + verify(mongoTemplate).save(process); + } + + @Test + public void testAddLink() + throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + + // when + service.addLinks(process, List.of(protocol), "protocols"); + + // then + assertThat(process.getProtocols().size()).isEqualTo(1); + assertThat(process.getProtocols().contains(protocol)).isTrue(); + verify(validationStateChangeService, times(0)) + .changeValidationState(protocol.getType(), protocol.getId(), ValidationState.DRAFT); + verify(validationStateChangeService) + .changeValidationState(process.getType(), process.getId(), ValidationState.DRAFT); + verify(mongoTemplate).save(process); + } + + @Test + public void testAddLinkNotGraphValid() + throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + submission.enactStateTransition(SubmissionState.METADATA_VALID); + + // when + service.addLinks(process, List.of(protocol), "protocols"); + + // then + assertThat(process.getProtocols().size()).isEqualTo(1); + assertThat(process.getProtocols().contains(protocol)).isTrue(); + verify(validationStateChangeService, times(0)) + .changeValidationState(protocol.getType(), protocol.getId(), ValidationState.DRAFT); + verify(validationStateChangeService, times(0)) + .changeValidationState(process.getType(), process.getId(), ValidationState.DRAFT); + verify(mongoTemplate).save(process); + } + + @Test + public void testRetry() + throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + // given + + when(validationStateChangeService.changeValidationState( + process.getType(), process.getId(), ValidationState.DRAFT)) + .thenThrow(new OptimisticLockingFailureException("Error")) + .thenThrow(new OptimisticLockingFailureException("Error")) + .thenReturn(process); + + // when + service.addLinks(process, List.of(protocol), "protocols"); + + // then + assertThat(process.getProtocols().size()).isEqualTo(1); + assertThat(process.getProtocols().contains(protocol)).isTrue(); + verify(validationStateChangeService, times(0)) + .changeValidationState(protocol.getType(), protocol.getId(), ValidationState.DRAFT); + verify(validationStateChangeService, times(3)) + .changeValidationState(process.getType(), process.getId(), ValidationState.DRAFT); + verify(mongoTemplate).save(process); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/core/service/MetadataUpdateServiceTest.java b/src/test/java/uk/ac/ebi/subs/ingest/core/service/MetadataUpdateServiceTest.java new file mode 100644 index 000000000..12e2a7ebc --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/core/service/MetadataUpdateServiceTest.java @@ -0,0 +1,151 @@ +package uk.ac.ebi.subs.ingest.core.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.data.mapping.context.PersistentEntities; +import org.springframework.data.rest.webmvc.mapping.Associations; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import uk.ac.ebi.subs.ingest.ProjectJson; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.patch.JsonPatcher; +import uk.ac.ebi.subs.ingest.patch.PatchService; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.state.ValidationState; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = {MetadataUpdateService.class, JsonPatcher.class, ObjectMapper.class}) +public class MetadataUpdateServiceTest { + @Autowired private MetadataUpdateService service; + + @MockBean private MetadataDifferService metadataDifferService; + + @MockBean private MetadataCrudService metadataCrudService; + + @Autowired private JsonPatcher jsonPatcher; + + @MockBean private PatchService patchService; + + @MockBean private ValidationStateChangeService validationStateChangeService; + + @MockBean PersistentEntities persistentEntities; + + @MockBean private Associations associations; + + @Test + public void testUpdateShouldSaveAndReturnUpdatedMetadata() { + // given: + JsonNode content = ProjectJson.fromTitle("Old Project Title").toObjectNode().get("content"); + Project project = new Project(content); + + ObjectNode patch = ProjectJson.fromTitle("New Project Title").toObjectNode(); + + when(metadataCrudService.save(any())).thenReturn(project); + + // when: + Project updatedProject = service.update(project, patch); + + // then: + assertThat(updatedProject).isEqualTo(project); + verify(metadataCrudService).save(project); + } + + @Test + public void testUpdateShouldSetStateToDraftWhenContentChanged() { + // given: + JsonNode content = ProjectJson.fromTitle("Old Project Title").toObjectNode().get("content"); + Project project = new Project(content); + + ObjectNode patch = ProjectJson.fromTitle("New Project Title").toObjectNode(); + + when(metadataCrudService.save(any())).thenReturn(project); + + // when: + Project updatedProject = service.update(project, patch); + + // then: + assertThat(updatedProject).isEqualTo(project); + verify(metadataCrudService).save(project); + verify(validationStateChangeService) + .changeValidationState(project.getType(), project.getId(), ValidationState.DRAFT); + } + + @Test + public void testUpdateShouldNotSetStateWhenContentIsUnchanged() { + // given: + JsonNode content = ProjectJson.fromTitle("Old Project Title").toObjectNode().get("content"); + Project project = new Project(content); + + ObjectNode patch = ProjectJson.fromTitle("Old Project Title").toObjectNode(); + + when(metadataCrudService.save(any())).thenReturn(project); + + // when: + Project updatedProject = service.update(project, patch); + + // then: + assertThat(updatedProject).isEqualTo(project); + verify(metadataCrudService).save(project); + verify(validationStateChangeService, never()) + .changeValidationState(project.getType(), project.getId(), ValidationState.DRAFT); + } + + @Test + public void testUpdateShouldNotSetStateWhenNoContent() { + // given: + JsonNode content = ProjectJson.fromTitle("Old Project Title").toObjectNode().get("content"); + Project project = new Project(content); + + ObjectMapper mapper = new ObjectMapper(); + ObjectNode patch = mapper.createObjectNode(); + patch.put("isInCatalogue", false); + + when(metadataCrudService.save(any())).thenReturn(project); + + // when: + Project updatedProject = service.update(project, patch); + + // then: + assertThat(updatedProject).isEqualTo(project); + verify(metadataCrudService).save(project); + verify(validationStateChangeService, never()) + .changeValidationState(project.getType(), project.getId(), ValidationState.DRAFT); + } + + @Test + public void testUpdateProjectWithSupplementaryFileShouldNotThrowRecursionError() { + // given: + JsonNode content = + ProjectJson.fromTitle("Project with supplementary file").toObjectNode().get("content"); + Project project = new Project(content); + + File supplementaryFile = new File(null, "fileName"); + supplementaryFile.setProject(project); + project.getSupplementaryFiles().add(supplementaryFile); + + ObjectNode patch = + ProjectJson.fromTitle("Updated project with supplementary file").toObjectNode(); + + when(metadataCrudService.save(any())).thenReturn(project); + + // when: + Project updatedProject = service.update(project, patch); + + // then: + assertThat(updatedProject).isEqualTo(project); + verify(metadataCrudService).save(project); + verify(validationStateChangeService) + .changeValidationState(project.getType(), project.getId(), ValidationState.DRAFT); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/core/service/strategy/BiomaterialCrudStrategyTest.java b/src/test/java/uk/ac/ebi/subs/ingest/core/service/strategy/BiomaterialCrudStrategyTest.java new file mode 100644 index 000000000..cd0512d89 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/core/service/strategy/BiomaterialCrudStrategyTest.java @@ -0,0 +1,40 @@ +package uk.ac.ebi.subs.ingest.core.service.strategy; + +import static org.mockito.Mockito.verify; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.core.service.strategy.impl.BiomaterialCrudStrategy; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = {BiomaterialCrudStrategy.class}) +public class BiomaterialCrudStrategyTest { + @Autowired private BiomaterialCrudStrategy biomaterialCrudStrategy; + + @MockBean private BiomaterialRepository biomaterialRepository; + @MockBean private MessageRouter messageRouter; + + private Biomaterial testBiomaterial; + + @BeforeEach + void setUp() { + testBiomaterial = new Biomaterial(null); + } + + @Test + public void testDeleteBiomaterial() { + // when + biomaterialCrudStrategy.deleteDocument(testBiomaterial); + // then + verify(biomaterialRepository).delete(testBiomaterial); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/core/service/strategy/FileCrudStrategyTest.java b/src/test/java/uk/ac/ebi/subs/ingest/core/service/strategy/FileCrudStrategyTest.java new file mode 100644 index 000000000..86aab5b88 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/core/service/strategy/FileCrudStrategyTest.java @@ -0,0 +1,62 @@ +package uk.ac.ebi.subs.ingest.core.service.strategy; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; + +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.core.service.strategy.impl.FileCrudStrategy; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = {FileCrudStrategy.class}) +public class FileCrudStrategyTest { + @Autowired private FileCrudStrategy fileCrudStrategy; + + @MockBean private FileRepository fileRepository; + @MockBean private ProjectRepository projectRepository; + @MockBean private MessageRouter messageRouter; + + private File testFile; + + @BeforeEach + void setUp() { + testFile = new File(null, "fileName"); + } + + @Test + public void testRemoveLinksFile() { + // given + Project projectWithFile = new Project(null); + projectWithFile.getSupplementaryFiles().add(testFile); + when(projectRepository.findBySupplementaryFilesContains(testFile)) + .thenReturn(Stream.of(projectWithFile)); + + // when + fileCrudStrategy.removeLinksToDocument(testFile); + + // then + assertThat(projectWithFile.getSupplementaryFiles()).isEmpty(); + verify(projectRepository).save(projectWithFile); + } + + @Test + public void testDeleteFile() { + // when + fileCrudStrategy.deleteDocument(testFile); + // then + verify(fileRepository).delete(testFile); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/core/service/strategy/ProcessCrudStrategyTest.java b/src/test/java/uk/ac/ebi/subs/ingest/core/service/strategy/ProcessCrudStrategyTest.java new file mode 100644 index 000000000..8df76df29 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/core/service/strategy/ProcessCrudStrategyTest.java @@ -0,0 +1,84 @@ +package uk.ac.ebi.subs.ingest.core.service.strategy; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; + +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.core.service.strategy.impl.ProcessCrudStrategy; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = {ProcessCrudStrategy.class}) +public class ProcessCrudStrategyTest { + @Autowired private ProcessCrudStrategy processCrudStrategy; + + @MockBean private BiomaterialRepository biomaterialRepository; + @MockBean private ProcessRepository processRepository; + @MockBean private FileRepository fileRepository; + @MockBean private MessageRouter messageRouter; + + private Process testProcess; + + @BeforeEach + void setUp() { + testProcess = new Process(null); + } + + @Test + public void testRemoveLinksProcess() { + // given + File inputFile = spy(new File(null, "inputFile")); + File derivedFile = spy(new File(null, "derivedFile")); + inputFile.getInputToProcesses().add(testProcess); + derivedFile.getDerivedByProcesses().add(testProcess); + when(fileRepository.findByInputToProcessesContains(testProcess)) + .thenReturn(Stream.of(inputFile)); + when(fileRepository.findByDerivedByProcessesContains(testProcess)) + .thenReturn(Stream.of(derivedFile)); + + Biomaterial inputBio = spy(new Biomaterial(null)); + Biomaterial derivedBio = spy(new Biomaterial(null)); + inputBio.getInputToProcesses().add(testProcess); + derivedBio.getDerivedByProcesses().add(testProcess); + when(biomaterialRepository.findByInputToProcessesContains(testProcess)) + .thenReturn(Stream.of(inputBio)); + when(biomaterialRepository.findByDerivedByProcessesContains(testProcess)) + .thenReturn(Stream.of(derivedBio)); + + // when + processCrudStrategy.removeLinksToDocument(testProcess); + + // then + assertThat(inputFile.getInputToProcesses()).isEmpty(); + assertThat(derivedFile.getDerivedByProcesses()).isEmpty(); + assertThat(inputBio.getInputToProcesses()).isEmpty(); + assertThat(derivedBio.getDerivedByProcesses()).isEmpty(); + verify(fileRepository).save(inputFile); + verify(fileRepository).save(derivedFile); + verify(biomaterialRepository).save(inputBio); + verify(biomaterialRepository).save(derivedBio); + } + + @Test + public void testDeleteProcess() { + // when + processCrudStrategy.deleteDocument(testProcess); + // then + verify(processRepository).delete(testProcess); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/core/service/strategy/ProjectCrudStrategyTest.java b/src/test/java/uk/ac/ebi/subs/ingest/core/service/strategy/ProjectCrudStrategyTest.java new file mode 100644 index 000000000..3c189277d --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/core/service/strategy/ProjectCrudStrategyTest.java @@ -0,0 +1,97 @@ +package uk.ac.ebi.subs.ingest.core.service.strategy; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; + +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.core.service.strategy.impl.ProjectCrudStrategy; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = {ProjectCrudStrategy.class}) +public class ProjectCrudStrategyTest { + @Autowired private ProjectCrudStrategy projectCrudStrategy; + + @MockBean private ProjectRepository projectRepository; + @MockBean private ProtocolRepository protocolRepository; + @MockBean private ProcessRepository processRepository; + @MockBean private FileRepository fileRepository; + @MockBean private BiomaterialRepository biomaterialRepository; + @MockBean private MessageRouter messageRouter; + + private Project testProject; + + @BeforeEach + void setUp() { + testProject = new Project(null); + } + + @Test + public void testRemoveLinksProject() { + // Given + Biomaterial biomaterialWithProject = new Biomaterial(null); + biomaterialWithProject.setProject(testProject); + biomaterialWithProject.getProjects().add(testProject); + when(biomaterialRepository.findByProject(testProject)) + .thenReturn(Stream.of(biomaterialWithProject)); + when(biomaterialRepository.findByProjectsContaining(testProject)) + .thenReturn(Stream.of(biomaterialWithProject)); + + File fileWithProject = new File(null, "fileWithProject"); + fileWithProject.setProject(testProject); + when(fileRepository.findByProject(testProject)).thenReturn(Stream.of(fileWithProject)); + + Process processWithProject = new Process(null); + processWithProject.setProject(testProject); + processWithProject.getProjects().add(testProject); + when(processRepository.findByProject(testProject)).thenReturn(Stream.of(processWithProject)); + when(processRepository.findByProjectsContaining(testProject)) + .thenReturn(Stream.of(processWithProject)); + + Protocol protocolWithProject = new Protocol(null); + protocolWithProject.setProject(testProject); + when(protocolRepository.findByProject(testProject)).thenReturn(Stream.of(protocolWithProject)); + + // when + projectCrudStrategy.removeLinksToDocument(testProject); + + // then + assertThat(biomaterialWithProject.getProject()).isNull(); + assertThat(biomaterialWithProject.getProjects()).isEmpty(); + assertThat(fileWithProject.getProject()).isNull(); + assertThat(processWithProject.getProject()).isNull(); + assertThat(processWithProject.getProjects()).isEmpty(); + assertThat(protocolWithProject.getProject()).isNull(); + verify(biomaterialRepository, times(2)).save(biomaterialWithProject); + verify(fileRepository).save(fileWithProject); + verify(processRepository, times(2)).save(processWithProject); + verify(protocolRepository).save(protocolWithProject); + } + + @Test + public void testDeleteProject() { + // when + projectCrudStrategy.deleteDocument(testProject); + // then + verify(projectRepository).delete(testProject); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/core/service/strategy/ProtocolCrudStrategyTest.java b/src/test/java/uk/ac/ebi/subs/ingest/core/service/strategy/ProtocolCrudStrategyTest.java new file mode 100644 index 000000000..9549c13d3 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/core/service/strategy/ProtocolCrudStrategyTest.java @@ -0,0 +1,62 @@ +package uk.ac.ebi.subs.ingest.core.service.strategy; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; + +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.core.service.strategy.impl.ProtocolCrudStrategy; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = {ProtocolCrudStrategy.class}) +public class ProtocolCrudStrategyTest { + @Autowired private ProtocolCrudStrategy protocolCrudStrategy; + + @MockBean private ProtocolRepository protocolRepository; + @MockBean private ProcessRepository processRepository; + @MockBean private MessageRouter messageRouter; + + private Protocol testProtocol; + + @BeforeEach + void setUp() { + testProtocol = new Protocol(null); + } + + @Test + public void testRemoveLinksProject() { + // given + Process processWithProtocol = new Process(null); + processWithProtocol.getProtocols().add(testProtocol); + when(processRepository.findByProtocolsContains(testProtocol)) + .thenReturn(Stream.of(processWithProtocol)); + + // when + protocolCrudStrategy.removeLinksToDocument(testProtocol); + + // then + assertThat(processWithProtocol.getProtocols()).isEmpty(); + verify(processRepository).save(processWithProtocol); + } + + @Test + public void testDeleteProject() { + // when + protocolCrudStrategy.deleteDocument(testProtocol); + // then + verify(protocolRepository).delete(testProtocol); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/core/web/SpringLinkGeneratorTest.java b/src/test/java/uk/ac/ebi/subs/ingest/core/web/SpringLinkGeneratorTest.java new file mode 100644 index 000000000..272362d3b --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/core/web/SpringLinkGeneratorTest.java @@ -0,0 +1,51 @@ +package uk.ac.ebi.subs.ingest.core.web; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.data.rest.core.mapping.ResourceMappings; +import org.springframework.data.rest.core.mapping.ResourceMetadata; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.bundle.BundleManifest; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.protocol.Protocol; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = {SpringLinkGenerator.class}) +public class SpringLinkGeneratorTest { + + @MockBean private ResourceMappings mappings; + + @Autowired private SpringLinkGenerator linkGenerator = new SpringLinkGenerator(); + + @Test + public void testCreateCallback() { + ResourceMetadata resourceMetadata = mock(ResourceMetadata.class); + when(resourceMetadata.getRel()).thenReturn("metadata"); + when(mappings.getMetadataFor(any())).thenReturn(resourceMetadata); + + // when: + String processCallback = linkGenerator.createCallback(Process.class, "df00e2"); + String biomaterialCallback = linkGenerator.createCallback(Biomaterial.class, "c80122"); + String fileCallback = linkGenerator.createCallback(File.class, "98dd90"); + String protocolCallback = linkGenerator.createCallback(Protocol.class, "846df1"); + String bmCallback = linkGenerator.createCallback(BundleManifest.class, "332fd9"); + + // then: + assertThat(processCallback).isEqualToIgnoringCase("/metadata/df00e2"); + assertThat(biomaterialCallback).isEqualToIgnoringCase("/metadata/c80122"); + assertThat(fileCallback).isEqualToIgnoringCase("/metadata/98dd90"); + assertThat(protocolCallback).isEqualToIgnoringCase("/metadata/846df1"); + assertThat(bmCallback).isEqualToIgnoringCase("/metadata/332fd9"); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/dataset/DatasetServiceTest.java b/src/test/java/uk/ac/ebi/subs/ingest/dataset/DatasetServiceTest.java new file mode 100644 index 000000000..edbdb13d9 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/dataset/DatasetServiceTest.java @@ -0,0 +1,244 @@ +package uk.ac.ebi.subs.ingest.dataset; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.*; + +import java.util.Objects; +import java.util.Optional; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.ApplicationContext; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.web.server.ResponseStatusException; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.dataset.util.UploadAreaUtil; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; + +@ExtendWith(SpringExtension.class) +@SpringBootTest( + classes = { + DatasetService.class, + DatasetRepository.class, + UploadAreaUtil.class, + BiomaterialRepository.class, + ProtocolRepository.class, + ProcessRepository.class, + FileRepository.class + }) +public class DatasetServiceTest { + @Autowired private ApplicationContext applicationContext; + + @Autowired private DatasetService datasetService; + + @MockBean private MongoTemplate mongoTemplate; + + @MockBean private DatasetRepository datasetRepository; + + @MockBean private ProtocolRepository protocolRepository; + + @MockBean private ProcessRepository processRepository; + + @MockBean private FileRepository fileRepository; + + @MockBean private BiomaterialRepository biomaterialRepository; + + @MockBean private DatasetEventHandler datasetEventHandler; + + @MockBean private MetadataCrudService metadataCrudService; + + @MockBean private MetadataUpdateService metadataUpdateService; + + @MockBean private UploadAreaUtil uploadAreaUtil; + + @BeforeEach + void setUp() { + applicationContext.getBeansWithAnnotation(MockBean.class).forEach(Mockito::reset); + Mockito.reset(metadataCrudService, datasetRepository, datasetEventHandler); + } + + @Nested + class DatasetRegistration { + + @Test + @DisplayName("Register Dataset - Success") + void registerSuccess() { + // given: + String content = "{\"name\": \"dataset\"}"; + Dataset dataset = new Dataset(content); + + // and: + Dataset persistentDataset = new Dataset(content); + doReturn(persistentDataset).when(datasetRepository).save(dataset); + + // when: + Dataset result = datasetService.register(dataset); + + // then: + verify(datasetRepository, times(1)).save(dataset); + assertThat(result).isEqualTo(persistentDataset); + verify(datasetEventHandler).registeredDataset(persistentDataset); + } + } + + @Nested + class DatasetUpdate { + @Test + @DisplayName("Update Dataset - Success") + void updateSuccess() { + // given: + String datasetId = "datasetId"; + ObjectNode patch = createUpdatePatch("Updated Dataset Name"); + Dataset existingDataset = new Dataset("{\"name\": \"dataset\"}"); + + // and: + when(datasetRepository.findById(existingDataset.getId())) + .thenReturn(Optional.of(existingDataset)); + when(metadataUpdateService.update(existingDataset, patch)).thenReturn(existingDataset); + + // when: + Dataset result = datasetService.update(existingDataset, patch); + + // then: + verify(datasetRepository).findById(existingDataset.getId()); + verify(metadataUpdateService).update(existingDataset, patch); + verify(datasetEventHandler).updatedDataset(existingDataset); + assertThat(result).isEqualTo(existingDataset); + } + + // Helper method to create an update patch + private ObjectNode createUpdatePatch(String updatedName) { + ObjectNode patch = JsonNodeFactory.instance.objectNode(); + patch.put("content", JsonNodeFactory.instance.objectNode().put("name", updatedName)); + return patch; + } + + @Test + @DisplayName("Update Dataset - Not Found") + void updateDatasetNotFound() { + // given: + Dataset nonExistentDataset = new Dataset("{\"name\": \"non existant dataset\"}"); + ObjectNode patch = createUpdatePatch("Updated Dataset Name"); + + // and: + when(datasetRepository.findById(nonExistentDataset.getId())).thenReturn(Optional.empty()); + + // when, then: + ResponseStatusException exception = + assertThrows( + ResponseStatusException.class, + () -> datasetService.update(nonExistentDataset, patch)); + assertThat(Objects.requireNonNull(exception.getMessage()).contains("404 NOT_FOUND")); + + // verify that other methods are not called + verify(metadataCrudService, never()).deleteDocument(any()); + verify(datasetEventHandler, never()).deletedDataset(any()); + } + } + + @Nested + class DatasetReplace { + + @Test + @DisplayName("Replace Dataset - Success") + void replaceSuccess() { + // given: + String datasetId = "datasetId"; + Dataset existingDataset = new Dataset("{\"name\": \"Existing Dataset Name\"}"); + Dataset updatedDataset = new Dataset("{\"name\": \"Updated Dataset Name\"}"); + + // and: + when(datasetRepository.findById(datasetId)).thenReturn(Optional.of(existingDataset)); + + // when: + Dataset result = datasetService.replace(datasetId, updatedDataset); + + // then: + verify(datasetRepository).findById(datasetId); + verify(datasetRepository).save(updatedDataset); // Verify save is called + verify(datasetEventHandler).updatedDataset(updatedDataset); + assertThat(result).isEqualTo(updatedDataset); + } + + @Test + @DisplayName("Replace Dataset - Not Found") + void replaceDatasetNotFound() { + // given: + String nonExistentDatasetId = "nonExistentId"; + Dataset updatedDataset = new Dataset("{\"name\": \"Updated Dataset Name\"}"); + + // and: + when(datasetRepository.findById(nonExistentDatasetId)).thenReturn(Optional.empty()); + + // when, then: + ResponseStatusException exception = + assertThrows( + ResponseStatusException.class, + () -> datasetService.replace(nonExistentDatasetId, updatedDataset)); + assertThat(Objects.requireNonNull(exception.getMessage()).contains("404 NOT_FOUND")); + + // verify that other methods are not called + verify(datasetRepository, never()).save(any()); + verify(datasetEventHandler, never()).updatedDataset(any()); + } + } + + @Nested + class DatasetDeletion { + @Test + @DisplayName("Delete Dataset - Success") + void deleteSuccess() { + // given: + String datasteId = "testDeleteId"; + String content = "{\"name\": \"delete dataset\"}"; + Dataset persistentDataset = new Dataset(content); + + // and: + when(datasetRepository.findById(datasteId)).thenReturn(Optional.of(persistentDataset)); + + // when: + datasetService.delete(datasteId, false); + + // then: + verify(metadataCrudService).deleteDocument(persistentDataset); + verify(datasetEventHandler).deletedDataset(datasteId); + } + + @Test + @DisplayName("Delete Dataset - Not Found") + void deleteDatasetNotFound() { + // given: + String nonExistentDatasetId = "nonExistentId"; + + // and: + when(datasetRepository.findById(nonExistentDatasetId)).thenReturn(Optional.empty()); + + // when, then: + ResponseStatusException exception = + assertThrows( + ResponseStatusException.class, + () -> datasetService.delete(nonExistentDatasetId, false)); + assertThat(Objects.requireNonNull(exception.getMessage()).contains("404 NOT_FOUND")); + + verify(metadataCrudService, never()).deleteDocument(any()); + verify(datasetEventHandler, never()).deletedDataset(any()); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/errors/SubmissionErrorControllerTest.java b/src/test/java/uk/ac/ebi/subs/ingest/errors/SubmissionErrorControllerTest.java new file mode 100644 index 000000000..8ef9288bf --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/errors/SubmissionErrorControllerTest.java @@ -0,0 +1,41 @@ +package uk.ac.ebi.subs.ingest.errors; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.mockito.Mockito.verify; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.data.web.PagedResourcesAssembler; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.errors.web.SubmissionErrorController; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = {SubmissionErrorController.class}) +public class SubmissionErrorControllerTest { + @Autowired private SubmissionErrorController controller; + + @MockBean SubmissionErrorService submissionErrorService; + + @MockBean private PagedResourcesAssembler pagedResourcesAssembler; + + @Test + public void testDeleteSubmissionErrors() { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + + // when: + ResponseEntity response = + controller.deleteSubmissionEnvelopeErrors(submissionEnvelope); + + // then + assertThat(response).isNotNull(); + assertThat(response.getStatusCode().value()).isEqualTo(204); + verify(submissionErrorService).deleteSubmissionEnvelopeErrors(submissionEnvelope); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/errors/SubmissionErrorServiceTest.java b/src/test/java/uk/ac/ebi/subs/ingest/errors/SubmissionErrorServiceTest.java new file mode 100644 index 000000000..c12400c23 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/errors/SubmissionErrorServiceTest.java @@ -0,0 +1,121 @@ +package uk.ac.ebi.subs.ingest.errors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +import java.net.URI; +import java.util.Collections; +import java.util.Random; +import java.util.UUID; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.zalando.problem.*; + +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = {SubmissionErrorService.class}) +public class SubmissionErrorServiceTest { + @MockBean private Pageable pageable; + @MockBean private SubmissionErrorRepository submissionErrorRepository; + @Autowired private SubmissionErrorService submissionErrorService; + + @Test + public void serviceCallsRepository() { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + // and: + when(submissionErrorRepository.findBySubmissionEnvelope( + any(SubmissionEnvelope.class), any(Pageable.class))) + .thenReturn(new PageImpl(Collections.emptyList())); + + // then: + assertThat(submissionErrorService.getErrorsFromEnvelope(submissionEnvelope, pageable)) + .isEmpty(); + } + + @Test + public void errorIsGivenEnvelope() { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + ArgumentCaptor insertedError = ArgumentCaptor.forClass(SubmissionError.class); + + // when: + SubmissionError submissionError = + submissionErrorService.addErrorToEnvelope(submissionEnvelope, randomProblem()); + + // then: + verify(submissionErrorRepository).insert(insertedError.capture()); + assertThat(insertedError.getValue().getSubmissionEnvelope()).isEqualTo(submissionEnvelope); + assertThat(insertedError.getValue()).isEqualTo(submissionError); + } + + @Test + public void problemHasInstanceRemoved() { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + ArgumentCaptor insertedError = ArgumentCaptor.forClass(SubmissionError.class); + + // when: + submissionErrorService.addErrorToEnvelope(submissionEnvelope, randomProblem()); + + // then: + verify(submissionErrorRepository).insert(insertedError.capture()); + assertThat(insertedError.getValue().getInstance()).isNull(); + } + + @Test + public void problemHasStatusRemoved() { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + ArgumentCaptor insertedError = ArgumentCaptor.forClass(SubmissionError.class); + + // when: + submissionErrorService.addErrorToEnvelope(submissionEnvelope, randomProblem()); + + // then: + verify(submissionErrorRepository).insert(insertedError.capture()); + assertThat(insertedError.getValue().getStatus()).isNull(); + } + + public static Problem randomProblem() { + Random random = new Random(); + StatusType status; + URI baseType = URI.create("http://test.ingest.data.humancellatlas.org/"); + String type; + if (random.nextBoolean()) { + status = Status.valueOf(400); + type = "Error"; + } else { + status = Status.valueOf(300); + type = "Warning"; + } + + return Problem.builder() + .withStatus(status) + .withType(baseType.resolve(type)) + .withTitle("Random " + type) + .withDetail(UUID.randomUUID().toString() + UUID.randomUUID().toString()) + .withInstance(baseType.resolve(type + "/" + UUID.randomUUID())) + .build(); + } + + @Test + public void deleteSubmissionEnvelopeErrors() { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + // when: + submissionErrorService.deleteSubmissionEnvelopeErrors(submissionEnvelope); + // then: + verify(submissionErrorRepository).deleteBySubmissionEnvelope(submissionEnvelope); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/exporter/DefaultExporterTest.java b/src/test/java/uk/ac/ebi/subs/ingest/exporter/DefaultExporterTest.java new file mode 100644 index 000000000..839af0bdd --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/exporter/DefaultExporterTest.java @@ -0,0 +1,334 @@ +package uk.ac.ebi.subs.ingest.exporter; + +import static java.util.stream.Collectors.toList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; +import static uk.ac.ebi.subs.ingest.export.destination.ExportDestinationName.DCP; + +import java.time.Instant; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import org.json.simple.JSONObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.stubbing.Answer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.bundle.BundleManifestRepository; +import uk.ac.ebi.subs.ingest.bundle.BundleManifestService; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.export.destination.ExportDestination; +import uk.ac.ebi.subs.ingest.export.entity.ExportEntityService; +import uk.ac.ebi.subs.ingest.export.job.ExportJob; +import uk.ac.ebi.subs.ingest.export.job.ExportJobRepository; +import uk.ac.ebi.subs.ingest.export.job.ExportJobService; +import uk.ac.ebi.subs.ingest.export.job.web.ExportJobRequest; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.process.ProcessService; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@ExtendWith(SpringExtension.class) +@SpringBootTest +public class DefaultExporterTest { + + @Autowired private Exporter exporter; + + @MockBean private ProcessService processService; + + @MockBean private MessageRouter messageRouter; + + @MockBean private MetadataCrudService metadataCrudService; + + @MockBean private ExportJobService exportJobService; + + @MockBean private ExportJobRepository exportJobRepository; + + @MockBean private ProcessRepository processRepository; + + @MockBean private ProjectRepository projectRepository; + + @MockBean private ExportEntityService exportEntityService; + + @MockBean private BundleManifestService bundleManifestService; + + @MockBean private BundleManifestRepository bundleManifestRepository; + + SubmissionEnvelope submissionEnvelope; + + Project project; + + Set assayIds; + + @BeforeEach + void setUp() { + // given: + submissionEnvelope = new SubmissionEnvelope(); + assayIds = mockProcessIds(2); + project = new Project(null); + project.setUuid(Uuid.newUuid()); + project.getSubmissionEnvelopes().add(submissionEnvelope); + + mockProcessSvcGetProcesses(submissionEnvelope, assayIds); + + // and: + Set receivedData = mockSendingManifestThroughMessageRouter(); + } + + private String projectUuid() { + return project.getUuid().getUuid().toString(); + } + + @Test + public void testExportManifests() { + // when: + Set receivedData = mockSendingManifestThroughMessageRouter(); + + exporter.exportManifests(submissionEnvelope); + + // then: + assertAllProcessIdsProcessed(submissionEnvelope, assayIds, receivedData); + + // and: + verify(messageRouter, times(assayIds.size())) + .sendManifestForExport(any(ExperimentProcess.class)); + } + + @Test + public void testExportDataSetsContextsAndCallsMessageRouter() { + // given + mockCreateExportJob(projectUuid()); + + // when + exporter.exportData(submissionEnvelope); + + // then + var insertCaptor = ArgumentCaptor.forClass(ExportJob.class); + var sendCaptor = ArgumentCaptor.forClass(ExportJob.class); + verify(exportJobRepository).insert(insertCaptor.capture()); + verify(messageRouter).sendSubmissionForDataExport(sendCaptor.capture(), any()); + + assertThat(insertCaptor.getValue().getDestination().getContext().get("projectUuid")) + .isEqualTo(projectUuid()); + assertThat(insertCaptor.getValue().getContext().get("dataFileTransfer")).isEqualTo(false); + assertThat(sendCaptor.getValue().getDestination().getContext().get("projectUuid")) + .isEqualTo(projectUuid()); + assertThat(sendCaptor.getValue().getContext().get("dataFileTransfer")).isEqualTo(false); + } + + @Test + public void testExportMetadataFromExportJob() { + // given: + mockProcessSave(); + ExportJob newExportJob = mockCreateExportJob(projectUuid()); + Set receivedData = mockSendingProcessThroughMessageRouter(); + + // when: + exporter.exportMetadata(newExportJob); + + // then: + assertAllProcessIdsProcessed(submissionEnvelope, assayIds, receivedData); + assertDcpVersionUpdated(receivedData, newExportJob.getCreatedDate()); + verify(processRepository, times(assayIds.size())).save(any(Process.class)); + verify(messageRouter, times(assayIds.size())) + .sendExperimentForExport(any(ExperimentProcess.class), any(ExportJob.class), any()); + } + + @Test + public void testGenerateSpreadsheetFromSubmission() { + // given + mockCreateExportJob(projectUuid()); + + // when: + exporter.generateSpreadsheet(submissionEnvelope); + + // then + var insertCaptor = ArgumentCaptor.forClass(ExportJob.class); + var saveCaptor = ArgumentCaptor.forClass(ExportJob.class); + var sendCaptor = ArgumentCaptor.forClass(ExportJob.class); + verify(exportJobRepository).insert(insertCaptor.capture()); + verify(exportJobRepository).save(saveCaptor.capture()); + verify(messageRouter).sendGenerateSpreadsheet(sendCaptor.capture(), any()); + + assertThat(insertCaptor.getValue().getDestination().getContext().get("projectUuid")) + .isEqualTo(projectUuid()); + assertThat(saveCaptor.getValue().getContext().get("spreadsheetGeneration")).isEqualTo(false); + assertThat(sendCaptor.getValue().getDestination().getContext().get("projectUuid")) + .isEqualTo(projectUuid()); + assertThat(sendCaptor.getValue().getContext().get("spreadsheetGeneration")).isEqualTo(false); + } + + @Test + public void testGenerateSpreadsheetSetsContextsAndCallsMessageRouter() { + // given + ExportJob newExportJob = mockCreateExportJob(projectUuid()); + + // when: + exporter.generateSpreadsheet(newExportJob); + + // then + var saveCaptor = ArgumentCaptor.forClass(ExportJob.class); + var sendCaptor = ArgumentCaptor.forClass(ExportJob.class); + verify(exportJobRepository).save(saveCaptor.capture()); + verify(messageRouter).sendGenerateSpreadsheet(sendCaptor.capture(), any()); + + assertThat(saveCaptor.getValue().getContext().get("spreadsheetGeneration")).isEqualTo(false); + assertThat(sendCaptor.getValue().getDestination().getContext().get("projectUuid")) + .isEqualTo(projectUuid()); + assertThat(sendCaptor.getValue().getContext().get("spreadsheetGeneration")).isEqualTo(false); + } + + private ExportJob mockCreateExportJob(String projectUuidUuid) { + var destinationContext = new JSONObject(); + destinationContext.put("projectUuid", projectUuidUuid); + + var exportJobContext = new JSONObject(); + exportJobContext.put("totalAssayCount", assayIds.size()); + exportJobContext.put("dataFileTransfer", false); + ExportJob newExportJob = + ExportJob.builder() + .submission(submissionEnvelope) + .destination(new ExportDestination(DCP, "v2", destinationContext)) + .context(exportJobContext) + .build(); + doReturn(newExportJob) + .when(exportJobService) + .createExportJob(any(SubmissionEnvelope.class), any(ExportJobRequest.class)); + doReturn(newExportJob).when(exportJobRepository).insert(any(ExportJob.class)); + doReturn(Stream.of(project)) + .when(projectRepository) + .findBySubmissionEnvelopesContains(any(SubmissionEnvelope.class)); + return newExportJob; + } + + private void assertAllProcessIdsProcessed( + SubmissionEnvelope submissionEnvelope, + Set assayIds, + Set receivedData) { + int expectedCount = 2; + assertThat(receivedData).hasSize(expectedCount); + assertUniqueIndexes(receivedData); + assertCorrectTotalCount(receivedData, expectedCount); + assertCorrectSubmissionEnvelope(receivedData, submissionEnvelope); + assertAllProcessesExported(assayIds, receivedData); + } + + private void mockProcessSave() { + when(processRepository.save(any(Process.class))) + .thenAnswer( + (Answer) + invocation -> { + Process process = invocation.getArgument(0); + return process; + }); + } + + private void mockProcessSvcGetProcesses( + SubmissionEnvelope submissionEnvelope, Set assayIds) { + when(processService.getProcesses(any())) + .thenAnswer( + (Answer>) + invocation -> { + List ids = invocation.getArgument(0); + return ids.stream() + .map( + id -> { + Process process = spy(new Process(null)); + doReturn(id).when(process).getId(); + process.setSubmissionEnvelope(submissionEnvelope); + return process; + }); + }); + + doReturn(assayIds).when(processService).findAssays(any(SubmissionEnvelope.class)); + } + + private Set mockProcessIds(int max) { + return IntStream.range(0, max) + .mapToObj(count -> UUID.randomUUID().toString()) + .collect(Collectors.toSet()); + } + + private Set mockSendingManifestThroughMessageRouter() { + final Set experimentProcess = new HashSet<>(); + Answer addToSet = + invocation -> { + experimentProcess.add(invocation.getArgument(0)); + return null; + }; + doAnswer(addToSet).when(messageRouter).sendManifestForExport(any(ExperimentProcess.class)); + return experimentProcess; + } + + private Set mockSendingProcessThroughMessageRouter() { + final Set experimentProcess = new HashSet<>(); + Answer addToSet = + invocation -> { + experimentProcess.add(invocation.getArgument(0)); + return null; + }; + doAnswer(addToSet) + .when(messageRouter) + .sendExperimentForExport(any(ExperimentProcess.class), any(ExportJob.class), any()); + return experimentProcess; + } + + private void assertUniqueIndexes(Set receivedData) { + List indexes = + receivedData.stream().map(ExperimentProcess::getIndex).collect(toList()); + assertThat(indexes).containsOnlyOnce(0, 1); + } + + private void assertDcpVersionUpdated(Set receivedData, Instant dcpVersion) { + receivedData.stream() + .map(ExperimentProcess::getProcess) + .forEach(process -> assertThat(process.getDcpVersion()).isEqualTo(dcpVersion)); + } + + private void assertCorrectTotalCount(Set receivedData, int expectedCount) { + receivedData.stream() + .forEach( + exporterData -> { + assertThat(exporterData.getTotalCount()).isEqualTo(expectedCount); + }); + } + + private void assertCorrectSubmissionEnvelope( + Set receivedData, SubmissionEnvelope submissionEnvelope) { + receivedData.forEach( + exporterData -> + assertThat(exporterData.getSubmissionEnvelope()).isEqualTo(submissionEnvelope)); + } + + private void assertAllProcessesExported( + Set assayIds, Set exporterData) { + + List sentProcesses = + exporterData.stream().map(ExperimentProcess::getProcess).collect(toList()); + + assertThat(sentProcesses.stream().map(Process::getId)).containsAll(assayIds); + } + + @Configuration + static class TestConfiguration { + + @Bean + Exporter defaultExporter() { + return new DefaultExporter(); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/file/FileTest.java b/src/test/java/uk/ac/ebi/subs/ingest/file/FileTest.java new file mode 100644 index 000000000..0f3b3467d --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/file/FileTest.java @@ -0,0 +1,87 @@ +package uk.ac.ebi.subs.ingest.file; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; + +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +public class FileTest { + Process process; + File file; + + @BeforeEach + void setUp() { + // given: + file = new File(null, "fileName"); + + process = spy(new Process(null)); + doReturn("fe89a0").when(process).getId(); + } + + @Test + public void testAddAsDerivedByProcess() { + // when: + file.addAsDerivedByProcess(process); + + // then: + assertThat(file.getDerivedByProcesses()).containsExactly(process); + } + + @Test + public void testAddDerivedByProcessdNoDuplication() { + // when: + file.addAsDerivedByProcess(process); + file.addAsDerivedByProcess(process); + + // then: + assertThat(file.getDerivedByProcesses()).hasSize(1); + } + + @Test + public void testAddToAnalysis() { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + process.setSubmissionEnvelope(submissionEnvelope); + + // when: + file.addToAnalysis(process); + + // then: + assertThat(file.getDerivedByProcesses()).contains(process); + assertThat(file.getSubmissionEnvelope()).isEqualTo(submissionEnvelope); + } + + @Test + public void testAddToAnalysisWhenFileAlreadyLinkedToSubmissionEnvelope() { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + process.setSubmissionEnvelope(submissionEnvelope); + file.setSubmissionEnvelope(submissionEnvelope); + + // when: + file.addToAnalysis(process); + + // then: + assertThat(file.getDerivedByProcesses()).contains(process); + assertThat(file.getSubmissionEnvelope()).isEqualTo(submissionEnvelope); + } + + @ParameterizedTest + @MethodSource("testFiles") + public void newFileHasDataFileUuidNotNull(File file) { + assertThat(file).extracting("dataFileUuid").doesNotContainNull(); + } + + private static Stream testFiles() { + return Stream.of(new File(), new File(null, "test-File")); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/messaging/MessageRouterTest.java b/src/test/java/uk/ac/ebi/subs/ingest/messaging/MessageRouterTest.java new file mode 100644 index 000000000..d996c1f95 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/messaging/MessageRouterTest.java @@ -0,0 +1,168 @@ +package uk.ac.ebi.subs.ingest.messaging; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.*; +import static uk.ac.ebi.subs.ingest.export.destination.ExportDestinationName.DCP; +import static uk.ac.ebi.subs.ingest.messaging.Constants.Exchanges.EXPORTER_EXCHANGE; +import static uk.ac.ebi.subs.ingest.messaging.Constants.Routing.MANIFEST_SUBMITTED; + +import java.time.Instant; + +import org.json.simple.JSONObject; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.rest.core.config.RepositoryRestConfiguration; +import org.springframework.data.rest.core.mapping.ResourceMappings; + +import uk.ac.ebi.subs.ingest.config.ConfigurationService; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.web.LinkGenerator; +import uk.ac.ebi.subs.ingest.export.destination.ExportDestination; +import uk.ac.ebi.subs.ingest.export.job.ExportJob; +import uk.ac.ebi.subs.ingest.exporter.ExperimentProcess; +import uk.ac.ebi.subs.ingest.messaging.model.ExportSubmissionMessage; +import uk.ac.ebi.subs.ingest.messaging.model.ManifestMessage; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@SpringBootTest +public class MessageRouterTest { + + @Autowired private MessageRouter messageRouter; + + @MockBean private MessageSender messageSender; + + @MockBean private ResourceMappings resourceMappings; + + @MockBean private RepositoryRestConfiguration config; + + @MockBean private LinkGenerator linkGenerator; + + @MockBean private ConfigurationService configurationService; + + @Test + public void testSendManifestForExport() { + // expect: + doTestSendForExport(MANIFEST_SUBMITTED); + } + + @Test + public void testSendSubmissionForDataExport() { + // given + var submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.setUuid(Uuid.newUuid()); + var project = new Project(null); + project.setUuid(Uuid.newUuid()); + project.getSubmissionEnvelopes().add(submissionEnvelope); + var exportJob = exportJob(submissionEnvelope, project); + var context = new JSONObject(); + + // when + messageRouter.sendSubmissionForDataExport(exportJob, context); + + // then + var argumentCaptor = ArgumentCaptor.forClass(ExportSubmissionMessage.class); + verify(messageSender) + .queueNewExportMessage(anyString(), anyString(), argumentCaptor.capture(), anyLong()); + verify(linkGenerator).createCallback(any(), anyString()); + + var capturedArgument = argumentCaptor.getValue(); + assertThat(capturedArgument.getExportJobId()).isEqualTo(exportJob.getId()); + assertThat(capturedArgument.getSubmissionUuid()) + .isEqualTo(submissionEnvelope.getUuid().getUuid().toString()); + assertThat(capturedArgument.getProjectUuid()).isEqualTo(project.getUuid().getUuid().toString()); + assertThat(capturedArgument.getContext()).isEqualTo(context); + } + + private ExportJob exportJob(SubmissionEnvelope submissionEnvelope, Project project) { + var destinationContext = new JSONObject(); + destinationContext.put("projectUuid", project.getUuid().getUuid().toString()); + + var exportJobContext = new JSONObject(); + exportJobContext.put("dataFileTransfer", false); + return ExportJob.builder() + .id("testExportJobId") + .submission(submissionEnvelope) + .destination(new ExportDestination(DCP, "v2", destinationContext)) + .context(exportJobContext) + .build(); + } + + private void doTestSendForExport(String routingKey) { + // given: + String processId = "78bbd9"; + Process process = spy(new Process(null)); + doReturn(processId).when(process).getId(); + + Uuid processUuid = Uuid.newUuid(); + process.setUuid(processUuid); + Instant version = Instant.now(); + process.setDcpVersion(version); + + // and: + String envelopeId = "87bcf3"; + SubmissionEnvelope submissionEnvelope = spy(new SubmissionEnvelope()); + doReturn(envelopeId).when(submissionEnvelope).getId(); + Uuid envelopeUuid = Uuid.newUuid(); + submissionEnvelope.setUuid(envelopeUuid); + + process.setSubmissionEnvelope(submissionEnvelope); + + // and: + ExperimentProcess exporterData = new ExperimentProcess(2, 4, process, submissionEnvelope, null); + + // and: + String callbackLink = "/processes/78bbd9"; + doReturn(callbackLink).when(linkGenerator).createCallback(any(Class.class), anyString()); + + // when: + messageRouter.sendManifestForExport(exporterData); + + // then: + ArgumentCaptor messageCaptor = ArgumentCaptor.forClass(ManifestMessage.class); + verify(messageSender) + .queueNewExportMessage( + eq(EXPORTER_EXCHANGE), eq(routingKey), messageCaptor.capture(), anyLong()); + + // and: + ManifestMessage submittedMessage = messageCaptor.getValue(); + assertThat(submittedMessage) + .extracting( + "documentId", + "documentUuid", + "callbackLink", + "documentType", + "envelopeId", + "envelopeUuid", + "index", + "total") + .containsExactly( + processId, + processUuid.getUuid().toString(), + callbackLink, + process.getClass().getSimpleName(), + envelopeId, + envelopeUuid.getUuid().toString(), + 2, + 4); + } + + @Configuration + static class TestConfiguration { + + @Bean + MessageRouter messageRouter() { + return new MessageRouter(); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/messaging/MessageServiceTest.java b/src/test/java/uk/ac/ebi/subs/ingest/messaging/MessageServiceTest.java new file mode 100644 index 000000000..6da01547c --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/messaging/MessageServiceTest.java @@ -0,0 +1,72 @@ +package uk.ac.ebi.subs.ingest.messaging; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.verify; + +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.amqp.rabbit.core.RabbitMessagingTemplate; +import org.springframework.context.annotation.Configuration; +import org.springframework.messaging.MessagingException; +import org.springframework.messaging.converter.MessageConversionException; + +public class MessageServiceTest { + + private RabbitMessagingTemplate rabbitMessagingTemplate = + Mockito.mock(RabbitMessagingTemplate.class); + + private MessageService messageService = new MessageService(rabbitMessagingTemplate); + + @Test + public void testPublish() { + // given: + Message message = new Message("exchange", "routingKey", "payload"); + + // when: + messageService.publish(message); + + // then: + verify(rabbitMessagingTemplate) + .convertAndSend(message.getExchange(), message.getRoutingKey(), message.getPayload()); + } + + @Test + public void testPublishMessageConversionException() { + // given: + Message message = new Message("", "", ""); + + // when: + doThrow(MessageConversionException.class) + .when(rabbitMessagingTemplate) + .convertAndSend("", "", ""); + + // then: + assertThatThrownBy( + () -> { + messageService.publish(message); + }) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Unable to convert payload"); + } + + @Test + public void testPublishMessagingException() { + // given: + Message message = new Message("", "", ""); + + // when: + doThrow(MessagingException.class).when(rabbitMessagingTemplate).convertAndSend("", "", ""); + + // then: + assertThatThrownBy( + () -> { + messageService.publish(message); + }) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("There was a problem sending message"); + } + + @Configuration + static class TestConfiguration {} +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/notifications/NotificationCoordinatorTest.java b/src/test/java/uk/ac/ebi/subs/ingest/notifications/NotificationCoordinatorTest.java new file mode 100644 index 000000000..fb2e353d4 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/notifications/NotificationCoordinatorTest.java @@ -0,0 +1,182 @@ +package uk.ac.ebi.subs.ingest.notifications; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.mockito.stubbing.Answer; + +import uk.ac.ebi.subs.ingest.notifications.exception.ProcessingException; +import uk.ac.ebi.subs.ingest.notifications.model.Checksum; +import uk.ac.ebi.subs.ingest.notifications.model.Notification; +import uk.ac.ebi.subs.ingest.notifications.model.NotificationState; +import uk.ac.ebi.subs.ingest.notifications.processors.NotificationProcessor; +import uk.ac.ebi.subs.ingest.notifications.sources.NotificationSource; +import uk.ac.ebi.subs.ingest.notifications.sources.impl.inmemory.InmemoryNotificationSource; + +public class NotificationCoordinatorTest { + + private Notification generateTestNotification(String checksumValue) { + return Notification.buildNew() + .metadata(new HashMap<>()) + .content("testcontent") + .checksum(new Checksum("testtype", checksumValue)) + .build(); + } + + private NotificationService mockNotificationService() { + NotificationService notificationService = mock(NotificationService.class); + + Answer mockChangeStateFn = + invocation -> { + Notification notification = invocation.getArgument(0); + NotificationState toState = invocation.getArgument(1); + return Notification.builder() + .checksum(notification.getChecksum()) + .content(notification.getContent()) + .metadata(notification.getMetadata()) + .notifyAt(notification.getNotifyAt()) + .state(toState) + .build(); + }; + + Mockito.doAnswer(mockChangeStateFn) + .when(notificationService) + .changeState(any(Notification.class), any(NotificationState.class)); + + return notificationService; + } + + @Test + public void testQueue() { + List testNotifications = + List.of(generateTestNotification("testvalue1"), generateTestNotification("testvalue2")); + NotificationSource testInmemorySource = new InmemoryNotificationSource(); + + NotificationService notificationService = mockNotificationService(); + Mockito.doReturn(testNotifications.stream()) + .when(notificationService) + .getUnhandledNotifications(); + + NotificationCoordinator notificationCoordinator = + new NotificationCoordinator( + Collections.emptyList(), testInmemorySource, notificationService); + + notificationCoordinator.queue(); + + Assertions.assertThat(testInmemorySource.stream().map(n -> n.getChecksum().getValue())) + .containsSequence( + testNotifications.stream() + .map(n -> n.getChecksum().getValue()) + .collect(Collectors.toList())); + } + + @Test + public void testProcess() { + NotificationSource mockInmemorySource = mock(NotificationSource.class); + Mockito.doReturn( + Stream.of(generateTestNotification("testvalue1"), generateTestNotification("testvalue2"))) + .when(mockInmemorySource) + .stream(); + + NotificationService notificationService = mockNotificationService(); + + NotificationProcessor mockHappyNotificationProcessor = mock(NotificationProcessor.class); + Mockito.doNothing().when(mockHappyNotificationProcessor).handle(any(Notification.class)); + + NotificationCoordinator notificationCoordinator = + new NotificationCoordinator( + List.of(mockHappyNotificationProcessor), mockInmemorySource, notificationService); + + notificationCoordinator.process(); + + Mockito.verify(notificationService, times(2)) + .changeState(any(Notification.class), eq(NotificationState.PROCESSING)); + + Mockito.verify(notificationService, times(2)) + .changeState(any(Notification.class), eq(NotificationState.PROCESSED)); + } + + @Test + public void testProcessingFailure() { + NotificationSource mockInmemorySource = mock(NotificationSource.class); + Mockito.doReturn( + Stream.of(generateTestNotification("testvalue1"), generateTestNotification("testvalue2"))) + .when(mockInmemorySource) + .stream(); + + NotificationService notificationService = mockNotificationService(); + + NotificationProcessor mockUnhappyNotificationProcessor = mock(NotificationProcessor.class); + Mockito.doThrow(new ProcessingException("")) + .when(mockUnhappyNotificationProcessor) + .handle(any(Notification.class)); + Mockito.doReturn(true) + .when(mockUnhappyNotificationProcessor) + .isEligible(any(Notification.class)); + + NotificationCoordinator notificationCoordinator = + new NotificationCoordinator( + List.of(mockUnhappyNotificationProcessor), mockInmemorySource, notificationService); + + notificationCoordinator.process(); + + Mockito.verify(notificationService, times(2)) + .changeState(any(Notification.class), eq(NotificationState.PROCESSING)); + + Mockito.verify(notificationService, times(2)) + .changeState(any(Notification.class), eq(NotificationState.FAILED)); + } + + /** + * Test behaviour when one processor succeeds, but another fails. Expect an overall failure + * outcome. + */ + @Test + public void testMultipleProcessors_OneFailing() { + NotificationSource fakeSource = new InmemoryNotificationSource(); + NotificationService notificationService = mockNotificationService(); + + NotificationProcessor mockHappyNotificationProcessor = mock(NotificationProcessor.class); + Mockito.doNothing().when(mockHappyNotificationProcessor).handle(any(Notification.class)); + Mockito.doReturn(true).when(mockHappyNotificationProcessor).isEligible(any(Notification.class)); + + NotificationProcessor mockUnhappyNotificationProcessor = mock(NotificationProcessor.class); + Mockito.doThrow(new ProcessingException("")) + .when(mockUnhappyNotificationProcessor) + .handle(any(Notification.class)); + Mockito.doReturn(true) + .when(mockUnhappyNotificationProcessor) + .isEligible(any(Notification.class)); + + NotificationCoordinator notificationCoordinator = + new NotificationCoordinator( + List.of(mockUnhappyNotificationProcessor, mockHappyNotificationProcessor), + fakeSource, + notificationService); + + fakeSource.supply( + List.of(generateTestNotification("testvalue1"), generateTestNotification("testvalue2"))); + notificationCoordinator.process(); + + Mockito.verify(notificationService, times(2)) + .changeState(any(Notification.class), eq(NotificationState.PROCESSING)); + + Mockito.verify(notificationService, times(2)) + .changeState(any(Notification.class), eq(NotificationState.FAILED)); + + Mockito.verify(notificationService, never()) + .changeState(any(Notification.class), eq(NotificationState.PROCESSED)); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/notifications/NotificationServicesTest.java b/src/test/java/uk/ac/ebi/subs/ingest/notifications/NotificationServicesTest.java new file mode 100644 index 000000000..e6aaefc10 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/notifications/NotificationServicesTest.java @@ -0,0 +1,125 @@ +package uk.ac.ebi.subs.ingest.notifications; + +import static org.assertj.core.api.Assertions.*; +import static org.mockito.AdditionalAnswers.returnsFirstArg; +import static org.mockito.Mockito.*; + +import java.util.HashMap; +import java.util.Optional; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.dao.DuplicateKeyException; + +import uk.ac.ebi.subs.ingest.notifications.exception.DuplicateNotification; +import uk.ac.ebi.subs.ingest.notifications.model.Checksum; +import uk.ac.ebi.subs.ingest.notifications.model.Notification; +import uk.ac.ebi.subs.ingest.notifications.model.NotificationRequest; +import uk.ac.ebi.subs.ingest.notifications.model.NotificationState; + +public class NotificationServicesTest { + private NotificationRepository notificationRepository = mock(NotificationRepository.class); + private NotificationService notificationService = new NotificationService(notificationRepository); + + @Test + public void testCreateNotification() { + Checksum testChecksum = new Checksum("testtype", "testvalue"); + NotificationRequest notificationRequest = + new NotificationRequest("testcontent", new HashMap<>(), testChecksum); + + Mockito.doReturn( + Notification.buildNew() + .metadata(new HashMap<>()) + .content("testcontent") + .checksum(testChecksum) + .build()) + .when(notificationRepository) + .save(any(Notification.class)); + + Notification createdNotification = notificationService.createNotification(notificationRequest); + + assertThat(createdNotification.getChecksum()).isEqualTo(testChecksum); + assertThat(createdNotification.getMetadata()).isEqualTo(new HashMap<>()); + assertThat(createdNotification.getState()).isEqualTo(NotificationState.PENDING); + assertThat(createdNotification.getContent()).isEqualTo("testcontent"); + } + + @Test + public void testCreateDuplicateNotification() { + Checksum testChecksum = new Checksum("testtype", "testvalue"); + NotificationRequest notificationRequest = + new NotificationRequest("testcontent", new HashMap<>(), testChecksum); + + Notification testExistingNotification = + Notification.buildNew() + .metadata(new HashMap<>()) + .content("testcontent") + .checksum(testChecksum) + .build(); + + Mockito.doThrow(new DuplicateKeyException("")) + .when(notificationRepository) + .save(any(Notification.class)); + + Mockito.doReturn(Optional.of(testExistingNotification)) + .when(notificationRepository) + .findByChecksum_Value("testvalue"); + + Assertions.assertThatExceptionOfType(DuplicateNotification.class) + .isThrownBy(() -> notificationService.createNotification(notificationRequest)); + } + + @Test + public void testRetrieveByChecksum() { + Checksum testChecksum = new Checksum("testtype", "testvalue"); + + Mockito.doReturn( + Optional.of( + Notification.buildNew() + .metadata(new HashMap<>()) + .content("testcontent") + .checksum(testChecksum) + .build())) + .when(notificationRepository) + .findByChecksum(testChecksum); + + Assertions.assertThat( + notificationService.retrieveForChecksum(testChecksum).orElseThrow().getChecksum()) + .isEqualTo(testChecksum); + } + + @Test + public void testChangeState() { + Checksum testChecksum = new Checksum("testtype", "testvalue"); + Notification testNotification = + Notification.buildNew() + .metadata(new HashMap<>()) + .content("testcontent") + .checksum(testChecksum) + .build(); + + Mockito.doAnswer(returnsFirstArg()).when(notificationRepository).save(any(Notification.class)); + + Notification changedState = + notificationService.changeState(testNotification, NotificationState.QUEUED); + assertThat(changedState.getState()).isEqualTo(NotificationState.QUEUED); + } + + @Test + public void testIllegalStateChange() { + Checksum testChecksum = new Checksum("testtype", "testvalue"); + Notification testNotification = + Notification.buildNew() + .metadata(new HashMap<>()) + .content("testcontent") + .checksum(testChecksum) + .build(); + + Mockito.doAnswer(returnsFirstArg()).when(notificationRepository).save(any(Notification.class)); + + Assertions.assertThatExceptionOfType(IllegalStateException.class) + .isThrownBy( + () -> notificationService.changeState(testNotification, NotificationState.PROCESSED)); + } +} diff --git a/src/test/java/org/humancellatlas/ingest/notifications/model/NotificationStateTest.java b/src/test/java/uk/ac/ebi/subs/ingest/notifications/model/NotificationStateTest.java similarity index 73% rename from src/test/java/org/humancellatlas/ingest/notifications/model/NotificationStateTest.java rename to src/test/java/uk/ac/ebi/subs/ingest/notifications/model/NotificationStateTest.java index 7b5cb60d2..a5e5f2170 100644 --- a/src/test/java/org/humancellatlas/ingest/notifications/model/NotificationStateTest.java +++ b/src/test/java/uk/ac/ebi/subs/ingest/notifications/model/NotificationStateTest.java @@ -1,9 +1,9 @@ -package org.humancellatlas.ingest.notifications.model; +package uk.ac.ebi.subs.ingest.notifications.model; -import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.*; -import static org.humancellatlas.ingest.notifications.model.NotificationState.*; +import static uk.ac.ebi.subs.ingest.notifications.model.NotificationState.*; +import org.junit.jupiter.api.Test; public class NotificationStateTest { @Test @@ -14,9 +14,8 @@ public void testLegalStateTransitions() { assertThat(PROCESSED.legalTransitions()).contains(FAILED); assertThat(FAILED.legalTransitions()).contains(QUEUED); - for(NotificationState state: NotificationState.values()) { + for (NotificationState state : NotificationState.values()) { assertThat(state.legalTransitions()).contains(FAILED); } } - } diff --git a/src/test/java/uk/ac/ebi/subs/ingest/notifications/processors/EmailNotificationsProcessorTest.java b/src/test/java/uk/ac/ebi/subs/ingest/notifications/processors/EmailNotificationsProcessorTest.java new file mode 100644 index 000000000..c6541801f --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/notifications/processors/EmailNotificationsProcessorTest.java @@ -0,0 +1,3 @@ +package uk.ac.ebi.subs.ingest.notifications.processors; + +public class EmailNotificationsProcessorTest {} diff --git a/src/test/java/org/humancellatlas/ingest/notifications/sources/InmemoryNotificationSourceTest.java b/src/test/java/uk/ac/ebi/subs/ingest/notifications/sources/InmemoryNotificationSourceTest.java similarity index 54% rename from src/test/java/org/humancellatlas/ingest/notifications/sources/InmemoryNotificationSourceTest.java rename to src/test/java/uk/ac/ebi/subs/ingest/notifications/sources/InmemoryNotificationSourceTest.java index 391209d97..87578a2a5 100644 --- a/src/test/java/org/humancellatlas/ingest/notifications/sources/InmemoryNotificationSourceTest.java +++ b/src/test/java/uk/ac/ebi/subs/ingest/notifications/sources/InmemoryNotificationSourceTest.java @@ -1,21 +1,23 @@ -package org.humancellatlas.ingest.notifications.sources; +package uk.ac.ebi.subs.ingest.notifications.sources; import java.util.HashMap; import java.util.List; + import org.assertj.core.api.Assertions; -import org.humancellatlas.ingest.notifications.model.Checksum; -import org.humancellatlas.ingest.notifications.model.Notification; -import org.humancellatlas.ingest.notifications.sources.impl.inmemory.InmemoryNotificationSource; import org.junit.jupiter.api.Test; +import uk.ac.ebi.subs.ingest.notifications.model.Checksum; +import uk.ac.ebi.subs.ingest.notifications.model.Notification; +import uk.ac.ebi.subs.ingest.notifications.sources.impl.inmemory.InmemoryNotificationSource; + public class InmemoryNotificationSourceTest { private Notification generateTestNotification(String checksumValue) { return Notification.buildNew() - .metadata(new HashMap<>()) - .content("testcontent") - .checksum(new Checksum("testtype", checksumValue)) - .build(); + .metadata(new HashMap<>()) + .content("testcontent") + .checksum(new Checksum("testtype", checksumValue)) + .build(); } @Test @@ -24,8 +26,9 @@ public void testSupply() { Notification testNotification2 = generateTestNotification("testvalue2"); NotificationSource testInmemorySource = new InmemoryNotificationSource(); - Assertions.assertThatCode(() -> testInmemorySource.supply(List.of(testNotification1, testNotification2))) - .doesNotThrowAnyException(); + Assertions.assertThatCode( + () -> testInmemorySource.supply(List.of(testNotification1, testNotification2))) + .doesNotThrowAnyException(); } @Test @@ -34,14 +37,11 @@ public void testStream() { Notification testNotification2 = generateTestNotification("testvalue2"); NotificationSource testInmemorySource = new InmemoryNotificationSource(); - testInmemorySource.supply(List.of(testNotification1, testNotification2)); - Assertions.assertThat(testInmemorySource.stream()) - .containsSequence(testNotification1, testNotification2); + .containsSequence(testNotification1, testNotification2); - Assertions.assertThat(testInmemorySource.stream()) - .isEmpty(); + Assertions.assertThat(testInmemorySource.stream()).isEmpty(); } } diff --git a/src/test/java/uk/ac/ebi/subs/ingest/process/ProcessServiceTest.java b/src/test/java/uk/ac/ebi/subs/ingest/process/ProcessServiceTest.java new file mode 100644 index 000000000..33fbc6828 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/process/ProcessServiceTest.java @@ -0,0 +1,103 @@ +package uk.ac.ebi.subs.ingest.process; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.*; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.bundle.BundleManifestRepository; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.state.MetadataDocumentEventHandler; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@ExtendWith(SpringExtension.class) +public class ProcessServiceTest { + + @Autowired private ProcessService service; + + @MockBean private SubmissionEnvelopeRepository submissionEnvelopeRepository; + @MockBean private ProcessRepository processRepository; + @MockBean private FileRepository fileRepository; + @MockBean private BiomaterialRepository biomaterialRepository; + @MockBean private BundleManifestRepository bundleManifestRepository; + @MockBean private ProjectRepository projectRepository; + @MockBean private MetadataDocumentEventHandler metadataDocumentEventHandler; + @MockBean private MetadataCrudService metadataCrudService; + @MockBean private MetadataUpdateService metadataUpdateService; + + String fileName = "ERR1630013.fastq.gz"; + File file; + Process analysis; + SubmissionEnvelope submissionEnvelope; + + @BeforeEach + void setUp() { + // Given: + file = spy(new File(null, fileName)); + analysis = new Process(null); + submissionEnvelope = new SubmissionEnvelope(); + analysis.setSubmissionEnvelope(submissionEnvelope); + } + + @Test + public void testAddFileToAnalysisProcess() { + // given: + doReturn(Collections.emptyList()) + .when(fileRepository) + .findBySubmissionEnvelopeAndFileName(any(SubmissionEnvelope.class), anyString()); + + // when: + Process result = service.addOutputFileToAnalysisProcess(analysis, file); + + // then: + assertThat(result).isEqualTo(analysis); + verify(file).addToAnalysis(analysis); + verify(fileRepository).save(file); + } + + @Test + public void testAddFileToAnalysisProcessWhenFileAlreadyExists() { + // given: + List persistentFiles = Arrays.asList(file); + doReturn(persistentFiles) + .when(fileRepository) + .findBySubmissionEnvelopeAndFileName(submissionEnvelope, fileName); + + // when: + Process result = service.addOutputFileToAnalysisProcess(analysis, file); + + // then: + assertThat(result).isEqualTo(analysis); + + // and: + verify(file).addToAnalysis(analysis); + verify(fileRepository).save(file); + } + + @Configuration + static class TestConfiguration { + + @Bean + ProcessService processService() { + return new ProcessService(); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/process/ProcessTest.java b/src/test/java/uk/ac/ebi/subs/ingest/process/ProcessTest.java new file mode 100644 index 000000000..a2bad7572 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/process/ProcessTest.java @@ -0,0 +1,62 @@ +package uk.ac.ebi.subs.ingest.process; + +public class ProcessTest { + + // @Test + // public void testIsAssaying() { + // //given: + // Process nonAssayingProcess = new Process(); + // + // //and: + // Process hasInputBiomaterial = new Process(); + // hasInputBiomaterial.addInput(new Biomaterial(null)); + // + // //and: + // Process hasDerivedFile = new Process(); + // hasDerivedFile.addDerivative(new File(null)); + // + // //and: + // Process assayingProcess = new Process(); + // assayingProcess.addInput(new Biomaterial(null)); + // assayingProcess.addDerivative(new File(null)); + // + // //expect: + // String notAssaying = "Expected Process to be NON assaying."; + // assertThat(nonAssayingProcess.isAssaying()).as(notAssaying).isFalse(); + // assertThat(hasInputBiomaterial.isAssaying()).as(notAssaying).isFalse(); + // assertThat(hasDerivedFile.isAssaying()).as(notAssaying).isFalse(); + // + // //and: + // assertThat(assayingProcess.isAssaying()).as("Expected Process to be + // assaying.").isTrue(); + // } + // + // @Test + // public void testIsAnalysis() { + // //given: + // Process nonAnalysis = new Process(); + // + // //and: + // Process hasInputFile = new Process(); + // hasInputFile.addInput(new File("input")); + // + // //and: + // Process hasDerivedFile = new Process(); + // hasDerivedFile.addDerivative(new File("output")); + // + // //and: + // Process analysis = new Process(); + // analysis.addInput(new File("input")); + // analysis.addDerivative(new File("output")); + // + // //then: + // String notAnalysis = "Expected Process to be non Analysis"; + // assertThat(nonAnalysis.isAnalysis()).as(notAnalysis).isFalse(); + // assertThat(hasInputFile.isAnalysis()).as(notAnalysis).isFalse(); + // assertThat(hasDerivedFile.isAnalysis()).as(notAnalysis).isFalse(); + // + // //then: + // assertThat(analysis.isAnalysis()).as("Expected Process to be Analysis.").isTrue(); + // } + +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/project/MetadataDocumentSerializationTest.java b/src/test/java/uk/ac/ebi/subs/ingest/project/MetadataDocumentSerializationTest.java new file mode 100644 index 000000000..9ea7ab816 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/project/MetadataDocumentSerializationTest.java @@ -0,0 +1,99 @@ +package uk.ac.ebi.subs.ingest.project; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; +import java.util.stream.Stream; + +import org.assertj.core.api.Assertions; +import org.junit.Ignore; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.json.JsonTest; +import org.springframework.boot.test.mock.mockito.MockBean; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; + +import uk.ac.ebi.subs.ingest.config.MigrationConfiguration; + +@JsonTest +class MetadataDocumentSerializationTest { + + @Autowired private ObjectMapper objectMapper; + + @MockBean + // NOTE: Adding MigrationConfiguration as a MockBean is needed + // as otherwise MigrationConfiguration won't be initialised. + private MigrationConfiguration migrationConfiguration; + + @BeforeEach + public void setup() { + SimpleModule module = new SimpleModule(); + module.addSerializer(DataAccessTypes.class, new DataAccessTypesJsonSerializer()); + module.addDeserializer(DataAccessTypes.class, new DataAccessTypesJsonDeserializer()); + objectMapper.registerModule(module); + } + + @Ignore + // TODO: this test is not working because it needs proper json input. + // @ParameterizedTest() + @MethodSource("testData") + public void testSerialization(Object someObject, String expectedJson) + throws JsonProcessingException { + String json = objectMapper.writeValueAsString(someObject); + Assertions.assertThat(json).isEqualTo(expectedJson); + } + + @ParameterizedTest() + @MethodSource("testData") + public void testDeserialization(Supplier objectSupplier, String expectedJson) + throws JsonProcessingException { + Object someObject = objectSupplier.get(); + Object deserialized = objectMapper.readValue(expectedJson, someObject.getClass()); + Comparator upToSeconds = Comparator.comparing(i -> i.truncatedTo(ChronoUnit.SECONDS)); + Assertions.assertThat(deserialized) + .usingComparatorForFields(upToSeconds, "contentLastModified") + .isEqualToComparingFieldByField(someObject); + } + + @ParameterizedTest() + @MethodSource("testData") + public void testSerializationThenDeserialization( + Supplier objectSupplier, String expectedJson) throws JsonProcessingException { + Object someObject = objectSupplier.get(); + String json = objectMapper.writeValueAsString(someObject); + Object deserialized = objectMapper.readValue(json, someObject.getClass()); + Assertions.assertThat(deserialized).isEqualTo(someObject); + } + + private static Stream testData() { + // The test objects are created during test execution so that the time fields will be close + // to the desrialization target objects. If it were not so, the timestamps would be seconds + // apart. + return Stream.of( + Arguments.of((Supplier) () -> DataAccessTypes.OPEN, "\"All fully open\""), + Arguments.of( + (Supplier) () -> new DataAccess(DataAccessTypes.OPEN), + "{\"type\":\"All fully open\",\"notes\":null}"), + Arguments.of( + (Supplier) + () -> { + Project project = new Project(new HashMap<>()); + // TODO: use project builder + ((Map) project.getContent()) + .put( + "dataAccess", + new ObjectToMapConverter().asMap(new DataAccess(DataAccessTypes.OPEN))); + return project; + }, + "{\"content\":{\"dataAccess\":{\"type\":\"All fully open\",\"notes\":null}}}")); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/project/ProjectBuilderTest.java b/src/test/java/uk/ac/ebi/subs/ingest/project/ProjectBuilderTest.java new file mode 100644 index 000000000..146fdedb2 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/project/ProjectBuilderTest.java @@ -0,0 +1,33 @@ +package uk.ac.ebi.subs.ingest.project; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.within; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +public class ProjectBuilderTest { + + @Test + public void testProjectBuilder() { + Project fromBuilder = Project.builder().emptyProject().withManagedAccess().build(); + + Project fromCtor = new Project(new HashMap<>()); + ((Map) fromCtor.getContent()) + .put( + "dataAccess", + new ObjectToMapConverter().asMap(new DataAccess(DataAccessTypes.MANAGED))); + + assertThat(fromBuilder.getContentLastModified()) + .isCloseTo(fromCtor.getContentLastModified(), within(1, ChronoUnit.SECONDS)); + Comparator upToMillies = Comparator.comparing(d -> d.truncatedTo(ChronoUnit.SECONDS)); + assertThat(fromBuilder) + .usingComparatorForFields(upToMillies, "contentLastModified") + .isEqualToIgnoringGivenFields(fromCtor, "uuid"); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/project/ProjectFilterQueryBuilderTest.java b/src/test/java/uk/ac/ebi/subs/ingest/project/ProjectFilterQueryBuilderTest.java new file mode 100644 index 000000000..02e015dfe --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/project/ProjectFilterQueryBuilderTest.java @@ -0,0 +1,51 @@ +package uk.ac.ebi.subs.ingest.project; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import uk.ac.ebi.subs.ingest.project.web.SearchFilter; +import uk.ac.ebi.subs.ingest.project.web.SearchType; + +public class ProjectFilterQueryBuilderTest { + @Test + void null_search_type_with_non_null_text() { + SearchFilter searchFilter = + SearchFilter.builder() + .search("project keyword") + .wranglingState(null) + .primaryWrangler(null) + .wranglingPriority(null) + .hasOfficialHcaPublication(null) + .identifyingOrganism(null) + .organOntology(null) + .minCellCount(null) + .maxCellCount(null) + .projectLabels(null) + .dcpReleaseNumber(null) + .dataAccess(null) + .searchType(null) + .build(); + + ProjectQueryBuilder.buildProjectsQuery(searchFilter); + // no exception thrown when searchType is null + } + + @ParameterizedTest + @CsvSource({ + "AllKeywords,k1 k2,\"k1\" \"k2\"", + "AnyKeyword,k1 k2,k1 k2", + "ExactMatch,k1 k2,\"k1 k2\"", + "null,k1 k2,k1 k2", + "AllKeywords,\"k1 k2\",\"k1 k2\"", + "AnyKeyword,k1 \"k2\" k3,k1 \"k2\" k3", + "ExactMatch,\"k1\" k2,\"k1\" k2", + }) + public void quoting_in_mongo_syntax_by_search_type( + String searchTypeStr, String input, String expected) { + SearchType searchType = searchTypeStr.equals("null") ? null : SearchType.valueOf(searchTypeStr); + SearchFilter searchFilter = SearchFilter.builder().search(input).searchType(searchType).build(); + Assertions.assertThat(ProjectQueryBuilder.formatSearchString(searchFilter)).isEqualTo(expected); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/project/ProjectLinkChangeListenerTest.java b/src/test/java/uk/ac/ebi/subs/ingest/project/ProjectLinkChangeListenerTest.java new file mode 100644 index 000000000..f1b2be644 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/project/ProjectLinkChangeListenerTest.java @@ -0,0 +1,40 @@ +package uk.ac.ebi.subs.ingest.project; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = ProjectLinkChangeListener.class) +class ProjectLinkChangeListenerTest { + @MockBean ProjectService projectService; + @Autowired ProjectLinkChangeListener projectLinkChangeListener; + + @Test + void test_whenUpdatingStatus_usingProjectService() { + // given + Project project = new Project(null); + project.setUuid(Uuid.newUuid()); + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + project.addToSubmissionEnvelopes(submissionEnvelope); + List submissions = List.of(submissionEnvelope); + + // then + projectLinkChangeListener.beforeLinkSave(project, submissions); + + // then + verify(projectService).updateWranglingState(eq(project), any()); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/project/ProjectServiceTest.java b/src/test/java/uk/ac/ebi/subs/ingest/project/ProjectServiceTest.java new file mode 100644 index 000000000..c334d0cfc --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/project/ProjectServiceTest.java @@ -0,0 +1,324 @@ +package uk.ac.ebi.subs.ingest.project; + +import static java.util.stream.Collectors.toList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; +import static uk.ac.ebi.subs.ingest.audit.AuditType.STATUS_UPDATED; +import static uk.ac.ebi.subs.ingest.project.WranglingState.ELIGIBLE; + +import java.util.*; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.ApplicationContext; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import uk.ac.ebi.subs.ingest.audit.AuditEntry; +import uk.ac.ebi.subs.ingest.audit.AuditEntryService; +import uk.ac.ebi.subs.ingest.bundle.BundleManifestRepository; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.dataset.DatasetRepository; +import uk.ac.ebi.subs.ingest.project.exception.NonEmptyProject; +import uk.ac.ebi.subs.ingest.schemas.Schema; +import uk.ac.ebi.subs.ingest.schemas.SchemaService; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@ExtendWith(SpringExtension.class) +@SpringBootTest( + classes = { + ProjectService.class, + ProjectRepository.class, + SubmissionEnvelopeRepository.class, + DatasetRepository.class + }) +public class ProjectServiceTest { + + @Autowired private ApplicationContext applicationContext; + + @Autowired private ProjectService projectService; + + @MockBean private MongoTemplate mongoTemplate; + + @MockBean private SubmissionEnvelopeRepository submissionEnvelopeRepository; + + @MockBean private ProjectRepository projectRepository; + + @MockBean private DatasetRepository datasetRepository; + + @MockBean private MetadataCrudService metadataCrudService; + + @MockBean private MetadataUpdateService metadataUpdateService; + + @MockBean private SchemaService schemaService; + + @MockBean private BundleManifestRepository bundleManifestRepository; + + @MockBean private ProjectEventHandler projectEventHandler; + + @MockBean private AuditEntryService auditEntryService; + + @BeforeEach + void setUp() { + applicationContext.getBeansWithAnnotation(MockBean.class).forEach(Mockito::reset); + } + + @Nested + class SubmissionEnvelopes { + Project project1; + Project project2; + Set submissionSet1; + Set submissionSet2; + + @BeforeEach + void setup() { + // given + project1 = spy(new Project(null)); + doReturn("project1").when(project1).getId(); + project1.setUuid(Uuid.newUuid()); + + submissionSet1 = new HashSet<>(); + IntStream.range(0, 3) + .mapToObj(Integer::toString) + .forEach( + id -> { + var sub = spy(new SubmissionEnvelope()); + doReturn(id).when(sub).getId(); + submissionSet1.add(sub); + }); + submissionSet1.forEach(project1::addToSubmissionEnvelopes); + + // and: + project2 = spy(new Project(null)); + doReturn("project2").when(project2).getId(); + project2.setUuid(project1.getUuid()); + + submissionSet2 = new HashSet<>(); + IntStream.range(10, 15) + .mapToObj(Integer::toString) + .forEach( + id -> { + var sub = spy(new SubmissionEnvelope()); + doReturn(id).when(sub).getId(); + submissionSet2.add(sub); + }); + submissionSet2.forEach(project2::addToSubmissionEnvelopes); + } + + @Test + @DisplayName("get all submissions") + void getFromAllCopiesOfProjects() { + // given + when(projectRepository.findByUuid(project1.getUuid())) + .thenReturn(Stream.of(project1, project2)); + + // when: + var submissionEnvelopes = projectService.getSubmissionEnvelopes(project1); + + // then: + assertThat(submissionEnvelopes).containsAll(submissionSet1).containsAll(submissionSet2); + } + + @Test + @DisplayName("no duplicate submissions") + void getFromAllCopiesOfProjectsNoDuplicates() { + // given + var project3 = spy(new Project(null)); + doReturn("project3").when(project3).getId(); + project3.setUuid(project1.getUuid()); + + submissionSet1.forEach(project3::addToSubmissionEnvelopes); + + var documentIds = new ArrayList(); + submissionSet1.forEach(submission -> documentIds.add(submission.getId())); + submissionSet2.forEach(submission -> documentIds.add(submission.getId())); + + // and: + when(projectRepository.findByUuid(project1.getUuid())) + .thenReturn(Stream.of(project1, project2, project3)); + + // when: + var submissionEnvelopes = projectService.getSubmissionEnvelopes(project1); + + // then: + var returnDocumentIds = new ArrayList(); + submissionEnvelopes.forEach(submission -> returnDocumentIds.add(submission.getId())); + + assertThat(returnDocumentIds).containsExactlyInAnyOrderElementsOf(documentIds); + } + } + + @Nested + class Registration { + + @Test + @DisplayName("success") + void succeed() { + // given: + String content = "{\"name\": \"project\"}"; + Project project = new Project(content); + + // and: + Project persistentProject = new Project(content); + doReturn(persistentProject).when(projectRepository).save(project); + + // when: + Project result = projectService.register(project); + + // then: + verify(projectRepository).save(project); + assertThat(result).isEqualTo(persistentProject); + verify(projectEventHandler).registeredProject(persistentProject); + } + } + + @Nested + class Deletion { + + @Test + @DisplayName("success") + void succeedForEmptyProject() throws Exception { + // given: + var project = new Project("{\"name\": \"test\"}"); + + // and: + var persistentProjects = + IntStream.range(0, 3).mapToObj(number -> new Project(null)).collect(toList()); + persistentProjects.forEach( + persistentProject -> BeanUtils.copyProperties(project, persistentProject)); + doReturn(persistentProjects.stream()).when(projectRepository).findByUuid(project.getUuid()); + + // when: + projectService.delete(project); + + // then: + persistentProjects.forEach( + persistentProject -> { + verify(metadataCrudService).deleteDocument(persistentProject); + verify(projectEventHandler).deletedProject(persistentProject); + }); + } + + @Test + @DisplayName("fails for non-empty Project") + void failForProjectWithSubmissions() { + // given: + var project = new Project(null); + + // and: copy of project with no submissions + var persistentEmptyProject = new Project(null); + BeanUtils.copyProperties(project, persistentEmptyProject); + + // and: copy of project with submissions + var persistentNonEmptyProject = new Project(null); + BeanUtils.copyProperties(project, persistentNonEmptyProject); + IntStream.range(0, 3) + .mapToObj(Integer::toString) + .forEach( + id -> { + SubmissionEnvelope sub = spy(new SubmissionEnvelope()); + doReturn(id).when(sub).getId(); + persistentNonEmptyProject.addToSubmissionEnvelopes(sub); + }); + + // and: + doReturn(Stream.of(persistentEmptyProject, persistentNonEmptyProject)) + .when(projectRepository) + .findByUuid(project.getUuid()); + + // expect: + Assertions.assertThatThrownBy(() -> projectService.delete(project)) + .isInstanceOf(NonEmptyProject.class); + } + } + + @Nested + class SuggestedProject { + + @Test + @DisplayName("Register a project") + void givenSuggestionCreatesProjectSuccessfully() { + // given: + ObjectMapper mapper = new ObjectMapper(); + ObjectNode suggestion = mapper.createObjectNode(); + suggestion.put("doi", "doi123"); + suggestion.put("name", "Test User"); + suggestion.put("email", "test@example.com"); + suggestion.put("comments", "This is a comment"); + + // and: + final String highLevelEntity = "type"; + Schema projectSchema = + new Schema( + highLevelEntity, + "2.0", + "project", + "project", + "project", + "mock.io/mock-schema-project"); + doReturn(projectSchema) + .when(schemaService) + .getLatestSchemaByEntityType(highLevelEntity, "project"); + + Map content = new HashMap<>(); + final String entityType = "project"; + content.put( + "describedBy", + schemaService.getLatestSchemaByEntityType(highLevelEntity, entityType).getSchemaUri()); + content.put("schema_type", entityType); + + Project persistentProject = new Project(content); + persistentProject.setWranglingState(WranglingState.NEW_SUGGESTION); + persistentProject.setWranglingNotes( + String.format( + "DOI: %s \nName: %s \nEmail: %s \nComments: %s", + suggestion.get("doi"), + suggestion.get("name"), + suggestion.get("email"), + suggestion.get("comments"))); + doReturn(persistentProject).when(projectRepository).save(any(Project.class)); + + // when: + Project result = projectService.createSuggestedProject(suggestion); + + // then: + assertThat(result).isEqualTo(persistentProject); + } + } + + @Nested + class ProjectUpdate { + + @Test + @DisplayName("state update adds a history entry") + void statusUpdatesAddsHistoryRecord() { + // given + Project project = new Project(null); + + // when + projectService.updateWranglingState(project, ELIGIBLE); + + // then + verify(auditEntryService) + .addAuditEntry(new AuditEntry(STATUS_UPDATED, any(), ELIGIBLE, project)); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/project/ProjectTest.java b/src/test/java/uk/ac/ebi/subs/ingest/project/ProjectTest.java new file mode 100644 index 000000000..d73913730 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/project/ProjectTest.java @@ -0,0 +1,81 @@ +package uk.ac.ebi.subs.ingest.project; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +public class ProjectTest { + @Test + public void testGetOpenSubmissionEnvelopes() { + // given: + SubmissionEnvelope openSubmissionEnvelope = new SubmissionEnvelope(); + openSubmissionEnvelope.enactStateTransition(SubmissionState.DRAFT); + + SubmissionEnvelope openSubmissionEnvelope2 = new SubmissionEnvelope(); + openSubmissionEnvelope2.enactStateTransition(SubmissionState.DRAFT); + openSubmissionEnvelope2.enactStateTransition(SubmissionState.METADATA_VALID); + openSubmissionEnvelope2.enactStateTransition(SubmissionState.SUBMITTED); + + Project project = new Project(null); + project.addToSubmissionEnvelopes(openSubmissionEnvelope); + project.addToSubmissionEnvelopes(openSubmissionEnvelope2); + + // when: + List openSubmissionEnvelopes = project.getOpenSubmissionEnvelopes(); + + // then: + assertThat(openSubmissionEnvelopes).hasSize(1); + } + + @Test + public void testGetOpenSubmissionEnvelopesNone() { + // given: + SubmissionEnvelope completeSubmission = new SubmissionEnvelope(); + completeSubmission.enactStateTransition(SubmissionState.DRAFT); + completeSubmission.enactStateTransition(SubmissionState.METADATA_VALID); + completeSubmission.enactStateTransition(SubmissionState.SUBMITTED); + completeSubmission.enactStateTransition(SubmissionState.PROCESSING); + completeSubmission.enactStateTransition(SubmissionState.COMPLETE); + + SubmissionEnvelope submittedSubmission = new SubmissionEnvelope(); + submittedSubmission.enactStateTransition(SubmissionState.DRAFT); + submittedSubmission.enactStateTransition(SubmissionState.METADATA_VALID); + submittedSubmission.enactStateTransition(SubmissionState.SUBMITTED); + + Project project = new Project(null); + project.addToSubmissionEnvelopes(submittedSubmission); + project.addToSubmissionEnvelopes(completeSubmission); + + // when: + List openSubmissionEnvelopes = project.getOpenSubmissionEnvelopes(); + + // then: + assertThat(openSubmissionEnvelopes).hasSize(0); + } + + @Test + public void testIsEditable() { + Project project = new Project(null); + assertThat(project.isEditable()).isTrue(); + + SubmissionEnvelope submissionOne = new SubmissionEnvelope(); + submissionOne.enactStateTransition(SubmissionState.METADATA_VALID); + SubmissionEnvelope submissionTwo = new SubmissionEnvelope(); + submissionTwo.enactStateTransition(SubmissionState.METADATA_INVALID); + project.addToSubmissionEnvelopes(submissionOne); + project.addToSubmissionEnvelopes(submissionTwo); + + assertThat(project.isEditable()).isTrue(); + + // submissionOne.enactStateTransition(SubmissionState.GRAPH_VALID); + // assertThat(project.isEditable()).isTrue(); + // + // submissionTwo.enactStateTransition(SubmissionState.GRAPH_VALID); + // assertThat(project.isEditable()).isFalse(); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/project/WranglingStateTest.java b/src/test/java/uk/ac/ebi/subs/ingest/project/WranglingStateTest.java new file mode 100644 index 000000000..b4ceac9b9 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/project/WranglingStateTest.java @@ -0,0 +1,51 @@ +package uk.ac.ebi.subs.ingest.project; + +import static junit.framework.TestCase.assertEquals; + +import java.io.IOException; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.json.JsonTest; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +@JsonTest +public class WranglingStateTest { + @Autowired private ObjectMapper objectMapper; + + @Test + public void testDeserialize() throws IOException { + assertEquals(objectMapper.readValue("\"New\"", WranglingState.class), WranglingState.NEW); + assertEquals( + objectMapper.readValue("\"Eligible\"", WranglingState.class), WranglingState.ELIGIBLE); + assertEquals( + objectMapper.readValue("\"Not eligible\"", WranglingState.class), + WranglingState.NOT_ELIGIBLE); + assertEquals( + objectMapper.readValue("\"In progress\"", WranglingState.class), + WranglingState.IN_PROGRESS); + assertEquals( + objectMapper.readValue("\"Stalled\"", WranglingState.class), WranglingState.STALLED); + assertEquals( + objectMapper.readValue("\"Submitted\"", WranglingState.class), WranglingState.SUBMITTED); + assertEquals( + objectMapper.readValue("\"Published in DCP\"", WranglingState.class), + WranglingState.PUBLISHED_IN_DCP); + assertEquals( + objectMapper.readValue("\"Deleted\"", WranglingState.class), WranglingState.DELETED); + } + + @Test + public void testSerialize() throws JsonProcessingException { + assertEquals(objectMapper.writeValueAsString(WranglingState.NEW), "\"New\""); + assertEquals(objectMapper.writeValueAsString(WranglingState.ELIGIBLE), "\"Eligible\""); + assertEquals(objectMapper.writeValueAsString(WranglingState.NOT_ELIGIBLE), "\"Not eligible\""); + assertEquals(objectMapper.writeValueAsString(WranglingState.IN_PROGRESS), "\"In progress\""); + assertEquals(objectMapper.writeValueAsString(WranglingState.STALLED), "\"Stalled\""); + assertEquals( + objectMapper.writeValueAsString(WranglingState.PUBLISHED_IN_DCP), "\"Published in DCP\""); + assertEquals(objectMapper.writeValueAsString(WranglingState.DELETED), "\"Deleted\""); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/protocol/ProtocolServiceTest.java b/src/test/java/uk/ac/ebi/subs/ingest/protocol/ProtocolServiceTest.java new file mode 100644 index 000000000..549a52ca6 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/protocol/ProtocolServiceTest.java @@ -0,0 +1,78 @@ +package uk.ac.ebi.subs.ingest.protocol; + +import static java.util.Arrays.asList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + +import java.util.Optional; + +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; + +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; + +@SpringBootTest(classes = ProtocolService.class) +public class ProtocolServiceTest { + + @Autowired private ProtocolService protocolService; + + @MockBean private MetadataCrudService metadataCrudService; + + @MockBean private MetadataUpdateService metadataUpdateService; + + @MockBean private ProtocolRepository protocolRepository; + + @MockBean private ProcessRepository processRepository; + + @MockBean private ProjectRepository projectRepository; + + @MockBean private SubmissionEnvelopeRepository submissionEnvelopeRepository; + + @Nested + class Submission { + + @Test + void determineLinking() { + // given: + SubmissionEnvelope submission = new SubmissionEnvelope(); + + // and: + Protocol linked = new Protocol("linked"); + Protocol notLinked = new Protocol("not linked"); + Pageable pageable = mock(Pageable.class); + doReturn(new PageImpl(asList(linked, notLinked))) + .when(protocolRepository) + .findBySubmissionEnvelope(submission, pageable); + + // and: + doReturn(Optional.of(new Process(null))) + .when(processRepository) + .findFirstByProtocolsContains(linked); + doReturn(Optional.empty()).when(processRepository).findFirstByProtocolsContains(notLinked); + + // when: + Page results = protocolService.retrieve(submission, pageable); + + // then: + assertThat(results).isNotNull(); + assertThat(results.getTotalElements()).isEqualTo(2); + + // and: + assertThat(linked.isLinked()).isTrue(); + assertThat(notLinked.isLinked()).isFalse(); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/query/QueryBuilderTest.java b/src/test/java/uk/ac/ebi/subs/ingest/query/QueryBuilderTest.java new file mode 100644 index 000000000..a85965e14 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/query/QueryBuilderTest.java @@ -0,0 +1,48 @@ +package uk.ac.ebi.subs.ingest.query; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; + +public class QueryBuilderTest { + @Test + public void testBuildOr() { + // given: + QueryBuilder queryBuilder = new QueryBuilder(); + + List metadataCriteriaList = new ArrayList<>(); + metadataCriteriaList.add(new MetadataCriteria("field", Operator.IS, "value")); + Query expectedQuery = new Query(); + Criteria criteria = Criteria.where("field").is("value"); + expectedQuery.addCriteria(new Criteria().orOperator(new Criteria[] {criteria})); + + // when: + Query actualQuery = queryBuilder.build(metadataCriteriaList, false); + + // then: + assertThat(actualQuery).isEqualTo(expectedQuery); + } + + @Test + public void testBuildAnd() { + // given: + QueryBuilder queryBuilder = new QueryBuilder(); + + List metadataCriteriaList = new ArrayList<>(); + metadataCriteriaList.add(new MetadataCriteria("field", Operator.IS, "value")); + Query expectedQuery = new Query(); + Criteria criteria = Criteria.where("field").is("value"); + expectedQuery.addCriteria(new Criteria().andOperator(new Criteria[] {criteria})); + + // when: + Query actualQuery = queryBuilder.build(metadataCriteriaList, true); + + // then: + assertThat(actualQuery).isEqualTo(expectedQuery); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/repository/ProcessRepositoryTest.java b/src/test/java/uk/ac/ebi/subs/ingest/repository/ProcessRepositoryTest.java new file mode 100644 index 000000000..d42186935 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/repository/ProcessRepositoryTest.java @@ -0,0 +1,4 @@ +package uk.ac.ebi.subs.ingest.repository; + +/** Created by rolando on 16/02/2018. */ +public class ProcessRepositoryTest {} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/schemas/SchemaServiceTest.java b/src/test/java/uk/ac/ebi/subs/ingest/schemas/SchemaServiceTest.java new file mode 100644 index 000000000..06995ddcb --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/schemas/SchemaServiceTest.java @@ -0,0 +1,179 @@ +package uk.ac.ebi.subs.ingest.schemas; + +import static java.util.Arrays.asList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doReturn; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.schemas.schemascraper.SchemaScraper; + +@ExtendWith(SpringExtension.class) +public class SchemaServiceTest { + + @Autowired private SchemaService schemaService; + + @MockBean private SchemaRepository schemaRepository; + + @MockBean private SchemaScraper schemaScraper; + + @Test + public void testGetLatestSchemasVersions() { + // given: + Schema version1_2_3 = createTestSchema("1.2.3", "process_core"); + Schema version1_2_4 = createTestSchema("1.2.4", "process_core"); + Schema version2_0 = createTestSchema("2.0", "process_core"); + Schema version11_1_1 = createTestSchema("11.1.1", "process_core"); + + // and: + List schemas = asList(version2_0, version1_2_3, version11_1_1, version1_2_4); + doReturn(schemas).when(schemaRepository).findAll(); + + // when: + List latestSchemas = schemaService.getLatestSchemas(); + + // then: + assertThat(latestSchemas).hasSize(1); + Schema latestSchema = latestSchemas.get(0); + assertThat(latestSchema.getSchemaVersion()).isEqualTo(version11_1_1.getSchemaVersion()); + } + + @Test + public void testGetLatestSchemasCount() { + // given: + Schema version1_2_3 = createTestSchema("1.2.3", "process_core"); + Schema version1_2_4 = createTestSchema("1.2.4", "process_core"); + Schema version2_0 = createTestSchema("2.0", "process_core"); + Schema version11_1_1 = createTestSchema("11.1.1", "process_core"); + Schema protocol_version1_2_3 = createTestSchema("1.2.3", "protocol"); + Schema biomaterial_version1_2_4 = createTestSchema("1.2.4", "biomaterial"); + Schema project_version2_0 = createTestSchema("2.0", "project"); + Schema project_version11_1_1 = createTestSchema("11.1.1", "project"); + + // and: + List schemas = + asList( + version2_0, + version1_2_3, + version11_1_1, + version1_2_4, + protocol_version1_2_3, + biomaterial_version1_2_4, + project_version2_0, + project_version11_1_1); + doReturn(schemas).when(schemaRepository).findAll(); + + // when: + List latestSchemas = schemaService.getLatestSchemas(); + + // then: + assertThat(latestSchemas).hasSize(4); + } + + @Test + public void testGetLatestSchemaByEntityTypeWithExistingType() { + // given: + final String projectEntityType = "project"; + final String oldSchemaVersion = "2.0"; + final String newSchemaVersion = "11.1.1"; + Schema protocol_version1_2_3 = createTestSchema(oldSchemaVersion, "protocol"); + Schema biomaterial_version1_2_4 = createTestSchema(newSchemaVersion, "biomaterial"); + Schema project_version2_0 = createTestSchema(oldSchemaVersion, projectEntityType); + Schema project_version11_1_1 = createTestSchema(newSchemaVersion, projectEntityType); + + // and: + List schemas = + asList( + project_version2_0, + protocol_version1_2_3, + project_version11_1_1, + biomaterial_version1_2_4); + doReturn(schemas).when(schemaRepository).findAll(); + + // when: + Schema latestSchema = schemaService.getLatestSchemaByEntityType("type", projectEntityType); + + // then: + assertThat(latestSchema.getConcreteEntity()).isEqualTo(projectEntityType); + assertThat(latestSchema.getSchemaVersion()).isEqualTo(newSchemaVersion); + } + + @Test + public void testGetLatestSchemaByEntityTypeWithNonExistingType() { + // given: + final String projectEntityType = "project"; + final String oldSchemaVersion = "2.0"; + final String newSchemaVersion = "11.1.1"; + Schema protocol_version1_2_3 = createTestSchema(oldSchemaVersion, "protocol"); + Schema biomaterial_version1_2_4 = createTestSchema(newSchemaVersion, "biomaterial"); + Schema project_version2_0 = createTestSchema(oldSchemaVersion, projectEntityType); + Schema project_version11_1_1 = createTestSchema(newSchemaVersion, projectEntityType); + + // and: + List schemas = + asList( + project_version2_0, + protocol_version1_2_3, + project_version11_1_1, + biomaterial_version1_2_4); + doReturn(schemas).when(schemaRepository).findAll(); + + // when: + Schema latestSchema = schemaService.getLatestSchemaByEntityType("type", "non_exists"); + + // then: + assertThat(latestSchema).isNull(); + } + + private Schema createTestSchema(String schemaVersion, String entityType) { + return new Schema( + "type", + schemaVersion, + entityType, + entityType, + entityType, + "http://schema.humancellatlas.org"); + } + + @Configuration + static class TestConfiguration { + + @Bean + SchemaService schemaService() { + return new SchemaService(); + } + } + + @Test + public void testGetLatestMorphicSchemasVersions() { + // given: + Schema version1_0_0 = createMorphicTestSchema("1.0.0", "biomaterial"); + Schema version0_9_0 = createMorphicTestSchema("0.9.0", "biomaterial"); + Schema version2_0_0 = createMorphicTestSchema("2.0.0", "biomaterial"); + + // and: + List schemas = asList(version2_0_0, version0_9_0, version1_0_0); + doReturn(schemas).when(schemaRepository).findAll(); + + // when: + List latestSchemas = schemaService.getLatestSchemas(); + + // then: + assertThat(latestSchemas).hasSize(1); + Schema latestSchema = latestSchemas.get(0); + assertThat(latestSchema.getSchemaVersion()).isEqualTo(version2_0_0.getSchemaVersion()); + } + + private Schema createMorphicTestSchema(String schemaVersion, String entityType) { + return new Schema( + "type", schemaVersion, entityType, "", entityType, "https://dev.schema.morphic.bio"); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/schemas/SchemaTest.java b/src/test/java/uk/ac/ebi/subs/ingest/schemas/SchemaTest.java new file mode 100644 index 000000000..4253bdbce --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/schemas/SchemaTest.java @@ -0,0 +1,86 @@ +package uk.ac.ebi.subs.ingest.schemas; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; + +import org.junit.jupiter.api.Test; + +public class SchemaTest { + + @Test + public void testCompareToSameVersion() { + // given: + Schema schema = createTestSchema("7.3.1"); + + // expect: + assertThat(schema.compareTo(schema)).isEqualTo(0); + } + + @Test + public void testCompareToOlderVersion() { + // given: + Schema schemaVersion10 = createTestSchema("10.9"); + Schema schemaVersion7 = createTestSchema("7.3.1"); + Schema schemaVersion6_7 = createTestSchema("6.7.3"); + Schema schemaVersion6_3 = createTestSchema("6.3.11"); + + // expect: + assertThat(schemaVersion10.compareTo(schemaVersion7)).isGreaterThan(0); + assertThat(schemaVersion7.compareTo(schemaVersion6_7)).isGreaterThan(0); + assertThat(schemaVersion6_7.compareTo(schemaVersion6_3)).isGreaterThan(0); + } + + @Test + public void testCompareToNewerVersion() { + // given: + Schema schemaVersion5 = createTestSchema("5"); + Schema schemaVersion5_1 = createTestSchema("5.1"); + Schema schemaVersion5_1_3 = createTestSchema("5.1.3"); + + // expect: + assertThat(schemaVersion5.compareTo(schemaVersion5_1)).isLessThan(0); + assertThat(schemaVersion5_1.compareTo(schemaVersion5_1_3)).isLessThan(0); + assertThat(schemaVersion5.compareTo(schemaVersion5_1_3)).isLessThan(0); + } + + @Test + public void testCompareDifferentSchemas() { + // given: + Schema biomaterialCore = + new Schema( + "core", + "5.9.10", + "biomaterial", + "", + "biomaterial_core", + "http://schema.humancellatlas.org"); + Schema processCore = createTestSchema("7.4.3"); + assumeThat(biomaterialCore.getConcreteEntity()) + .isNotEqualToIgnoringCase(processCore.getConcreteEntity()); + + // expect: + assertThat(biomaterialCore.compareTo(processCore)).isLessThan(0); + assertThat(processCore.compareTo(biomaterialCore)).isGreaterThan(0); + } + + private Schema createTestSchema(String schemaVersion) { + return new Schema( + "core", schemaVersion, "process", "", "process_core", "http://schema.humancellatlas.org"); + } + + @Test + public void testCompareMorphicSchemas() { + // given: + Schema morphicSchemaVersion1_0_0 = createMorphicTestSchema("1.0.0"); + Schema morphicSchemaVersion2_0_0 = createMorphicTestSchema("2.0.0"); + + // expect: + assertThat(morphicSchemaVersion1_0_0.compareTo(morphicSchemaVersion2_0_0)).isLessThan(0); + assertThat(morphicSchemaVersion2_0_0.compareTo(morphicSchemaVersion1_0_0)).isGreaterThan(0); + } + + private Schema createMorphicTestSchema(String schemaVersion) { + return new Schema( + "type", schemaVersion, "biomaterial", "", "biomaterial", "https://dev.schema.morphic.bio"); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/security/AccountServiceTest.java b/src/test/java/uk/ac/ebi/subs/ingest/security/AccountServiceTest.java new file mode 100644 index 000000000..7c0e092dd --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/security/AccountServiceTest.java @@ -0,0 +1,86 @@ +package uk.ac.ebi.subs.ingest.security; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assumptions.assumeThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.verify; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.security.exception.DuplicateAccount; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = {DefaultAccountService.class}) +public class AccountServiceTest { + + @Autowired private AccountService accountService; + + @MockBean private AccountRepository accountRepository; + + @Nested + @DisplayName("Registration") + class Registration { + + @Test + void success() { + // given: + String providerReference = "67fe90"; + Account account = new Account(providerReference); + assumeThat(account.getRoles()).isEmpty(); + + // and: + String name = "Juan dela Cruz"; + account.setName(name); + + // and: + Account persistentAccount = new Account("773b471", providerReference); + persistentAccount.setName(name); + doReturn(persistentAccount).when(accountRepository).save(any(Account.class)); + + // when: + Account result = accountService.register(account); + + // then: + assertThat(result).isEqualTo(persistentAccount); + var accountCaptor = ArgumentCaptor.forClass(Account.class); + verify(accountRepository).save(accountCaptor.capture()); + + // and: + var savedAccount = accountCaptor.getValue(); + assertThat(savedAccount) + .extracting(Account::getProviderReference, Account::getName) + .containsExactly(providerReference, name); + assertThat(savedAccount.getRoles()).containsOnly(Role.CONTRIBUTOR); + } + + @Test + void duplicateAccount() { + // given: + String providerReference = "84cd01b"; + Account account = new Account(providerReference); + + // and: + Account persistentAccount = new Account("72b1c9e", providerReference); + doReturn(persistentAccount) + .when(accountRepository) + .findByProviderReference(providerReference); + + // expect: + assertThatThrownBy( + () -> { + accountService.register(account); + }) + .isInstanceOf(DuplicateAccount.class); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/security/AccountTest.java b/src/test/java/uk/ac/ebi/subs/ingest/security/AccountTest.java new file mode 100644 index 000000000..a34c893bf --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/security/AccountTest.java @@ -0,0 +1,25 @@ +package uk.ac.ebi.subs.ingest.security; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +public class AccountTest { + + @Nested + class Guest { + + @Test + void ensureNoNullFields() { + // expect: + assertThat(Account.GUEST).hasNoNullFieldsOrProperties(); + } + + @Test + void ensureGuestRole() { + // expect: + assertThat(Account.GUEST.getRoles()).containsOnly(Role.GUEST); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/security/JwtGenerator.java b/src/test/java/uk/ac/ebi/subs/ingest/security/JwtGenerator.java new file mode 100644 index 000000000..22ca3e012 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/security/JwtGenerator.java @@ -0,0 +1,109 @@ +package uk.ac.ebi.subs.ingest.security; + +import static java.util.Map.entry; + +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.security.interfaces.RSAPrivateKey; +import java.security.interfaces.RSAPublicKey; +import java.util.*; + +import javax.annotation.Nullable; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.JWTCreator; +import com.auth0.jwt.algorithms.Algorithm; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import uk.ac.ebi.subs.ingest.security.authn.oidc.UserInfo; + +public class JwtGenerator { + + public static final String DEFAULT_ISSUER = "https://humancellatlas.auth0.com"; + public static final String DEFAULT_KEY_ID = + "MDc2OTM3ODI4ODY2NUU5REVGRDVEM0MyOEYwQTkzNDZDRDlEQzNBRQ"; + + public static final String OIDC_ISS = "iss"; + public static final String OIDC_SUB = "sub"; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + private final KeyPair keyPair; + + private final String issuer; + + public JwtGenerator() { + this(DEFAULT_ISSUER); + } + + /** + * Creates an instance with a pre-defined default issuer. This issuer >will be overridden + * by "iss" claim during when generating the JWT if it is set. + * + * @param issuer + */ + // The decision to allow "iss" claim to override the issuer field is so that UserInfo can be + // encoded with little + // modifications to this utility class. + public JwtGenerator(@Nullable String issuer) { + KeyPairGenerator keyGenerator = getKeyPairGenerator(); + this.keyPair = keyGenerator.generateKeyPair(); + this.issuer = Optional.ofNullable(issuer).orElse(DEFAULT_ISSUER); + } + + private KeyPairGenerator getKeyPairGenerator() { + try { + KeyPairGenerator keyGenerator = KeyPairGenerator.getInstance("RSA"); + keyGenerator.initialize(2048); + return keyGenerator; + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + public RSAPublicKey getPublicKey() { + return (RSAPublicKey) keyPair.getPublic(); + } + + public String generate() { + return generate(null, null, null); + } + + public String generate(Map claims) { + return generate(null, null, claims); + } + + public String generate( + @Nullable String keyId, @Nullable String subject, @Nullable Map claims) { + var kid = Optional.ofNullable(keyId); + Map header = Map.ofEntries(entry("kid", kid.orElse(DEFAULT_KEY_ID))); + + Map allClaims = new HashMap<>(); + Optional.ofNullable(claims).ifPresent(allClaims::putAll); + JWTCreator.Builder builder = + JWT.create() + .withHeader(header) + .withIssuer(Optional.ofNullable(allClaims.get(OIDC_ISS)).orElse(issuer)) + .withSubject( + Optional.ofNullable(subject) + .or(() -> Optional.ofNullable(allClaims.get(OIDC_SUB))) + .orElse(UUID.randomUUID().toString())); + + Arrays.asList(OIDC_ISS, OIDC_SUB).forEach(allClaims::remove); + allClaims.forEach(builder::withClaim); + + var rsa256 = Algorithm.RSA256(null, (RSAPrivateKey) keyPair.getPrivate()); + return builder.sign(rsa256); + } + + public String generateWithSubject(String subject) { + return generate(null, subject, null); + } + + public String encode(UserInfo userInfo) { + var claims = objectMapper.convertValue(userInfo, new TypeReference>() {}); + return generate(null, null, claims); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/security/JwtVerifierResolverTest.java b/src/test/java/uk/ac/ebi/subs/ingest/security/JwtVerifierResolverTest.java new file mode 100644 index 000000000..1bcb50d2c --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/security/JwtVerifierResolverTest.java @@ -0,0 +1,180 @@ +package uk.ac.ebi.subs.ingest.security; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + +import java.security.interfaces.RSAPublicKey; + +import org.junit.jupiter.api.Test; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.interfaces.DecodedJWT; +import com.auth0.jwt.interfaces.JWTVerifier; + +import uk.ac.ebi.subs.ingest.security.authn.provider.gcp.GcpJwkVault; +import uk.ac.ebi.subs.ingest.security.common.jwk.DelegatingJwtVerifier; +import uk.ac.ebi.subs.ingest.security.common.jwk.JwtVerifierResolver; + +public class JwtVerifierResolverTest { + + @Test + public void testResolveForJwt() { + // given: + JwtGenerator jwtGenerator = new JwtGenerator("issuerFromToken"); + RSAPublicKey publicKey = jwtGenerator.getPublicKey(); + + // and: + String audience = "https://dev.data.humancellatlas.org/"; + GcpJwkVault jwkVault = mock(GcpJwkVault.class); + doReturn(publicKey).when(jwkVault).getPublicKey(any(DecodedJWT.class)); + + // and: + JwtVerifierResolver jwtVerifierResolver = new JwtVerifierResolver(jwkVault, audience, null); + + // and: given the token + String jwt = jwtGenerator.generate(); + DecodedJWT token = JWT.decode(jwt); + + // when: + JWTVerifier verifier = jwtVerifierResolver.resolve(jwt); + + // then: + assertThat(verifier).isNotNull(); + + // and: inspect using verifier with extended interface as a work around + assertThat(verifier).isInstanceOf(DelegatingJwtVerifier.class); + DelegatingJwtVerifier delegatingVerifier = (DelegatingJwtVerifier) verifier; + assertThat(delegatingVerifier) + .extracting("audience", "issuer") + .containsExactly(audience, token.getIssuer()); + } + + @Test + public void testResolveForJwtWithIssuer() { + // given: + String issuerFromToken = "issuerFromToken"; + JwtGenerator jwtGenerator = new JwtGenerator(issuerFromToken); + RSAPublicKey publicKey = jwtGenerator.getPublicKey(); + + // and: + String audience = "https://dev.data.humancellatlas.org/"; + String issuer = "auth0"; + GcpJwkVault jwkVault = mock(GcpJwkVault.class); + doReturn(publicKey).when(jwkVault).getPublicKey(any(DecodedJWT.class)); + + // and: + JwtVerifierResolver jwtVerifierResolver = new JwtVerifierResolver(jwkVault, audience, issuer); + + // and: given the token + String jwt = jwtGenerator.generate(); + + // when: + JWTVerifier verifier = jwtVerifierResolver.resolve(jwt); + + // then: + assertThat(verifier).isNotNull(); + + // and: inspect using verifier with extended interface as a work around + assertThat(verifier).isInstanceOf(DelegatingJwtVerifier.class); + DelegatingJwtVerifier delegatingVerifier = (DelegatingJwtVerifier) verifier; + assertThat(delegatingVerifier) + .extracting("audience", "issuer") + .containsExactly(audience, issuer); + } + + @Test + public void testResolveForJwtWithNoAudience() { + // given: + JwtGenerator jwtGenerator = new JwtGenerator(); + RSAPublicKey publicKey = jwtGenerator.getPublicKey(); + + // and: + String issuer = "auth0"; + GcpJwkVault jwkVault = mock(GcpJwkVault.class); + doReturn(publicKey).when(jwkVault).getPublicKey(any(DecodedJWT.class)); + + // and: + JwtVerifierResolver jwtVerifierResolver = new JwtVerifierResolver(jwkVault, null, issuer); + + // and: given the token + String jwt = jwtGenerator.generate(); + + // when: + JWTVerifier verifier = jwtVerifierResolver.resolve(jwt); + + // then: + assertThat(verifier).isNotNull(); + + // and: inspect using verifier with extended interface as a work around + assertThat(verifier).isInstanceOf(DelegatingJwtVerifier.class); + DelegatingJwtVerifier delegatingVerifier = (DelegatingJwtVerifier) verifier; + assertThat(delegatingVerifier).extracting("audience", "issuer").containsExactly(null, issuer); + } + + @Test + public void testResolveForJwtWithNoAudienceAndNoIssuer() { + // given: + String issuerFromToken = "issuerFromToken"; + JwtGenerator jwtGenerator = new JwtGenerator(issuerFromToken); + RSAPublicKey publicKey = jwtGenerator.getPublicKey(); + + // and: + String issuer = "auth0"; + GcpJwkVault jwkVault = mock(GcpJwkVault.class); + doReturn(publicKey).when(jwkVault).getPublicKey(any(DecodedJWT.class)); + + // and: + JwtVerifierResolver jwtVerifierResolver = new JwtVerifierResolver(jwkVault, null, null); + + // and: given the token + String jwt = jwtGenerator.generate(); + DecodedJWT token = JWT.decode(jwt); + + // when: + JWTVerifier verifier = jwtVerifierResolver.resolve(jwt); + + // then: + assertThat(verifier).isNotNull(); + + // and: inspect using verifier with extended interface as a work around + assertThat(verifier).isInstanceOf(DelegatingJwtVerifier.class); + DelegatingJwtVerifier delegatingVerifier = (DelegatingJwtVerifier) verifier; + assertThat(delegatingVerifier) + .extracting("audience", "issuer") + .containsExactly(null, token.getIssuer()); + } + + @Test + public void testResolveForJwtWithAudienceAndNoIssuer() { + // given: + JwtGenerator jwtGenerator = new JwtGenerator("issuerFromToken"); + RSAPublicKey publicKey = jwtGenerator.getPublicKey(); + + // and: + String audience = "https://dev.data.humancellatlas.org/"; + GcpJwkVault jwkVault = mock(GcpJwkVault.class); + doReturn(publicKey).when(jwkVault).getPublicKey(any(DecodedJWT.class)); + + // and: + JwtVerifierResolver jwtVerifierResolver = new JwtVerifierResolver(jwkVault, audience, null); + + // and: given the token + String jwt = jwtGenerator.generate(); + DecodedJWT token = JWT.decode(jwt); + + // when: + JWTVerifier verifier = jwtVerifierResolver.resolve(jwt); + + // then: + assertThat(verifier).isNotNull(); + + // and: inspect using verifier with extended interface as a work around + assertThat(verifier).isInstanceOf(DelegatingJwtVerifier.class); + DelegatingJwtVerifier delegatingVerifier = (DelegatingJwtVerifier) verifier; + assertThat(delegatingVerifier) + .extracting("audience", "issuer") + .containsExactly(audience, token.getIssuer()); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/security/RowLevelFilterSecuritySpelExpressionTest.java b/src/test/java/uk/ac/ebi/subs/ingest/security/RowLevelFilterSecuritySpelExpressionTest.java new file mode 100644 index 000000000..6aec4fd1d --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/security/RowLevelFilterSecuritySpelExpressionTest.java @@ -0,0 +1,25 @@ +package uk.ac.ebi.subs.ingest.security; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +class RowLevelFilterSecuritySpelExpressionTest { + @Test + public void testSpelContainsWithAPrefix() { + ExampleDocument document = new ExampleDocument(); + document.listA = Set.of("ROLE_b", "ROLE_c"); + String spelExpression = "#x.listA.contains('ROLE_'+#y.toString())"; + assertThat( + new SpelHelper() + .parseExpression(List.of("x", "y"), List.of(document, "c"), spelExpression)) + .isTrue(); + } + + class ExampleDocument { + public Set listA; + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/security/UserAuditingTest.java b/src/test/java/uk/ac/ebi/subs/ingest/security/UserAuditingTest.java new file mode 100644 index 000000000..f98e64353 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/security/UserAuditingTest.java @@ -0,0 +1,68 @@ +package uk.ac.ebi.subs.ingest.security; + +import static java.util.Arrays.asList; +import static org.assertj.core.api.Java6Assertions.assertThat; +import static org.mockito.Mockito.*; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContext; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.core.context.SecurityContextImpl; + +public class UserAuditingTest { + + private UserAuditing userAuditing; + + @BeforeEach + void setUp() { + userAuditing = new UserAuditing(); + } + + @Nested + @DisplayName("getCurrentAuditor") + class GetCurrentAuditor { + + @Test + void accountTypePrincipal() { + // given: + String providerReference = "6700ed52"; + Account userAccount = new Account(providerReference, "elixir-1"); + + // and: + Authentication authentication = mock(Authentication.class); + doReturn(userAccount).when(authentication).getPrincipal(); + when(authentication.isAuthenticated()).thenReturn(true); + + // and: + SecurityContext securityContext = new SecurityContextImpl(authentication); + SecurityContextHolder.setContext(securityContext); + + // when: + String auditor = userAuditing.getCurrentAuditor().orElseThrow(); + + // then: + assertThat(auditor).isEqualTo(providerReference); + } + + @Test + void nonAccountTypePrincipal() { + // given: + String principal = "jdelacruz"; + Authentication authentication = + new UsernamePasswordAuthenticationToken(principal, "pas$w0rd", asList(Role.CONTRIBUTOR)); + SecurityContextImpl securityContext = new SecurityContextImpl(authentication); + SecurityContextHolder.setContext(securityContext); + + // when: + String auditor = userAuditing.getCurrentAuditor().orElseThrow(); + + // then: + assertThat(auditor).isEqualTo(principal); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/security/authn/oidc/OpenIdAuthenticationTest.java b/src/test/java/uk/ac/ebi/subs/ingest/security/authn/oidc/OpenIdAuthenticationTest.java new file mode 100644 index 000000000..c1b0bc1ff --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/security/authn/oidc/OpenIdAuthenticationTest.java @@ -0,0 +1,139 @@ +package uk.ac.ebi.subs.ingest.security.authn.oidc; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; + +import java.util.Collection; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.GrantedAuthority; + +import uk.ac.ebi.subs.ingest.security.Account; +import uk.ac.ebi.subs.ingest.security.Role; + +public class OpenIdAuthenticationTest { + + private final String subjectId = "73985cc"; + private final UserInfo userInfo = new UserInfo(subjectId, ""); + + private Account account; + private Authentication authentication; + + @BeforeEach + void setUp() { + account = new Account(subjectId); + authentication = new OpenIdAuthentication(account); + ((OpenIdAuthentication) authentication).authenticateWith(userInfo); + } + + @Nested + @DisplayName("Authentication") + class AuthenticationTest { + + private OpenIdAuthentication authentication; + + @BeforeEach + void setUp() { + authentication = new OpenIdAuthentication(account); + } + + @Test + public void successful() { + // when: + authentication.authenticateWith(userInfo); + + // expect: + assertThat(authentication.isAuthenticated()).isTrue(); + assertThat(authentication.getCredentials()).isEqualTo(userInfo); + } + + @Test + public void noPrincipalAsGuest() { + // given: + authentication = new OpenIdAuthentication((Account) null); + + // when: + authentication.authenticateWith(userInfo); + + // expect: + assertThat(authentication.isAuthenticated()).isTrue(); + assertThat(authentication.getPrincipal()).isEqualTo(Account.GUEST); + assertThat(authentication.getCredentials()).isEqualTo(userInfo); + } + + @Test + public void nonMatchingSubjectId() { + // given: + String anotherSubjectId = "82909a1"; + UserInfo anotherUserInfo = new UserInfo(anotherSubjectId, ""); + assumeThat(anotherSubjectId).isNotEqualTo(subjectId); + + // when: + authentication.authenticateWith(anotherUserInfo); + + // then: + assertThat(authentication.isAuthenticated()).isFalse(); + assertThat(authentication.getCredentials()).isEqualTo(anotherUserInfo); + } + + @Test + public void authenticatedGuest() { + // given: + var authentication = new OpenIdAuthentication((Account) null); + + // when: + authentication.authenticateWith(userInfo); + + // then: + assertThat(authentication.isAuthenticated()).isTrue(); + assertThat(authentication.getCredentials()).isEqualTo(userInfo); + } + } + + @Test + public void testGetPrincipal() { + // expect: + assertThat(authentication.getPrincipal()).isEqualTo(account); + } + + @Test + public void testGetCredentials() { + // expect: + assertThat(authentication.getCredentials()).isEqualTo(userInfo); + } + + @Test + public void testGetAuthorities() { + // given: + account.addRole(Role.CONTRIBUTOR); + + // expect: + // this assignment is to work around the weirdness with generic type that I couldn't figure out + Collection authorities = + (Collection) authentication.getAuthorities(); + assertThat(authorities).containsExactly(Role.CONTRIBUTOR); + } + + @Test + public void testGetName() { + // expect: + assertThat(authentication.getName()).isEqualTo(subjectId); + } + + @Test + public void testGetDetails() { + // expect: + assertThat(authentication.getDetails()).isEqualTo(userInfo); + } + + @Test + public void ensureNonNullPrincipal() { + // expect: + Authentication authentication = new OpenIdAuthentication((Account) null); + assertThat(authentication.getPrincipal()).isSameAs(Account.GUEST); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/security/authn/oidc/UserInfoTest.java b/src/test/java/uk/ac/ebi/subs/ingest/security/authn/oidc/UserInfoTest.java new file mode 100644 index 000000000..6316ac79f --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/security/authn/oidc/UserInfoTest.java @@ -0,0 +1,24 @@ +package uk.ac.ebi.subs.ingest.security.authn.oidc; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import uk.ac.ebi.subs.ingest.security.Account; + +public class UserInfoTest { + + @Test + public void convertToAccount() { + // given: + String subjectId = "723b4001"; + String name = "Jean Valjean"; + UserInfo userInfo = new UserInfo(subjectId, name); + + // when: + Account account = userInfo.toAccount(); + + // then: + assertThat(account).extracting("providerReference", "name").containsExactly(subjectId, name); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/auth0/TestUserWhitelist.java b/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/auth0/TestUserWhitelist.java new file mode 100644 index 000000000..672b5ad31 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/auth0/TestUserWhitelist.java @@ -0,0 +1,29 @@ +package uk.ac.ebi.subs.ingest.security.authn.provider.auth0; + +import static java.util.Arrays.asList; +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import uk.ac.ebi.subs.ingest.security.authn.provider.gcp.GcpDomainWhiteList; + +public class TestUserWhitelist { + + @Test + public void testLists() { + // given: + GcpDomainWhiteList userWhiteList = + new GcpDomainWhiteList("trusteddomain.com", "friendlypeople.net"); + + // expect: + asList( + "goodguy@trusteddomain.com", + "upstandinglass@friendlypeople.net", + "cooldude@friendlypeople.net") + .forEach(email -> assertThat(userWhiteList.lists(email)).isTrue()); + + // and: + asList("maninavan@shadycharacters.tv", "suspicious@darkcorner.xyz") + .forEach(email -> assertThat(userWhiteList.lists(email)).isFalse()); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/auth0/UserJwtAuthenticationProviderTest.java b/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/auth0/UserJwtAuthenticationProviderTest.java new file mode 100644 index 000000000..58cf0e41a --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/auth0/UserJwtAuthenticationProviderTest.java @@ -0,0 +1,120 @@ +package uk.ac.ebi.subs.ingest.security.authn.provider.auth0; + +import static java.util.Map.entry; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + +import java.util.Map; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.springframework.security.authentication.AuthenticationProvider; +import org.springframework.security.core.Authentication; + +import com.auth0.spring.security.api.JwtAuthenticationProvider; +import com.auth0.spring.security.api.authentication.PreAuthenticatedAuthenticationJsonWebToken; + +import uk.ac.ebi.subs.ingest.security.JwtGenerator; +import uk.ac.ebi.subs.ingest.security.authn.provider.gcp.GcpDomainWhiteList; +import uk.ac.ebi.subs.ingest.security.exception.InvalidUserGroup; + +public class UserJwtAuthenticationProviderTest { + + @Nested + @DisplayName("authentication") + class AuthenticationTest { + private JwtGenerator jwtGenerator = new JwtGenerator(); + + @Test + @DisplayName("authentication succeeds") + public void testAuthenticate() { + // given: JWT authentication + String userEmail = "trustedfellow@friendlysite.com"; + Map claims = + Map.ofEntries(entry("https://auth.data.humancellatlas.org/group", "hca")); + + String jwt = jwtGenerator.generate(null, userEmail, claims); + Authentication jwtAuthentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); + + // and: + JwtAuthenticationProvider delegate = mock(JwtAuthenticationProvider.class); + doReturn(jwtAuthentication).when(delegate).authenticate(any(Authentication.class)); + + // and: + GcpDomainWhiteList userWhitelist = mock(GcpDomainWhiteList.class); + doReturn(true).when(userWhitelist).lists(anyString()); + + // and: + AuthenticationProvider authenticationProvider = new UserJwtAuthenticationProvider(delegate); + + // when: + Authentication authentication = authenticationProvider.authenticate(jwtAuthentication); + + // then: + assertThat(authentication).extracting("principal").containsExactly(userEmail); + } + + @Test + @DisplayName("no user group") + public void testForNoUserGroup() { + // given: JWT authentication + String userGroup = "null"; + String jwt = jwtGenerator.generateWithSubject(userGroup); + Authentication authentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); + + // and: + JwtAuthenticationProvider delegate = mock(JwtAuthenticationProvider.class); + doReturn(authentication).when(delegate).authenticate(any(Authentication.class)); + + // and: + GcpDomainWhiteList userWhitelist = mock(GcpDomainWhiteList.class); + doReturn(false).when(userWhitelist).lists(anyString()); + + // and: + AuthenticationProvider authenticationProvider = new UserJwtAuthenticationProvider(delegate); + + // expect: + assertThatThrownBy( + () -> { + authenticationProvider.authenticate(authentication); + }) + .isExactlyInstanceOf(InvalidUserGroup.class) + .hasMessageContaining(userGroup); + } + + @Test + @DisplayName("invalid user group") + public void testForInvalidUserGroup() { + // given: JWT authentication + String userGroup = "public"; + Map claims = + Map.ofEntries(entry("https://auth.data.humancellatlas.org/group", userGroup)); + String jwt = jwtGenerator.generate(null, userGroup, claims); + Authentication authentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); + + // and: + JwtAuthenticationProvider delegate = mock(JwtAuthenticationProvider.class); + doReturn(authentication).when(delegate).authenticate(any(Authentication.class)); + + // and: + GcpDomainWhiteList userWhitelist = mock(GcpDomainWhiteList.class); + doReturn(false).when(userWhitelist).lists(anyString()); + + // and: + AuthenticationProvider authenticationProvider = new UserJwtAuthenticationProvider(delegate); + + // expect: + assertThatThrownBy( + () -> { + authenticationProvider.authenticate(authentication); + }) + .isExactlyInstanceOf(InvalidUserGroup.class) + .hasMessageContaining(userGroup); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/elixir/ElixirAaiAuthenticationProviderTest.java b/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/elixir/ElixirAaiAuthenticationProviderTest.java new file mode 100644 index 000000000..e800c9af4 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/elixir/ElixirAaiAuthenticationProviderTest.java @@ -0,0 +1,219 @@ +package uk.ac.ebi.subs.ingest.security.authn.provider.elixir; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assumptions.assumeThat; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; +import static uk.ac.ebi.subs.ingest.security.ElixirConfig.ELIXIR; + +import org.junit.jupiter.api.*; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.autoconfigure.web.client.AutoConfigureWebClient; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.security.authentication.AuthenticationProvider; +import org.springframework.security.core.Authentication; + +import com.auth0.jwt.exceptions.JWTVerificationException; +import com.auth0.jwt.interfaces.DecodedJWT; +import com.auth0.jwt.interfaces.JWTVerifier; +import com.auth0.spring.security.api.authentication.PreAuthenticatedAuthenticationJsonWebToken; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import uk.ac.ebi.subs.ingest.security.Account; +import uk.ac.ebi.subs.ingest.security.AccountRepository; +import uk.ac.ebi.subs.ingest.security.JwtGenerator; +import uk.ac.ebi.subs.ingest.security.authn.oidc.UserInfo; +import uk.ac.ebi.subs.ingest.security.common.jwk.JwtVerifierResolver; +import uk.ac.ebi.subs.ingest.security.exception.JwtVerificationFailed; + +@SpringBootTest(classes = {ElixirAaiAuthenticationProviderTest.Config.class}) +@AutoConfigureWebClient +public class ElixirAaiAuthenticationProviderTest { + + @Configuration + @Import(ElixirAaiAuthenticationProvider.class) + static class Config {} + + private MockWebServer mockBackEnd; + + @MockBean private JWTVerifier jwtVerifier; + + @MockBean + @Qualifier(ELIXIR) + private JwtVerifierResolver jwtVerifierResolver; + + @MockBean private AccountRepository accountRepository; + + @Autowired private AuthenticationProvider authenticationProvider; + + @Nested + @DisplayName("Authenticate") + class AuthenticationTests { + + private ObjectMapper objectMapper = new ObjectMapper(); + + @BeforeEach + public void setUp() throws Exception { + mockBackEnd = new MockWebServer(); + mockBackEnd.start(); + + doReturn(jwtVerifier).when(jwtVerifierResolver).resolve(anyString()); + String baseUrl = String.format("http://localhost:%s", mockBackEnd.getPort()); + doReturn(baseUrl).when(jwtVerifierResolver).getIssuer(); + } + + @AfterEach + public void tearDown() throws Exception { + mockBackEnd.shutdown(); + } + + @Test + @DisplayName("success") + public void testAuthenticate() throws Exception { + // given: JWT + String subject = "johndoe@elixirdomain.tld"; + UserInfo userInfo = new UserInfo(subject, "name", "pref", "giv", "fam", "email@ebi.ac.uk"); + JwtGenerator jwtGenerator = new JwtGenerator("elixir"); + String jwt = jwtGenerator.encode(userInfo); + + // and: given a JWT Authentication + var jwtAuthentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); + assumeThat(jwtAuthentication).isNotNull(); + + // and: given JWT Verifier will verify token successfully + DecodedJWT token = mock(DecodedJWT.class); + doReturn(jwt).when(token).getToken(); + doReturn(token).when(jwtVerifier).verify(jwtAuthentication.getToken()); + + // and: given account with same provider reference will be found + Account account = new Account("73bbc45", subject); + doReturn(account).when(accountRepository).findByProviderReference(subject); + + // and: Elixir user info will be returned + mockBackEnd.enqueue( + new MockResponse() + .setBody(objectMapper.writeValueAsString(userInfo)) + .addHeader("Content-Type", "application/json")); + + // when: + Authentication authentication = authenticationProvider.authenticate(jwtAuthentication); + + // then: + assertThat(authentication).extracting("authenticated", "principal"); + assertThat(authentication.isAuthenticated()).isTrue(); + assertThat(authentication.getPrincipal()).isEqualTo(account); + assertCorrectRequest(jwtAuthentication.getToken()); + } + + @Test + @DisplayName("no account") + public void testForNoAccount() throws Exception { + // given: JWT + String subject = "johndoe@elixirdomain.tld"; + UserInfo userInfo = new UserInfo(subject, "name", "pref", "giv", "fam", "email@ebi.ac.uk"); + String jwt = new JwtGenerator("elixir").encode(userInfo); + + // and: given a JWT Authentication + var jwtAuthentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); + assumeThat(jwtAuthentication).isNotNull(); + + // and: given JWT Verifier will verify token successfully + DecodedJWT token = mock(DecodedJWT.class); + doReturn(jwt).when(token).getToken(); + doReturn(token).when(jwtVerifier).verify(jwtAuthentication.getToken()); + + // and: Elixir user info will be returned + mockBackEnd.enqueue( + new MockResponse() + .setBody(objectMapper.writeValueAsString(userInfo)) + .addHeader("Content-Type", "application/json")); + + // and: no matching records in the database + doReturn(null).when(accountRepository).findByProviderReference(anyString()); + + // when: + Authentication authentication = authenticationProvider.authenticate(jwtAuthentication); + + // then: + assertThat(authentication).isNotNull(); + assertThat(authentication.getPrincipal()).isEqualTo(Account.GUEST); + assertCorrectRequest(jwtAuthentication.getToken()); + + // and: + assertThat(authentication.getCredentials()).isInstanceOf(UserInfo.class); + UserInfo credentials = (UserInfo) authentication.getCredentials(); + assertThat(credentials).isEqualToComparingFieldByField(userInfo); + } + + private void assertCorrectRequest(String token) throws InterruptedException { + RecordedRequest request = mockBackEnd.takeRequest(); + assertThat(request.getMethod()).isEqualToIgnoringCase("GET"); + String bearerToken = String.format("Bearer %s", token); + assertThat(request.getHeaders().get("Authorization")).isEqualTo(bearerToken); + } + + @Test + @DisplayName("valid user email") + public void testForValidUserEmail() throws JsonProcessingException { + // given: + UserInfo userInfo = new UserInfo("subject", "name", "pref", "giv", "fam", "email@embl.ac.uk"); + mockBackEnd.enqueue( + new MockResponse() + .setBody(objectMapper.writeValueAsString(userInfo)) + .addHeader("Content-Type", "application/json")); + + // and: + JwtGenerator jwtGenerator = new JwtGenerator("elixir"); + String jwt = jwtGenerator.generate(); + var jwtAuthentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); + + DecodedJWT token = mock(DecodedJWT.class); + doReturn(jwt).when(token).getToken(); + doReturn(token).when(jwtVerifier).verify(jwtAuthentication.getToken()); + Account account = mock(Account.class); + doReturn(account).when(accountRepository).findByProviderReference("sub"); + + // when: + Authentication auth = authenticationProvider.authenticate(jwtAuthentication); + + // then: + assertThat(auth).isNotNull(); + } + + @Test + @DisplayName("verification failed") + public void testForFailedVerification() throws JsonProcessingException { + // given: + UserInfo userInfo = new UserInfo("subject", "name", "pref", "giv", "fam", "email@ebi.ac.uk"); + mockBackEnd.enqueue( + new MockResponse() + .setBody(objectMapper.writeValueAsString(userInfo)) + .addHeader("Content-Type", "application/json")); + + // and: given a JWT Authentication + JwtGenerator jwtGenerator = new JwtGenerator("elixir"); + String jwt = jwtGenerator.generateWithSubject("sub"); + var jwtAuthentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); + + // and: JWT verifier will fail + Exception verificationFailed = new JWTVerificationException("verification failed"); + doThrow(verificationFailed).when(jwtVerifier).verify(jwtAuthentication.getToken()); + + // expect: + assertThatThrownBy( + () -> { + authenticationProvider.authenticate(jwtAuthentication); + }) + .isInstanceOf(JwtVerificationFailed.class); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/elixir/ElixirJwkVaultTest.java b/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/elixir/ElixirJwkVaultTest.java new file mode 100644 index 000000000..8a5b710f4 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/elixir/ElixirJwkVaultTest.java @@ -0,0 +1,47 @@ +package uk.ac.ebi.subs.ingest.security.authn.provider.elixir; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + +import org.junit.jupiter.api.Test; + +import com.auth0.jwk.Jwk; +import com.auth0.jwk.UrlJwkProvider; +import com.auth0.jwt.JWT; + +import uk.ac.ebi.subs.ingest.security.JwtGenerator; +import uk.ac.ebi.subs.ingest.security.common.jwk.JwkVault; +import uk.ac.ebi.subs.ingest.security.common.jwk.UrlJwkProviderResolver; + +public class ElixirJwkVaultTest { + + @Test + public void testGetPublicKey() throws Exception { + // given: JWT + String issuer = "https://login.elixir-czech.org/oidc"; + JwtGenerator generator = new JwtGenerator(issuer); + var jwt = generator.generate(); + + Jwk jwk = mock(Jwk.class); + doReturn(generator.getPublicKey()).when(jwk).getPublicKey(); + + // and: + UrlJwkProvider urlJwkProvider = mock(UrlJwkProvider.class); + doReturn(jwk).when(urlJwkProvider).get(JwtGenerator.DEFAULT_KEY_ID); + + // and: + UrlJwkProviderResolver urlJwkProviderResolver = mock(UrlJwkProviderResolver.class); + doReturn(urlJwkProvider).when(urlJwkProviderResolver).resolve(); + + // and: + JwkVault jwkVault = new ElixirJwkVault(urlJwkProviderResolver); + + // when: + var token = JWT.decode(jwt); + var publicKey = jwkVault.getPublicKey(token); + + // then: + assertThat(publicKey).isNotNull(); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/gcp/GcpJwkVaultTest.java b/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/gcp/GcpJwkVaultTest.java new file mode 100644 index 000000000..6cfb2f1eb --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/gcp/GcpJwkVaultTest.java @@ -0,0 +1,53 @@ +package uk.ac.ebi.subs.ingest.security.authn.provider.gcp; + +import static java.util.Map.entry; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import com.auth0.jwk.Jwk; +import com.auth0.jwk.UrlJwkProvider; +import com.auth0.jwt.JWT; + +import uk.ac.ebi.subs.ingest.security.JwtGenerator; +import uk.ac.ebi.subs.ingest.security.common.jwk.JwkVault; +import uk.ac.ebi.subs.ingest.security.common.jwk.UrlJwkProviderResolver; + +public class GcpJwkVaultTest { + + @Test + public void testGetPublicKeyForJwt() throws Exception { + // given: JWT + String issuer = "https://humancellatlas.auth0.com"; + JwtGenerator generator = new JwtGenerator(issuer); + var customClaims = + Map.ofEntries(entry("https://auth.data.humancellatlas.org/email", "sample@domain.tld")); + var jwt = generator.generate(customClaims); + + // and: JWK from remote service + Jwk jwk = mock(Jwk.class); + doReturn(generator.getPublicKey()).when(jwk).getPublicKey(); + + // and: + UrlJwkProvider urlJwkProvider = mock(UrlJwkProvider.class); + doReturn(jwk).when(urlJwkProvider).get(JwtGenerator.DEFAULT_KEY_ID); + + // and: + UrlJwkProviderResolver urlJwkProviderResolver = mock(UrlJwkProviderResolver.class); + doReturn(urlJwkProvider).when(urlJwkProviderResolver).resolve(issuer); + + // and: GoogleServiceJwkVault + JwkVault jwkVault = new GcpJwkVault(urlJwkProviderResolver); + + // when: + var token = JWT.decode(jwt); + var publicKey = jwkVault.getPublicKey(token); + + // then: + assertThat(publicKey).isNotNull(); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/gcp/GoogleServiceJwtAuthenticationProviderTest.java b/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/gcp/GoogleServiceJwtAuthenticationProviderTest.java new file mode 100644 index 000000000..c48942396 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/security/authn/provider/gcp/GoogleServiceJwtAuthenticationProviderTest.java @@ -0,0 +1,121 @@ +package uk.ac.ebi.subs.ingest.security.authn.provider.gcp; + +import static java.util.Map.entry; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assumptions.assumeThat; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.springframework.security.authentication.AuthenticationProvider; +import org.springframework.security.core.Authentication; + +import com.auth0.jwt.exceptions.JWTVerificationException; +import com.auth0.jwt.interfaces.JWTVerifier; +import com.auth0.spring.security.api.authentication.PreAuthenticatedAuthenticationJsonWebToken; + +import uk.ac.ebi.subs.ingest.security.JwtGenerator; +import uk.ac.ebi.subs.ingest.security.common.jwk.JwtVerifierResolver; +import uk.ac.ebi.subs.ingest.security.exception.JwtVerificationFailed; +import uk.ac.ebi.subs.ingest.security.exception.UnlistedJwtIssuer; + +public class GoogleServiceJwtAuthenticationProviderTest { + + @Nested + @DisplayName("Authenticate") + class AuthenticationTests { + private JWTVerifier jwtVerifier; + + private JwtVerifierResolver jwtVerifierResolver; + + private GcpDomainWhiteList projectWhitelist; + + @BeforeEach + public void setUp() { + jwtVerifier = mock(JWTVerifier.class); + jwtVerifierResolver = mock(JwtVerifierResolver.class); + doReturn(jwtVerifier).when(jwtVerifierResolver).resolve(anyString()); + + projectWhitelist = mock(GcpDomainWhiteList.class); + doReturn(true).when(projectWhitelist).lists("sample@domain.tld"); + } + + @Test + @DisplayName("success") + public void testAuthenticate() { + // given: JWT + Map claims = + Map.ofEntries(entry("https://auth.data.humancellatlas.org/group", "public")); + String keyId = "MDc2OTM3ODI4ODY2NUU5REVGRDVEM0MyOEYwQTkzNDZDRDlEQzNBRQ"; + String subject = "johndoe@somedomain.tld"; + + JwtGenerator jwtGenerator = new JwtGenerator("sample@domain.tld"); + String jwt = jwtGenerator.generate(keyId, subject, claims); + + // and: given a JWT Authentication + Authentication jwtAuthentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); + assumeThat(jwtAuthentication).isNotNull(); + + // and: + AuthenticationProvider authenticationProvider = + new GoogleServiceJwtAuthenticationProvider(projectWhitelist, jwtVerifierResolver); + + // when: + Authentication authentication = authenticationProvider.authenticate(jwtAuthentication); + + // then: + assertThat(authentication).isNotNull(); + } + + @Test + @DisplayName("unlisted issuer") + public void testForUnlistedIssuer() { + // given: + AuthenticationProvider authenticationProvider = + new GoogleServiceJwtAuthenticationProvider(projectWhitelist, jwtVerifierResolver); + + // and: + JwtGenerator jwtGenerator = new JwtGenerator("sample@otherdomain.tld"); + String jwt = jwtGenerator.generate(); + Authentication jwtAuthentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); + + // expect: + assertThatThrownBy( + () -> { + authenticationProvider.authenticate(jwtAuthentication); + }) + .isInstanceOf(UnlistedJwtIssuer.class) + .hasMessageContaining("sample@otherdomain.tld"); + } + + @Test + @DisplayName("verification failed") + public void testForFailedVerification() { + // given: + AuthenticationProvider authenticationProvider = + new GoogleServiceJwtAuthenticationProvider(projectWhitelist, jwtVerifierResolver); + + // and: + Exception verificationFailed = new JWTVerificationException("verification failed"); + doThrow(verificationFailed).when(jwtVerifier).verify(anyString()); + + // and: + JwtGenerator jwtGenerator = new JwtGenerator("sample@domain.tld"); + String jwt = jwtGenerator.generate(); + Authentication jwtAuthentication = PreAuthenticatedAuthenticationJsonWebToken.usingToken(jwt); + + // expect: + assertThatThrownBy( + () -> { + authenticationProvider.authenticate(jwtAuthentication); + }) + .isInstanceOf(JwtVerificationFailed.class); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/security/common/jwk/UrlJwkProviderResolverTest.java b/src/test/java/uk/ac/ebi/subs/ingest/security/common/jwk/UrlJwkProviderResolverTest.java new file mode 100644 index 000000000..8acd1e08a --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/security/common/jwk/UrlJwkProviderResolverTest.java @@ -0,0 +1,28 @@ +package uk.ac.ebi.subs.ingest.security.common.jwk; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import com.auth0.jwk.UrlJwkProvider; + +public class UrlJwkProviderResolverTest { + + @Test + public void testResolve() { + // given: + String baseUrl = "https://sample.service.tld"; + UrlJwkProviderResolver resolver = new UrlJwkProviderResolver(baseUrl); + + // when: + String relativePath = "issuer.service.tld"; + UrlJwkProvider provider = resolver.resolve(relativePath); + + // then: + assertThat(provider).isNotNull(); + + // and: inspect assigned URL through sub-class interface as a work-around + var url = ((RemoteJwkProvider) provider).getUrl(); + assertThat(url.toString()).isEqualTo("https://sample.service.tld/issuer.service.tld"); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/security/web/AuthenticationControllerTest.java b/src/test/java/uk/ac/ebi/subs/ingest/security/web/AuthenticationControllerTest.java new file mode 100644 index 000000000..6adc22975 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/security/web/AuthenticationControllerTest.java @@ -0,0 +1,198 @@ +package uk.ac.ebi.subs.ingest.security.web; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.authentication; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static uk.ac.ebi.subs.ingest.security.ElixirConfig.ELIXIR; +import static uk.ac.ebi.subs.ingest.security.GcpConfig.GCP; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.HttpStatus; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.security.authentication.AuthenticationProvider; +import org.springframework.security.core.Authentication; +import org.springframework.security.test.context.support.WithMockUser; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.web.context.WebApplicationContext; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import uk.ac.ebi.subs.ingest.security.Account; +import uk.ac.ebi.subs.ingest.security.AccountService; +import uk.ac.ebi.subs.ingest.security.Role; +import uk.ac.ebi.subs.ingest.security.authn.oidc.OpenIdAuthentication; +import uk.ac.ebi.subs.ingest.security.authn.oidc.UserInfo; +import uk.ac.ebi.subs.ingest.security.exception.DuplicateAccount; + +@WebMvcTest(AuthenticationController.class) +@AutoConfigureMockMvc(printOnlyOnFailure = false) +public class AuthenticationControllerTest { + + private static final String BASE_PATH = "/auth"; + + @Autowired private WebApplicationContext applicationContext; + + @Autowired private MockMvc webApp; + + @MockBean(name = GCP) + private AuthenticationProvider gcp; + + @MockBean(name = ELIXIR) + private AuthenticationProvider elixir; + + @MockBean(name = "COGNITO") + private AuthenticationProvider cognito; + + @MockBean private AccountService accountService; + + @Nested + @DisplayName("Registration") + class Registration { + + private static final String PATH = "/auth/registration"; + + @Test + void byAuthenticatedGuest() throws Exception { + // given: + String subjectId = "cf12881b"; + UserInfo userInfo = new UserInfo(subjectId, "Jane Doe"); + Authentication authentication = new OpenIdAuthentication(null, userInfo); + + // and: + String accountId = "b4912b3"; + Account persistentAccount = new Account(accountId, subjectId); + doReturn(persistentAccount).when(accountService).register(any(Account.class)); + + // when: + MvcResult result = + webApp.perform(post(PATH).with(authentication(authentication)).with(csrf())).andReturn(); + + // then: + MockHttpServletResponse response = result.getResponse(); + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + + // and: + ObjectMapper objectMapper = new ObjectMapper(); + var resultingAccount = objectMapper.readValue(response.getContentAsString(), Account.class); + assertThat(resultingAccount.getId()).isEqualTo(accountId); + assertCorrectRegisteredAccount(userInfo); + } + + private void assertCorrectRegisteredAccount(UserInfo userInfo) { + var accountCaptor = ArgumentCaptor.forClass(Account.class); + verify(accountService).register(accountCaptor.capture()); + + var registeredAccount = accountCaptor.getValue(); + assertThat(registeredAccount) + .extracting("providerReference", "name") + .containsExactly(userInfo.getSubjectId(), userInfo.getName()); + assertThat(registeredAccount.getRoles()).isEmpty(); + } + + @Test + @WithMockUser(roles = {"CONTRIBUTOR"}) + void byRegisteredUser() throws Exception { + // expect: + webApp.perform(post(PATH)).andExpect(status().isForbidden()); + } + + /* + Similar scenario to byRegisteredUser but somehow the Account was either, + 1) unrecognised and so was treated as an authenticated Guest, or + 2) Account was erroneously assigned the Guest role. + Essentially, we want to handle duplicated subject id in our system. + */ + @Test + void byUnrecognisedRegisteredUser() throws Exception { + // given: + UserInfo userInfo = new UserInfo("cc9a9a1", ""); + Authentication authentication = new OpenIdAuthentication(userInfo); + + // and: + doThrow(new DuplicateAccount()).when(accountService).register(any(Account.class)); + + // expect: + webApp + .perform(post(PATH).with(authentication(authentication)).with(csrf())) + .andExpect(status().isConflict()); + } + + @Test + void byAnonymousUser() throws Exception { + // expect: + webApp.perform(post(PATH)).andExpect(status().isUnauthorized()); + } + } + + @Nested + @DisplayName("Account Retrieval") + class AccountRetrieval { + + private final String PATH = String.format("%s/account", BASE_PATH); + + @Test + void registeredUser() throws Exception { + // given: + String accountId = "bcdde10"; + String subjectId = "67135cc"; + + // and: + Account account = new Account(accountId, subjectId); + account.addRole(Role.CONTRIBUTOR); + + // and: + UserInfo credentials = new UserInfo(subjectId, ""); + Authentication authentication = new OpenIdAuthentication(account, credentials); + + // when: + MvcResult result = webApp.perform(get(PATH).with(authentication(authentication))).andReturn(); + + // then: + MockHttpServletResponse response = result.getResponse(); + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + assertCorrectAccountDetails(response, accountId, subjectId); + } + + private void assertCorrectAccountDetails( + MockHttpServletResponse response, String accountId, String subjectId) throws Exception { + ObjectMapper objectMapper = new ObjectMapper(); + Account retrievedAccount = + objectMapper.readValue(response.getContentAsString(), Account.class); + assertThat(retrievedAccount) + .extracting("id", "providerReference") + .containsExactly(accountId, subjectId); + assertThat(retrievedAccount.getRoles()).containsExactly(Role.CONTRIBUTOR); + } + + @Test + void authenticatedGuest() throws Exception { + // given: + UserInfo userInfo = new UserInfo("82ffab9", ""); + Authentication authentication = new OpenIdAuthentication(userInfo); + + // expect: + webApp + .perform(get(PATH).with(authentication(authentication))) + .andExpect(status().isNotFound()); + } + + @Test + void unknownGuest() throws Exception { + // expect: + webApp.perform(get(PATH)).andExpect(status().isUnauthorized()); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/stagingjob/StagingJobServiceTest.java b/src/test/java/uk/ac/ebi/subs/ingest/stagingjob/StagingJobServiceTest.java new file mode 100644 index 000000000..9904c995c --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/stagingjob/StagingJobServiceTest.java @@ -0,0 +1,66 @@ +package uk.ac.ebi.subs.ingest.stagingjob; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.mockito.Mockito.*; + +import java.util.UUID; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.springframework.dao.DuplicateKeyException; + +import uk.ac.ebi.subs.ingest.stagingjob.StagingJobService.JobAlreadyRegisteredException; + +public class StagingJobServiceTest { + + private StagingJobRepository stagingJobRepository = mock(StagingJobRepository.class); + private StagingJobService stagingJobService = new StagingJobService(stagingJobRepository); + + @BeforeEach + public void setUp() { + reset(stagingJobRepository); + } + + @Nested + class Registration { + + @Test + public void validJob() { + // given: + UUID stagingAreaUUid = UUID.randomUUID(); + String fileName = "test_1.fastq.gz"; + String metadataUuid = UUID.randomUUID().toString(); + StagingJob stagingJob = new StagingJob(stagingAreaUUid, metadataUuid, fileName); + + // and: + StagingJob persistentJob = spy(stagingJob); + doReturn("_generated_id_1").when(persistentJob).getId(); + doReturn(persistentJob).when(stagingJobRepository).save(any()); + + // when: + StagingJob resultingJob = stagingJobService.register(stagingJob); + + // then: + verify(stagingJobRepository).save(stagingJob); + assertThat(resultingJob).isEqualTo(persistentJob); + } + + @Test + public void duplicateJob() { + // given: + UUID stagingAreaUuid = UUID.randomUUID(); + String metadataUuid = UUID.randomUUID().toString(); + String fileName = "test.fastq.gz"; + StagingJob stagingJob = new StagingJob(stagingAreaUuid, metadataUuid, fileName); + + // and: + doThrow(new DuplicateKeyException("duplicate key")).when(stagingJobRepository).save(any()); + + // expect : + assertThatExceptionOfType(JobAlreadyRegisteredException.class) + .isThrownBy(() -> stagingJobService.register(stagingJob)); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobControllerTest.java b/src/test/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobControllerTest.java new file mode 100644 index 000000000..7e053befa --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/stagingjob/web/StagingJobControllerTest.java @@ -0,0 +1,66 @@ +package uk.ac.ebi.subs.ingest.stagingjob.web; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.*; + +import java.util.UUID; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.data.mapping.PersistentEntity; +import org.springframework.data.rest.webmvc.PersistentEntityResource; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; + +import uk.ac.ebi.subs.ingest.stagingjob.StagingJob; +import uk.ac.ebi.subs.ingest.stagingjob.StagingJobService; + +@ExtendWith(MockitoExtension.class) +public class StagingJobControllerTest { + + private StagingJobService stagingJobService; + + private StagingJobController controller; + + @BeforeEach + public void setUp(@Mock StagingJobService stagingJobService) { + this.stagingJobService = stagingJobService; + controller = new StagingJobController(stagingJobService); + } + + @Test + public void createStagingJob() { + // given: + StagingJob stagingJob = new StagingJob(UUID.randomUUID(), "file_1.json"); + StagingJob persistentJob = spy(stagingJob); + given(stagingJobService.register(any(StagingJob.class))).willReturn(persistentJob); + + // and: + PersistentEntityResourceAssembler resourceAssembler = + mock(PersistentEntityResourceAssembler.class); + given(resourceAssembler.toFullResource(any())) + .willAnswer( + invocation -> { + Object entity = invocation.getArgument(0); + return PersistentEntityResource.build(entity, mock(PersistentEntity.class)).build(); + }); + + // when: + ResponseEntity response = controller.createStagingJob(stagingJob, resourceAssembler); + + // then: + verify(stagingJobService).register(any(StagingJob.class)); + + // and: + assertThat(response).isNotNull().extracting("status").containsExactly(HttpStatus.OK); + assertThat(response.getBody()).isInstanceOf(PersistentEntityResource.class); + PersistentEntityResource responseBody = (PersistentEntityResource) response.getBody(); + assertThat(responseBody.getContent()).isEqualTo(persistentJob); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/state/MetadataDocumentEventHandlerTest.java b/src/test/java/uk/ac/ebi/subs/ingest/state/MetadataDocumentEventHandlerTest.java new file mode 100644 index 000000000..c9d96ab3d --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/state/MetadataDocumentEventHandlerTest.java @@ -0,0 +1,35 @@ +package uk.ac.ebi.subs.ingest.state; + +import static org.mockito.Mockito.*; + +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +public class MetadataDocumentEventHandlerTest { + + private MessageRouter messageRouter = mock(MessageRouter.class); + MetadataDocumentEventHandler handler = new MetadataDocumentEventHandler(messageRouter); + + @Test + public void testHandleCreateDocumentsWithoutSubmissionEnvelope() { + Biomaterial biomaterial = new Biomaterial(null); + handler.handleMetadataDocumentCreate(biomaterial); + Mockito.verify(messageRouter, times(1)).routeValidationMessageFor(biomaterial); + // Mockito.verify(messageRouter, times(1)).routeStateTrackingUpdateMessageFor(biomaterial); + } + + @Test + public void testHandleCreateDocumentsWithSubmissionEnvelope() { + Project project = new Project(null); + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + project.getSubmissionEnvelopes().add(submissionEnvelope); + handler.handleMetadataDocumentCreate(project); + Mockito.verify(messageRouter, times(1)).routeValidationMessageFor(project); + // Mockito.verify(messageRouter, times(1)).routeStateTrackingUpdateMessageFor(project); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/state/MetadataStateChangeListenerTest.java b/src/test/java/uk/ac/ebi/subs/ingest/state/MetadataStateChangeListenerTest.java new file mode 100644 index 000000000..16e7f57eb --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/state/MetadataStateChangeListenerTest.java @@ -0,0 +1,37 @@ +package uk.ac.ebi.subs.ingest.state; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; + +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.data.mongodb.core.mapping.event.AfterSaveEvent; +import org.springframework.data.mongodb.core.mapping.event.BeforeConvertEvent; + +import uk.ac.ebi.subs.ingest.core.MetadataDocument; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.project.Project; + +public class MetadataStateChangeListenerTest { + private MessageRouter messageRouter = mock(MessageRouter.class); + + MetadataStateChangeListener metadataDocumentMongoEventListener = + new MetadataStateChangeListener(messageRouter); + + @Test + public void testOnBeforeConvert() { + Project project = new Project(null); + metadataDocumentMongoEventListener.onBeforeConvert(new BeforeConvertEvent(project, "project")); + assertThat(project.getUuid()).isNotNull(); + assertThat(project.getDcpVersion()).isEqualTo(project.getSubmissionDate()); + } + + @Test + public void testOnAfterSave() { + Project project = new Project(null); + AfterSaveEvent afterSaveEvent = mock(AfterSaveEvent.class); + doReturn(project).when(afterSaveEvent).getSource(); + metadataDocumentMongoEventListener.onAfterSave(afterSaveEvent); + Mockito.verify(messageRouter, times(1)).routeValidationMessageFor(project); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/state/ValidationStateTest.java b/src/test/java/uk/ac/ebi/subs/ingest/state/ValidationStateTest.java new file mode 100644 index 000000000..692e5008f --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/state/ValidationStateTest.java @@ -0,0 +1,39 @@ +package uk.ac.ebi.subs.ingest.state; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +import java.util.stream.Stream; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.json.JsonTest; +import org.springframework.boot.test.json.JacksonTester; + +import uk.ac.ebi.subs.ingest.file.ValidationReport; + +@JsonTest +public class ValidationStateTest { + @Autowired private JacksonTester json; + + private static Stream provideStatesForTestFromJSON() { + return Stream.of( + Arguments.of(ValidationState.INVALID, "invalid"), + Arguments.of(ValidationState.INVALID, "Invalid"), + Arguments.of(ValidationState.INVALID, "INVALID"), + Arguments.of(ValidationState.VALID, "valid"), + Arguments.of(ValidationState.VALID, "Valid"), + Arguments.of(ValidationState.VALID, "VALID"), + Arguments.of(ValidationState.VALIDATING, "validating"), + Arguments.of(ValidationState.VALIDATING, "Validating"), + Arguments.of(ValidationState.VALIDATING, "VALIDATING")); + } + + @ParameterizedTest + @MethodSource("provideStatesForTestFromJSON") + public void testFromJSON(ValidationState expected, String given) throws Exception { + var jsonValue = String.format("{ \"validationState\": \"%s\" }", given); + assertThat(json.parse(jsonValue).getObject().getValidationState()).isEqualTo(expected); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/study/StudyControllerProfileTest.java b/src/test/java/uk/ac/ebi/subs/ingest/study/StudyControllerProfileTest.java new file mode 100644 index 000000000..c59df604d --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/study/StudyControllerProfileTest.java @@ -0,0 +1,132 @@ +package uk.ac.ebi.subs.ingest.study; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.*; + +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.env.Environment; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.hateoas.Resource; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import uk.ac.ebi.subs.ingest.study.web.StudyController; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = {StudyController.class, StudyRepository.class}) +@Disabled("Disabled until the morphic profile is active") +public class StudyControllerProfileTest { + + @MockBean private StudyService studyService; + + @MockBean private StudyRepository studyRepository; + + @MockBean private PersistentEntityResourceAssembler assembler; + + @Autowired private StudyController studyController; + + @MockBean private Environment environment; + + @BeforeEach + void setUp() { + Mockito.reset(studyService, assembler); + } + + @Nested + class StudyUpdate { + @Test + @DisplayName("Update Study - True Morphic Profile") + public void testUpdateStudyMorphicProfile() { + // given: + String studyId = "studyId"; + ObjectNode patch = createUpdatePatch("Updated Study Name"); + // Study updatedStudy = new Study("{\"name\": \"study\"}"); + Study newStudy = new Study("Schema URL", "1.0", "Generic", "{\"name\": \"study\"}"); + Study updatedStudy = new Study("Schema URL", "1.0", "Generic", "{\"name\": \"study\"}"); + + // and: mock the environment to simulate the "morphic" profile being active + when(environment.getActiveProfiles()).thenReturn(new String[] {"morphic"}); + + // when: + when(studyService.update(newStudy, patch)).thenReturn(updatedStudy); + ResponseEntity> response = + studyController.updateStudy(newStudy.getId(), patch, assembler); + + // then: + assertEquals(HttpStatus.OK, response.getStatusCode()); + verify(assembler).toFullResource(updatedStudy); + } + + @Test + @DisplayName("Update Study - False Morphic Profile") + public void testUpdateStudyNonMorphicProfile() { + // given: + Study newStudy = new Study("Schema URL", "1.0", "Generic", "{\"name\": \"study\"}"); + ObjectNode patch = createUpdatePatch("Updated Study Name"); + + // and: mock the environment to simulate the "morphic" profile being active + when(environment.getActiveProfiles()).thenReturn(new String[] {"non-morphic"}); + + // when: + ResponseEntity> response = + studyController.updateStudy(newStudy.getId(), patch, assembler); + + // then: + assertEquals(HttpStatus.NOT_FOUND, response.getStatusCode()); + verify(studyService, never()).update(any(), any()); + verify(assembler, never()).toFullResource(any()); + } + + private ObjectNode createUpdatePatch(String updatedName) { + ObjectNode patch = JsonNodeFactory.instance.objectNode(); + patch.put("content", JsonNodeFactory.instance.objectNode().put("name", updatedName)); + return patch; + } + } + + @Nested + class StudyDeletion { + @Test + @DisplayName("Delete Study - True Morphic Profile") + public void testDeleteStudyMorphicProfile() { + // given : + String studyId = "studyId"; + + // and: mock the environment to simulate the "morphic" profile being active + when(environment.getActiveProfiles()).thenReturn(new String[] {"morphic"}); + + // when: + ResponseEntity response = studyController.deleteStudy(studyId); + + // then: + assertEquals(HttpStatus.NO_CONTENT, response.getStatusCode()); + verify(studyService).delete(studyId); + } + + @Test + @DisplayName("Delete Study - False Morphic Profile") + public void testDeleteStudyNonMorphicProfile() { + // given : + String studyId = "studyId"; + + // and: mock the environment to simulate the "morphic" profile being active + when(environment.getActiveProfiles()).thenReturn(new String[] {"non-morphic"}); + + // when: + ResponseEntity response = studyController.deleteStudy(studyId); + + // then: + assertEquals(HttpStatus.NOT_FOUND, response.getStatusCode()); + verify(studyService, never()).delete(any()); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/study/StudyServiceTest.java b/src/test/java/uk/ac/ebi/subs/ingest/study/StudyServiceTest.java new file mode 100644 index 000000000..67fe8874e --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/study/StudyServiceTest.java @@ -0,0 +1,345 @@ +package uk.ac.ebi.subs.ingest.study; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.*; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.ApplicationContext; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.web.server.ResponseStatusException; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.dataset.DatasetRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = {StudyService.class, StudyRepository.class, DatasetRepository.class}) +public class StudyServiceTest { + + @Autowired private ApplicationContext applicationContext; + + @Autowired private StudyService studyService; + + @MockBean private MongoTemplate mongoTemplate; + + @MockBean private StudyRepository studyRepository; + + @MockBean private DatasetRepository datasetRepository; + + @MockBean private StudyEventHandler studyEventHandler; + + @MockBean private MetadataCrudService metadataCrudService; + + @MockBean private MetadataUpdateService metadataUpdateService; + + @BeforeEach + void setUp() { + applicationContext.getBeansWithAnnotation(MockBean.class).forEach(Mockito::reset); + Mockito.reset(metadataCrudService, studyRepository, studyEventHandler); + } + + @Nested + class SubmissionEnvelopes { + Study study1; + Study study2; + Set submissionSet1; + Set submissionSet2; + + @BeforeEach + void setup() { + // given + study1 = spy(new Study("Schema URL", "1.0", "Generic", null)); + doReturn("study1").when(study1).getId(); + study1.setUuid(Uuid.newUuid()); + + submissionSet1 = new HashSet<>(); + IntStream.range(0, 3) + .mapToObj(Integer::toString) + .forEach( + id -> { + var sub = spy(new SubmissionEnvelope()); + doReturn(id).when(sub).getId(); + submissionSet1.add(sub); + }); + submissionSet1.forEach(study1::addToSubmissionEnvelopes); + + // and: + study2 = spy(new Study("Schema URL-2", "2.0", "Generic", null)); + doReturn("study2").when(study2).getId(); + study2.setUuid(study1.getUuid()); + + submissionSet2 = new HashSet<>(); + IntStream.range(10, 15) + .mapToObj(Integer::toString) + .forEach( + id -> { + var sub = spy(new SubmissionEnvelope()); + doReturn(id).when(sub).getId(); + submissionSet2.add(sub); + }); + submissionSet2.forEach(study2::addToSubmissionEnvelopes); + } + + @Test + @DisplayName("get all submissions") + void getFromAllCopiesOfStudies() { + // given + when(studyRepository.findByUuid(study1.getUuid())).thenReturn(Stream.of(study1, study2)); + + // when: + var submissionEnvelopes = studyService.getSubmissionEnvelopes(study1); + + // then: + assertThat(submissionEnvelopes).containsAll(submissionSet1).containsAll(submissionSet2); + } + + @Test + @DisplayName("no duplicate submissions") + void getFromAllCopiesOfStudiesNoDuplicates() { + // given + var study3 = spy(new Study("Schema URL", "1.0", "Generic", null)); + doReturn("study3").when(study3).getId(); + study3.setUuid(study1.getUuid()); + + submissionSet1.forEach(study3::addToSubmissionEnvelopes); + + var documentIds = new ArrayList(); + submissionSet1.forEach(submission -> documentIds.add(submission.getId())); + submissionSet2.forEach(submission -> documentIds.add(submission.getId())); + + // and: + when(studyRepository.findByUuid(study1.getUuid())) + .thenReturn(Stream.of(study1, study2, study3)); + + // when: + var submissionEnvelopes = studyService.getSubmissionEnvelopes(study1); + + // then: + var returnDocumentIds = new ArrayList(); + submissionEnvelopes.forEach(submission -> returnDocumentIds.add(submission.getId())); + + assertThat(returnDocumentIds).containsExactlyInAnyOrderElementsOf(documentIds); + } + } + + @Nested + class StudyRegistration { + + @Test + @DisplayName("Register Study - Success") + void registerSuccess() { + // given: + String content = "{\"name\": \"study\"}"; + Study study = new Study("Schema URL", "1.0", "Generic", "{\"name\": \"study\"}"); + + // and: + Study persistentStudy = new Study("Schema URL", "1.0", "Generic", "{\"name\": \"study\"}"); + doReturn(persistentStudy).when(studyRepository).save(study); + + // when: + Study result = studyService.register(study); + + // then: + verify(studyRepository, times(1)).save(study); + assertThat(result).isEqualTo(persistentStudy); + verify(studyEventHandler).registeredStudy(persistentStudy); + } + + @Test + @DisplayName("Register Study - Ensure Descriptive Fields Are Managed Correctly") + void shouldHandleDescriptiveFieldsCorrectly() { + // Initialize studies with descriptive fields + Study study = + new Study( + "Schema URL", "1.0", "Study", "{\"type\": \"study\", \"content\": \"Details\"}"); + + // Test retrieval of descriptive fields + assertEquals("Schema URL", study.getDescribedBy()); + assertEquals("1.0", study.getSchemaVersion()); + assertEquals("Study", study.getSchemaType()); + + // Test setting and getting new values + study.setDescribedBy("Updated Schema URL"); + study.setSchemaVersion("1.1"); + study.setSchemaType("Protocol"); + + assertEquals("Updated Schema URL", study.getDescribedBy()); + assertEquals("1.1", study.getSchemaVersion()); + assertEquals("Protocol", study.getSchemaType()); + } + } + + @Nested + class StudyUpdate { + @Test + @DisplayName("Update Study - Success") + void updateSuccess() { + // given: + String studyId = "studyId"; + ObjectNode patch = createUpdatePatch("Updated Study Name"); + Study existingStudy = mock(Study.class); + + // and: + when(existingStudy.getId()).thenReturn(studyId); + when(studyRepository.findById(studyId)).thenReturn(Optional.of(existingStudy)); + when(metadataUpdateService.update(existingStudy, patch)).thenReturn(existingStudy); + + // when: + Study result = studyService.update(existingStudy, patch); + + // then: + verify(studyRepository).findById(studyId); + verify(metadataUpdateService).update(existingStudy, patch); + verify(studyEventHandler).updatedStudy(existingStudy); + assertThat(result).isEqualTo(existingStudy); + } + + // Helper method to create an update patch + private ObjectNode createUpdatePatch(String updatedName) { + ObjectNode patch = JsonNodeFactory.instance.objectNode(); + patch.put("content", JsonNodeFactory.instance.objectNode().put("name", updatedName)); + return patch; + } + + @Test + @DisplayName("Update Study - Not Found") + void updateStudyNotFound() { + // given: + String nonExistentStudyId = "nonExistentId"; + ObjectNode patch = createUpdatePatch("Updated Study Name"); + + // and: + when(studyRepository.findById(nonExistentStudyId)).thenReturn(Optional.empty()); + + // when, then: + ResponseStatusException exception = + assertThrows( + ResponseStatusException.class, + () -> + studyService.update( + new Study("Schema URL", "1.0", "Generic", "{\"name\": \"study\"}"), patch)); + assertThat("404 NOT_FOUND").isEqualTo(exception.getMessage()); + + // verify that other methods are not called + verify(metadataCrudService, never()).deleteDocument(any()); + verify(studyEventHandler, never()).deletedStudy(any()); + } + } + + @Nested + class StudyReplace { + + @Test + @DisplayName("Replace Study - Success") + void replaceSuccess() { + // given: + String studyId = "studyId"; + Study existingStudy = + new Study("ExistingSchema URL", "1.0", "Generic", "{\"name\": \"Existing Study Name\"}"); + Study updatedStudy = + new Study("UpdatedSchema URL", "1.1", "Specific", "{\"name\": \"Updated Study Name\"}"); + // Study existingStudy = new Study("{\"name\": \"Existing Study Name\"}"); + // Study updatedStudy = new Study("{\"name\": \"Updated Study Name\"}"); + + // and: + when(studyRepository.findById(studyId)).thenReturn(Optional.of(existingStudy)); + + // when: + Study result = studyService.replace(studyId, updatedStudy); + + // then: + verify(studyRepository).findById(studyId); + verify(studyRepository).save(updatedStudy); // Verify save is called + verify(studyEventHandler).updatedStudy(updatedStudy); + assertThat(result).isEqualTo(updatedStudy); + } + + @Test + @DisplayName("Replace Study - Not Found") + void replaceStudyNotFound() { + // given: + String nonExistentStudyId = "nonExistentId"; + Study updatedStudy = + new Study("Schema URL", "1.1", "Specific", "{\"name\": \"Updated Study Name\"}"); + + // and: + when(studyRepository.findById(nonExistentStudyId)).thenReturn(Optional.empty()); + + // when, then: + ResponseStatusException exception = + assertThrows( + ResponseStatusException.class, + () -> studyService.replace(nonExistentStudyId, updatedStudy)); + assertThat("404 NOT_FOUND").isEqualTo(exception.getMessage()); + + // verify that other methods are not called + verify(studyRepository, never()).save(any()); + verify(studyEventHandler, never()).updatedStudy(any()); + } + } + + @Nested + class StudyDeletion { + @Test + @DisplayName("Delete Study - Success") + void deleteSuccess() { + // given: + String studyId = "testDeleteId"; + // String content = "{\"name\": \"delete study\"}"; + Study persistentStudy = + new Study("Schema URL", "1.1", "Specific", "{\"name\": \"Updated Study Name\"}"); + + // and: + when(studyRepository.findById(studyId)).thenReturn(Optional.of(persistentStudy)); + + // when: + studyService.delete(studyId); + + // then: + verify(metadataCrudService).deleteDocument(persistentStudy); + verify(studyEventHandler).deletedStudy(studyId); + } + + @Test + @DisplayName("Delete Study - Not Found") + void deleteStudyNotFound() { + // given: + String nonExistentStudyId = "nonExistentId"; + + // and: + when(studyRepository.findById(nonExistentStudyId)).thenReturn(Optional.empty()); + + // when, then: + ResponseStatusException exception = + assertThrows( + ResponseStatusException.class, () -> studyService.delete(nonExistentStudyId)); + assertThat("404 NOT_FOUND").isEqualTo(exception.getMessage()); + + verify(metadataCrudService, never()).deleteDocument(any()); + verify(studyEventHandler, never()).deletedStudy(any()); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/study/StudyTest.java b/src/test/java/uk/ac/ebi/subs/ingest/study/StudyTest.java new file mode 100644 index 000000000..2aeddbb96 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/study/StudyTest.java @@ -0,0 +1,90 @@ +package uk.ac.ebi.subs.ingest.study; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; + +public class StudyTest { + @Test + public void testGetOpenSubmissionEnvelopes() { + // given: + SubmissionEnvelope openSubmissionEnvelope = new SubmissionEnvelope(); + openSubmissionEnvelope.enactStateTransition(SubmissionState.DRAFT); + + SubmissionEnvelope openSubmissionEnvelope2 = new SubmissionEnvelope(); + openSubmissionEnvelope2.enactStateTransition(SubmissionState.DRAFT); + openSubmissionEnvelope2.enactStateTransition(SubmissionState.METADATA_VALID); + openSubmissionEnvelope2.enactStateTransition(SubmissionState.SUBMITTED); + + Study study = new Study("Schema URL", "1.1", "Specific", "{\"name\": \"Updated Study Name\"}"); + study.addToSubmissionEnvelopes(openSubmissionEnvelope); + study.addToSubmissionEnvelopes(openSubmissionEnvelope2); + + // when: + List openSubmissionEnvelopes = study.getOpenSubmissionEnvelopes(); + + // then: + assertThat(openSubmissionEnvelopes).hasSize(1); + } + + @Test + public void testGetOpenSubmissionEnvelopesNone() { + // given: + SubmissionEnvelope completeSubmission = new SubmissionEnvelope(); + completeSubmission.enactStateTransition(SubmissionState.DRAFT); + completeSubmission.enactStateTransition(SubmissionState.METADATA_VALID); + completeSubmission.enactStateTransition(SubmissionState.SUBMITTED); + completeSubmission.enactStateTransition(SubmissionState.PROCESSING); + completeSubmission.enactStateTransition(SubmissionState.COMPLETE); + + SubmissionEnvelope submittedSubmission = new SubmissionEnvelope(); + submittedSubmission.enactStateTransition(SubmissionState.DRAFT); + submittedSubmission.enactStateTransition(SubmissionState.METADATA_VALID); + submittedSubmission.enactStateTransition(SubmissionState.SUBMITTED); + + Study study = new Study("Schema URL", "1.1", "Specific", "{\"name\": \"Updated Study Name\"}"); + study.addToSubmissionEnvelopes(submittedSubmission); + study.addToSubmissionEnvelopes(completeSubmission); + + // when: + List openSubmissionEnvelopes = study.getOpenSubmissionEnvelopes(); + + // then: + assertThat(openSubmissionEnvelopes).hasSize(0); + } + + @Test + public void testIsEditable() { + Study study = new Study("Schema URL", "1.1", "Specific", "{\"name\": \"Updated Study Name\"}"); + assertThat(study.isEditable()).isTrue(); + + SubmissionEnvelope submissionOne = new SubmissionEnvelope(); + submissionOne.enactStateTransition(SubmissionState.METADATA_VALID); + SubmissionEnvelope submissionTwo = new SubmissionEnvelope(); + submissionTwo.enactStateTransition(SubmissionState.METADATA_INVALID); + study.addToSubmissionEnvelopes(submissionOne); + study.addToSubmissionEnvelopes(submissionTwo); + + assertThat(study.isEditable()).isTrue(); + + submissionOne.enactStateTransition(SubmissionState.PROCESSING); + assertThat(study.isEditable()).isFalse(); + + submissionOne.enactStateTransition(SubmissionState.DRAFT); + assertThat(study.isEditable()).isTrue(); + + submissionTwo.enactStateTransition(SubmissionState.SUBMITTED); + assertThat(study.isEditable()).isFalse(); + + // submissionOne.enactStateTransition(SubmissionState.GRAPH_VALID); + // assertThat(study.isEditable()).isTrue(); + // + // submissionTwo.enactStateTransition(SubmissionState.GRAPH_VALID); + // assertThat(study.isEditable()).isFalse(); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeServiceTest.java b/src/test/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeServiceTest.java new file mode 100644 index 000000000..99800bdd6 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeServiceTest.java @@ -0,0 +1,435 @@ +package uk.ac.ebi.subs.ingest.submission; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.stream.Stream; + +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.domain.*; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.biomaterial.Biomaterial; +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.bundle.BundleManifestRepository; +import uk.ac.ebi.subs.ingest.core.Uuid; +import uk.ac.ebi.subs.ingest.core.service.MetadataCrudService; +import uk.ac.ebi.subs.ingest.core.service.MetadataUpdateService; +import uk.ac.ebi.subs.ingest.errors.SubmissionErrorRepository; +import uk.ac.ebi.subs.ingest.exporter.Exporter; +import uk.ac.ebi.subs.ingest.file.File; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.patch.PatchRepository; +import uk.ac.ebi.subs.ingest.process.Process; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.project.Project; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.project.ProjectService; +import uk.ac.ebi.subs.ingest.project.WranglingState; +import uk.ac.ebi.subs.ingest.protocol.Protocol; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; +import uk.ac.ebi.subs.ingest.state.SubmissionState; +import uk.ac.ebi.subs.ingest.state.SubmitAction; +import uk.ac.ebi.subs.ingest.state.ValidationState; +import uk.ac.ebi.subs.ingest.submissionmanifest.SubmissionManifestRepository; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = {SubmissionEnvelopeService.class}) +@AutoConfigureMockMvc(printOnlyOnFailure = false) +public class SubmissionEnvelopeServiceTest { + @Autowired private SubmissionEnvelopeService service; + + @MockBean private MessageRouter messageRouter; + + @MockBean private Exporter exporter; + + @MockBean private MetadataCrudService metadataCrudService; + + @MockBean private MetadataUpdateService metadataUpdateService; + + @MockBean private ExecutorService executorService; + + @MockBean private SubmissionEnvelopeRepository submissionEnvelopeRepository; + + @MockBean private SubmissionEnvelopeCreateHandler submissionEnvelopeCreateHandler; + + @MockBean private SubmissionManifestRepository submissionManifestRepository; + + @MockBean private BundleManifestRepository bundleManifestRepository; + + @MockBean private ProjectRepository projectRepository; + + @MockBean private ProcessRepository processRepository; + + @MockBean private ProtocolRepository protocolRepository; + + @MockBean private FileRepository fileRepository; + + @MockBean private BiomaterialRepository biomaterialRepository; + + @MockBean private PatchRepository patchRepository; + + @MockBean private SubmissionErrorRepository submissionErrorRepository; + + @MockBean ProjectService projectService; + + @Configuration + static class TestConfiguration {} + + @Test + public void testContentLastUpdated() { + // given + SubmissionEnvelope submission = mock(SubmissionEnvelope.class); + Project project = mock(Project.class); + Biomaterial biomaterial = mock(Biomaterial.class); + Process process = mock(Process.class); + File file = mock(File.class); + + PageRequest request = PageRequest.of(0, 1, new Sort(Sort.Direction.DESC, "updateDate")); + when(projectRepository.findBySubmissionEnvelopesContaining(submission, request)) + .thenReturn(new PageImpl<>(List.of(project), request, 1)); + when(protocolRepository.findBySubmissionEnvelope(submission, request)).thenReturn(Page.empty()); + when(biomaterialRepository.findBySubmissionEnvelope(submission, request)) + .thenReturn(new PageImpl<>(List.of(biomaterial), request, 1)); + when(processRepository.findBySubmissionEnvelope(submission, request)) + .thenReturn(new PageImpl<>(List.of(process), request, 1)); + when(fileRepository.findBySubmissionEnvelope(submission, request)) + .thenReturn(new PageImpl<>(List.of(file), request, 1)); + + Instant now = Instant.now(); + Instant yesterday = now.minus(1, ChronoUnit.DAYS); + when(project.getUpdateDate()).thenReturn(yesterday); + when(biomaterial.getUpdateDate()).thenReturn(yesterday); + when(process.getUpdateDate()).thenReturn(yesterday); + when(file.getUpdateDate()).thenReturn(now); + + // when + Optional lastUpdateDate = service.getSubmissionContentLastUpdated(submission); + + // then + assertThat(lastUpdateDate.isPresent()).isTrue(); + assertThat(lastUpdateDate.get().toString()).isEqualTo(now.toString()); + } + + @Test + public void testContentLastUpdatedEmptySubmission() { + // given + SubmissionEnvelope submission = mock(SubmissionEnvelope.class); + + PageRequest request = PageRequest.of(0, 1, new Sort(Sort.Direction.DESC, "updateDate")); + when(projectRepository.findBySubmissionEnvelopesContaining(submission, request)) + .thenReturn(Page.empty()); + when(protocolRepository.findBySubmissionEnvelope(submission, request)).thenReturn(Page.empty()); + when(biomaterialRepository.findBySubmissionEnvelope(submission, request)) + .thenReturn(Page.empty()); + when(processRepository.findBySubmissionEnvelope(submission, request)).thenReturn(Page.empty()); + when(fileRepository.findBySubmissionEnvelope(submission, request)).thenReturn(Page.empty()); + + // when + Optional lastUpdateDate = service.getSubmissionContentLastUpdated(submission); + + // then + assertThat(lastUpdateDate.isPresent()).isFalse(); + } + + @Test + public void testDeleteSubmission() { + // given SubmissionEnvelope + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.setUuid(Uuid.newUuid()); + + // given metadata within the SubmissionEnvelope + Biomaterial testBiomaterial = + new Biomaterial(Map.ofEntries(Map.entry("key", UUID.randomUUID()))); + Protocol testProtocol = new Protocol(Map.ofEntries(Map.entry("key", UUID.randomUUID()))); + Process testProcess = new Process(Map.ofEntries(Map.entry("key", UUID.randomUUID()))); + + testProcess.setSubmissionEnvelope(submissionEnvelope); + testProtocol.setSubmissionEnvelope(submissionEnvelope); + testBiomaterial.setSubmissionEnvelope(submissionEnvelope); + + // given metadata outside the SubmissionEnvelope + Biomaterial testOutsideBiomaterial = + new Biomaterial(Map.ofEntries(Map.entry("key", UUID.randomUUID()))); + File testOutsideFile = new File(Map.ofEntries(Map.entry("key", UUID.randomUUID())), ""); + Process testOutsideProcess = new Process(Map.ofEntries(Map.entry("key", UUID.randomUUID()))); + + // given links to metadata outside of the SubmissionEnvelope + testOutsideBiomaterial.getInputToProcesses().add(testProcess); + testOutsideFile.getDerivedByProcesses().add(testProcess); + testOutsideProcess.getProtocols().add(testProtocol); + + // given File + File file = new File(null, "testFile.txt"); + file.setSubmissionEnvelope(submissionEnvelope); + + // given Project + Project project = new Project(new Object()); + project.setUuid(Uuid.newUuid()); + project.addToSubmissionEnvelopes(submissionEnvelope); + assertThat(project.getSubmissionEnvelopes()).contains(submissionEnvelope); + + // given SupplementaryFile + project.getSupplementaryFiles().add(file); + assertThat(project.getSupplementaryFiles()).contains(file); + + // given ProjectRepository + List projectList = new ArrayList<>(); + projectList.add(project); + when(projectRepository.findBySubmissionEnvelopesContaining(any(), any())) + .thenReturn(new PageImpl<>(projectList, Pageable.unpaged(), 1)); + + when(projectRepository.findBySubmissionEnvelopesContains(any())).thenReturn(Stream.of(project)); + + when(projectRepository.findBySupplementaryFilesContains(any())).thenReturn(Stream.of(project)); + + // when + when(processRepository.findBySubmissionEnvelope(submissionEnvelope)) + .thenReturn(Stream.of(testProcess)); + when(biomaterialRepository.findBySubmissionEnvelope(submissionEnvelope)) + .thenReturn(Stream.of(testBiomaterial)); + when(protocolRepository.findBySubmissionEnvelope(submissionEnvelope)) + .thenReturn(Stream.of(testProtocol)); + when(fileRepository.findBySubmissionEnvelope(submissionEnvelope)).thenReturn(Stream.of(file)); + + when(biomaterialRepository.findByInputToProcessesContains(testProcess)) + .thenReturn(Stream.of(testOutsideBiomaterial)); + when(fileRepository.findByDerivedByProcessesContains(testProcess)) + .thenReturn(Stream.of(testOutsideFile)); + when(processRepository.findByProtocolsContains(testProtocol)) + .thenReturn(Stream.of(testOutsideProcess)); + + service.deleteSubmission(submissionEnvelope, false); + + // then: + verify(metadataCrudService).removeLinksToDocument(testProcess); + verify(metadataCrudService).removeLinksToDocument(testProtocol); + verify(metadataCrudService).removeLinksToDocument(file); + + verify(biomaterialRepository).deleteBySubmissionEnvelope(submissionEnvelope); + verify(processRepository).deleteBySubmissionEnvelope(submissionEnvelope); + verify(protocolRepository).deleteBySubmissionEnvelope(submissionEnvelope); + verify(fileRepository).deleteBySubmissionEnvelope(submissionEnvelope); + verify(bundleManifestRepository) + .deleteByEnvelopeUuid(submissionEnvelope.getUuid().getUuid().toString()); + verify(patchRepository).deleteBySubmissionEnvelope(submissionEnvelope); + verify(submissionManifestRepository).deleteBySubmissionEnvelope(submissionEnvelope); + verify(submissionErrorRepository).deleteBySubmissionEnvelope(submissionEnvelope); + + verify(projectRepository).findBySubmissionEnvelopesContains(submissionEnvelope); + assertThat(project.getSubmissionEnvelopes()).isEmpty(); + verify(projectRepository, atLeastOnce()).save(project); + verify(submissionEnvelopeRepository).delete(submissionEnvelope); + } + + @ParameterizedTest + // @EnumSource( + // value = SubmissionState.class, + // names = { + // "PENDING", + // "DRAFT", + // "METADATA_VALIDATING", + // "METADATA_VALID", + // "METADATA_INVALID", + // "GRAPH_VALIDATION_REQUESTED", + // "GRAPH_VALIDATING", + // "GRAPH_VALID", + // "GRAPH_INVALID", + // "SUBMITTED", + // "PROCESSING", + // "ARCHIVING", + // "ARCHIVED", + // "EXPORTING", + // "EXPORTED", + // "CLEANUP", + // "COMPLETE" + // }) + @EnumSource( + value = SubmissionState.class, + names = { + "PENDING", + "DRAFT", + "METADATA_VALID", + "METADATA_INVALID", + "GRAPH_VALID", + "GRAPH_INVALID", + "SUBMITTED", + "PROCESSING", + "ARCHIVING", + "ARCHIVED", + "EXPORTING", + "EXPORTED", + "CLEANUP", + "COMPLETE" + }) + public void testRedundantHandleEnvelopeStateUpdateRequest(SubmissionState state) { + // Given + var submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.enactStateTransition(state); + + // When + service.handleEnvelopeStateUpdateRequest(submissionEnvelope, state); + + // Then + // no errors + } + + @Nested + @DisplayName("SubmitRequestTests") + class SubmitRequestTests { + SubmissionEnvelope submissionEnvelope; + Project project; + + @BeforeEach + public void setup() { + submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALID); + project = new Project(null); + project.setValidationState(ValidationState.VALID); + when(projectRepository.findBySubmissionEnvelopesContains(any())) + .thenReturn(Stream.of(project)); + } + + @Test + public void testSubmissionBlocked() { + // given: + submissionEnvelope.enactStateTransition(SubmissionState.METADATA_VALID); + + // when + Throwable exception = + assertThrows( + RuntimeException.class, + () -> service.handleSubmitRequest(submissionEnvelope, List.of(SubmitAction.EXPORT))); + + // then: + assertErrorMessageContains(exception, "without a graph valid state"); + } + + @Test + @Disabled("Not yet applicable to MorPhiC") + public void testSubmissionUnblocked() { + // when + service.handleSubmitRequest(submissionEnvelope, List.of(SubmitAction.EXPORT)); + + // then: + verify(submissionEnvelopeRepository).save(submissionEnvelope); + } + + @Test + @Disabled("Not yet applicable to MorPhiC") + public void testGraphValidationErrorsCleared() { + // given envelope: + submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_INVALID); + + // given metadata within the SubmissionEnvelope + Biomaterial testBiomaterial = + new Biomaterial(Map.ofEntries(Map.entry("key", UUID.randomUUID()))); + Protocol testProtocol = new Protocol(Map.ofEntries(Map.entry("key", UUID.randomUUID()))); + Process testProcess = new Process(Map.ofEntries(Map.entry("key", UUID.randomUUID()))); + File testFile = new File(null, "testFile.txt"); + + testProcess.setSubmissionEnvelope(submissionEnvelope); + testProtocol.setSubmissionEnvelope(submissionEnvelope); + testBiomaterial.setSubmissionEnvelope(submissionEnvelope); + testFile.setSubmissionEnvelope(submissionEnvelope); + + // given graph validation errors on the metadata + testBiomaterial.setGraphValidationErrors(Arrays.asList("test1", "test2")); + testProcess.setGraphValidationErrors(Arrays.asList("test1", "test2")); + testProtocol.setGraphValidationErrors(Arrays.asList("test1", "test2")); + testFile.setGraphValidationErrors(Arrays.asList("test1", "test2")); + + // when + when(biomaterialRepository.findBySubmissionEnvelope(any())) + .thenReturn(Stream.of(testBiomaterial)); + when(processRepository.findBySubmissionEnvelope(any())).thenReturn(Stream.of(testProcess)); + when(protocolRepository.findBySubmissionEnvelope(any())).thenReturn(Stream.of(testProtocol)); + when(fileRepository.findBySubmissionEnvelope(any())).thenReturn(Stream.of(testFile)); + + service.handleEnvelopeStateUpdateRequest(submissionEnvelope, SubmissionState.GRAPH_VALID); + submissionEnvelope.enactStateTransition(SubmissionState.GRAPH_VALID); + // then: + assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(SubmissionState.GRAPH_VALID); + assertThat(testBiomaterial.getGraphValidationErrors()).isEqualTo(new ArrayList<>()); + assertThat(testProcess.getGraphValidationErrors()).isEqualTo(new ArrayList<>()); + assertThat(testProtocol.getGraphValidationErrors()).isEqualTo(new ArrayList<>()); + assertThat(testFile.getGraphValidationErrors()).isEqualTo(new ArrayList<>()); + } + + @Test + public void testSubmissionInvalidProject() { + // given: + project.setValidationState(ValidationState.INVALID); + + // when + Throwable exception = + assertThrows( + RuntimeException.class, + () -> service.handleSubmitRequest(submissionEnvelope, List.of(SubmitAction.EXPORT))); + + // then: + assertErrorMessageContains(exception, "cannot be submitted when the project is invalid"); + } + + @Test + public void testSubmissionNoProject() { + // given: + when(projectRepository.findBySubmissionEnvelopesContains(any())).thenReturn(Stream.empty()); + + // when + Throwable exception = + assertThrows( + RuntimeException.class, + () -> service.handleSubmitRequest(submissionEnvelope, List.of(SubmitAction.EXPORT))); + + // then: + assertErrorMessageContains(exception, "cannot be submitted without a project"); + } + + @Test + void testExportedEventUpdatesHistory() { + // given + // submission from setUp() + // when + service.handleCommitExported(submissionEnvelope); + + // then + verify(projectService).updateWranglingState(project, WranglingState.SUBMITTED); + } + + private void assertErrorMessageContains(Throwable exception, String s) { + assertThat(exception.getMessage()).contains(s); + verify(submissionEnvelopeRepository, never()).save(submissionEnvelope); + } + } + + @Nested + @DisplayName("StateUpdateRequestTests") + class StateUpdateRequestTests { + SubmissionEnvelope submissionEnvelope; + HashSet submitActions; + + @BeforeEach + public void setup() { + submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.enactStateTransition(SubmissionState.SUBMITTED); + submitActions = new HashSet<>(); + submissionEnvelope.setSubmitActions(submitActions); + } + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeTest.java b/src/test/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeTest.java new file mode 100644 index 000000000..cf3f1709f --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/submission/SubmissionEnvelopeTest.java @@ -0,0 +1,221 @@ +package uk.ac.ebi.subs.ingest.submission; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import uk.ac.ebi.subs.ingest.state.SubmissionState; + +public class SubmissionEnvelopeTest { + + @Test + public void testAllowedSubmissionStateTransitions() { + List states = getAllowedStates(SubmissionState.PENDING); + assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.DRAFT)); + } + + @Test + public void testAllowedSubmissionStateTransitionsForDraft() { + List states = getAllowedStates(SubmissionState.DRAFT); + assertThat(states) + .containsExactlyInAnyOrderElementsOf( + List.of(SubmissionState.METADATA_VALID, SubmissionState.METADATA_INVALID)); + } + + @Test + public void testAllowedSubmissionStateTransitionsForMetadataValidating() { + List states = getAllowedStates(SubmissionState.METADATA_VALID); + assertThat(states) + .containsExactlyInAnyOrderElementsOf( + List.of( + SubmissionState.DRAFT, + SubmissionState.METADATA_INVALID, + SubmissionState.GRAPH_VALID, + SubmissionState.GRAPH_INVALID)); + } + + @Test + public void testAllowedSubmissionStateTransitionsForMetadataValid() { + List states = getAllowedStates(SubmissionState.METADATA_VALID); + assertThat(states) + .containsExactlyInAnyOrderElementsOf( + List.of( + SubmissionState.DRAFT, + SubmissionState.GRAPH_VALID, + SubmissionState.METADATA_INVALID, + SubmissionState.GRAPH_INVALID)); + } + + @Test + public void testAllowedSubmissionStateTransitionsForMetadataInvalid() { + List states = getAllowedStates(SubmissionState.METADATA_INVALID); + assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.DRAFT)); + } + + @Test + public void testAllowedSubmissionStateTransitionsForGraphValidationRequested() { + List states = getAllowedStates(SubmissionState.GRAPH_VALID); + assertThat(states) + .containsExactlyInAnyOrderElementsOf( + List.of(SubmissionState.SUBMITTED, SubmissionState.DRAFT)); + } + + @Test + public void testAllowedSubmissionStateTransitionsForGraphValidating() { + List states = getAllowedStates(SubmissionState.GRAPH_VALID); + assertThat(states) + .containsExactlyInAnyOrderElementsOf( + List.of(SubmissionState.SUBMITTED, SubmissionState.DRAFT)); + } + + @Test + public void testAllowedSubmissionStateTransitionsForGraphValid() { + List states = getAllowedStates(SubmissionState.GRAPH_VALID); + assertThat(states) + .containsExactlyInAnyOrderElementsOf( + List.of(SubmissionState.DRAFT, SubmissionState.SUBMITTED)); + } + + @Test + public void testAllowedSubmissionStateTransitionsForGraphInvalid() { + List states = getAllowedStates(SubmissionState.GRAPH_INVALID); + assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.DRAFT)); + } + + @Test + public void testAllowedSubmissionStateTransitionsForSubmitted() { + List states = getAllowedStates(SubmissionState.SUBMITTED); + assertThat(states) + .containsExactlyInAnyOrderElementsOf( + List.of(SubmissionState.PROCESSING, SubmissionState.EXPORTING)); + } + + @Test + public void testAllowedSubmissionStateTransitionsForProcessing() { + List states = getAllowedStates(SubmissionState.PROCESSING); + assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.ARCHIVING)); + } + + @Test + public void testAllowedSubmissionStateTransitionsForArchiving() { + List states = getAllowedStates(SubmissionState.ARCHIVING); + assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.ARCHIVED)); + } + + @Test + public void testAllowedSubmissionStateTransitionsForArchived() { + List states = getAllowedStates(SubmissionState.ARCHIVED); + assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.EXPORTING)); + } + + @Test + public void testAllowedSubmissionStateTransitionsForExported() { + List states = getAllowedStates(SubmissionState.EXPORTED); + assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.CLEANUP)); + } + + @Test + public void testAllowedSubmissionStateTransitionsForCleanup() { + List states = getAllowedStates(SubmissionState.CLEANUP); + assertThat(states).containsExactlyInAnyOrderElementsOf(List.of(SubmissionState.COMPLETE)); + } + + @Test + public void testAllowedSubmissionStateTransitionsForComplete() { + List states = getAllowedStates(SubmissionState.COMPLETE); + assertThat(states).isEmpty(); + } + + @ParameterizedTest + @EnumSource( + value = SubmissionState.class, + names = { + /*"PENDING",*/ + "EXPORTING", + "PROCESSING", + "CLEANUP", + "ARCHIVED", + "SUBMITTED" + }) + public void testIsNotEditable(SubmissionState state) { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.enactStateTransition(state); + assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(state); + + // then: + assertThat(submissionEnvelope.isEditable()).isFalse(); + } + + @ParameterizedTest + @EnumSource( + value = SubmissionState.class, + names = { + "METADATA_VALID", + "METADATA_INVALID", + "EXPORTED", + "GRAPH_VALID", + "GRAPH_INVALID", + "COMPLETE", + "DRAFT", + "ARCHIVING" + }) + public void testIsEditable(SubmissionState state) { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.enactStateTransition(state); + assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(state); + + // then: + assertThat(submissionEnvelope.isEditable()).isTrue(); + } + + @ParameterizedTest + @EnumSource( + value = SubmissionState.class, + names = {"EXPORTING", "PROCESSING", "ARCHIVED", "SUBMITTED"}) + public void testCannotAddTo(SubmissionState state) { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.enactStateTransition(state); + assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(state); + + // then: + assertThat(submissionEnvelope.isSystemEditable()).isFalse(); + } + + @ParameterizedTest + @EnumSource( + value = SubmissionState.class, + names = { + "PENDING", + "METADATA_VALID", + "METADATA_INVALID", + "EXPORTED", + "GRAPH_VALID", + "GRAPH_INVALID", + "COMPLETE", + "DRAFT", + "ARCHIVING", + "CLEANUP" + }) + public void testCanAddTo(SubmissionState state) { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.enactStateTransition(state); + assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(state); + + // then: + assertThat(submissionEnvelope.isSystemEditable()).isTrue(); + } + + private List getAllowedStates(SubmissionState state) { + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.enactStateTransition(state); + return submissionEnvelope.allowedSubmissionStateTransitions(); + } +} diff --git a/src/test/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionControllerTest.java b/src/test/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionControllerTest.java new file mode 100644 index 000000000..7e3d54bd2 --- /dev/null +++ b/src/test/java/uk/ac/ebi/subs/ingest/submission/web/SubmissionControllerTest.java @@ -0,0 +1,189 @@ +package uk.ac.ebi.subs.ingest.submission.web; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static uk.ac.ebi.subs.ingest.state.SubmissionState.*; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.rest.webmvc.PersistentEntityResourceAssembler; +import org.springframework.data.web.PagedResourcesAssembler; +import org.springframework.http.HttpEntity; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import uk.ac.ebi.subs.ingest.biomaterial.BiomaterialRepository; +import uk.ac.ebi.subs.ingest.bundle.BundleManifestRepository; +import uk.ac.ebi.subs.ingest.dataset.DatasetRepository; +import uk.ac.ebi.subs.ingest.exporter.Exporter; +import uk.ac.ebi.subs.ingest.file.FileRepository; +import uk.ac.ebi.subs.ingest.messaging.MessageRouter; +import uk.ac.ebi.subs.ingest.process.ProcessRepository; +import uk.ac.ebi.subs.ingest.process.ProcessService; +import uk.ac.ebi.subs.ingest.project.ProjectRepository; +import uk.ac.ebi.subs.ingest.protocol.ProtocolRepository; +import uk.ac.ebi.subs.ingest.protocol.ProtocolService; +import uk.ac.ebi.subs.ingest.study.StudyRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelope; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeRepository; +import uk.ac.ebi.subs.ingest.submission.SubmissionEnvelopeService; +import uk.ac.ebi.subs.ingest.submission.SubmissionStateMachineService; +import uk.ac.ebi.subs.ingest.submissionmanifest.SubmissionManifestRepository; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = {SubmissionController.class}) +public class SubmissionControllerTest { + + @Autowired private SubmissionController controller; + + @MockBean private Exporter exporter; + + @MockBean private SubmissionEnvelopeService submissionEnvelopeService; + @MockBean private ProcessService processService; + @MockBean private ProtocolService protocolService; + + @MockBean private SubmissionEnvelopeRepository submissionEnvelopeRepository; + @MockBean private FileRepository fileRepository; + @MockBean private ProjectRepository projectRepository; + @MockBean private ProtocolRepository protocolRepository; + @MockBean private BiomaterialRepository biomaterialRepository; + @MockBean private ProcessRepository processRepository; + @MockBean private StudyRepository studyRepository; + @MockBean private DatasetRepository datasetRepository; + @MockBean private BundleManifestRepository bundleManifestRepository; + @MockBean private SubmissionManifestRepository submissionManifestRepository; + @MockBean private PagedResourcesAssembler pagedResourcesAssembler; + @MockBean private SubmissionStateMachineService submissionStateMachineService; + @MockBean private MessageRouter messageRouter; + + @Test + public void testEnactSubmitEnvelope() { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + assertThat(submissionEnvelope.getSubmissionState()).isNotEqualTo(SUBMITTED); + + // and: + PersistentEntityResourceAssembler resourceAssembler = + mock(PersistentEntityResourceAssembler.class); + + // when: + HttpEntity response = controller.enactSubmitEnvelope(submissionEnvelope, resourceAssembler); + + // then: + assertThat(response).isNotNull(); + assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(SUBMITTED); + verify(submissionEnvelopeRepository).save(submissionEnvelope); + verify(submissionEnvelopeService).handleCommitSubmit(submissionEnvelope); + } + + @Test + public void testDeleteSubmissionEnvelopeWithoutForce() { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + + // when: + HttpEntity response = controller.forceDeleteSubmission(submissionEnvelope, false); + + // then: + assertThat(response).isNotNull(); + verify(submissionEnvelopeService).deleteSubmission(submissionEnvelope, false); + } + + @Test + public void testDeleteSubmissionEnvelopeWithForce() { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + + // when: + HttpEntity response = controller.forceDeleteSubmission(submissionEnvelope, true); + + // then: + assertThat(response).isNotNull(); + verify(submissionEnvelopeService).deleteSubmission(submissionEnvelope, true); + } + + @Test + public void testDraftStateTransition() { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.enactStateTransition(GRAPH_VALID); + assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(GRAPH_VALID); + + // and: + PersistentEntityResourceAssembler resourceAssembler = + mock(PersistentEntityResourceAssembler.class); + + // When: + HttpEntity response = controller.enactDraftEnvelope(submissionEnvelope, resourceAssembler); + + // then: + assertThat(response).isNotNull(); + assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(DRAFT); + } + + /*@Test + public void testHappyValidationPath() { + // given: + SubmissionEnvelope submissionEnvelope = new SubmissionEnvelope(); + submissionEnvelope.enactStateTransition(DRAFT); + assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(DRAFT); + + // and: + PersistentEntityResourceAssembler resourceAssembler = + mock(PersistentEntityResourceAssembler.class); + + // Test metadata validation happy path + // Metadata validation is triggered when documents are added to the submission + // so no endpoints fro requesting the state tracker to change the submission state + + // draft -> metadata validating + HttpEntity response = + controller.enactValidatingEnvelope(submissionEnvelope, resourceAssembler); + assertThat(response).isNotNull(); + assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(METADATA_VALIDATING); + + // metadata validating -> metadata valid + response = controller.enactValidEnvelope(submissionEnvelope, resourceAssembler); + assertThat(response).isNotNull(); + assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(METADATA_VALID); + + // Test graph validation happy path + // endpoints for requesting the state tracker to change the submission state are used here + + // metadata valid -> graph validation requested + HttpEntity requestResponse = + controller.requestGraphValidation(submissionEnvelope, resourceAssembler); + assertThat(requestResponse).isNotNull(); + verify(submissionEnvelopeService) + .handleEnvelopeStateUpdateRequest(submissionEnvelope, GRAPH_VALIDATION_REQUESTED); + HttpEntity enactResponse = + controller.enactGraphValidationRequested(submissionEnvelope, resourceAssembler); + assertThat(enactResponse).isNotNull(); + assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(GRAPH_VALIDATION_REQUESTED); + + // graph validation requested -> graph validating + requestResponse = controller.requestGraphValidating(submissionEnvelope, resourceAssembler); + assertThat(requestResponse).isNotNull(); + verify(submissionEnvelopeService) + .handleEnvelopeStateUpdateRequest(submissionEnvelope, GRAPH_VALIDATING); + enactResponse = controller.enactGraphValidating(submissionEnvelope, resourceAssembler); + assertThat(enactResponse).isNotNull(); + assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(GRAPH_VALIDATING); + + // graph validating -> graph valid + requestResponse = controller.requestGraphValid(submissionEnvelope, resourceAssembler); + assertThat(requestResponse).isNotNull(); + verify(submissionEnvelopeService) + .handleEnvelopeStateUpdateRequest(submissionEnvelope, GRAPH_VALID); + enactResponse = controller.enactGraphValid(submissionEnvelope, resourceAssembler); + assertThat(enactResponse).isNotNull(); + assertThat(submissionEnvelope.getSubmissionState()).isEqualTo(GRAPH_VALID); + }*/ + + @Configuration + static class TestConfiguration {} +} diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties index b5c526f88..b95f06590 100644 --- a/src/test/resources/application.properties +++ b/src/test/resources/application.properties @@ -1,2 +1,8 @@ -AUTH_ISSUER=https://domain.tld/issuer -GCP_JWK_PROVIDER_BASE_URL=https://gcpjwk.domain.tld/ \ No newline at end of file +SCHEMA_BASE_URI='' +GCP_PROJECT_WHITELIST='' +SVC_AUTH_AUDIENCE='' +USR_AUTH_AUDIENCE='' +AUTH_ISSUER=http://domain.tld/issuer +GCP_JWK_PROVIDER_BASE_URL=http://domain.tld + +logging.level.org.springframework.security=DEBUG