feat(backend): Return pipelineVersion in get-released-data (#3433)

* Add 'pipeline_version' to SequenceEntriesView * Add 'pipelineVersion' to RawProcessedData * Add 'pipelineVersion' to get-released-data return values * Add 'pipelineVersion' to _common-metadata.tpl * Test fix * Add missing comma * Update schema documentation based on migration changes * trigger preview * fixes * Update schema documentation based on migration changes * trigger preview --------- Co-authored-by: GitHub Action <action@github.com>
loculus-project · Dec 12, 2024 · ee6ec02 · ee6ec02
1 parent 3c6510b
commit ee6ec02
Show file tree

Hide file tree

Showing 8 changed files with 61 additions and 1 deletion.
diff --git a/backend/docs/db/schema.sql b/backend/docs/db/schema.sql
@@ -464,6 +464,11 @@ CREATE VIEW public.sequence_entries_view AS
     sepd.finished_processing_at,
     sepd.processed_data,
     (sepd.processed_data || em.joint_metadata) AS joint_metadata,
+        CASE
+            WHEN se.is_revocation THEN ( SELECT current_processing_pipeline.version
+               FROM public.current_processing_pipeline)
+            ELSE sepd.pipeline_version
+        END AS pipeline_version,
     sepd.errors,
     sepd.warnings,
         CASE

diff --git a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt
@@ -128,6 +128,7 @@ open class ReleasedDataModel(
                 ("versionStatus" to TextNode(versionStatus.name)),
                 ("dataUseTerms" to TextNode(currentDataUseTerms.type.name)),
                 ("dataUseTermsRestrictedUntil" to restrictedDataUseTermsUntil),
+                ("pipelineVersion" to LongNode(rawProcessedData.pipelineVersion)),
             ) +
             if (rawProcessedData.isRevocation) {
                 mapOf("versionComment" to TextNode(rawProcessedData.versionComment))

diff --git a/backend/src/main/kotlin/org/loculus/backend/service/submission/SequenceEntriesView.kt b/backend/src/main/kotlin/org/loculus/backend/service/submission/SequenceEntriesView.kt
@@ -46,6 +46,7 @@ object SequenceEntriesView : Table(SEQUENCE_ENTRIES_VIEW_NAME) {
     val versionCommentColumn = varchar("version_comment", 255).nullable()
     val errorsColumn = jacksonSerializableJsonb<List<PreprocessingAnnotation>>("errors").nullable()
     val warningsColumn = jacksonSerializableJsonb<List<PreprocessingAnnotation>>("warnings").nullable()
+    val pipelineVersionColumn = long("pipeline_version").nullable()
 
     override val primaryKey = PrimaryKey(accessionColumn, versionColumn)
 

diff --git a/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt b/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt
@@ -583,6 +583,7 @@ class SubmissionDatabaseService(
             SequenceEntriesView.submittedAtTimestampColumn,
             SequenceEntriesView.releasedAtTimestampColumn,
             SequenceEntriesView.submissionIdColumn,
+            SequenceEntriesView.pipelineVersionColumn,
             DataUseTermsTable.dataUseTermsTypeColumn,
             DataUseTermsTable.restrictedUntilColumn,
         )
@@ -610,6 +611,7 @@ class SubmissionDatabaseService(
                     null -> emptyProcessedDataProvider.provide(organism)
                     else -> processedDataPostprocessor.retrieveFromStoredValue(processedData, organism)
                 },
+                pipelineVersion = it[SequenceEntriesView.pipelineVersionColumn]!!,
                 submittedAtTimestamp = it[SequenceEntriesView.submittedAtTimestampColumn],
                 releasedAtTimestamp = it[SequenceEntriesView.releasedAtTimestampColumn]!!,
                 dataUseTerms = DataUseTerms.fromParameters(
@@ -1206,5 +1208,6 @@ data class RawProcessedData(
     val releasedAtTimestamp: LocalDateTime,
     val submissionId: String,
     val processedData: ProcessedData<GeneticSequence>,
+    val pipelineVersion: Long,
     val dataUseTerms: DataUseTerms,
 ) : AccessionVersionInterface
diff --git a/...c/main/resources/db/migration/V1.9__update_sequence_entries_view_add_pipeline_version.sql b/...c/main/resources/db/migration/V1.9__update_sequence_entries_view_add_pipeline_version.sql
@@ -0,0 +1,41 @@
+drop view if exists sequence_entries_view;
+
+create view sequence_entries_view as
+select
+    se.*,
+    sepd.started_processing_at,
+    sepd.finished_processing_at,
+    sepd.processed_data as processed_data,
+    sepd.processed_data || em.joint_metadata as joint_metadata,
+    case
+        when se.is_revocation then (select version from current_processing_pipeline)
+        else sepd.pipeline_version
+    end as pipeline_version,
+    sepd.errors,
+    sepd.warnings,
+    case
+        when se.released_at is not null then 'APPROVED_FOR_RELEASE'
+        when se.is_revocation then 'PROCESSED'
+        when sepd.processing_status = 'IN_PROCESSING' then 'IN_PROCESSING'
+        when sepd.processing_status = 'PROCESSED' then 'PROCESSED'
+        else 'RECEIVED'
+    end as status,
+    case
+        when sepd.processing_status = 'IN_PROCESSING' then null
+        when sepd.errors is not null and jsonb_array_length(sepd.errors) > 0 then 'HAS_ERRORS'
+        when sepd.warnings is not null and jsonb_array_length(sepd.warnings) > 0 then 'HAS_WARNINGS'
+        else 'NO_ISSUES'
+    end as processing_result
+from
+    sequence_entries se
+    left join sequence_entries_preprocessed_data sepd on
+        se.accession = sepd.accession
+        and se.version = sepd.version
+        and sepd.pipeline_version = (select version from current_processing_pipeline)
+    left join external_metadata_view em on
+        se.accession = em.accession
+        and se.version = em.version;
+
+update sequence_entries_preprocessed_data
+set processing_status = 'PROCESSED'
+where processing_status in ('HAS_ERRORS', 'FINISHED');
diff --git a/.../src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt b/.../src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt
@@ -50,6 +50,7 @@ import org.loculus.backend.controller.DEFAULT_GROUP_CHANGED
 import org.loculus.backend.controller.DEFAULT_GROUP_NAME
 import org.loculus.backend.controller.DEFAULT_GROUP_NAME_CHANGED
 import org.loculus.backend.controller.DEFAULT_ORGANISM
+import org.loculus.backend.controller.DEFAULT_PIPELINE_VERSION
 import org.loculus.backend.controller.DEFAULT_USER_NAME
 import org.loculus.backend.controller.EndpointTest
 import org.loculus.backend.controller.datauseterms.DataUseTermsControllerClient
@@ -143,7 +144,7 @@ class GetReleasedDataEndpointTest(
         responseBody.forEach {
             val id = it.metadata["accession"]!!.asText()
             val version = it.metadata["version"]!!.asLong()
-            assertThat(version, `is`(1L))
+            assertThat(version, `is`(1))
 
             val expectedMetadata = defaultProcessedData.metadata + mapOf(
                 "accession" to TextNode(id),
@@ -157,6 +158,7 @@ class GetReleasedDataEndpointTest(
                 "releasedDate" to TextNode(currentDate),
                 "submittedDate" to TextNode(currentDate),
                 "dataUseTermsRestrictedUntil" to NullNode.getInstance(),
+                "pipelineVersion" to IntNode(DEFAULT_PIPELINE_VERSION.toInt()),
             )
 
             for ((key, value) in it.metadata) {
@@ -294,6 +296,7 @@ class GetReleasedDataEndpointTest(
                     value,
                     `is`(TextNode("This is a test revocation")),
                 )
+                "pipelineVersion" -> assertThat(value, `is`(IntNode(DEFAULT_PIPELINE_VERSION.toInt())))
 
                 else -> assertThat("value for $key", value, `is`(NullNode.instance))
             }
@@ -367,6 +370,7 @@ class GetReleasedDataEndpointTest(
                 .andGetGroupId()
 
             SequenceEntriesTable.batchInsert(accessionVersions.shuffled()) { (accession, version) ->
+                this[SequenceEntriesTable.isRevocationColumn] = true
                 this[SequenceEntriesTable.accessionColumn] = accession
                 this[SequenceEntriesTable.versionColumn] = version
                 this[SequenceEntriesTable.groupIdColumn] = submittingGroupId

diff --git a/backend/src/test/kotlin/org/loculus/backend/utils/EarliestReleaseDateFinderTest.kt b/backend/src/test/kotlin/org/loculus/backend/utils/EarliestReleaseDateFinderTest.kt
@@ -71,6 +71,7 @@ fun row(
     groupId = 0,
     groupName = "foo",
     dataUseTerms = DataUseTerms.Open,
+    pipelineVersion = 0,
 )
 
 // Notes:

diff --git a/kubernetes/loculus/templates/_common-metadata.tpl b/kubernetes/loculus/templates/_common-metadata.tpl
@@ -107,6 +107,10 @@ fields:
     type: string
     displayName: Version comment
     header: Submission details
+  - name: pipelineVersion
+    type: int
+    notSearchable: true
+    hideOnSequenceDetailsPage: true
 {{- end}}
 
 {{/* Patches schema by adding to it and overwriting overlapping fields by the value in metadataAdd*/}}