Remove sources from schema and consolidate its URL field into caseReference

data-serving/data-service/package.json

@@ -59,7 +59,7 @@
     "dotenv": "^8.2.0",
     "envalid": "^6.0.2",
     "express": "^4.17.1",
-    "express-openapi-validator": "^3.16.2",
+    "express-openapi-validator": "^3.16.4",
     "express-validator": "^6.6.0",
     "mongodb": "^3.5.9",
     "mongoose": "^5.9.20",

data-serving/data-service/schemas/cases.index.json

@@ -11,5 +11,5 @@
     "location.place": "text",
     "location.name": "text",
     "pathogen.name": "text",
-    "sources.url": "text"
+    "caseReference.sourceUrl": "text"
 }

data-serving/data-service/schemas/cases.schema.json

@@ -21,7 +21,26 @@
           },
           "sourceEntryId": {
             "bsonType": "string"
-          }
+          },
+          "sourceUrl": {
+            "bsonType": "string"
+          },
+          "additionalSources": {
+            "bsonType": "array",
+            "uniqueItems": true,
+            "items": {
+              "bsonType": "object",
+              "additionalProperties": false,
+              "properties": {
+                "_id": {
+                  "bsonType": "objectId"
+                },
+                "sourceUrl": {
+                  "bsonType": "string"
+                }
+              }
+            }
+          } 
         }
       },
       "demographics": {
@@ -360,28 +379,6 @@
           }
         }
       },
-      "sources": {
-        "bsonType": "array",
-        "uniqueItems": true,
-        "items": {
-          "bsonType": "object",
-          "additionalProperties": false,
-          "properties": {
-            "_id": {
-              "bsonType": "objectId"
-            },
-            "id": {
-              "bsonType": "string"
-            },
-            "url": {
-              "bsonType": "string"
-            },
-            "other": {
-              "bsonType": "string"
-            }
-          }
-        }
-      },
       "pathogens": {
         "bsonType": "array",
         "uniqueItems": true,

data-serving/data-service/src/model/case-reference.ts

@@ -6,9 +6,30 @@ export const caseReferenceSchema = new mongoose.Schema({
         required: true,
     },
     sourceEntryId: String,
+    sourceUrl: {
+        type: String,
+        required: true,
+    },
+    additionalSources: [
+        {
+            sourceUrl: String,
+        },
+    ],
 });
 
 export type CaseReferenceDocument = mongoose.Document & {
+    /** Foreign key to the sources collection. */
     sourceId: string;
+
+    /** The original id of the case in the source.  */
     sourceEntryId: string;
+
+    /** The URL of the source of the case data at the time of ingestion. */
+    sourceUrl: string;
+
+    additionalSources: [
+        {
+            sourceUrl: string;
+        },
+    ];
 };

data-serving/data-service/src/model/case.ts

@@ -12,7 +12,6 @@ import {
     RevisionMetadataDocument,
     revisionMetadataSchema,
 } from './revision-metadata';
-import { SourceDocument, sourceSchema } from './source';
 import { TransmissionDocument, transmissionSchema } from './transmission';
 import { TravelHistoryDocument, travelHistorySchema } from './travel-history';
 
@@ -21,7 +20,10 @@ import mongoose from 'mongoose';
 
 const caseSchema = new mongoose.Schema(
     {
-        caseReference: caseReferenceSchema,
+        caseReference: {
+            type: caseReferenceSchema,
+            required: true,
+        },
         demographics: demographicsSchema,
         events: {
             type: [eventSchema],
@@ -37,19 +39,11 @@ const caseSchema = new mongoose.Schema(
         location: locationSchema,
         revisionMetadata: {
             type: revisionMetadataSchema,
-            required: 'Must include revision metadata',
+            required: true,
         },
         notes: String,
         pathogens: [pathogenSchema],
         preexistingConditions: dictionarySchema,
-        sources: {
-            type: [sourceSchema],
-            required: true,
-            validate: {
-                validator: (sources: [SourceDocument]) => sources.length > 0,
-                message: 'Must include one or more sources',
-            },
-        },
         symptoms: dictionarySchema,
         transmission: transmissionSchema,
         travelHistory: travelHistorySchema,
@@ -82,7 +76,6 @@ type CaseDocument = mongoose.Document & {
     notes: string;
     pathogens: [PathogenDocument];
     preexistingConditions: DictionaryDocument;
-    sources: [SourceDocument];
     symptoms: DictionaryDocument;
     transmission: TransmissionDocument;
     travelHistory: TravelHistoryDocument;

data-serving/data-service/src/model/pathogen.ts

@@ -1,5 +1,3 @@
-import { SourceDocument, sourceSchema } from './source';
-
 import mongoose from 'mongoose';
 import { positiveIntFieldInfo } from './positive-int';
 

data-serving/data-service/test/controllers/case.test.ts

@@ -167,7 +167,11 @@ describe('PUT', () => {
         const res = await request(app)
             .put('/api/cases')
             .send({
-                caseReference: { sourceId: sourceId, sourceEntryId: entryId },
+                caseReference: {
+                    sourceId: sourceId,
+                    sourceEntryId: entryId,
+                    sourceUrl: 'cdc.gov',
+                },
                 notes: newNotes,
             })
             .expect('Content-Type', /json/)
@@ -197,7 +201,11 @@ describe('PUT', () => {
         return request(app)
             .put('/api/cases')
             .send({
-                caseReference: { sourceId: sourceId, sourceEntryId: entryId },
+                caseReference: {
+                    sourceId: sourceId,
+                    sourceEntryId: entryId,
+                    sourceUrl: 'cdc.gov',
+                },
                 location: {},
             })
             .expect(422);

data-serving/data-service/test/model/case-reference.test.ts

@@ -22,6 +22,16 @@ describe('validate', () => {
             expect(e.name).toBe(Error.ValidationError.name);
         });
     });
+
+    it('a caseReference document without sourceUrl is invalid', async () => {
+        const missingSourceUrl = { ...minimalModel };
+        delete missingSourceUrl.sourceUrl;
+
+        return new CaseReference(missingSourceUrl).validate((e) => {
+            expect(e.name).toBe(Error.ValidationError.name);
+        });
+    });
+
     it('a minimal caseReference document is valid', async () => {
         return new CaseReference(minimalModel).validate();
     });

data-serving/data-service/test/model/case.test.ts

@@ -5,8 +5,11 @@ import minimalEvent from './data/event.minimal.json';
 import minimalModel from './data/case.minimal.json';
 
 describe('validate', () => {
-    it('model without sources is invalid', async () => {
-        return new Case({ ...minimalModel, sources: [] }).validate((e) => {
+    it('model without caseReference is invalid', async () => {
+        const noCaseReference = { ...minimalModel };
+        delete noCaseReference.caseReference;
+
+        return new Case({ ...noCaseReference }).validate((e) => {
             expect(e.name).toBe(Error.ValidationError.name);
         });
     });
@@ -28,10 +31,10 @@ describe('validate', () => {
     });
 
     it('model without revision metadata is invalid', async () => {
-        const noRevisionmetadata = { ...minimalModel };
-        delete noRevisionmetadata.revisionMetadata;
+        const noRevisionMetadata = { ...minimalModel };
+        delete noRevisionMetadata.revisionMetadata;
 
-        return new Case(noRevisionmetadata).validate((e) => {
+        return new Case(noRevisionMetadata).validate((e) => {
             expect(e.name).toBe(Error.ValidationError.name);
         });
     });

data-serving/data-service/test/model/data/case-reference.full.json

@@ -1,4 +1,13 @@
 {
     "sourceId": "sourceId",
-    "sourceEntryId": "entryId"
+    "sourceEntryId": "entryId",
+    "sourceUrl": "cdc.gov",
+    "additionalSources": [
+        {
+            "sourceUrl": "twitter.com/a-tweet"
+        },
+        {
+            "sourceUrl": "news.org/an-article"
+        }
+    ]
 }

data-serving/data-service/test/model/data/case-reference.minimal.json

@@ -1,3 +1,4 @@
 {
-    "sourceId": "sourceId"
+    "sourceId": "sourceId",
+    "sourceUrl": "cdc.gov"
 }

data-serving/data-service/test/model/data/case.full.json

@@ -1,7 +1,16 @@
 {
     "caseReference": {
         "sourceId": "sourceId",
-        "sourceEntryId": "entryId"
+        "sourceEntryId": "entryId",
+        "sourceUrl": "cdc.gov",
+        "additionalSources": [
+            {
+                "sourceUrl": "twitter.com/a-tweet"
+            },
+            {
+                "sourceUrl": "news.org/an-article"
+            }
+        ]
     },
     "demographics": {
         "ageRange": {

data-serving/data-service/test/model/data/case.minimal.json

@@ -1,4 +1,8 @@
 {
+    "caseReference": {
+        "sourceId": "sourceId",
+        "sourceUrl": "cdc.gov"
+    },
     "events": [
         {
             "name": "confirmed",
@@ -8,20 +12,15 @@
             }
         }
     ],
-    "sources": [
-        {
-            "url": "cdc.gov"
-        }
-    ],
+    "location": {
+        "country": "China",
+        "geoResolution": "Country"
+    },
     "revisionMetadata": {
         "revisionNumber": 0,
         "creationMetadata": {
             "curator": "abc123",
             "date": "2020-01-03"
         }
-    },
-    "location": {
-        "country": "China",
-        "geoResolution": "Country"
     }
 }