Add a MongoDB index on case reference UUID fields

data-serving/data-service/schemas/cases.caserefindex.json

@@ -0,0 +1,4 @@
+{
+    "caseReference.sourceId": -1,
+    "caseReference.sourceEntryId": -1
+}

data-serving/scripts/data-pipeline/convert_and_import_latest_data.sh

@@ -13,17 +13,19 @@ db='covid19'
 collection='cases'
 sample_rate=1
 schema_path="$SCRIPT_PATH/../../data-service/schemas/cases.schema.json"
-index_path="$SCRIPT_PATH/../../data-service/schemas/cases.index.json"
+text_index_path="$SCRIPT_PATH/../../data-service/schemas/cases.textindex.json"
+caseref_index_path="$SCRIPT_PATH/../../data-service/schemas/cases.caserefindex.json"
 
-while getopts :m:d:c:r:s flag
+while getopts :m:d:c:r:s:ti:ci flag
 do
     case "${flag}" in
         m) mongodb_connection_string=${OPTARG};;
         d) db=${OPTARG};;
         c) collection=${OPTARG};;
         r) sample_rate=${OPTARG};;
         s) schema_path=${OPTARG};;
-        i) index_path=${OPTARG};;
+        ti) text_index_path=${OPTARG};;
+        ci) caseref_index_path=${OPTARG};;
         ?) echo $USAGE; exit 1
     esac
 done
@@ -66,7 +68,8 @@ function setup_db() {
         DB=$db \
         COLL=$collection \
         SCHEMA=$schema_path \
-        INDEX=$index_path \
+        TEXTINDEX=$text_index_path \
+        CASEREFINDEX=$caseref_index_path \
         npm run --prefix $SCRIPT_PATH/../setup-db setup
 }
 
@@ -92,7 +95,8 @@ main() {
     [-c] collection: $collection
     [-r] sample rate: $sample_rate
     [-s] schema path: $schema_path
-    [-i] index path: $index_path"
+    [-ti] text index path: $text_index_path
+    [-ci] case ref index path: $text_index_path"
 
     fetch_latest_data
     convert_data

data-serving/scripts/setup-db/package.json

@@ -6,8 +6,8 @@
   "scripts": {
     "build": "tsc",
     "lint": "tsc --noEmit && eslint '*/**/*.{js,ts,tsx}' --quiet --fix",
-    "setup": "npm install && tsc && mongo $CONN --eval \"let args={connectionString: '$CONN', databaseName: '$DB', collectionName: '$COLL', schemaPath: '$SCHEMA', indexPath: '$INDEX', deleteAllDocuments: '$DELETE_ALL_DOCUMENTS'}\" dist/index.js",
-    "setup-cases": "npm install && tsc && mongo --eval \"let args={databaseName: 'covid19', collectionName: 'cases', schemaPath: './../../data-service/schemas/cases.schema.json', indexPath: './../../data-service/schemas/cases.index.json', deleteAllDocuments: '$DELETE_ALL_DOCUMENTS'}\" dist/index.js"
+    "setup": "npm install && tsc && mongo $CONN --eval \"let args={connectionString: '$CONN', databaseName: '$DB', collectionName: '$COLL', schemaPath: '$SCHEMA', textIndexPath: '$TEXTINDEX', caseRefIndexPath: '$CASEREFINDEX', deleteAllDocuments: '$DELETE_ALL_DOCUMENTS'}\" dist/index.js",
+    "setup-cases": "npm install && tsc && mongo --eval \"let args={databaseName: 'covid19', collectionName: 'cases', schemaPath: './../../data-service/schemas/cases.schema.json', textIndexPath: './../../data-service/schemas/cases.textindex.json', caseRefIndexPath: './../../data-service/schemas/cases.caserefindex.json', deleteAllDocuments: '$DELETE_ALL_DOCUMENTS'}\" dist/index.js"
   },
   "repository": {
     "type": "git",

data-serving/scripts/setup-db/src/index.ts

@@ -5,7 +5,8 @@ interface SetupDatabaseParameters {
     databaseName: string;
     collectionName: string;
     schemaPath: string;
-    indexPath: string;
+    textIndexPath: string;
+    caseRefIndexPath: string;
     /** If not specified, deletes only imported documents. Defaults to false. */
     deleteAllDocuments: boolean;
 }
@@ -15,15 +16,19 @@ const setupDatabase = async ({
     databaseName,
     collectionName,
     schemaPath,
-    indexPath,
+    textIndexPath,
+    caseRefIndexPath,
     deleteAllDocuments = false,
 }: SetupDatabaseParameters): Promise<void> => {
     try {
         const schema = JSON.parse(await cat(schemaPath));
         print(`Read schema from ${schemaPath}`);
 
-        const index = JSON.parse(await cat(indexPath));
-        print(`Read index from ${indexPath}`);
+        const textIndex = JSON.parse(await cat(textIndexPath));
+        print(`Read text index from ${textIndexPath}`);
+
+        const caseRefIndex = JSON.parse(await cat(caseRefIndexPath));
+        print(`Read caseRef index from ${caseRefIndexPath}`);
 
         // Connect to the default MongoDb instance.
         const connection: Connection = new Mongo(connectionString);
@@ -68,11 +73,22 @@ const setupDatabase = async ({
         }
 
         print('Dropping indexes 👇');
-        const indexName = `${collectionName}Idx`;
-        await collection.dropIndex(indexName);
+        const textIndexName = `${collectionName}Idx`;
+        await collection.dropIndex(textIndexName);
+        const caseRefIndexName = `${collectionName}CaseRefIdx`;
+        await collection.dropIndex(caseRefIndexName);
 
         print('Creating indexes 👆');
-        await collection.createIndex(index, { name: indexName });
+        await collection.createIndex(textIndex, { name: textIndexName });
+        await collection.createIndex(caseRefIndex, {
+            name: caseRefIndexName,
+            unique: true,
+            partialFilterExpression: {
+                'caseReference.sourceEntryId': {
+                    $exists: true,
+                },
+            },
+        });
         print('Done 👍');
 
         // Print some stats -- for fun and confirmation!

data-serving/scripts/setup-db/src/types/mongo-cli.d.ts

@@ -15,7 +15,7 @@ declare function quit(): void;
 interface Collection {
     remove: (query: object) => Promise<CommandResult>;
     stats: () => Promise<CollectionStats>;
-    createIndex: (spec: object, options: { name: string }) => Promise<CommandResult>;
+    createIndex: (spec: object, options: { name: string, unique?: boolean, partialFilterExpression?: any }) => Promise<CommandResult>;
     dropIndex: (name: string) => Promise<CommandResult>;
 }