Merge pull request #680 from globaldothealth/mongo-bulk-upsert

Add batchUpsert method to data service case API

data-serving/data-service/api/openapi.yaml

@@ -140,9 +140,27 @@ paths:
           $ref: "#/components/responses/207BatchValidateCaseResponse"
         "500":
           $ref: "#/components/responses/500"
+  /cases/batchUpsert:
+    post:
+      summary: Batch upserts cases
+      operationId: batchUpsertCases
+      requestBody:
+        description: Cases to upsert
+        required: true
+        content:
+          application/json:
+            schema:
+                $ref: "#/components/schemas/CaseArray"
+      responses:
+        "207":
+          $ref: "#/components/responses/207BatchUpsertCaseResponse"
+        "422":
+          $ref: "#/components/responses/422"
+        "500":
+          $ref: "#/components/responses/500"
   /cases/symptoms:
     get:
-      summary: Lists most frequently used sypmtoms
+      summary: Lists most frequently used symptoms
       operationId: listSymptoms
       parameters:
         - name: limit
@@ -253,6 +271,20 @@ components:
               - message
       required:
         - errors
+    BatchUpsertCaseResponse:
+      description: Response to batch upsert case API requests
+      properties:
+        createdCaseIds:
+          type: array
+          items:
+            type: string
+        updatedCaseIds:
+          type: array
+          items:
+            type: string
+      required:
+        - createdCaseIds
+        - updatedCaseIds
     Case:
       description: A single line-list case.
       properties:
@@ -541,6 +573,12 @@ components:
         application/json:
           schema:
             $ref: '#/components/schemas/BatchValidateCaseResponse'
+    "207BatchUpsertCaseResponse":
+      description: Multi-status
+      content:
+        application/json:
+          schema:
+            $ref: '#/components/schemas/BatchUpsertCaseResponse'
     "400":
       description: Malformed request
     "403":

data-serving/data-service/src/controllers/case.ts

@@ -140,6 +140,114 @@ export const batchValidate = async (
     }
 };
 
+/**
+ * Find IDs of existing cases that have {caseReference.sourceId,
+ * caseReference.sourceEntryId} combinations matching any cases in the provided
+ * request.
+ *
+ * This is used in batchUpsert. Background:
+ *
+ *   While MongoDB does return IDs of created documents, it doesn't do so
+ *   for modified documents (e.g. cases updated via upsert calls). In
+ *   order to (necessarily) provide that information, we'll query existing
+ *   cases, filtering on provided case reference data, in order to provide
+ *   an accurate list of updated case IDs.
+ */
+const findCaseIdsWithCaseReferenceData = async (
+    req: Request,
+): Promise<string[]> => {
+    const providedCaseReferenceData = req.body.cases
+        .filter(
+            // Case data should be validated prior to this point.
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            (c: any) =>
+                c.caseReference?.sourceId && c.caseReference?.sourceEntryId,
+        )
+        .map((c: any) => {
+            return {
+                'caseReference.sourceId': c.caseReference.sourceId,
+                'caseReference.sourceEntryId': c.caseReference.sourceEntryId,
+            };
+        });
+    const result =
+        providedCaseReferenceData.length > 0
+            ? await Case.find()
+                  .select('_id')
+                  .or(providedCaseReferenceData)
+                  .lean()
+                  .exec()
+            : [];
+    return result.map((res) => String(res['_id']));
+};
+
+/**
+ * Batch upserts cases.
+ *
+ * Handles HTTP POST /api/cases/batchUpsert.
+ *
+ * Note that this method is _not_ atomic, and that validation _should_ be
+ * performed prior to invocation. Upserted cases are not validated, and while
+ * any validation issues for created cases will cause the API to return 422,
+ * all provided cases without validation errors will be written.
+ *
+ * TODO: Wrap batchValidate in this method.
+ */
+export const batchUpsert = async (
+    req: Request,
+    res: Response,
+): Promise<void> => {
+    try {
+        const toBeUpsertedCaseIds = await findCaseIdsWithCaseReferenceData(req);
+        const bulkWriteResult = await Case.bulkWrite(
+            // Case data should be validated prior to this point.
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            req.body.cases.map((c: any) => {
+                if (
+                    c.caseReference?.sourceId &&
+                    c.caseReference?.sourceEntryId
+                ) {
+                    return {
+                        updateOne: {
+                            filter: {
+                                'caseReference.sourceId':
+                                    c.caseReference.sourceId,
+                                'caseReference.sourceEntryId':
+                                    c.caseReference.sourceEntryId,
+                            },
+                            update: { $set: { c } },
+                            upsert: true,
+                        },
+                    };
+                } else {
+                    return {
+                        insertOne: {
+                            document: c,
+                        },
+                    };
+                }
+            }),
+        );
+        res.status(207).json({
+            // Types are a little goofy here. We're grabbing the string ID from
+            // what MongoDB returns, which is data in the form of:
+            //   { index0 (string): _id0 (ObjectId), ..., indexN: _idN}
+            createdCaseIds: Object.entries(bulkWriteResult.insertedIds)
+                .concat(Object.entries(bulkWriteResult.upsertedIds))
+                .map((kv) => String(kv[1])),
+            updatedCaseIds: toBeUpsertedCaseIds,
+        });
+        return;
+    } catch (err) {
+        if (err.name === 'ValidationError') {
+            res.status(422).json(err.message);
+            return;
+        }
+        console.warn(err);
+        res.status(500).json(err.message);
+        return;
+    }
+};
+
 /**
  * Update a specific case.
  *

data-serving/data-service/src/index.ts

@@ -62,6 +62,7 @@ new OpenApiValidator({
         apiRouter.get('/cases/symptoms', caseController.listSymptoms);
         apiRouter.post('/cases', setRevisionMetadata, caseController.create);
         apiRouter.post('/cases/batchValidate', caseController.batchValidate);
+        apiRouter.post('/cases/batchUpsert', caseController.batchUpsert);
         apiRouter.put('/cases', setRevisionMetadata, caseController.upsert);
         apiRouter.put(
             '/cases/:id([a-z0-9]{24})',

data-serving/data-service/test/controllers/case.test.ts

@@ -1,6 +1,7 @@
 import { Case } from '../../src/model/case';
 import { MongoMemoryServer } from 'mongodb-memory-server';
 import app from './../../src/index';
+import fullCase from './../model/data/case.full.json';
 import minimalCase from './../model/data/case.minimal.json';
 import mongoose from 'mongoose';
 import request from 'supertest';
@@ -229,6 +230,40 @@ describe('POST', () => {
         expect(await Case.collection.countDocuments()).toEqual(0);
         expect(res.body._id).not.toHaveLength(0);
     });
+    it('batch upsert with no body should return 415', () => {
+        return request(app).post('/api/cases/batchUpsert').expect(415);
+    });
+    it('batch upsert with no cases should return 400', () => {
+        return request(app).post('/api/cases/batchUpsert').send({}).expect(400);
+    });
+    it('batch upsert with only valid cases should return 207 with IDs', async () => {
+        const newCaseWithoutEntryId = new Case(minimalCase);
+        const newCaseWithEntryId = new Case(fullCase);
+        newCaseWithEntryId.caseReference.sourceEntryId = 'newId';
+
+        const existingCaseWithEntryId = new Case(fullCase);
+        await existingCaseWithEntryId.save();
+        existingCaseWithEntryId.notes = 'new notes';
+
+        const res = await request(app)
+            .post('/api/cases/batchUpsert')
+            .send({
+                cases: [
+                    newCaseWithoutEntryId,
+                    newCaseWithEntryId,
+                    existingCaseWithEntryId,
+                ],
+            })
+            .expect(207);
+        expect(res.body.createdCaseIds).toHaveLength(2);
+        expect(res.body.updatedCaseIds).toHaveLength(1);
+    });
+    it('batch upsert with any invalid case should return 422', async () => {
+        await request(app)
+            .post('/api/cases/batchUpsert')
+            .send({ cases: [minimalCase, invalidRequest] })
+            .expect(422);
+    });
     it('batch validate with no body should return 415', () => {
         return request(app).post('/api/cases/batchValidate').expect(415);
     });