Merge pull request #4099 from beyondessential/dev

Sima create instance
beyondessential · Aug 19, 2022 · 8308b06 · 8308b06
2 parents c2fd4de + 6881ba5
commit 8308b06
Show file tree

Hide file tree

Showing 142 changed files with 4,605 additions and 1,242 deletions.
diff --git a/package.json b/package.json
@@ -51,9 +51,11 @@
   "devDependencies": {
     "@babel/cli": "^7.11.6",
     "@babel/node": "^7.10.5",
+    "@babel/plugin-transform-runtime": "^7.18.10",
     "@beyondessential/eslint-config-jest": "^1.0.0",
     "@beyondessential/eslint-config-js": "^1.1.1",
     "@beyondessential/eslint-config-ts": "^1.3.2",
+    "babel-eslint": "^10.1.0",
     "babel-jest": "^27.0.6",
     "concurrently": "^5.2.0",
     "cypress-dotenv": "^1.2.2",

diff --git a/packages/admin-panel-server/package.json b/packages/admin-panel-server/package.json
@@ -19,7 +19,6 @@
     "build-dev": "npm run build",
     "lint": "yarn package:lint:ts",
     "lint:fix": "yarn lint --fix",
-    "prestart": "npm run build",
     "start": "node dist",
     "start-dev": "yarn package:start:backend-start-dev 9994 -ts",
     "test": "yarn package:test",

diff --git a/packages/admin-panel/src/pages/resources/DataSourcesPage.js b/packages/admin-panel/src/pages/resources/DataSourcesPage.js
@@ -26,7 +26,7 @@ const DataSourceConfigView = row => {
     .map(([key, value]) => (
       <React.Fragment key={key}>
         <dt style={localStyles.config.dt}>{key}:</dt>
-        <dd>{value.toString()}</dd>
+        <dd>{value ? value.toString() : '""'}</dd>
       </React.Fragment>
     ));
 
@@ -77,9 +77,11 @@ const DATA_ELEMENT_FIELDS = [
       default: '{}',
       getJsonFieldSchema: () => [
         {
-          label: 'Regional Server (Choose "No" if stored on country specific server)',
-          fieldName: 'isDataRegional',
-          type: 'boolean',
+          label: 'DHIS Server',
+          fieldName: 'dhisInstanceCode',
+          optionsEndpoint: 'dhisInstances',
+          optionLabelKey: 'dhisInstances.code',
+          optionValueKey: 'dhisInstances.code',
         },
         {
           label: 'Data element code',
@@ -112,9 +114,11 @@ const DATA_GROUP_FIELDS = [
       default: '{}',
       getJsonFieldSchema: () => [
         {
-          label: 'Regional Server (Choose "No" if stored on country specific server)',
-          fieldName: 'isDataRegional',
-          type: 'boolean',
+          label: 'DHIS Server',
+          fieldName: 'dhisInstanceCode',
+          optionsEndpoint: 'dhisInstances',
+          optionLabelKey: 'dhisInstances.code',
+          optionValueKey: 'dhisInstances.code',
         },
       ],
     },

diff --git a/packages/admin-panel/src/pages/resources/SurveysPage.js b/packages/admin-panel/src/pages/resources/SurveysPage.js
@@ -94,8 +94,8 @@ const SURVEY_COLUMNS = [
           editConfig: {
             options: SERVICE_TYPES,
             setFieldsOnChange: (newValue, currentRecord) => {
-              const { isDataRegional = true } = currentRecord['data_group.config'];
-              const config = newValue === 'dhis' ? { isDataRegional } : {};
+              const { dhisInstanceCode = 'regional' } = currentRecord['data_group.config'];
+              const config = newValue === 'dhis' ? { dhisInstanceCode } : {};
               return { 'data_group.config': config };
             },
           },
@@ -109,10 +109,11 @@ const SURVEY_COLUMNS = [
               recordData['data_group.service_type'] === 'dhis'
                 ? [
                     {
-                      label:
-                        'Stored On Regional Server (Choose "No" if stored on country specific server)',
-                      fieldName: 'isDataRegional',
-                      type: 'boolean',
+                      label: 'DHIS Server',
+                      fieldName: 'dhisInstanceCode',
+                      optionsEndpoint: 'dhisInstances',
+                      optionLabelKey: 'dhisInstances.code',
+                      optionValueKey: 'dhisInstances.code',
                     },
                   ]
                 : [],

diff --git a/packages/central-server/package.json b/packages/central-server/package.json
@@ -18,12 +18,12 @@
     "build-dev": "npm run build",
     "lint": "yarn package:lint:js",
     "lint:fix": "yarn lint --fix",
-    "prestart": "npm run build",
     "start": "node dist",
     "start-dev": "yarn package:start:backend-start-dev 9999",
     "start-verbose": "LOG_LEVEL=debug yarn start-dev",
     "test": "yarn workspace @tupaia/database check-test-database-exists && DB_NAME=tupaia_test NODE_ENV=test mocha",
-    "test:coverage": "cross-env NODE_ENV=test nyc mocha"
+    "test:coverage": "cross-env NODE_ENV=test nyc mocha",
+    "create-meditrak-sync-view": "node dist/createMeditrakSyncView.js"
   },
   "dependencies": {
     "@babel/polyfill": "^7.0.0",

diff --git a/packages/central-server/src/apiV2/changesMetadata.js b/packages/central-server/src/apiV2/changesMetadata.js
@@ -0,0 +1,30 @@
+/**
+ * Tupaia
+ * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd
+ */
+
+import { respond } from '@tupaia/utils';
+import { buildPermissionsBasedMeditrakSyncQuery } from './utilities';
+import { allowNoPermissions } from '../permissions';
+
+/**
+ * Permissions based sync metadata
+ * {
+ *   changeCount: number of changes since last sync
+ *   countries: countries included in the sync
+ *   permissionGroups: permissions groups included in the sync
+ * }
+ * Responds to GET requests to the /changes/metadata endpoint
+ */
+export async function changesMetadata(req, res) {
+  const { models } = req;
+
+  await req.assertPermissions(allowNoPermissions);
+
+  const { query, countries, permissionGroups } = await buildPermissionsBasedMeditrakSyncQuery(req, {
+    select: 'count(*)',
+  });
+  const queryResult = await query.executeOnDatabase(models.database);
+  const changeCount = parseInt(queryResult[0].count);
+  respond(res, { changeCount, countries, permissionGroups });
+}
diff --git a/packages/central-server/src/apiV2/countChanges/LegacyCountChangesHandler.js b/packages/central-server/src/apiV2/countChanges/LegacyCountChangesHandler.js
@@ -5,7 +5,7 @@
 
 import { getKeysSortedByValues, respond, UnauthenticatedError } from '@tupaia/utils';
 import { getUniversalTypes } from '../../database/utilities';
-import { fetchRequestingMeditrakDevice, getChangesFilter } from '../utilities';
+import { fetchRequestingMeditrakDevice, buildMeditrakSyncQuery } from '../utilities';
 
 const MAX_FAILS_BEFORE_LOG_OUT = 2;
 const MAX_FAILS_BEFORE_TYPE_EXCLUSION = 5;
@@ -135,11 +135,16 @@ export class LegacyCountChangesHandler {
     const { models, query } = this.req;
 
     const universalTypes = getUniversalTypes(models);
-    const filter = await getChangesFilter({
-      ...this.req,
-      query: { ...query, recordTypes: universalTypes.join(',') },
-    });
-    const changesCount = await models.meditrakSyncQueue.count(filter);
+
+    const { query: dbQuery } = await buildMeditrakSyncQuery(
+      {
+        ...this.req,
+        query: { ...query, recordTypes: universalTypes.join(',') },
+      },
+      { select: 'count(*)' },
+    );
+    const result = await dbQuery.executeOnDatabase(models.database);
+    const changesCount = parseInt(result[0].count);
 
     return changesCount > 0;
   }
@@ -163,8 +168,9 @@ export class LegacyCountChangesHandler {
 
   async handle() {
     await this.handleSetUp();
-    const filter = await getChangesFilter(this.req);
-    const changeCount = await this.req.models.meditrakSyncQueue.count(filter);
+    const { query } = await buildMeditrakSyncQuery(this.req, { select: 'count(*)' });
+    const queryResult = await query.executeOnDatabase(this.req.models.database);
+    const changeCount = parseInt(queryResult[0].count);
     respond(this.res, { changeCount });
   }
 }
diff --git a/packages/central-server/src/apiV2/countChanges/countChanges.js b/packages/central-server/src/apiV2/countChanges/countChanges.js
@@ -4,15 +4,15 @@
  */
 
 import { respond, DatabaseError, UnauthenticatedError } from '@tupaia/utils';
-import { getChangesFilter } from '../utilities';
+import { buildMeditrakSyncQuery } from '../utilities';
 import { LegacyCountChangesHandler } from './LegacyCountChangesHandler';
 import { allowNoPermissions } from '../../permissions';
 
 const handleNonLegacyRequest = async (req, res) => {
   const { models } = req;
-
-  const filter = await getChangesFilter(req);
-  const changeCount = await models.meditrakSyncQueue.count(filter);
+  const { query } = await buildMeditrakSyncQuery(req, { select: 'count(*)' });
+  const queryResult = await query.executeOnDatabase(models.database);
+  const changeCount = parseInt(queryResult[0].count);
   respond(res, { changeCount });
 };
 

diff --git a/packages/central-server/src/apiV2/getChanges.js b/packages/central-server/src/apiV2/getChanges.js
@@ -3,17 +3,24 @@
  * Copyright (c) 2017 Beyond Essential Systems Pty Ltd
  */
 
+import keyBy from 'lodash.keyby';
+import groupBy from 'lodash.groupby';
 import { respond, DatabaseError } from '@tupaia/utils';
 import { TYPES } from '@tupaia/database';
-import { getChangesFilter, getColumnsForMeditrakApp } from './utilities';
+import {
+  supportsPermissionsBasedSync,
+  buildMeditrakSyncQuery,
+  buildPermissionsBasedMeditrakSyncQuery,
+  getColumnsForMeditrakApp,
+} from './utilities';
 import { allowNoPermissions } from '../permissions';
 
 const MAX_CHANGES_RETURNED = 100;
 
 /**
  * Gets the record ready to sync down to a sync client, transforming any properties as required
  */
-async function getRecordForSync(record) {
+function getRecordForSync(record) {
   const recordWithoutNulls = {};
   // Remove null entries to a) save bandwidth and b) remain consistent with previous mongo based db
   // which simply had no key for undefined properties, whereas postgres uses null
@@ -34,45 +41,67 @@ async function getRecordForSync(record) {
  */
 export async function getChanges(req, res) {
   const { database, models } = req;
-  const { limit = MAX_CHANGES_RETURNED, offset = 0 } = req.query;
+  const { limit = MAX_CHANGES_RETURNED, offset = 0, appVersion } = req.query;
 
   await req.assertPermissions(allowNoPermissions);
 
   try {
-    const filter = await getChangesFilter(req);
-    const changes = await database.find(TYPES.MEDITRAK_SYNC_QUEUE, filter, {
-      sort: ['change_time'],
+    const queryBuilder = supportsPermissionsBasedSync(appVersion)
+      ? buildPermissionsBasedMeditrakSyncQuery
+      : buildMeditrakSyncQuery;
+    const { query } = await queryBuilder(req, {
+      select: (await models.meditrakSyncQueue.fetchFieldNames()).join(', '),
+      sort: 'change_time ASC',
       limit,
       offset,
     });
-    const changesToSend = await Promise.all(
-      changes.map(async change => {
-        const {
-          type: action,
-          record_type: recordType,
-          record_id: recordId,
-          change_time: timestamp,
-        } = change;
-        const columns = await getColumnsForMeditrakApp(models.getModelForDatabaseType(recordType));
-        const changeObject = { action, recordType, timestamp };
-        if (action === 'delete') {
-          changeObject.record = { id: recordId };
-          if (recordType === TYPES.GEOGRAPHICAL_AREA) {
-            // TODO LEGACY Deal with this bug on app end for v3 api
-            changeObject.recordType = 'area';
-          }
+    const changes = await query.executeOnDatabase(database);
+    const changesByRecordType = groupBy(changes, 'record_type');
+    const recordTypesToSync = Object.keys(changesByRecordType);
+    const columnNamesByRecordType = Object.fromEntries(
+      await Promise.all(
+        recordTypesToSync.map(async recordType => [
+          recordType,
+          await getColumnsForMeditrakApp(models.getModelForDatabaseType(recordType)),
+        ]),
+      ),
+    );
+    const changeRecords = (
+      await Promise.all(
+        Object.entries(changesByRecordType).map(async ([recordType, changesForType]) => {
+          const changeIds = changesForType.map(change => change.record_id);
+          const columns = columnNamesByRecordType[recordType];
+          return database.find(recordType, { id: changeIds }, { lean: true, columns });
+        }),
+      )
+    ).flat();
+    const changeRecordsById = keyBy(changeRecords, 'id');
+
+    const changesToSend = changes.map(change => {
+      const {
+        type: action,
+        record_type: recordType,
+        record_id: recordId,
+        change_time: timestamp,
+      } = change;
+      const changeObject = { action, recordType, timestamp };
+      if (action === 'delete') {
+        changeObject.record = { id: recordId };
+        if (recordType === TYPES.GEOGRAPHICAL_AREA) {
+          // TODO LEGACY Deal with this bug on app end for v3 api
+          changeObject.recordType = 'area';
+        }
+      } else {
+        const record = changeRecordsById[recordId];
+        if (!record) {
+          const errorMessage = `Couldn't find record type ${recordType} with id ${recordId}`;
+          changeObject.error = { error: errorMessage };
         } else {
-          const record = await database.findById(recordType, recordId, { lean: true, columns });
-          if (!record) {
-            const errorMessage = `Couldn't find record type ${recordType} with id ${recordId}`;
-            changeObject.error = { error: errorMessage };
-          } else {
-            changeObject.record = await getRecordForSync(record);
-          }
+          changeObject.record = getRecordForSync(record);
         }
-        return changeObject;
-      }),
-    );
+      }
+      return changeObject;
+    });
     respond(res, changesToSend);
     return;
   } catch (error) {

diff --git a/packages/central-server/src/apiV2/import/importEntities/getOrCreateParentEntity.js b/packages/central-server/src/apiV2/import/importEntities/getOrCreateParentEntity.js
@@ -64,7 +64,7 @@ export async function getOrCreateParentEntity(
     };
     district = await transactingModels.geographicalArea.findOrCreate(districtObject);
     const defaultMetadata = {
-      dhis: { isDataRegional: true },
+      dhis: { dhisInstanceCode: 'regional' },
       openStreetMaps: { id: districtOsmId },
     };
     const districtEntityMetadata = await getEntityMetadata(
@@ -96,7 +96,7 @@ export async function getOrCreateParentEntity(
       level_name: 'Sub District',
     };
     const defaultMetadata = {
-      dhis: { isDataRegional: true },
+      dhis: { dhisInstanceCode: 'regional' },
       openStreetMaps: { id: subDistrictOsmId },
     };
     const subDistrictEntityMetadata = await getEntityMetadata(

diff --git a/packages/central-server/src/apiV2/import/importEntities/updateCountryEntities.js b/packages/central-server/src/apiV2/import/importEntities/updateCountryEntities.js
@@ -84,7 +84,7 @@ export async function updateCountryEntities(
     type: transactingModels.entity.types.WORLD,
   });
   const defaultMetadata = {
-    dhis: { isDataRegional: true },
+    dhis: { dhisInstanceCode: 'regional' },
   };
   const countryEntityMetadata = await getEntityMetadata(
     transactingModels,

diff --git a/packages/central-server/src/apiV2/index.js b/packages/central-server/src/apiV2/index.js
@@ -18,6 +18,7 @@ import { useRouteHandler } from './RouteHandler';
 import { exportRoutes } from './export';
 import { importRoutes } from './import';
 import { authenticate } from './authenticate';
+import { changesMetadata } from './changesMetadata';
 import { countChanges } from './countChanges';
 import { getChanges } from './getChanges';
 import { BESAdminCreateHandler } from './CreateHandler';
@@ -144,6 +145,7 @@ apiV2.use('/import', importRoutes);
  * GET routes
  */
 apiV2.get('/changes/count', catchAsyncErrors(countChanges));
+apiV2.get('/changes/metadata', catchAsyncErrors(changesMetadata));
 apiV2.get('/changes', catchAsyncErrors(getChanges));
 apiV2.get('/socialFeed', catchAsyncErrors(getSocialFeed));
 apiV2.get('/me', useRouteHandler(GETUserForMe));
@@ -212,6 +214,7 @@ apiV2.get('/clinics/:recordId?', useRouteHandler(GETClinics));
 apiV2.get('/facilities/:recordId?', useRouteHandler(GETClinics));
 apiV2.get('/geographicalAreas/:recordId?', useRouteHandler(GETGeographicalAreas));
 apiV2.get('/reports/:recordId?', useRouteHandler(GETReports));
+apiV2.get('/dhisInstances/:recordId?', useRouteHandler(BESAdminGETHandler));
 
 /**
  * POST routes