From 812286c1266b96cd7429bcbc3a2bbeea79a7f13c Mon Sep 17 00:00:00 2001 From: Rohan Port <59544282+rohan-bes@users.noreply.github.com> Date: Tue, 23 Aug 2022 13:55:17 +1000 Subject: [PATCH 01/14] RN-623: Hide dot when exporting line charts (#4107) --- packages/ui-components/src/components/Chart/LineChart.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/ui-components/src/components/Chart/LineChart.js b/packages/ui-components/src/components/Chart/LineChart.js index 32b31e27f6..8099cfaa93 100644 --- a/packages/ui-components/src/components/Chart/LineChart.js +++ b/packages/ui-components/src/components/Chart/LineChart.js @@ -23,6 +23,7 @@ export const LineChart = ({ dot, }) => { const defaultColor = isExporting ? DARK_BLUE : BLUE; + const showDot = isExporting ? false : dot; // Always hide when exporting as it doesn't look nice return ( {isExporting && exportWithLabels && ( Date: Tue, 23 Aug 2022 17:45:14 +1000 Subject: [PATCH 02/14] RN-423: Add 'Sync Groups' to the Admin Panel (kobo sync link part 1) (#4088) * Refactor: Move survey code in sync group table * Refactor: rename action for clarity * Minor: remove obsolete code * Add CRUD for data service data groups --- packages/admin-panel/src/editor/actions.js | 6 +- packages/admin-panel/src/editor/constants.js | 2 +- packages/admin-panel/src/editor/reducer.js | 4 +- .../src/pages/resources/IndicatorsPage.js | 11 - .../src/pages/resources/SyncGroupsPage.js | 81 ++++++++ .../admin-panel/src/pages/resources/index.js | 1 + packages/admin-panel/src/routes.js | 6 + packages/central-server/src/apiV2/index.js | 5 + .../src/apiV2/syncGroups/CreateSyncGroups.js | 26 +++ .../src/apiV2/syncGroups/DeleteSyncGroups.js | 17 ++ .../src/apiV2/syncGroups/EditSyncGroups.js | 21 ++ .../src/apiV2/syncGroups/GETSyncGroups.js | 20 ++ .../syncGroups/assertSyncGroupPermissions.js | 75 +++++++ .../src/apiV2/syncGroups/index.js | 9 + .../constructNewRecordValidationRules.js | 6 + .../central-server/src/kobo/manualKoBoSync.js | 4 +- .../src/kobo/startSyncWithKoBo.js | 190 +++++++++++------- .../services/kobo/KoBoService.fixtures.js | 3 +- .../src/services/kobo/KoBoService.js | 10 +- .../src/services/kobo/KoBoTranslator.js | 2 - packages/data-broker/src/utils.js | 2 +- ...LinkSyncServiceToSurvey-modifies-schema.js | 50 +++++ ...eToDataServiceSyncGroup-modifies-schema.js | 51 +++++ .../src/modelClasses/DataServiceSyncGroup.js | 18 ++ .../{SyncServiceLog.js => SyncGroupLog.js} | 8 +- .../database/src/modelClasses/SyncService.js | 30 --- packages/database/src/modelClasses/index.js | 6 +- packages/database/src/runPostMigration.js | 1 + packages/database/src/types.js | 3 +- .../src/validation/validatorFunctions.js | 4 +- 30 files changed, 526 insertions(+), 146 deletions(-) create mode 100644 packages/admin-panel/src/pages/resources/SyncGroupsPage.js create mode 100644 packages/central-server/src/apiV2/syncGroups/CreateSyncGroups.js create mode 100644 packages/central-server/src/apiV2/syncGroups/DeleteSyncGroups.js create mode 100644 packages/central-server/src/apiV2/syncGroups/EditSyncGroups.js create mode 100644 packages/central-server/src/apiV2/syncGroups/GETSyncGroups.js create mode 100644 packages/central-server/src/apiV2/syncGroups/assertSyncGroupPermissions.js create mode 100644 packages/central-server/src/apiV2/syncGroups/index.js create mode 100644 packages/database/src/migrations/20220630013151-LinkSyncServiceToSurvey-modifies-schema.js create mode 100644 packages/database/src/migrations/20220812055419-MoveSyncServiceToDataServiceSyncGroup-modifies-schema.js rename packages/database/src/modelClasses/{SyncServiceLog.js => SyncGroupLog.js} (57%) delete mode 100644 packages/database/src/modelClasses/SyncService.js diff --git a/packages/admin-panel/src/editor/actions.js b/packages/admin-panel/src/editor/actions.js index 7e6a01e392..527aaa818a 100644 --- a/packages/admin-panel/src/editor/actions.js +++ b/packages/admin-panel/src/editor/actions.js @@ -11,7 +11,7 @@ import { EDITOR_DISMISS, EDITOR_ERROR, EDITOR_FIELD_EDIT, - EDITOR_OPEN_CREATOR, + EDITOR_OPEN, } from './constants'; import { convertSearchTermToFilter, makeSubstitutionsInString } from '../utilities'; @@ -68,7 +68,7 @@ export const openBulkEditModal = ( }); dispatch({ - type: EDITOR_OPEN_CREATOR, + type: EDITOR_OPEN, fields, recordData: {}, endpoint: bulkUpdateEndpoint, @@ -126,7 +126,7 @@ export const openEditModal = ({ editEndpoint, fields }, recordId) => async ( }); dispatch({ - type: EDITOR_OPEN_CREATOR, + type: EDITOR_OPEN, fields, recordData: {}, endpoint: editEndpoint, diff --git a/packages/admin-panel/src/editor/constants.js b/packages/admin-panel/src/editor/constants.js index c098c25522..1e7c5732d2 100644 --- a/packages/admin-panel/src/editor/constants.js +++ b/packages/admin-panel/src/editor/constants.js @@ -10,7 +10,7 @@ export const EDITOR_DATA_EDIT_SUCCESS = 'EDITOR_DATA_EDIT_SUCCESS'; export const EDITOR_DISMISS = 'EDITOR_DISMISS'; export const EDITOR_ERROR = 'EDITOR_ERROR'; export const EDITOR_FIELD_EDIT = 'EDITOR_FIELD_EDIT'; -export const EDITOR_OPEN_CREATOR = 'EDITOR_OPEN_CREATOR'; +export const EDITOR_OPEN = 'EDITOR_OPEN'; export const DATA_CHANGE_ACTIONS = { start: EDITOR_DATA_EDIT_BEGIN, diff --git a/packages/admin-panel/src/editor/reducer.js b/packages/admin-panel/src/editor/reducer.js index 769a2668e0..1f42d5a969 100644 --- a/packages/admin-panel/src/editor/reducer.js +++ b/packages/admin-panel/src/editor/reducer.js @@ -12,7 +12,7 @@ import { EDITOR_DISMISS, EDITOR_ERROR, EDITOR_FIELD_EDIT, - EDITOR_OPEN_CREATOR, + EDITOR_OPEN, } from './constants'; const defaultState = { @@ -52,7 +52,7 @@ const stateChanges = { } return defaultState; // If no error, dismiss the whole modal and clear its state }, - [EDITOR_OPEN_CREATOR]: payload => payload, + [EDITOR_OPEN]: payload => payload, [EDITOR_FIELD_EDIT]: ({ fieldKey, newValue }, { editedFields }) => ({ editedFields: { ...editedFields, diff --git a/packages/admin-panel/src/pages/resources/IndicatorsPage.js b/packages/admin-panel/src/pages/resources/IndicatorsPage.js index 0005401885..25c891b4b3 100644 --- a/packages/admin-panel/src/pages/resources/IndicatorsPage.js +++ b/packages/admin-panel/src/pages/resources/IndicatorsPage.js @@ -31,17 +31,6 @@ const FIELDS = [ editConfig: { type: 'jsonEditor', default: '{ "formula": "", "aggregation": { "" : "" } }', - getJsonFieldSchema: () => [ - { - label: 'Formula', - fieldName: 'formula', - }, - { - label: 'Aggregation', - fieldName: 'aggregation', - type: 'object', - }, - ], }, }, ]; diff --git a/packages/admin-panel/src/pages/resources/SyncGroupsPage.js b/packages/admin-panel/src/pages/resources/SyncGroupsPage.js new file mode 100644 index 0000000000..5b6de30d0d --- /dev/null +++ b/packages/admin-panel/src/pages/resources/SyncGroupsPage.js @@ -0,0 +1,81 @@ +/** + * Tupaia MediTrak + * Copyright (c) 2017 Beyond Essential Systems Pty Ltd + */ + +import React from 'react'; +import PropTypes from 'prop-types'; +import { ResourcePage } from './ResourcePage'; + +const SERVICE_TYPES = [{ label: 'Kobo', value: 'kobo' }]; + +const FIELDS = [ + { + Header: 'Survey Code', + source: 'data_group_code', + }, + { + Header: 'Service Type', + source: 'service_type', + editConfig: { + options: SERVICE_TYPES, + }, + }, + { + Header: 'Config', + source: 'config', + type: 'jsonTooltip', + editConfig: { + type: 'jsonEditor', + default: '{}', + }, + }, +]; + +const COLUMNS = [ + ...FIELDS, + { + Header: 'Edit', + type: 'edit', + source: 'id', + actionConfig: { + editEndpoint: 'dataServiceSyncGroups', + fields: [...FIELDS], + }, + }, + { + Header: 'Delete', + source: 'id', + type: 'delete', + actionConfig: { + endpoint: 'dataServiceSyncGroups', + }, + }, +]; + +const EDIT_CONFIG = { + title: 'Edit Sync Group', +}; + +const CREATE_CONFIG = { + title: 'Add Sync Group', + actionConfig: { + editEndpoint: 'dataServiceSyncGroups', + fields: FIELDS, + }, +}; + +export const SyncGroupsPage = ({ getHeaderEl }) => ( + +); + +SyncGroupsPage.propTypes = { + getHeaderEl: PropTypes.func.isRequired, +}; diff --git a/packages/admin-panel/src/pages/resources/index.js b/packages/admin-panel/src/pages/resources/index.js index 11b06682f1..15705a65e4 100644 --- a/packages/admin-panel/src/pages/resources/index.js +++ b/packages/admin-panel/src/pages/resources/index.js @@ -30,3 +30,4 @@ export { DashboardItemsPage } from './DashboardItemsPage'; export { DashboardRelationsPage } from './DashboardRelationsPage'; export { LegacyReportsPage } from './LegacyReportsPage'; export { ProjectsPage } from './ProjectsPage'; +export { SyncGroupsPage } from './SyncGroupsPage'; diff --git a/packages/admin-panel/src/routes.js b/packages/admin-panel/src/routes.js index 541969ff36..5022f693c6 100644 --- a/packages/admin-panel/src/routes.js +++ b/packages/admin-panel/src/routes.js @@ -30,6 +30,7 @@ import { DataElementsPage, DataGroupsPage, ProjectsPage, + SyncGroupsPage, } from './pages/resources'; export const ROUTES = [ @@ -68,6 +69,11 @@ export const ROUTES = [ to: '/survey-responses', component: SurveyResponsesPage, }, + { + label: 'Sync Groups', + to: '/sync-groups', + component: SyncGroupsPage, + }, ], }, { diff --git a/packages/central-server/src/apiV2/index.js b/packages/central-server/src/apiV2/index.js index 8c09edb3ca..282dfaddf6 100644 --- a/packages/central-server/src/apiV2/index.js +++ b/packages/central-server/src/apiV2/index.js @@ -107,6 +107,7 @@ import { EditMapOverlayVisualisation, GETMapOverlayVisualisations, } from './mapOverlayVisualisations'; +import { GETSyncGroups, EditSyncGroups, CreateSyncGroups, DeleteSyncGroups } from './syncGroups'; // quick and dirty permission wrapper for open endpoints const allowAnyone = routeHandler => (req, res, next) => { @@ -212,6 +213,7 @@ apiV2.get('/facilities/:recordId?', useRouteHandler(GETClinics)); apiV2.get('/geographicalAreas/:recordId?', useRouteHandler(GETGeographicalAreas)); apiV2.get('/reports/:recordId?', useRouteHandler(GETReports)); apiV2.get('/dhisInstances/:recordId?', useRouteHandler(BESAdminGETHandler)); +apiV2.get('/dataServiceSyncGroups/:recordId?', useRouteHandler(GETSyncGroups)); /** * POST routes @@ -243,6 +245,7 @@ apiV2.post('/dashboardVisualisations', useRouteHandler(CreateDashboardVisualisat apiV2.post('/mapOverlayVisualisations', useRouteHandler(CreateMapOverlayVisualisation)); apiV2.post('/mapOverlayGroupRelations', useRouteHandler(CreateMapOverlayGroupRelation)); apiV2.post('/syncFromService', allowAnyone(manualKoBoSync)); +apiV2.post('/dataServiceSyncGroups', useRouteHandler(CreateSyncGroups)); /** * PUT routes @@ -274,6 +277,7 @@ apiV2.put('/mapOverlayGroupRelations/:recordId', useRouteHandler(EditMapOverlayG apiV2.put('/indicators/:recordId', useRouteHandler(BESAdminEditHandler)); apiV2.put('/projects/:recordId', useRouteHandler(BESAdminEditHandler)); apiV2.put('/me', catchAsyncErrors(editUser)); +apiV2.put('/dataServiceSyncGroups/:recordId', useRouteHandler(EditSyncGroups)); /** * DELETE routes @@ -302,6 +306,7 @@ apiV2.delete( useRouteHandler(DeleteMapOverlayGroupRelations), ); apiV2.delete('/indicators/:recordId', useRouteHandler(BESAdminDeleteHandler)); +apiV2.delete('/dataServiceSyncGroups/:recordId', useRouteHandler(DeleteSyncGroups)); apiV2.use(handleError); // error handler must come last diff --git a/packages/central-server/src/apiV2/syncGroups/CreateSyncGroups.js b/packages/central-server/src/apiV2/syncGroups/CreateSyncGroups.js new file mode 100644 index 0000000000..da1e28c501 --- /dev/null +++ b/packages/central-server/src/apiV2/syncGroups/CreateSyncGroups.js @@ -0,0 +1,26 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd + */ + +import { CreateHandler } from '../CreateHandler'; +import { + assertAnyPermissions, + assertAdminPanelAccess, + assertBESAdminAccess, +} from '../../permissions'; + +export class CreateSyncGroups extends CreateHandler { + async assertUserHasAccess() { + await this.assertPermissions( + assertAnyPermissions( + [assertBESAdminAccess, assertAdminPanelAccess], + 'You need either BES Admin or Tupaia Admin Panel access to create a Sync Group', + ), + ); + } + + async createRecord() { + return this.insertRecord(); + } +} diff --git a/packages/central-server/src/apiV2/syncGroups/DeleteSyncGroups.js b/packages/central-server/src/apiV2/syncGroups/DeleteSyncGroups.js new file mode 100644 index 0000000000..086919526d --- /dev/null +++ b/packages/central-server/src/apiV2/syncGroups/DeleteSyncGroups.js @@ -0,0 +1,17 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd + */ + +import { DeleteHandler } from '../DeleteHandler'; +import { assertAnyPermissions, assertBESAdminAccess } from '../../permissions'; +import { assertSyncGroupEditPermissions } from './assertSyncGroupPermissions'; + +export class DeleteSyncGroups extends DeleteHandler { + async assertUserHasAccess() { + const syncGroupChecker = accessPolicy => + assertSyncGroupEditPermissions(accessPolicy, this.models, this.recordId); + + await this.assertPermissions(assertAnyPermissions([assertBESAdminAccess, syncGroupChecker])); + } +} diff --git a/packages/central-server/src/apiV2/syncGroups/EditSyncGroups.js b/packages/central-server/src/apiV2/syncGroups/EditSyncGroups.js new file mode 100644 index 0000000000..278e7b2e14 --- /dev/null +++ b/packages/central-server/src/apiV2/syncGroups/EditSyncGroups.js @@ -0,0 +1,21 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd + */ + +import { EditHandler } from '../EditHandler'; +import { assertAnyPermissions, assertBESAdminAccess } from '../../permissions'; +import { assertSyncGroupEditPermissions } from './assertSyncGroupPermissions'; + +export class EditSyncGroups extends EditHandler { + async assertUserHasAccess() { + const syncGroupChecker = accessPolicy => + assertSyncGroupEditPermissions(accessPolicy, this.models, this.recordId); + + await this.assertPermissions(assertAnyPermissions([assertBESAdminAccess, syncGroupChecker])); + } + + async editRecord() { + await this.updateRecord(); + } +} diff --git a/packages/central-server/src/apiV2/syncGroups/GETSyncGroups.js b/packages/central-server/src/apiV2/syncGroups/GETSyncGroups.js new file mode 100644 index 0000000000..abc61245fc --- /dev/null +++ b/packages/central-server/src/apiV2/syncGroups/GETSyncGroups.js @@ -0,0 +1,20 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd + */ + +import { GETHandler } from '../GETHandler'; +import { assertAdminPanelAccess } from '../../permissions'; +import { createSyncGroupDBFilter } from './assertSyncGroupPermissions'; + +export class GETSyncGroups extends GETHandler { + permissionsFilteredInternally = true; + + async assertUserHasAccess() { + await this.assertPermissions(assertAdminPanelAccess); + } + + async getPermissionsFilter(criteria, options) { + return createSyncGroupDBFilter(this.accessPolicy, this.models, criteria, options); + } +} diff --git a/packages/central-server/src/apiV2/syncGroups/assertSyncGroupPermissions.js b/packages/central-server/src/apiV2/syncGroups/assertSyncGroupPermissions.js new file mode 100644 index 0000000000..bc3f880ed0 --- /dev/null +++ b/packages/central-server/src/apiV2/syncGroups/assertSyncGroupPermissions.js @@ -0,0 +1,75 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd + */ + +import { QUERY_CONJUNCTIONS, TYPES } from '@tupaia/database'; +import { assertSurveyEditPermissions } from '../surveys/assertSurveyPermissions'; +import { hasBESAdminAccess } from '../../permissions'; +import { fetchCountryIdsByPermissionGroupId, mergeMultiJoin } from '../utilities'; + +const { RAW } = QUERY_CONJUNCTIONS; + +export const assertSyncGroupEditPermissions = async (accessPolicy, models, syncGroupId) => { + const syncGroup = await models.dataServiceSyncGroup.findById(syncGroupId); + if (!syncGroup) { + throw new Error(`No Sync Group exists with id ${syncGroupId}`); + } + + const dataGroup = await models.dataGroup.findOne({ code: syncGroup.data_group_code }); + if (!dataGroup) { + throw new Error(`Sync Group is not linked to an existing Data Group`); + } + + const survey = await models.survey.findOne({ data_group_id: dataGroup.id }); + if (!survey) { + throw new Error(`No Survey found for Data Group used by Sync Group`); + } + + return assertSurveyEditPermissions(accessPolicy, models, survey.id); +}; + +export const createSyncGroupDBFilter = async (accessPolicy, models, criteria, options) => { + const dbConditions = { ...criteria }; + const dbOptions = { ...options }; + + if (hasBESAdminAccess(accessPolicy)) { + return { dbConditions, dbOptions }; + } + + const countryIdsByPermissionGroupId = await fetchCountryIdsByPermissionGroupId( + accessPolicy, + models, + ); + + dbOptions.multiJoin = mergeMultiJoin( + [ + { + joinWith: TYPES.DATA_GROUP, + joinCondition: [ + `${TYPES.DATA_GROUP}.code`, + `${TYPES.DATA_SERVICE_SYNC_GROUP}.data_group_code`, + ], + }, + { + joinWith: TYPES.SURVEY, + joinCondition: [`${TYPES.SURVEY}.data_group_id`, `${TYPES.DATA_GROUP}.id`], + }, + ], + dbOptions.multiJoin, + ); + + dbConditions[RAW] = { + sql: ` + ( + survey.country_ids + && + ARRAY( + SELECT TRIM('"' FROM JSON_ARRAY_ELEMENTS(?::JSON->survey.permission_group_id)::TEXT) + ) + )`, + parameters: JSON.stringify(countryIdsByPermissionGroupId), + }; + + return { dbConditions, dbOptions }; +}; diff --git a/packages/central-server/src/apiV2/syncGroups/index.js b/packages/central-server/src/apiV2/syncGroups/index.js new file mode 100644 index 0000000000..bc24f83165 --- /dev/null +++ b/packages/central-server/src/apiV2/syncGroups/index.js @@ -0,0 +1,9 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd + */ + +export { DeleteSyncGroups } from './DeleteSyncGroups'; +export { EditSyncGroups } from './EditSyncGroups'; +export { GETSyncGroups } from './GETSyncGroups'; +export { CreateSyncGroups } from './CreateSyncGroups'; diff --git a/packages/central-server/src/apiV2/utilities/constructNewRecordValidationRules.js b/packages/central-server/src/apiV2/utilities/constructNewRecordValidationRules.js index 51b0407057..769c54c87f 100644 --- a/packages/central-server/src/apiV2/utilities/constructNewRecordValidationRules.js +++ b/packages/central-server/src/apiV2/utilities/constructNewRecordValidationRules.js @@ -186,6 +186,12 @@ export const constructForSingle = (models, recordType) => { code: [hasContent], name: [hasContent], }; + case TYPES.DATA_SERVICE_SYNC_GROUP: + return { + data_group_code: [constructRecordExistsWithField(models.survey, 'code')], + service_type: [constructIsOneOf(Object.values(models.dataServiceSyncGroup.SERVICE_TYPES))], + config: [hasContent], + }; default: throw new ValidationError(`${recordType} is not a valid POST endpoint`); } diff --git a/packages/central-server/src/kobo/manualKoBoSync.js b/packages/central-server/src/kobo/manualKoBoSync.js index 0b8acf5e52..b753c46964 100644 --- a/packages/central-server/src/kobo/manualKoBoSync.js +++ b/packages/central-server/src/kobo/manualKoBoSync.js @@ -9,10 +9,10 @@ import { syncWithKoBo } from './startSyncWithKoBo'; export async function manualKoBoSync(req, res) { const { models } = req; - const { serviceCode } = req.query; + const { syncGroupCode } = req.query; const dataBroker = new DataBroker(); - await syncWithKoBo(models, dataBroker, serviceCode); + await syncWithKoBo(models, dataBroker, syncGroupCode); respond(res, { message: 'KoBo sync triggered' }); } diff --git a/packages/central-server/src/kobo/startSyncWithKoBo.js b/packages/central-server/src/kobo/startSyncWithKoBo.js index 81a952f96c..cb7d4b7755 100644 --- a/packages/central-server/src/kobo/startSyncWithKoBo.js +++ b/packages/central-server/src/kobo/startSyncWithKoBo.js @@ -6,98 +6,134 @@ import keyBy from 'lodash.keyby'; import { DataBroker } from '@tupaia/data-broker'; import { generateId } from '@tupaia/database'; +import winston from '../log'; const PERIOD_BETWEEN_SYNCS = 10 * 60 * 1000; // 10 minutes between syncs const SERVICE_TYPE = 'kobo'; -export async function startSyncWithKoBo(models) { - if (process.env.KOBO_SYNC_DISABLE === 'true') { - // eslint-disable-next-line no-console - console.log('KoBo sync is disabled'); - } else { - const dataBroker = new DataBroker(); - const koboSyncServices = await models.syncService.find({ service_type: SERVICE_TYPE }); - koboSyncServices.forEach(service => - setInterval(() => syncWithKoBo(models, dataBroker, service.code), PERIOD_BETWEEN_SYNCS), - ); - } -} - -export async function syncWithKoBo(models, dataBroker, serviceCode) { - const syncService = await models.syncService.findOne({ +const writeKoboDataToTupaia = async (transactingModels, koboData, syncGroupCode) => { + const dataServiceSyncGroup = await transactingModels.dataServiceSyncGroup.findOne({ service_type: SERVICE_TYPE, - code: serviceCode, + code: syncGroupCode, + }); + const apiUser = await transactingModels.apiClient.findOne({ + username: process.env.KOBO_API_USERNAME, }); - if (!syncService) { - throw new Error(`No KoBo sync service with the code ${serviceCode} exists`); - } - - // Pull data from KoBo - const koboData = await dataBroker.pull( - { - code: syncService.config.koboSurveys, - type: dataBroker.getDataSourceTypes().SYNC_GROUP, - }, - { - startSubmissionTime: syncService.sync_cursor, - }, - ); - - const apiUser = await models.apiClient.findOne({ username: process.env.KOBO_API_USERNAME }); if (!apiUser) { throw new Error('Cannot find API client user for KoBo'); } - // Create new survey_responses - let newSyncTime = syncService.sync_cursor; - await models.wrapInTransaction(async transactingModels => { - for (const koboSyncResponse of koboData) { - for (const [surveyCode, responses] of Object.entries(koboSyncResponse)) { - const survey = await transactingModels.survey.findOne({ code: surveyCode }); - - for (const responseData of responses) { - if (responseData.eventDate > newSyncTime) { - newSyncTime = responseData.eventDate; - } - const entity = await transactingModels.entity.findOne({ code: responseData.orgUnit }); - if (!entity) { - await syncService.log( - `Skipping KoBo sync for record id ${responseData.event}: unknown entity ${responseData.orgUnit}`, - ); - continue; - } - - const surveyResponse = await transactingModels.surveyResponse.create({ - id: generateId(), - survey_id: survey.id, - user_id: apiUser.user_account_id, - assessor_name: responseData.assessor || 'KoBo Integration', - entity_id: entity.id, - start_time: responseData.eventDate, - end_time: responseData.eventDate, - data_time: responseData.eventDate, - }); - - const questions = await transactingModels.question.find({ - code: Object.keys(responseData.dataValues), - }); - const questionByCode = keyBy(questions, 'code'); - - await transactingModels.answer.createMany( - Object.entries(responseData.dataValues).map(([questionCode, answer]) => ({ - id: generateId(), - type: questionByCode[questionCode].type, - survey_response_id: surveyResponse.id, - question_id: questionByCode[questionCode].id, - text: answer, - })), + let newSyncTime = dataServiceSyncGroup.sync_cursor; + let numberOfSurveyResponsesCreated = 0; + for (const koboSyncResponse of koboData) { + for (const [surveyCode, responses] of Object.entries(koboSyncResponse)) { + const survey = await transactingModels.survey.findOne({ code: surveyCode }); + + for (const responseData of responses) { + if (responseData.eventDate > newSyncTime) { + newSyncTime = responseData.eventDate; + } + const entity = await transactingModels.entity.findOne({ code: responseData.orgUnit }); + if (!entity) { + await dataServiceSyncGroup.log( + `Skipping KoBo sync for record id ${responseData.event}: unknown entity ${responseData.orgUnit}`, ); + continue; } + + const surveyResponse = await transactingModels.surveyResponse.create({ + id: generateId(), + survey_id: survey.id, + user_id: apiUser.user_account_id, + assessor_name: responseData.assessor || 'KoBo Integration', + entity_id: entity.id, + start_time: responseData.eventDate, + end_time: responseData.eventDate, + data_time: responseData.eventDate, + }); + + const questions = await transactingModels.question.find({ + code: Object.keys(responseData.dataValues), + }); + const questionByCode = keyBy(questions, 'code'); + + await transactingModels.answer.createMany( + Object.entries(responseData.dataValues).map(([questionCode, answer]) => ({ + id: generateId(), + type: questionByCode[questionCode].type, + survey_response_id: surveyResponse.id, + question_id: questionByCode[questionCode].id, + text: answer, + })), + ); + + numberOfSurveyResponsesCreated += 1; } } + } + + return { newSyncTime, numberOfSurveyResponsesCreated }; +}; + +export async function syncWithKoBo(models, dataBroker, syncGroupCode) { + const dataServiceSyncGroup = await models.dataServiceSyncGroup.findOne({ + service_type: SERVICE_TYPE, + code: syncGroupCode, }); - // Update sync cursor - await models.syncService.update({ id: syncService.id }, { sync_cursor: newSyncTime }); + if (!dataServiceSyncGroup) { + throw new Error(`No KoBo sync group with the code ${syncGroupCode} exists`); + } + + try { + // Pull data from KoBo + const koboData = await dataBroker.pull( + { + code: syncGroupCode, + type: dataBroker.getDataSourceTypes().SYNC_GROUP, + }, + { + startSubmissionTime: dataServiceSyncGroup.sync_cursor, + }, + ); + + await models.wrapInTransaction(async transactingModels => { + // Create new survey_responses in Tupaia + const { newSyncTime, numberOfSurveyResponsesCreated } = await writeKoboDataToTupaia( + transactingModels, + koboData, + syncGroupCode, + ); + + // Update sync cursor + await transactingModels.dataServiceSyncGroup.update( + { id: dataServiceSyncGroup.id }, + { sync_cursor: newSyncTime }, + ); + + await dataServiceSyncGroup.log( + `Sync successful, ${numberOfSurveyResponsesCreated} survey responses created`, + ); + }); + } catch (e) { + // Swallow errors when processing kobo data + await dataServiceSyncGroup.log(`ERROR: ${e.message}`); + winston.error(e.message); + } +} + +export async function startSyncWithKoBo(models) { + if (process.env.KOBO_SYNC_DISABLE === 'true') { + // eslint-disable-next-line no-console + console.log('KoBo sync is disabled'); + } else { + const dataBroker = new DataBroker(); + const koboDataServiceSyncGroups = await models.dataServiceSyncGroup.find({ + service_type: SERVICE_TYPE, + }); + koboDataServiceSyncGroups.forEach(dssg => + setInterval(() => syncWithKoBo(models, dataBroker, dssg.code), PERIOD_BETWEEN_SYNCS), + ); + } } diff --git a/packages/data-broker/src/__tests__/services/kobo/KoBoService.fixtures.js b/packages/data-broker/src/__tests__/services/kobo/KoBoService.fixtures.js index 4616f119b0..53220ac05f 100644 --- a/packages/data-broker/src/__tests__/services/kobo/KoBoService.fixtures.js +++ b/packages/data-broker/src/__tests__/services/kobo/KoBoService.fixtures.js @@ -80,9 +80,10 @@ export const MOCK_QUESTION_ANSWER_MAP = { }; export const MOCK_DATA_SOURCE = { + data_group_code: 'xyz', + service_type: 'kobo', config: { koboSurveyCode: 'abc', - internalSurveyCode: 'xyz', entityQuestionCode: 'entity', questionMapping: MOCK_QUESTION_ANSWER_MAP, }, diff --git a/packages/data-broker/src/services/kobo/KoBoService.js b/packages/data-broker/src/services/kobo/KoBoService.js index 6cbe58518e..a944b2e371 100644 --- a/packages/data-broker/src/services/kobo/KoBoService.js +++ b/packages/data-broker/src/services/kobo/KoBoService.js @@ -41,19 +41,19 @@ export class KoBoService extends Service { }; pullSyncGroups = async (dataSources, options) => { - const resultsByInternalCode = {}; + const resultsByDataGroupCode = {}; + for (const source of dataSources) { const results = await this.api.fetchKoBoSubmissions(source.config?.koboSurveyCode, options); - resultsByInternalCode[ - source.config?.internalSurveyCode - ] = await this.translator.translateKoBoResults( + + resultsByDataGroupCode[source.data_group_code] = await this.translator.translateKoBoResults( results, source.config?.questionMapping, source.config?.entityQuestionCode, ); } - return resultsByInternalCode; + return resultsByDataGroupCode; }; async pullMetadata() { diff --git a/packages/data-broker/src/services/kobo/KoBoTranslator.js b/packages/data-broker/src/services/kobo/KoBoTranslator.js index 1d8b53f470..7825f5e386 100644 --- a/packages/data-broker/src/services/kobo/KoBoTranslator.js +++ b/packages/data-broker/src/services/kobo/KoBoTranslator.js @@ -3,8 +3,6 @@ * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd */ -import keyBy from 'lodash.keyby'; - export class KoBoTranslator { constructor(models) { this.models = models; diff --git a/packages/data-broker/src/utils.js b/packages/data-broker/src/utils.js index 8f0cda0f72..b093546339 100644 --- a/packages/data-broker/src/utils.js +++ b/packages/data-broker/src/utils.js @@ -9,7 +9,7 @@ const SYNC_GROUP = 'syncGroup'; export const DATA_SOURCE_TYPES = { DATA_ELEMENT, DATA_GROUP, - SYNC_GROUP, + SYNC_GROUP, // A SYNC_GROUP is an extension to a DATA_GROUP, where a copy of the data is stored in one place e.g. tupaia but is synced from somewhere else }; /** diff --git a/packages/database/src/migrations/20220630013151-LinkSyncServiceToSurvey-modifies-schema.js b/packages/database/src/migrations/20220630013151-LinkSyncServiceToSurvey-modifies-schema.js new file mode 100644 index 0000000000..f8ddd67406 --- /dev/null +++ b/packages/database/src/migrations/20220630013151-LinkSyncServiceToSurvey-modifies-schema.js @@ -0,0 +1,50 @@ +'use strict'; + +var dbm; +var type; +var seed; + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ +exports.setup = function (options, seedLink) { + dbm = options.dbmigrate; + type = dbm.dataType; + seed = seedLink; +}; + +exports.up = async function (db) { + await db.runSql( + `DROP TRIGGER IF EXISTS data_service_sync_group_trigger ON data_service_sync_group`, + ); + + // Add new code column which should match current syncGroupCode + await db.runSql(`ALTER TABLE data_service_sync_group ADD COLUMN data_group_code text`); + + await db.runSql(` + UPDATE data_service_sync_group + SET + data_group_code = trim(both '"' from config->>'internalSurveyCode'), + code = trim(both '"' from config->>'internalSurveyCode'), + config = config - 'internalSurveyCode' + `); + // Overwriting old code to match data_group_code as a new convention (saves needing to have a made up code) + // not mandatory however, and can be changed to something else if wished + + // Make data_group_code NOT NULL + await db.runSql(`ALTER TABLE data_service_sync_group ALTER COLUMN data_group_code SET NOT NULL`); +}; + +exports.down = async function (db) { + await db.runSql(` + UPDATE data_service_sync_group + SET + config = config || jsonb_build_object('internalSurveyCode', data_group_code) + `); + await db.runSql(`ALTER TABLE data_service_sync_group DROP COLUMN data_group_code`); +}; + +exports._meta = { + version: 1, +}; diff --git a/packages/database/src/migrations/20220812055419-MoveSyncServiceToDataServiceSyncGroup-modifies-schema.js b/packages/database/src/migrations/20220812055419-MoveSyncServiceToDataServiceSyncGroup-modifies-schema.js new file mode 100644 index 0000000000..34d527a0e7 --- /dev/null +++ b/packages/database/src/migrations/20220812055419-MoveSyncServiceToDataServiceSyncGroup-modifies-schema.js @@ -0,0 +1,51 @@ +'use strict'; + +const { updateValues, findSingleRecord } = require('../utilities'); + +var dbm; +var type; +var seed; + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ +exports.setup = function (options, seedLink) { + dbm = options.dbmigrate; + type = dbm.dataType; + seed = seedLink; +}; + +exports.up = async function (db) { + // Add sync_cursor column to data_service_sync_group + await db.runSql( + `ALTER TABLE data_service_sync_group ADD COLUMN sync_cursor text DEFAULT '1970-01-01T00:00:00.000Z'`, + ); + + // Copy sync_cursor over to data_service_sync_group + const syncServices = (await db.runSql('SELECT * from sync_service')).rows; + for (let i = 0; i < syncServices.length; i++) { + const { sync_cursor: syncCursor, service_type: serviceType } = syncServices[i]; + await updateValues( + db, + 'data_service_sync_group', + { sync_cursor: syncCursor }, + { service_type: serviceType }, + ); + } + + await db.runSql(`ALTER TABLE sync_service_log RENAME TO sync_group_log;`); + await db.runSql(`ALTER TABLE sync_group_log RENAME COLUMN service_code TO sync_group_code;`); + + await db.runSql(`DROP TABLE sync_service;`); + + return null; +}; + +exports.down = function (db) { + return null; +}; + +exports._meta = { + version: 1, +}; diff --git a/packages/database/src/modelClasses/DataServiceSyncGroup.js b/packages/database/src/modelClasses/DataServiceSyncGroup.js index ff7695f4ca..7b65e7d6cc 100644 --- a/packages/database/src/modelClasses/DataServiceSyncGroup.js +++ b/packages/database/src/modelClasses/DataServiceSyncGroup.js @@ -3,15 +3,33 @@ * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd */ +import winston from 'winston'; import { DatabaseModel } from '../DatabaseModel'; import { DatabaseType } from '../DatabaseType'; import { TYPES } from '../types'; +import { generateId } from '../utilities'; + +const SERVICE_TYPES = { + KOBO: 'kobo', +}; class DataServiceSyncGroupType extends DatabaseType { static databaseType = TYPES.DATA_SERVICE_SYNC_GROUP; + + async log(message) { + winston.info(`${this.code} SYNC_GROUP_LOG: ${message}`); + await this.otherModels.syncGroupLog.create({ + id: generateId(), + sync_group_code: this.code, + service_type: this.service_type, + log_message: message, + }); + } } export class DataServiceSyncGroupModel extends DatabaseModel { + SERVICE_TYPES = SERVICE_TYPES; + get DatabaseTypeClass() { return DataServiceSyncGroupType; } diff --git a/packages/database/src/modelClasses/SyncServiceLog.js b/packages/database/src/modelClasses/SyncGroupLog.js similarity index 57% rename from packages/database/src/modelClasses/SyncServiceLog.js rename to packages/database/src/modelClasses/SyncGroupLog.js index 797c3db201..b38201a1c2 100644 --- a/packages/database/src/modelClasses/SyncServiceLog.js +++ b/packages/database/src/modelClasses/SyncGroupLog.js @@ -7,12 +7,12 @@ import { DatabaseModel } from '../DatabaseModel'; import { DatabaseType } from '../DatabaseType'; import { TYPES } from '../types'; -class SyncServiceLogType extends DatabaseType { - static databaseType = TYPES.SYNC_SERVICE_LOG; +class SyncGroupLogType extends DatabaseType { + static databaseType = TYPES.SYNC_GROUP_LOG; } -export class SyncServiceLogModel extends DatabaseModel { +export class SyncGroupLogModel extends DatabaseModel { get DatabaseTypeClass() { - return SyncServiceLogType; + return SyncGroupLogType; } } diff --git a/packages/database/src/modelClasses/SyncService.js b/packages/database/src/modelClasses/SyncService.js deleted file mode 100644 index 55cc90ab2c..0000000000 --- a/packages/database/src/modelClasses/SyncService.js +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Tupaia - * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd - */ - -import winston from 'winston'; -import { DatabaseModel } from '../DatabaseModel'; -import { DatabaseType } from '../DatabaseType'; -import { TYPES } from '../types'; -import { generateId } from '../utilities'; - -class SyncServiceType extends DatabaseType { - static databaseType = TYPES.SYNC_SERVICE; - - async log(message) { - winston.info(`${this.code} SYNC_SERVICE_LOG: ${message}`); - await this.otherModels.syncServiceLog.create({ - id: generateId(), - service_code: this.code, - service_type: this.service_type, - log_message: message, - }); - } -} - -export class SyncServiceModel extends DatabaseModel { - get DatabaseTypeClass() { - return SyncServiceType; - } -} diff --git a/packages/database/src/modelClasses/index.js b/packages/database/src/modelClasses/index.js index 5bd508f9a5..7f00c3be78 100644 --- a/packages/database/src/modelClasses/index.js +++ b/packages/database/src/modelClasses/index.js @@ -47,8 +47,7 @@ import { SurveyResponseModel } from './SurveyResponse'; import { SurveyResponseCommentModel } from './SurveyResponseComment'; import { SurveyScreenComponentModel } from './SurveyScreenComponent'; import { SurveyScreenModel } from './SurveyScreen'; -import { SyncServiceModel } from './SyncService'; -import { SyncServiceLogModel } from './SyncServiceLog'; +import { SyncGroupLogModel } from './SyncGroupLog'; import { UserEntityPermissionModel } from './UserEntityPermission'; import { UserModel } from './User'; import { UserSessionModel } from './UserSession'; @@ -103,8 +102,7 @@ export const modelClasses = { SurveyResponseComment: SurveyResponseCommentModel, SurveyScreen: SurveyScreenModel, SurveyScreenComponent: SurveyScreenComponentModel, - SyncService: SyncServiceModel, - SyncServiceLog: SyncServiceLogModel, + SyncGroupLog: SyncGroupLogModel, User: UserModel, UserEntityPermission: UserEntityPermissionModel, UserSession: UserSessionModel, diff --git a/packages/database/src/runPostMigration.js b/packages/database/src/runPostMigration.js index b78903b231..0bda28b64f 100644 --- a/packages/database/src/runPostMigration.js +++ b/packages/database/src/runPostMigration.js @@ -20,6 +20,7 @@ const EXCLUDED_TABLES_FROM_TRIGGER_CREATION = [ 'lesmis_session', 'admin_panel_session', 'analytics', + 'data_service_sync_group', // config is too large for triggers ]; // tables that should only have records created and deleted, and will throw an error if an update is diff --git a/packages/database/src/types.js b/packages/database/src/types.js index 503885b21a..a1f76e56cd 100644 --- a/packages/database/src/types.js +++ b/packages/database/src/types.js @@ -56,8 +56,7 @@ export const TYPES = { SURVEY_SCREEN_COMPONENT: 'survey_screen_component', SURVEY_SCREEN: 'survey_screen', SURVEY: 'survey', - SYNC_SERVICE: 'sync_service', - SYNC_SERVICE_LOG: 'sync_service_log', + SYNC_GROUP_LOG: 'sync_group_log', USER_ACCOUNT: 'user_account', USER_ENTITY_PERMISSION: 'user_entity_permission', USER_SESSION: 'userSession', diff --git a/packages/utils/src/validation/validatorFunctions.js b/packages/utils/src/validation/validatorFunctions.js index dc981b8228..747947235e 100644 --- a/packages/utils/src/validation/validatorFunctions.js +++ b/packages/utils/src/validation/validatorFunctions.js @@ -135,7 +135,9 @@ export const isValidPassword = password => { export const constructIsOneOf = options => value => { if (!options.includes(value)) { - throw new ValidationError(`${value} is not an accepted value`); + throw new ValidationError( + `${value} is not an accepted value. Accepted values: "${options.join('", "')}"`, + ); } }; From 53e94c43617cbe7c76bfe8475d1d5fb7f47200ed Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 25 Aug 2022 10:21:41 +1000 Subject: [PATCH 03/14] MAUI-1156 Delete some PG facilities (#4100) Co-authored-by: Sima-BES <87400368+Sima-BES@users.noreply.github.com> --- ...055530-DeletePgFacilities-modifies-data.js | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 packages/database/src/migrations/20220819055530-DeletePgFacilities-modifies-data.js diff --git a/packages/database/src/migrations/20220819055530-DeletePgFacilities-modifies-data.js b/packages/database/src/migrations/20220819055530-DeletePgFacilities-modifies-data.js new file mode 100644 index 0000000000..a18a5348bf --- /dev/null +++ b/packages/database/src/migrations/20220819055530-DeletePgFacilities-modifies-data.js @@ -0,0 +1,45 @@ +'use strict'; + +var dbm; +var type; +var seed; + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ +exports.setup = function (options, seedLink) { + dbm = options.dbmigrate; + type = dbm.dataType; + seed = seedLink; +}; + +const ENTITIES_TO_BE_DELETED = [ + 'PG_f-2OD-0Z2-Z5RZ', + 'PG_f-456-ZEQ-IEST', + 'PG_f-578-6F8-1MY4', + 'PG_f-A47-M56-IGMD', + 'PG_f-AFX-8US-AXM7', + 'PG_f-FFA-YDF-FW40', + 'PG_f-PBQ-1ES-5C4J', + 'PG_f-Q4W-YX7-LNGT', + 'PG_f-RX3-L7X-ZUFN', + 'PG_f-TCH-M7C-86TR', +]; + +exports.up = async function (db) { + await db.runSql( + `DELETE FROM survey_response WHERE entity_id IN (SELECT id FROM entity WHERE code IN ('${ENTITIES_TO_BE_DELETED.join( + `','`, + )}'))`, + ); + await db.runSql(`DELETE FROM entity WHERE code IN ('${ENTITIES_TO_BE_DELETED.join(`','`)}')`); +}; + +exports.down = function (db) { + return null; +}; + +exports._meta = { + version: 1, +}; From 39144e84da88e816185dca6ff20becaeb5d31abd Mon Sep 17 00:00:00 2001 From: Rohan Port <59544282+rohan-bes@users.noreply.github.com> Date: Thu, 25 Aug 2022 10:56:32 +1000 Subject: [PATCH 04/14] Minor renames in PDF export code (#4116) * Renamed instances of Pdf with PDF - To maintain consistency with naming convention - Rule seems to be: use same case chars for pdf/PDF --- .../lesmis-server/src/routes/PDFExportRoute.ts | 4 ++-- .../src/{PDFExporter.ts => downloadPageAsPDF.ts} | 14 +++++++------- packages/tsutils/src/index.ts | 2 +- .../src/export/PDFExportHandler.js | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) rename packages/tsutils/src/{PDFExporter.ts => downloadPageAsPDF.ts} (81%) diff --git a/packages/lesmis-server/src/routes/PDFExportRoute.ts b/packages/lesmis-server/src/routes/PDFExportRoute.ts index 12229fb07c..1986d9e5db 100644 --- a/packages/lesmis-server/src/routes/PDFExportRoute.ts +++ b/packages/lesmis-server/src/routes/PDFExportRoute.ts @@ -5,7 +5,7 @@ */ import { Request, Response, NextFunction } from 'express'; import { Route } from '@tupaia/server-boilerplate'; -import { downloadPageAsPdf } from '@tupaia/tsutils'; +import { downloadPageAsPDF } from '@tupaia/tsutils'; type Body = { pdfPageUrl: string; @@ -28,7 +28,7 @@ export class PDFExportRoute extends Route { const { pdfPageUrl } = this.req.body; const { cookie, host: cookieDomain } = this.req.headers; - const buffer = await downloadPageAsPdf(pdfPageUrl, cookie, cookieDomain); + const buffer = await downloadPageAsPDF(pdfPageUrl, cookie, cookieDomain); this.res.set({ 'Content-Type': 'application/pdf', 'Content-Length': buffer.length, diff --git a/packages/tsutils/src/PDFExporter.ts b/packages/tsutils/src/downloadPageAsPDF.ts similarity index 81% rename from packages/tsutils/src/PDFExporter.ts rename to packages/tsutils/src/downloadPageAsPDF.ts index fceeba61b6..4e2750039d 100644 --- a/packages/tsutils/src/PDFExporter.ts +++ b/packages/tsutils/src/downloadPageAsPDF.ts @@ -1,7 +1,7 @@ import cookie from 'cookie'; import puppeteer from 'puppeteer'; -const verifyPdfPageUrl = (pdfPageUrl: string): string => { +const verifyPDFPageUrl = (pdfPageUrl: string): string => { const lesmisValidDomains = ['lesmis.la', 'www.lesmis.la']; if (!pdfPageUrl || typeof pdfPageUrl !== 'string') { throw new Error(`'pdfPageUrl' should be provided in request body, got: ${pdfPageUrl}`); @@ -19,8 +19,8 @@ const verifyPdfPageUrl = (pdfPageUrl: string): string => { const buildParams = (pdfPageUrl: string, userCookie: string, cookieDomain: string | undefined) => { const cookies = cookie.parse(userCookie || ''); - const verifiedPdfPageUrl = verifyPdfPageUrl(pdfPageUrl); - const location = new URL(verifiedPdfPageUrl); + const verifiedPDFPageUrl = verifyPDFPageUrl(pdfPageUrl); + const location = new URL(verifiedPDFPageUrl); const finalisedCookieObjects = Object.keys(cookies).map(name => ({ name, domain: cookieDomain, @@ -28,7 +28,7 @@ const buildParams = (pdfPageUrl: string, userCookie: string, cookieDomain: strin httpOnly: true, value: cookies[name], })); - return { verifiedPdfPageUrl, cookies: finalisedCookieObjects }; + return { verifiedPDFPageUrl, cookies: finalisedCookieObjects }; }; /** @@ -37,20 +37,20 @@ const buildParams = (pdfPageUrl: string, userCookie: string, cookieDomain: strin * @param cookieDomain the domain of cookie, required when setting up cookie in page * @returns pdf buffer */ -export const downloadPageAsPdf = async ( +export const downloadPageAsPDF = async ( pdfPageUrl: string, userCookie = '', cookieDomain: string | undefined, ) => { let browser; let buffer; - const { cookies, verifiedPdfPageUrl } = buildParams(pdfPageUrl, userCookie, cookieDomain); + const { cookies, verifiedPDFPageUrl } = buildParams(pdfPageUrl, userCookie, cookieDomain); try { browser = await puppeteer.launch(); const page = await browser.newPage(); await page.setCookie(...cookies); - await page.goto(verifiedPdfPageUrl, { timeout: 60000, waitUntil: 'networkidle0' }); + await page.goto(verifiedPDFPageUrl, { timeout: 60000, waitUntil: 'networkidle0' }); buffer = await page.pdf({ format: 'a4', printBackground: true, diff --git a/packages/tsutils/src/index.ts b/packages/tsutils/src/index.ts index b92628d5c2..5153655aff 100644 --- a/packages/tsutils/src/index.ts +++ b/packages/tsutils/src/index.ts @@ -1,3 +1,3 @@ export * from './validation'; export * from './types'; -export * from './PDFExporter'; +export * from './downloadPageAsPDF'; diff --git a/packages/web-config-server/src/export/PDFExportHandler.js b/packages/web-config-server/src/export/PDFExportHandler.js index 8286119992..683ee66e86 100644 --- a/packages/web-config-server/src/export/PDFExportHandler.js +++ b/packages/web-config-server/src/export/PDFExportHandler.js @@ -1,4 +1,4 @@ -import { downloadPageAsPdf } from '@tupaia/tsutils'; +import { downloadPageAsPDF } from '@tupaia/tsutils'; import { convertToCDNHost } from '@tupaia/utils'; export const PDFExportHandler = async (req, res) => { @@ -6,7 +6,7 @@ export const PDFExportHandler = async (req, res) => { const { cookie, host, via } = req.headers; const cookieDomain = via && via.includes('cloudfront.net') ? convertToCDNHost(host) : host; - const buffer = await downloadPageAsPdf(pdfPageUrl, cookie, cookieDomain); + const buffer = await downloadPageAsPDF(pdfPageUrl, cookie, cookieDomain); res.set({ 'Content-Type': 'application/pdf', 'Content-Length': buffer.length, From 76c315dd4dca9ba68fa51656eee90ff99e981c33 Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 25 Aug 2022 15:00:13 +1000 Subject: [PATCH 05/14] RN-526 superset api (#4097) * Add superset_instance table and type * Minor: comments * Improve error logging * Add SupersetInstance model * Minor: allow 1 in DB_VERBOSE flag * Add package @tupaia/superset-api * Add superset service * Better visibilityCriteria in admin-panel * Add superset as an option in admin panel * Style: consistent label style * Add superset to DATA_SOURCE_SERVICE_TYPES * Add supersetChartId to possible config * Fix type of request * Mark arg optional * PR fixes * Add tests, explicit supersetItemCode --- .eslintrc | 1 + packages/admin-panel/src/editor/Editor.js | 5 +- .../src/importExport/ImportModal.js | 10 +- .../src/pages/resources/DataSourcesPage.js | 56 ++++++- .../src/pages/resources/SurveysPage.js | 3 + packages/admin-panel/src/utilities/index.js | 1 + .../src/utilities/visibilityCriteria.js | 14 ++ .../src/widgets/InputField/JsonInputField.js | 78 ++++++--- .../src/database/models/DataElement.js | 2 +- packages/data-broker/package.json | 1 + .../data-broker/src/__mocks__/superset-api.js | 8 + .../superset/SupersetService.stubs.js | 95 +++++++++++ .../services/superset/SupersetService.test.js | 101 ++++++++++++ .../data-broker/src/services/createService.js | 3 + .../services/indicator/IndicatorService.js | 3 +- .../src/services/superset/SupersetService.js | 153 ++++++++++++++++++ .../src/services/superset/getSupersetApi.js | 24 +++ .../src/services/superset/index.js | 0 packages/database/src/TupaiaDatabase.js | 2 +- ...teSupersetInstanceTable-modifies-schema.js | 35 ++++ .../database/src/modelClasses/DataElement.js | 1 + .../src/modelClasses/SupersetInstance.js | 18 +++ packages/database/src/modelClasses/index.js | 3 + packages/database/src/types.js | 1 + packages/devops/ci/tupaia-ci-cd.Dockerfile | 3 + packages/report-server/.env.example | 4 +- .../src/connections/ApiConnection.ts | 13 +- packages/superset-api/jest.config.ts | 11 ++ packages/superset-api/package.json | 24 +++ packages/superset-api/src/SupersetApi.ts | 114 +++++++++++++ .../src/__tests__/example.test.ts | 10 ++ packages/superset-api/src/index.ts | 6 + packages/superset-api/src/types.ts | 43 +++++ packages/superset-api/tsconfig-build.json | 10 ++ packages/superset-api/tsconfig.json | 7 + packages/utils/src/errors.js | 11 +- packages/utils/src/request.js | 11 +- scripts/bash/getInternalDependencies.sh | 2 +- tupaia-packages.code-workspace | 4 + yarn.lock | 10 ++ 40 files changed, 833 insertions(+), 68 deletions(-) create mode 100644 packages/admin-panel/src/utilities/visibilityCriteria.js create mode 100644 packages/data-broker/src/__mocks__/superset-api.js create mode 100644 packages/data-broker/src/__tests__/services/superset/SupersetService.stubs.js create mode 100644 packages/data-broker/src/__tests__/services/superset/SupersetService.test.js create mode 100644 packages/data-broker/src/services/superset/SupersetService.js create mode 100644 packages/data-broker/src/services/superset/getSupersetApi.js create mode 100644 packages/data-broker/src/services/superset/index.js create mode 100644 packages/database/src/migrations/20220811014552-CreateSupersetInstanceTable-modifies-schema.js create mode 100644 packages/database/src/modelClasses/SupersetInstance.js create mode 100644 packages/superset-api/jest.config.ts create mode 100644 packages/superset-api/package.json create mode 100644 packages/superset-api/src/SupersetApi.ts create mode 100644 packages/superset-api/src/__tests__/example.test.ts create mode 100644 packages/superset-api/src/index.ts create mode 100644 packages/superset-api/src/types.ts create mode 100644 packages/superset-api/tsconfig-build.json create mode 100644 packages/superset-api/tsconfig.json diff --git a/.eslintrc b/.eslintrc index 4a1d6eaf93..830522103f 100644 --- a/.eslintrc +++ b/.eslintrc @@ -44,6 +44,7 @@ "packages/psss-server/**", "packages/report-server/**", "packages/server-boilerplate/**", + "packages/supserset-api/**", "packages/tsutils/**" ], "extends": "@beyondessential/ts", diff --git a/packages/admin-panel/src/editor/Editor.js b/packages/admin-panel/src/editor/Editor.js index 6e0bcc5ce1..ec21dfabea 100644 --- a/packages/admin-panel/src/editor/Editor.js +++ b/packages/admin-panel/src/editor/Editor.js @@ -6,6 +6,7 @@ import React from 'react'; import PropTypes from 'prop-types'; import { InputField } from '../widgets'; +import { checkVisibilityCriteriaAreMet } from '../utilities'; export const Editor = ({ fields, recordData, onEditField }) => { const onInputChange = (inputKey, inputValue, editConfig = {}) => { @@ -42,9 +43,7 @@ export const Editor = ({ fields, recordData, onEditField }) => { // show or hide a field based on another field's value if (visibilityCriteria) { - return Object.entries(visibilityCriteria).every( - ([key, value]) => recordData[key] === value, - ); + return checkVisibilityCriteriaAreMet(visibilityCriteria, recordData); } return true; diff --git a/packages/admin-panel/src/importExport/ImportModal.js b/packages/admin-panel/src/importExport/ImportModal.js index 74d0015b3b..badbfea1d5 100644 --- a/packages/admin-panel/src/importExport/ImportModal.js +++ b/packages/admin-panel/src/importExport/ImportModal.js @@ -19,6 +19,7 @@ import { import { ModalContentProvider, InputField } from '../widgets'; import { useApi } from '../utilities/ApiProvider'; import { DATA_CHANGE_REQUEST, DATA_CHANGE_SUCCESS, DATA_CHANGE_ERROR } from '../table/constants'; +import { checkVisibilityCriteriaAreMet } from '../utilities'; const STATUS = { IDLE: 'idle', @@ -119,15 +120,6 @@ export const ImportModalComponent = React.memo( ? 'Request timed out, but may have still succeeded. Please wait 2 minutes and check to see if the data has changed' : errorMessage; - const checkVisibilityCriteriaAreMet = visibilityCriteria => { - if (!visibilityCriteria) { - return true; // no visibility criteria to meet, fine to display - } - return Object.entries(visibilityCriteria).every( - ([parameterKey, requiredValue]) => values[parameterKey] === requiredValue, - ); - }; - const renderButtons = useCallback(() => { switch (status) { case STATUS.TIMEOUT: diff --git a/packages/admin-panel/src/pages/resources/DataSourcesPage.js b/packages/admin-panel/src/pages/resources/DataSourcesPage.js index d7dd511fe5..1436c27bac 100644 --- a/packages/admin-panel/src/pages/resources/DataSourcesPage.js +++ b/packages/admin-panel/src/pages/resources/DataSourcesPage.js @@ -7,6 +7,37 @@ import React from 'react'; import PropTypes from 'prop-types'; import { ResourcePage } from './ResourcePage'; +const SERVICE_TYPE_OPTIONS = [ + { + label: 'Data Lake', + value: 'data-lake', + }, + { + label: 'DHIS', + value: 'dhis', + }, + { + label: 'Indicator', + value: 'indicator', + }, + { + label: 'Kobo', + value: 'kobo', + }, + { + label: 'Superset', + value: 'superset', + }, + { + label: 'Tupaia', + value: 'tupaia', + }, + { + label: 'Weather', + value: 'weather', + }, +]; + const localStyles = { config: { dt: { @@ -61,20 +92,23 @@ const DATA_SOURCE_FIELDS = [ source: 'code', }, { - Header: 'Service Type', + Header: 'Data Service', source: 'service_type', - editConfig: { default: 'dhis' }, + editConfig: { default: 'dhis', options: SERVICE_TYPE_OPTIONS }, }, ]; const DATA_ELEMENT_FIELDS = [ ...DATA_SOURCE_FIELDS, { - Header: 'Config', + Header: 'Data Service Configuration', source: 'config', Cell: DataSourceConfigView, editConfig: { type: 'json', default: '{}', + visibilityCriteria: { + service_type: values => ['dhis', 'superset'].includes(values.service_type), + }, getJsonFieldSchema: () => [ { label: 'DHIS Server', @@ -82,14 +116,22 @@ const DATA_ELEMENT_FIELDS = [ optionsEndpoint: 'dhisInstances', optionLabelKey: 'dhisInstances.code', optionValueKey: 'dhisInstances.code', + visibilityCriteria: { service_type: 'dhis' }, }, { label: 'Data element code', fieldName: 'dataElementCode', + visibilityCriteria: { service_type: 'dhis' }, }, { label: 'Category option combo code', fieldName: 'categoryOptionCombo', + visibilityCriteria: { service_type: 'dhis' }, + }, + { + label: 'Superset Chart ID', + fieldName: 'supersetChartId', + visibilityCriteria: { service_type: 'superset' }, }, ], }, @@ -106,12 +148,15 @@ const DATA_ELEMENT_FIELDS = [ const DATA_GROUP_FIELDS = [ ...DATA_SOURCE_FIELDS, { - Header: 'Config', + Header: 'Data Service Configuration', source: 'config', Cell: DataSourceConfigView, editConfig: { type: 'json', default: '{}', + visibilityCriteria: { + service_type: 'dhis', + }, getJsonFieldSchema: () => [ { label: 'DHIS Server', @@ -119,6 +164,7 @@ const DATA_GROUP_FIELDS = [ optionsEndpoint: 'dhisInstances', optionLabelKey: 'dhisInstances.code', optionValueKey: 'dhisInstances.code', + visibilityCriteria: { service_type: 'dhis' }, }, ], }, @@ -138,7 +184,7 @@ export const DataGroupsPage = ({ getHeaderEl }) => ( columns: [...DATA_ELEMENT_FIELDS, ...getButtonsConfig(DATA_ELEMENT_FIELDS, 'dataElement')], }, ]} - editConfig={{ title: 'Edit Data Source' }} + editConfig={{ title: 'Edit Data Group' }} createConfig={{ title: 'New Data Group', actionConfig: { diff --git a/packages/admin-panel/src/pages/resources/SurveysPage.js b/packages/admin-panel/src/pages/resources/SurveysPage.js index 65a80a8058..e32b244d50 100644 --- a/packages/admin-panel/src/pages/resources/SurveysPage.js +++ b/packages/admin-panel/src/pages/resources/SurveysPage.js @@ -105,6 +105,9 @@ const SURVEY_COLUMNS = [ source: 'data_group.config', editConfig: { type: 'json', + visibilityCriteria: { + 'data_group.service_type': 'dhis', + }, getJsonFieldSchema: (_, { recordData }) => recordData['data_group.service_type'] === 'dhis' ? [ diff --git a/packages/admin-panel/src/utilities/index.js b/packages/admin-panel/src/utilities/index.js index 4ef65c9cac..e0c48c4f7a 100644 --- a/packages/admin-panel/src/utilities/index.js +++ b/packages/admin-panel/src/utilities/index.js @@ -10,3 +10,4 @@ export { makeSubstitutionsInString } from './makeSubstitutionsInString'; export { usePortalWithCallback } from './usePortalWithCallback'; export * from './pretty'; export * from './useDebounce'; +export { checkVisibilityCriteriaAreMet } from './visibilityCriteria'; diff --git a/packages/admin-panel/src/utilities/visibilityCriteria.js b/packages/admin-panel/src/utilities/visibilityCriteria.js new file mode 100644 index 0000000000..ee9fdb0f66 --- /dev/null +++ b/packages/admin-panel/src/utilities/visibilityCriteria.js @@ -0,0 +1,14 @@ +/* + * Tupaia + * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd + */ + +export const checkVisibilityCriteriaAreMet = (visibilityCriteria, values) => { + if (!visibilityCriteria) { + return true; // no visibility criteria to meet, fine to display + } + return Object.entries(visibilityCriteria).every(([parameterKey, requiredValue]) => { + if (typeof requiredValue === 'function') return requiredValue(values, parameterKey); + return values[parameterKey] === requiredValue; + }); +}; diff --git a/packages/admin-panel/src/widgets/InputField/JsonInputField.js b/packages/admin-panel/src/widgets/InputField/JsonInputField.js index 053b3ab557..674a4624a2 100644 --- a/packages/admin-panel/src/widgets/InputField/JsonInputField.js +++ b/packages/admin-panel/src/widgets/InputField/JsonInputField.js @@ -10,6 +10,7 @@ import Card from '@material-ui/core/Card'; import CardContent from '@material-ui/core/CardContent'; import Typography from '@material-ui/core/Typography'; import { InputField } from './InputField'; +import { checkVisibilityCriteriaAreMet } from '../../utilities'; const getJsonFieldValues = value => { if (value) { @@ -38,8 +39,26 @@ const Container = styled.div` margin-bottom: 20px; `; +const CardLabel = styled.label` + color: rgb(111, 123, 130); + font-size: 15px; + font-weight: 400; + line-height: 18px; + display: block; + margin-bottom: 3px; +`; + export const JsonInputField = props => { - const { onChange, value, getJsonFieldSchema, disabled, label, secondaryLabel, variant } = props; + const { + onChange, + value, + getJsonFieldSchema, + disabled, + label, + secondaryLabel, + variant, + recordData, + } = props; const jsonFieldValues = getJsonFieldValues(value); const jsonFieldSchema = getJsonFieldSchema(value, props); const CardVariant = variant === 'grey' ? GreyCard : Card; @@ -51,34 +70,39 @@ export const JsonInputField = props => { return ( - - {label} - + {label} {secondaryLabel && {secondaryLabel}} - {jsonFieldSchema.map( - ({ - label: fieldLabel, - fieldName, - secondaryLabel: fieldSecondaryLabel, - type = DEFAULT_FIELD_TYPE, - csv, - ...inputFieldProps - }) => ( - onFieldValueChange(inputKey, fieldValue, csv)} - disabled={disabled} - type={type} - {...inputFieldProps} - /> - ), - )} + {jsonFieldSchema + .filter(({ visibilityCriteria }) => { + if (visibilityCriteria) { + return checkVisibilityCriteriaAreMet(visibilityCriteria, recordData); + } + return true; + }) + .map( + ({ + label: fieldLabel, + fieldName, + secondaryLabel: fieldSecondaryLabel, + type = DEFAULT_FIELD_TYPE, + csv, + ...inputFieldProps + }) => ( + onFieldValueChange(inputKey, fieldValue, csv)} + disabled={disabled} + type={type} + {...inputFieldProps} + /> + ), + )} @@ -93,6 +117,7 @@ JsonInputField.propTypes = { disabled: PropTypes.bool, secondaryLabel: PropTypes.string, variant: PropTypes.string, + recordData: PropTypes.object, }; JsonInputField.defaultProps = { @@ -100,4 +125,5 @@ JsonInputField.defaultProps = { disabled: false, secondaryLabel: null, variant: null, + recordData: {}, }; diff --git a/packages/central-server/src/database/models/DataElement.js b/packages/central-server/src/database/models/DataElement.js index 1a187bfe59..b5a079d95c 100644 --- a/packages/central-server/src/database/models/DataElement.js +++ b/packages/central-server/src/database/models/DataElement.js @@ -8,7 +8,7 @@ import { DataElementModel as CommonDataElementModel, } from '@tupaia/database'; -export const DATA_SOURCE_SERVICE_TYPES = ['dhis', 'tupaia', 'data-lake']; +export const DATA_SOURCE_SERVICE_TYPES = ['dhis', 'tupaia', 'data-lake', 'superset']; const getSurveyDateCode = surveyCode => `${surveyCode}SurveyDate`; diff --git a/packages/data-broker/package.json b/packages/data-broker/package.json index 477f433de1..82813f7909 100644 --- a/packages/data-broker/package.json +++ b/packages/data-broker/package.json @@ -29,6 +29,7 @@ "@tupaia/dhis-api": "1.0.0", "@tupaia/indicators": "1.0.0", "@tupaia/kobo-api": "1.0.0", + "@tupaia/superset-api": "1.0.0", "@tupaia/utils": "1.0.0", "@tupaia/weather-api": "1.0.0", "case": "^1.5.5", diff --git a/packages/data-broker/src/__mocks__/superset-api.js b/packages/data-broker/src/__mocks__/superset-api.js new file mode 100644 index 0000000000..4b16ac7822 --- /dev/null +++ b/packages/data-broker/src/__mocks__/superset-api.js @@ -0,0 +1,8 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd + */ + +const supersetApi = jest.createMockFromModule('@tupaia/superset-api'); + +module.exports = supersetApi; diff --git a/packages/data-broker/src/__tests__/services/superset/SupersetService.stubs.js b/packages/data-broker/src/__tests__/services/superset/SupersetService.stubs.js new file mode 100644 index 0000000000..d0a5112f2a --- /dev/null +++ b/packages/data-broker/src/__tests__/services/superset/SupersetService.stubs.js @@ -0,0 +1,95 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd + */ + +import { createModelsStub as baseCreateModelsStub } from '@tupaia/database'; +import { createJestMockInstance } from '@tupaia/utils'; + +export const DATA_ELEMENTS = { + ITEM_1: { + code: 'ITEM_1', + service_type: 'superset', + config: { + supersetInstanceCode: 'SUPERSET_INSTANCE_A', + supersetChartId: 123, + }, + }, + ITEM_2_CUSTOM_CODE: { + code: 'ITEM_2_CUSTOM_CODE', + service_type: 'superset', + config: { + supersetInstanceCode: 'SUPERSET_INSTANCE_A', + supersetChartId: 123, + supersetItemCode: 'ITEM_2', + }, + }, + DE_NO_CHART_ID: { + code: 'DE_NO_CHART_ID', + service_type: 'superset', + config: { + supersetInstanceCode: 'SUPERSET_INSTANCE_A', + }, + }, +}; + +const SUPERSET_INSTANCES = [ + { + code: 'SUPERSET_INSTANCE_A', + config: { baseUrl: 'http://localhost/' }, + }, + { + code: 'SUPERSET_INSTANCE_B', + config: { baseUrl: 'http://localhost/' }, + }, +]; + +export const SUPERSET_CHART_DATA_RESPONSE = { + result: [ + { + data: [ + { + item_code: 'ITEM_1', + store_code: 'STORE_1', + value: 1, + date: '2020-01-01', + }, + { + item_code: 'ITEM_1', + store_code: 'STORE_2', + value: 2, + date: '2020-01-01', + }, + { + item_code: 'ITEM_2', + store_code: 'STORE_1', + value: 3, + date: '2020-01-01', + }, + { + item_code: 'ITEM_2', + store_code: 'STORE_2', + value: 4, + date: '2020-01-01', + }, + ], + }, + ], +}; + +export const createModelsStub = () => { + return baseCreateModelsStub({ + dataElement: { + records: Object.values(DATA_ELEMENTS), + }, + supersetInstance: { + records: SUPERSET_INSTANCES, + }, + }); +}; + +export const createApiStub = () => { + return createJestMockInstance('@tupaia/superset-api', 'SupersetApi', { + chartData: () => SUPERSET_API_RESPONSE, + }); +}; diff --git a/packages/data-broker/src/__tests__/services/superset/SupersetService.test.js b/packages/data-broker/src/__tests__/services/superset/SupersetService.test.js new file mode 100644 index 0000000000..fa11f49f12 --- /dev/null +++ b/packages/data-broker/src/__tests__/services/superset/SupersetService.test.js @@ -0,0 +1,101 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd + */ + +import { + createModelsStub, + DATA_ELEMENTS, + SUPERSET_CHART_DATA_RESPONSE, +} from './SupersetService.stubs'; +import { SupersetService } from '../../../services/superset/SupersetService'; + +const models = createModelsStub(); +const mockApi = { + chartData: () => SUPERSET_CHART_DATA_RESPONSE, +}; +jest.mock('@tupaia/superset-api', () => ({ + SupersetApi: jest.fn().mockImplementation(() => mockApi), +})); + +describe('SupersetService', () => { + let supersetService; + + beforeEach(() => { + supersetService = new SupersetService(models); + }); + + describe('push()', () => { + it('throws an error', () => expect(supersetService.push()).toBeRejectedWith('not supported')); + }); + + describe('delete()', () => { + it('throws an error', () => expect(supersetService.delete()).toBeRejectedWith('not supported')); + }); + + describe('pull()', () => { + describe('pullAnalytics()', () => { + it('pulls', () => + expect(supersetService.pull([DATA_ELEMENTS.ITEM_1], 'dataElement')).resolves.toEqual({ + metadata: { + dataElementCodeToName: {}, + }, + results: [ + { + dataElement: 'ITEM_1', + organisationUnit: 'STORE_1', + period: '20200101', + value: 1, + }, + { + dataElement: 'ITEM_1', + organisationUnit: 'STORE_2', + period: '20200101', + value: 2, + }, + ], + })); + + it('uses supersetItemCode as a fallback', () => + expect( + supersetService.pull([DATA_ELEMENTS.ITEM_2_CUSTOM_CODE], 'dataElement'), + ).resolves.toEqual({ + metadata: expect.anything(), + results: [ + { + dataElement: 'ITEM_2_CUSTOM_CODE', + organisationUnit: 'STORE_1', + period: '20200101', + value: 3, + }, + { + dataElement: 'ITEM_2_CUSTOM_CODE', + organisationUnit: 'STORE_2', + period: '20200101', + value: 4, + }, + ], + })); + + it('throws if supersetChartId not set', () => + expect( + supersetService.pull([DATA_ELEMENTS.DE_NO_CHART_ID], 'dataElement'), + ).toBeRejectedWith('Data Element DE_NO_CHART_ID missing supersetChartId')); + }); + + describe('pullEvents()', () => { + it('throws an error', () => + expect(supersetService.pull({}, 'dataGroup')).toBeRejectedWith('not supported')); + }); + + describe('pullSyncGroups()', () => { + it('throws an error', () => + expect(supersetService.pull({}, 'syncGroup')).toBeRejectedWith('not supported')); + }); + }); + + describe('pullMetadata()', () => { + it('throws an error', () => + expect(supersetService.pullMetadata()).toBeRejectedWith('not supported')); + }); +}); diff --git a/packages/data-broker/src/services/createService.js b/packages/data-broker/src/services/createService.js index e78aa71967..c59a223d1f 100644 --- a/packages/data-broker/src/services/createService.js +++ b/packages/data-broker/src/services/createService.js @@ -14,6 +14,7 @@ import { DhisService } from './dhis'; import { IndicatorService } from './indicator'; import { WeatherService } from './weather/WeatherService'; import { KoBoService } from './kobo/KoBoService'; +import { SupersetService } from './superset/SupersetService'; export const createService = (models, type, dataBroker) => { switch (type) { @@ -29,6 +30,8 @@ export const createService = (models, type, dataBroker) => { return new WeatherService(models, new WeatherApi()); case 'kobo': return new KoBoService(models, new KoBoApi()); + case 'superset': + return new SupersetService(models); default: throw new Error(`Invalid service type: ${type}`); } diff --git a/packages/data-broker/src/services/indicator/IndicatorService.js b/packages/data-broker/src/services/indicator/IndicatorService.js index 9c21a445c8..2b3921a646 100644 --- a/packages/data-broker/src/services/indicator/IndicatorService.js +++ b/packages/data-broker/src/services/indicator/IndicatorService.js @@ -38,8 +38,7 @@ export class IndicatorService extends Service { return { results: await this.api.buildAnalytics(indicatorCodes, options), - // TODO: either implement properly in #tupaia-backlog/1153, - // or remove entirely in #tupaia-backlog/issues/1154 + // TODO: either implement properly in #NOT-521 or remove entirely in #NOT-522 metadata: { dataElementCodeToName: {} }, }; } diff --git a/packages/data-broker/src/services/superset/SupersetService.js b/packages/data-broker/src/services/superset/SupersetService.js new file mode 100644 index 0000000000..6f98751a30 --- /dev/null +++ b/packages/data-broker/src/services/superset/SupersetService.js @@ -0,0 +1,153 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd + */ +import moment from 'moment'; +import { Service } from '../Service'; +import { getSupersetApiInstance } from './getSupersetApi'; + +export class SupersetService extends Service { + /** + * @param {ModelRegistry} models + */ + constructor(models) { + super(models); + this.pullers = { + [this.dataSourceTypes.DATA_ELEMENT]: this.pullAnalytics.bind(this), + [this.dataSourceTypes.DATA_GROUP]: this.pullEvents.bind(this), + [this.dataSourceTypes.SYNC_GROUP]: this.pullSyncGroups.bind(this), + }; + } + + async push() { + throw new Error('Data push is not supported in SupersetService'); + } + + async delete() { + throw new Error('Data deletion is not supported in SupersetService'); + } + + async pull(dataSources, type, options) { + const puller = this.pullers[type]; + return puller(dataSources, options); + } + + async pullMetadata() { + throw new Error('pullMetadata is not supported in SupersetService'); + } + + /** + * @param {DataElement[]} dataSources + * @param {} options + * @private + */ + async pullAnalytics(dataSources, options) { + let mergedResults = []; + for (const [supersetInstanceCode, instanceDataSources] of Object.entries( + this.groupBySupersetInstanceCode(dataSources), + )) { + const supersetInstance = await this.models.supersetInstance.findOne({ + code: supersetInstanceCode, + }); + if (!supersetInstance) + throw new Error(`No superset instance found with code "${supersetInstanceCode}"`); + const api = await getSupersetApiInstance(this.models, supersetInstance); + for (const [chartId, chartDataSources] of Object.entries( + this.groupByChartId(instanceDataSources), + )) { + const results = await this.pullForApiForChart(api, chartId, chartDataSources); + mergedResults = mergedResults.concat(results); + } + } + + return { + results: mergedResults, + // TODO: either implement properly in #NOT-521 or remove entirely in #NOT-522 + metadata: { dataElementCodeToName: {} }, + }; + } + + /** + * @param {SupersetApi} api + * @param {string} chartId + * @param {DataElement[]} dataElements + * @return {Promise} analytic results + * @private + */ + async pullForApiForChart(api, chartId, dataElements) { + const response = await api.chartData(chartId); + const { data } = response.result[0]; + + const results = []; + for (const datum of data) { + const { item_code: itemCode, store_code: storeCode, value, date } = datum; + + const dataElement = dataElements.find( + de => de.code === itemCode || de.config.supersetItemCode === itemCode, + ); + if (!dataElement) continue; // unneeded data + + results.push({ + dataElement: dataElement.code, + organisationUnit: storeCode, + period: moment(date).format('YYYYMMDD'), + value, + }); + } + return results; + } + + /** + * @param {DataElement[]} dataSources + * @return {Object} + */ + groupBySupersetInstanceCode(dataSources) { + const dataSourcesBySupersetInstanceCode = {}; + for (const dataSource of dataSources) { + const { config } = dataSource; + const { supersetInstanceCode } = config; + if (!supersetInstanceCode) { + throw new Error(`Data Element ${dataSource.code} missing supersetInstanceCode`); + } + if (!dataSourcesBySupersetInstanceCode[supersetInstanceCode]) { + dataSourcesBySupersetInstanceCode[supersetInstanceCode] = []; + } + dataSourcesBySupersetInstanceCode[supersetInstanceCode].push(dataSource); + } + return dataSourcesBySupersetInstanceCode; + } + + /** + * @param {DataElement[]} dataSources + * @return {Object} + */ + groupByChartId(dataSources) { + const dataSourcesByChartId = {}; + for (const dataSource of dataSources) { + const { config } = dataSource; + const { supersetChartId } = config; + if (!supersetChartId) { + throw new Error(`Data Element ${dataSource.code} missing supersetChartId`); + } + if (!dataSourcesByChartId[supersetChartId]) { + dataSourcesByChartId[supersetChartId] = []; + } + dataSourcesByChartId[supersetChartId].push(dataSource); + } + return dataSourcesByChartId; + } + + /** + * @private + */ + async pullEvents() { + throw new Error('pullEvents is not supported in SupersetService'); + } + + /** + * @private + */ + async pullSyncGroups() { + throw new Error('pullSyncGroups is not supported in SupersetService'); + } +} diff --git a/packages/data-broker/src/services/superset/getSupersetApi.js b/packages/data-broker/src/services/superset/getSupersetApi.js new file mode 100644 index 0000000000..b5f036f4c1 --- /dev/null +++ b/packages/data-broker/src/services/superset/getSupersetApi.js @@ -0,0 +1,24 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd + */ + +import { SupersetApi } from '@tupaia/superset-api'; + +const instances = {}; + +/** + * @param {} models + * @param {SupersetInstance} supersetInstance + * @return {Promise} + */ +export const getSupersetApiInstance = async (models, supersetInstance) => { + const { code: serverName, config } = supersetInstance; + const { baseUrl, insecure } = config; + + if (!instances[serverName]) { + instances[serverName] = new SupersetApi(serverName, baseUrl, insecure); + } + + return instances[serverName]; +}; diff --git a/packages/data-broker/src/services/superset/index.js b/packages/data-broker/src/services/superset/index.js new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/database/src/TupaiaDatabase.js b/packages/database/src/TupaiaDatabase.js index 5c59dc5cce..8acaff760a 100644 --- a/packages/database/src/TupaiaDatabase.js +++ b/packages/database/src/TupaiaDatabase.js @@ -534,7 +534,7 @@ function buildQuery(connection, queryConfig, where = {}, options = {}) { query.returning('*'); } - if (process.env.DB_VERBOSE === 'true') { + if (process.env.DB_VERBOSE === 'true' || process.env.DB_VERBOSE === '1') { winston.info(query.toString()); } diff --git a/packages/database/src/migrations/20220811014552-CreateSupersetInstanceTable-modifies-schema.js b/packages/database/src/migrations/20220811014552-CreateSupersetInstanceTable-modifies-schema.js new file mode 100644 index 0000000000..719bd1084a --- /dev/null +++ b/packages/database/src/migrations/20220811014552-CreateSupersetInstanceTable-modifies-schema.js @@ -0,0 +1,35 @@ +'use strict'; + +var dbm; +var type; +var seed; + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ +exports.setup = function (options, seedLink) { + dbm = options.dbmigrate; + type = dbm.dataType; + seed = seedLink; +}; + +exports.up = async function (db) { + await db.runSql(` + CREATE TABLE superset_instance ( + id TEXT PRIMARY KEY, + code TEXT NOT NULL UNIQUE, + config JSONB NOT NULL + ) + `); + + await db.runSql(`ALTER TYPE service_type ADD VALUE IF NOT EXISTS 'superset';`); +}; + +exports.down = async function (db) { + await db.runSql(`DROP TABLE superset_instance`); +}; + +exports._meta = { + version: 1, +}; diff --git a/packages/database/src/modelClasses/DataElement.js b/packages/database/src/modelClasses/DataElement.js index d9d2b5e5db..4058d7a305 100644 --- a/packages/database/src/modelClasses/DataElement.js +++ b/packages/database/src/modelClasses/DataElement.js @@ -19,6 +19,7 @@ const CONFIG_SCHEMA_BY_SERVICE = { categoryOptionCombo: {}, dataElementCode: {}, dhisInstanceCode: { default: 'regional', allowNull: true }, + supersetChartId: {}, }, [SERVICE_TYPES.TUPAIA]: {}, [SERVICE_TYPES.INDICATOR]: {}, diff --git a/packages/database/src/modelClasses/SupersetInstance.js b/packages/database/src/modelClasses/SupersetInstance.js new file mode 100644 index 0000000000..266be3ea06 --- /dev/null +++ b/packages/database/src/modelClasses/SupersetInstance.js @@ -0,0 +1,18 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd + */ + +import { DatabaseType } from '../DatabaseType'; +import { TYPES } from '../types'; +import { DatabaseModel } from '../DatabaseModel'; + +export class SupersetInstanceType extends DatabaseType { + static databaseType = TYPES.SUPERSET_INSTANCE; +} + +export class SupersetInstanceModel extends DatabaseModel { + get DatabaseTypeClass() { + return SupersetInstanceType; + } +} diff --git a/packages/database/src/modelClasses/index.js b/packages/database/src/modelClasses/index.js index 7f00c3be78..b01676aee2 100644 --- a/packages/database/src/modelClasses/index.js +++ b/packages/database/src/modelClasses/index.js @@ -53,6 +53,7 @@ import { UserModel } from './User'; import { UserSessionModel } from './UserSession'; import { DataServiceEntityModel } from './DataServiceEntity'; import { DhisInstanceModel } from './DhisInstance'; +import { SupersetInstanceModel } from './SupersetInstance'; // export all models to be used in constructing a ModelRegistry export const modelClasses = { @@ -96,6 +97,7 @@ export const modelClasses = { Question: QuestionModel, RefreshToken: RefreshTokenModel, Report: ReportModel, + SupersetInstance: SupersetInstanceModel, Survey: SurveyModel, SurveyGroup: SurveyGroupModel, SurveyResponse: SurveyResponseModel, @@ -138,3 +140,4 @@ export { SurveyScreenComponentModel } from './SurveyScreenComponent'; export { SurveyScreenModel } from './SurveyScreen'; export { UserEntityPermissionModel } from './UserEntityPermission'; export { UserModel, UserType } from './User'; +export { SupersetInstanceModel } from './SupersetInstance'; diff --git a/packages/database/src/types.js b/packages/database/src/types.js index a1f76e56cd..a45345ba90 100644 --- a/packages/database/src/types.js +++ b/packages/database/src/types.js @@ -49,6 +49,7 @@ export const TYPES = { QUESTION: 'question', REFRESH_TOKEN: 'refresh_token', REPORT: 'report', + SUPERSET_INSTANCE: 'superset_instance', SETTING: 'setting', SURVEY_GROUP: 'survey_group', SURVEY_RESPONSE: 'survey_response', diff --git a/packages/devops/ci/tupaia-ci-cd.Dockerfile b/packages/devops/ci/tupaia-ci-cd.Dockerfile index 7567aa177b..ad9640bce2 100644 --- a/packages/devops/ci/tupaia-ci-cd.Dockerfile +++ b/packages/devops/ci/tupaia-ci-cd.Dockerfile @@ -65,6 +65,8 @@ RUN mkdir -p ./packages/indicators COPY packages/indicators/package.json ./packages/indicators RUN mkdir -p ./packages/kobo-api COPY packages/kobo-api/package.json ./packages/kobo-api +RUN mkdir -p ./packages/superset-api +COPY packages/superset-api/package.json ./packages/superset-api RUN mkdir -p ./packages/lesmis COPY packages/lesmis/package.json ./packages/lesmis RUN mkdir -p ./packages/lesmis-server @@ -117,6 +119,7 @@ COPY packages/ui-components/. ./packages/ui-components COPY packages/weather-api/. ./packages/weather-api COPY packages/server-boilerplate/. ./packages/server-boilerplate COPY packages/kobo-api/. ./packages/kobo-api +COPY packages/superset-api/. ./packages/superset-api COPY ./tsconfig* ./ ## build internal dependencies diff --git a/packages/report-server/.env.example b/packages/report-server/.env.example index f76589202a..55a98d0aee 100644 --- a/packages/report-server/.env.example +++ b/packages/report-server/.env.example @@ -15,4 +15,6 @@ ENTITY_API_URL= DATA_LAKE_DB_NAME= DATA_LAKE_DB_PASSWORD= DATA_LAKE_DB_URL= -DATA_LAKE_DB_USER= \ No newline at end of file +DATA_LAKE_DB_USER= +SUPERSET_API_USERNAME= +SUPERSET_API_PASSWORD= diff --git a/packages/server-boilerplate/src/connections/ApiConnection.ts b/packages/server-boilerplate/src/connections/ApiConnection.ts index bc4b550111..224b35ffde 100644 --- a/packages/server-boilerplate/src/connections/ApiConnection.ts +++ b/packages/server-boilerplate/src/connections/ApiConnection.ts @@ -8,17 +8,6 @@ import { fetchWithTimeout, verifyResponseStatus, stringifyQuery } from '@tupaia/ import { QueryParameters, RequestBody } from '../types'; import { AuthHandler } from './types'; -interface FetchHeaders { - Authorization: string; - 'Content-Type'?: string; -} - -interface FetchConfig { - method: string; - headers: FetchHeaders; - body?: string; -} - /** * @deprecated use @tupaia/api-client */ @@ -53,7 +42,7 @@ export class ApiConnection { body?: RequestBody, ) { const queryUrl = stringifyQuery(this.baseUrl, endpoint, queryParameters); - const fetchConfig: FetchConfig = { + const fetchConfig: RequestInit = { method: requestMethod || 'GET', headers: { Authorization: await this.authHandler.getAuthHeader(), diff --git a/packages/superset-api/jest.config.ts b/packages/superset-api/jest.config.ts new file mode 100644 index 0000000000..6e2ce361fd --- /dev/null +++ b/packages/superset-api/jest.config.ts @@ -0,0 +1,11 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd + */ + +import baseConfig from '../../jest.config-ts.json'; + +module.exports = async () => ({ + ...baseConfig, + rootDir: '.', +}); diff --git a/packages/superset-api/package.json b/packages/superset-api/package.json new file mode 100644 index 0000000000..e6715deddc --- /dev/null +++ b/packages/superset-api/package.json @@ -0,0 +1,24 @@ +{ + "name": "@tupaia/superset-api", + "version": "1.0.0", + "private": true, + "description": "Fetches data from a Superset API, in the form of analytics", + "repository": { + "type": "git", + "url": "git+https://github.com/beyondessential/tupaia.git", + "directory": "packages/superset-api" + }, + "author": "Beyond Essential Systems (https://beyondessential.com.au)", + "main": "dist/index.js", + "scripts": { + "build": "rm -rf dist && npm run --prefix ../../ package:build:ts", + "build-dev": "npm run build", + "lint": "yarn package:lint:ts", + "lint:fix": "yarn lint --fix", + "test": "yarn package:test" + }, + "dependencies": { + "@tupaia/utils": "1.0.0", + "winston": "^3.3.3" + } +} diff --git a/packages/superset-api/src/SupersetApi.ts b/packages/superset-api/src/SupersetApi.ts new file mode 100644 index 0000000000..9278b1a2f3 --- /dev/null +++ b/packages/superset-api/src/SupersetApi.ts @@ -0,0 +1,114 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd + */ +import { fetchWithTimeout } from '@tupaia/utils'; +import { + ChartDataResponseSchema, + SecurityLoginRequestBodySchema, + SecurityLoginResponseBodySchema, +} from './types'; +import { Agent as HttpsAgent } from 'https'; +import winston from 'winston'; + +const MAX_RETRIES = 1; +const MAX_FETCH_WAIT_TIME = 45 * 1000; // 45 seconds + +export class SupersetApi { + protected serverName: string; + protected baseUrl: string; + protected insecure: boolean; + protected insecureAgent: HttpsAgent; + protected accessToken: string | null = null; + + public constructor(serverName: string, baseUrl: string, insecure: boolean = false) { + if (!serverName) throw new Error('Argument serverName required'); + if (!baseUrl) throw new Error('Argument baseUrl required'); + this.serverName = serverName; + this.baseUrl = baseUrl; + this.insecure = insecure; + this.insecureAgent = new HttpsAgent({ rejectUnauthorized: false }); + } + + public async chartData(chartId: number): Promise { + return this.fetch(`${this.baseUrl}/api/v1/chart/${chartId}/data/`); + } + + protected async fetch(url: string, numRetries = 0): Promise { + if (numRetries > MAX_RETRIES) { + throw new Error(`Superset exceeded max retries (${MAX_RETRIES}). Failed to fetch ${url}`); + } + + if (!this.accessToken) { + await this.refreshAccessToken(); + return this.fetch(url, numRetries + 1); + } + + const fetchConfig: any = { + method: 'GET', + headers: { + Authorization: `Bearer ${this.accessToken}`, + 'Content-Type': 'application/json', + }, + }; + if (this.insecure) fetchConfig.agent = this.insecureAgent; + winston.info(`Superset fetch ${this.insecure ? '(insecure) ' : ' '}${url}`); + + const result = await fetchWithTimeout(url, fetchConfig, MAX_FETCH_WAIT_TIME); + + if (result.status !== 200) { + const bodyText = await result.text(); + + if (result.status === 422) { + winston.info(`Superset Auth error, response: ${bodyText}`); + await this.refreshAccessToken(); + return this.fetch(url, numRetries + 1); + } + + throw new Error( + `Error response from Superset API. Status: ${result.status}, body: ${bodyText}`, + ); + } + + return await result.json(); + } + + protected async refreshAccessToken() { + const getServerVariable = (variableName: string) => + process.env[`${this.serverName.toUpperCase()}_${variableName}`] || + process.env[variableName] || + ''; + + const username = getServerVariable('SUPERSET_API_USERNAME'); + const password = getServerVariable('SUPERSET_API_PASSWORD'); + + const body: SecurityLoginRequestBodySchema = { + username, + password, + provider: 'db', + refresh: true, + }; + + const url = `${this.baseUrl}/api/v1/security/login`; + const fetchConfig: any = { + method: 'POST', + body: JSON.stringify(body), + headers: { 'Content-Type': 'application/json' }, + }; + if (this.insecure) fetchConfig.agent = this.insecureAgent; + + winston.info(`Superset refresh access token ${this.insecure ? '(insecure) ' : ' '}${url}`); + const result = await fetchWithTimeout(url, fetchConfig, MAX_FETCH_WAIT_TIME); + + if (result.status !== 200) { + const bodyText = await result.text(); + throw new Error( + `Superset failed to refresh access token. Status: ${result.status}, body: ${bodyText}`, + ); + } + + const resultBody: SecurityLoginResponseBodySchema = await result.json(); + + this.accessToken = resultBody.access_token; + } +} diff --git a/packages/superset-api/src/__tests__/example.test.ts b/packages/superset-api/src/__tests__/example.test.ts new file mode 100644 index 0000000000..1524ec184e --- /dev/null +++ b/packages/superset-api/src/__tests__/example.test.ts @@ -0,0 +1,10 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd + */ + +describe('Example test', () => { + it('passes a stub test', () => { + expect(1).toBe(1); + }); +}); diff --git a/packages/superset-api/src/index.ts b/packages/superset-api/src/index.ts new file mode 100644 index 0000000000..8657e2a166 --- /dev/null +++ b/packages/superset-api/src/index.ts @@ -0,0 +1,6 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd + */ + +export { SupersetApi } from './SupersetApi'; diff --git a/packages/superset-api/src/types.ts b/packages/superset-api/src/types.ts new file mode 100644 index 0000000000..2d36ecc2f6 --- /dev/null +++ b/packages/superset-api/src/types.ts @@ -0,0 +1,43 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd + */ + +// From https://superset.apache.org/docs/api/ +export type ChartDataResponseSchema = { + result: ChartDataResponseResult[]; +}; + +// From https://superset.apache.org/docs/api/ +export type ChartDataResponseResult = { + annotation_data?: Record; + applied_filters: any[]; + cache_key?: string; + cache_timeout?: number; + cached_dttm?: string; + colnames: string[]; + coltypes: number[]; + data: any[]; + error?: string; + from_dttm?: number; + is_cached: boolean; + query: string; + rejected_filters: any[]; + stacktrace?: string; + status: 'stopped' | 'failed' | 'pending' | 'running' | 'scheduled' | 'success' | 'timed_out'; + to_dttm?: number; +}; + +// From https://superset.apache.org/docs/api/ +export type SecurityLoginRequestBodySchema = { + password: string; + provider: 'db' | 'ldap'; + refresh: boolean; + username: string; +}; + +// From https://superset.apache.org/docs/api/ +export type SecurityLoginResponseBodySchema = { + access_token: string; + refresh_token: string; +}; diff --git a/packages/superset-api/tsconfig-build.json b/packages/superset-api/tsconfig-build.json new file mode 100644 index 0000000000..d5c844ecde --- /dev/null +++ b/packages/superset-api/tsconfig-build.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "declaration": true, + "noEmit": false + }, + "include": ["src/**/*"], + "exclude": ["src/__tests__"] +} diff --git a/packages/superset-api/tsconfig.json b/packages/superset-api/tsconfig.json new file mode 100644 index 0000000000..6d72982c06 --- /dev/null +++ b/packages/superset-api/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig-ts.json", + "compilerOptions": { + "noEmit": true + }, + "include": ["src/**/*", "jest.config.ts"] +} diff --git a/packages/utils/src/errors.js b/packages/utils/src/errors.js index 64afe0183f..3675b917ec 100644 --- a/packages/utils/src/errors.js +++ b/packages/utils/src/errors.js @@ -11,9 +11,12 @@ import { respond } from './respond'; * this may change to saving the error info to the database, notifying the admin, or similar */ class LoggedError extends Error { - constructor(message) { + constructor(message, originalError = null) { super(message); this.message = message; + if (originalError) { + winston.error('Original error:', { stack: originalError.stack }); + } winston.error(this.message, { stack: this.stack }); } } @@ -23,8 +26,8 @@ class LoggedError extends Error { * the appropriate http status code */ export class RespondingError extends LoggedError { - constructor(message, statusCode, extraFields = {}) { - super(message); + constructor(message, statusCode, extraFields = {}, originalError = null) { + super(message, originalError); this.statusCode = statusCode; this.extraFields = extraFields; this.respond = res => respond(res, { error: this.message, ...extraFields }, statusCode); @@ -46,7 +49,7 @@ export class DatabaseError extends RespondingError { export class InternalServerError extends RespondingError { constructor(error) { - super(`Internal server error: ${error.message}`, 500); + super(`Internal server error: ${error.message}`, 500, undefined, error); } } diff --git a/packages/utils/src/request.js b/packages/utils/src/request.js index 9528b00a8f..33eb4f1ec8 100644 --- a/packages/utils/src/request.js +++ b/packages/utils/src/request.js @@ -22,9 +22,7 @@ export const stringifyQuery = (baseUrl, endpoint, queryParams) => { const urlAndEndpoint = baseUrl ? `${baseUrl}/${endpoint}` : endpoint; - return queryParamsString - ? `${urlAndEndpoint}?${queryParamsString}` - : `${urlAndEndpoint}`; + return queryParamsString ? `${urlAndEndpoint}?${queryParamsString}` : `${urlAndEndpoint}`; }; /** @@ -44,6 +42,13 @@ const createTimeoutPromise = maxWaitTime => { }); return { promise, cleanup }; }; + +/** + * @param {string} url + * @param {} [config] + * @param {number} [maxWaitTime] + * @return {Promise} + */ export const fetchWithTimeout = async (url, config, maxWaitTime = DEFAULT_MAX_WAIT_TIME) => { const { cleanup, promise: timeoutPromise } = createTimeoutPromise(maxWaitTime); try { diff --git a/scripts/bash/getInternalDependencies.sh b/scripts/bash/getInternalDependencies.sh index 05f624a3d8..a2d77845ca 100755 --- a/scripts/bash/getInternalDependencies.sh +++ b/scripts/bash/getInternalDependencies.sh @@ -12,7 +12,7 @@ dependencies_already_visited=($@) # if no package.json entrypoint is specified, just return all internal dependencies if [ -z ${package_path} ]; then - echo "ui-components" "tsutils" "utils" "access-policy" "admin-panel" "aggregator" "api-client" "auth" "database" "data-api" "data-broker" "data-lake-api" "dhis-api" "expression-parser" "indicators" "weather-api" "server-boilerplate" "kobo-api" + echo "ui-components" "tsutils" "utils" "access-policy" "admin-panel" "aggregator" "api-client" "auth" "database" "data-api" "data-broker" "data-lake-api" "dhis-api" "expression-parser" "indicators" "weather-api" "server-boilerplate" "kobo-api" "superset-api" exit 0 fi diff --git a/tupaia-packages.code-workspace b/tupaia-packages.code-workspace index 6e19831382..248e6cb347 100644 --- a/tupaia-packages.code-workspace +++ b/tupaia-packages.code-workspace @@ -104,6 +104,10 @@ "name": "server-boilerplate", "path": "packages/server-boilerplate" }, + { + "name": "superset-api", + "path": "packages/superset-api" + }, { "name": "ui-components", "path": "packages/ui-components" diff --git a/yarn.lock b/yarn.lock index 4ae6e65db6..1c751db472 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5183,6 +5183,7 @@ __metadata: "@tupaia/dhis-api": 1.0.0 "@tupaia/indicators": 1.0.0 "@tupaia/kobo-api": 1.0.0 + "@tupaia/superset-api": 1.0.0 "@tupaia/utils": 1.0.0 "@tupaia/weather-api": 1.0.0 case: ^1.5.5 @@ -5597,6 +5598,15 @@ __metadata: languageName: unknown linkType: soft +"@tupaia/superset-api@1.0.0, @tupaia/superset-api@workspace:packages/superset-api": + version: 0.0.0-use.local + resolution: "@tupaia/superset-api@workspace:packages/superset-api" + dependencies: + "@tupaia/utils": 1.0.0 + winston: ^3.3.3 + languageName: unknown + linkType: soft + "@tupaia/tsutils@1.0.0, @tupaia/tsutils@workspace:packages/tsutils": version: 0.0.0-use.local resolution: "@tupaia/tsutils@workspace:packages/tsutils" From a499bf536f645e6f74a00724c7137938eeef27d1 Mon Sep 17 00:00:00 2001 From: Chris Pollard Date: Fri, 26 Aug 2022 15:52:43 +1000 Subject: [PATCH 06/14] MAUI-1042: Add Household Head Attribute (#4039) * MAUI-1042: Add household_head attribute hook * Hook adds custom attribute to entity * Improve attribute hook, add test * Add entityAttributeHouseholdHead to question Co-authored-by: Andrew --- .../src/database/models/Answer.js | 1 + .../central-server/src/hooks/constants.js | 6 + .../src/hooks/entityAttribute.js | 17 +++ packages/central-server/src/hooks/index.js | 2 + packages/central-server/src/hooks/registry.js | 10 +- .../src/tests/hooks/questionHooks.test.js | 116 ++++++++++++++++++ ...okHouseholdHeadToQuestion-modifies-data.js | 35 ++++++ 7 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 packages/central-server/src/hooks/constants.js create mode 100644 packages/central-server/src/hooks/entityAttribute.js create mode 100644 packages/database/src/migrations/20220819002505-AddEntityAttributeHookHouseholdHeadToQuestion-modifies-data.js diff --git a/packages/central-server/src/database/models/Answer.js b/packages/central-server/src/database/models/Answer.js index 1b579d63b1..7c718c8a32 100644 --- a/packages/central-server/src/database/models/Answer.js +++ b/packages/central-server/src/database/models/Answer.js @@ -114,6 +114,7 @@ class AnswerType extends DatabaseType { answer: this, models: this.otherModels, surveyResponse, + hookName: hookId, }), `${hookId}:${this.id}`, ); diff --git a/packages/central-server/src/hooks/constants.js b/packages/central-server/src/hooks/constants.js new file mode 100644 index 0000000000..d75777f6fb --- /dev/null +++ b/packages/central-server/src/hooks/constants.js @@ -0,0 +1,6 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd + */ + +export const ENTITY_ATTRIBUTE_HOOK_PREFIX = 'entityAttribute'; diff --git a/packages/central-server/src/hooks/entityAttribute.js b/packages/central-server/src/hooks/entityAttribute.js new file mode 100644 index 0000000000..a398426414 --- /dev/null +++ b/packages/central-server/src/hooks/entityAttribute.js @@ -0,0 +1,17 @@ +import { snake } from 'case'; +import { ENTITY_ATTRIBUTE_HOOK_PREFIX } from './constants'; + +export async function entityAttribute({ answer, surveyResponse, hookName }) { + const entity = await surveyResponse.entity(); + + if (!entity) { + throw new Error('Invalid entity'); + } + + const attributeKey = snake(hookName.substring(ENTITY_ATTRIBUTE_HOOK_PREFIX.length)); + const attributeValue = answer.text; + + entity.attributes = { ...entity.attributes, [attributeKey]: attributeValue }; + + await entity.save(); +} diff --git a/packages/central-server/src/hooks/index.js b/packages/central-server/src/hooks/index.js index 364e0b0ea0..0d295006e4 100644 --- a/packages/central-server/src/hooks/index.js +++ b/packages/central-server/src/hooks/index.js @@ -1,4 +1,5 @@ import { entityImage } from './entityImage'; +import { entityAttribute } from './entityAttribute'; import { entityCoordinates } from './entityCoordinates'; import * as entityCreators from './entityCreate'; import { deduplicateQuestion } from './deduplicateQuestion'; @@ -7,6 +8,7 @@ import { registerHook } from './registry'; function registerAllHooks() { Object.entries({ entityImage, + entityAttribute, entityCoordinates, ...entityCreators, deduplicateQuestion, diff --git a/packages/central-server/src/hooks/registry.js b/packages/central-server/src/hooks/registry.js index e4f62bef2e..09bb3d3a76 100644 --- a/packages/central-server/src/hooks/registry.js +++ b/packages/central-server/src/hooks/registry.js @@ -1,3 +1,5 @@ +import { ENTITY_ATTRIBUTE_HOOK_PREFIX } from './constants'; + const hookRegistry = {}; export function registerHook(name, callback) { @@ -5,8 +7,14 @@ export function registerHook(name, callback) { return callback; } +const isEntityAttribute = name => { + return name.startsWith(ENTITY_ATTRIBUTE_HOOK_PREFIX); +}; + export function getHook(name) { - const hook = hookRegistry[name]; + const hook = isEntityAttribute(name) + ? hookRegistry[ENTITY_ATTRIBUTE_HOOK_PREFIX] + : hookRegistry[name]; if (!hook) { throw new Error('No such hook: ', name); diff --git a/packages/central-server/src/tests/hooks/questionHooks.test.js b/packages/central-server/src/tests/hooks/questionHooks.test.js index fb6f2d7160..4a90138fbf 100644 --- a/packages/central-server/src/tests/hooks/questionHooks.test.js +++ b/packages/central-server/src/tests/hooks/questionHooks.test.js @@ -5,6 +5,8 @@ import { TestableApp } from '../testUtilities'; import { registerHook } from '../../hooks'; const ENTITY_ID = generateTestId(); +const ENTITY2_ID = generateTestId(); +const ENTITY3_ID = generateTestId(); const GENERIC_SURVEY_ID = generateTestId(); const ENTITY_CREATION_SURVEY_ID = generateTestId(); @@ -19,6 +21,7 @@ const SURVEYS = [ { code: 'TEST_backdate-test', type: 'Text', hook: 'backdateTestHook' }, { code: 'TEST_whole-survey-a', type: 'Text', hook: 'wholeSurvey' }, { code: 'TEST_whole-survey-b', type: 'Text' }, + { code: 'TEST_attribute', type: 'Text', hook: 'entityAttributeBananaPopulation' }, ], }, { @@ -73,6 +76,22 @@ describe('Question hooks', () => { type: models.entity.types.FACILITY, }); + await models.entity.create({ + id: ENTITY2_ID, + code: ENTITY2_ID, + name: 'test entity', + type: models.entity.types.FACILITY, + attributes: { test_attribute: 'test_value' }, + }); + + await models.entity.create({ + id: ENTITY3_ID, + code: ENTITY3_ID, + name: 'test entity', + type: models.entity.types.FACILITY, + attributes: { banana_population: '1000' }, + }); + await buildAndInsertSurveys(models, SURVEYS); }); @@ -258,6 +277,103 @@ describe('Question hooks', () => { expect(beforeData).to.deep.equal(afterData); }); + describe('Adding an entity attribute', () => { + it("Should add a custom attribute to entity's attributes when no attributes currently exist", async () => { + const TEST_BANANA_POPULATION_VALUE = '1000'; + + const beforeEntity = await models.entity.findById(ENTITY_ID); + expect(beforeEntity.attributes).to.be.an('object'); + + // submit a survey response + await app.post('surveyResponse', { + body: { + survey_id: GENERIC_SURVEY_ID, + entity_id: ENTITY_ID, + timestamp: 123, + answers: { + TEST_attribute: TEST_BANANA_POPULATION_VALUE, + }, + }, + }); + + const newValue = { banana_population: TEST_BANANA_POPULATION_VALUE }; + await database.waitForAllChangeHandlers(); + + const entity = await models.entity.findById(ENTITY_ID); + expect(entity.attributes).to.deep.equal(newValue); + + // should be otherwise unchanged + const { attributes: beforeAttributes, ...beforeData } = await beforeEntity.getData(); + const { attributes: afterAttributes, ...afterData } = await entity.getData(); + expect(beforeData).to.deep.equal(afterData); + }); + + it("Should overwrite an existing attribute's value with the new value", async () => { + const AFTER_BANANA_POPULATION_VALUE = '5000'; + + const beforeEntity = await models.entity.findById(ENTITY3_ID); + expect(beforeEntity.attributes).to.deep.equal({ banana_population: '1000' }); + + // submit a survey response + await app.post('surveyResponse', { + body: { + survey_id: GENERIC_SURVEY_ID, + entity_id: ENTITY3_ID, + timestamp: 123, + answers: { + TEST_attribute: AFTER_BANANA_POPULATION_VALUE, + }, + }, + }); + + const newValue = { + banana_population: AFTER_BANANA_POPULATION_VALUE, + }; + await database.waitForAllChangeHandlers(); + + const entity = await models.entity.findById(ENTITY3_ID); + expect(entity.attributes).to.deep.equal(newValue); + + // should be otherwise unchanged + const { attributes: beforeAttributes, ...beforeData } = await beforeEntity.getData(); + const { attributes: afterAttributes, ...afterData } = await entity.getData(); + expect(beforeData).to.deep.equal(afterData); + }); + + it("Should add custom attribute to entity's attributes when there are existing attributes", async () => { + const TEST_BANANA_POPULATION_VALUE = '2000'; + + const beforeEntity = await models.entity.findById(ENTITY2_ID); + expect(beforeEntity.attributes).to.deep.equal({ test_attribute: 'test_value' }); + + // submit a survey response + await app.post('surveyResponse', { + body: { + survey_id: GENERIC_SURVEY_ID, + entity_id: ENTITY2_ID, + timestamp: 123, + answers: { + TEST_attribute: TEST_BANANA_POPULATION_VALUE, + }, + }, + }); + + const newValue = { + test_attribute: 'test_value', + banana_population: TEST_BANANA_POPULATION_VALUE, + }; + await database.waitForAllChangeHandlers(); + + const entity = await models.entity.findById(ENTITY2_ID); + expect(entity.attributes).to.deep.equal(newValue); + + // should be otherwise unchanged + const { attributes: beforeAttributes, ...beforeData } = await beforeEntity.getData(); + const { attributes: afterAttributes, ...afterData } = await entity.getData(); + expect(beforeData).to.deep.equal(afterData); + }); + }); + describe('Entity creation', () => { it('Should create an entity', async () => { const TEST_URL = 'https://facilities.com/test-dynamic.jpg'; diff --git a/packages/database/src/migrations/20220819002505-AddEntityAttributeHookHouseholdHeadToQuestion-modifies-data.js b/packages/database/src/migrations/20220819002505-AddEntityAttributeHookHouseholdHeadToQuestion-modifies-data.js new file mode 100644 index 0000000000..363f50b96e --- /dev/null +++ b/packages/database/src/migrations/20220819002505-AddEntityAttributeHookHouseholdHeadToQuestion-modifies-data.js @@ -0,0 +1,35 @@ +'use strict'; + +import { updateValues, codeToId } from '../utilities'; + +var dbm; +var type; +var seed; + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ + +exports.setup = function (options, seedLink) { + dbm = options.dbmigrate; + type = dbm.dataType; + seed = seedLink; +}; + +const QUESTION_CODE = 'PW_EH01_003'; +const QUESTION_TABLE = 'question'; +const HOOK = 'entityAttributeHouseholdHead'; + +exports.up = async function (db) { + const questionId = await codeToId(db, 'question', QUESTION_CODE); + await updateValues(db, QUESTION_TABLE, { hook: HOOK }, { id: questionId }); +}; + +exports.down = function (db) { + return null; +}; + +exports._meta = { + version: 1, +}; From ae9c7133389243c9ecf8ca506d10ca1f2090fe63 Mon Sep 17 00:00:00 2001 From: Chris Pollard Date: Fri, 26 Aug 2022 17:18:59 +1000 Subject: [PATCH 07/14] MAUI-1064: Enable Editing of Entity Name in Admin Panel (#4044) * Add put route for entity name * Add test, secure entities put backend, update title config * Add title to dispatch, remove editConfig * Update title config * Remove unused prop ConfirmDelteModalComponent * Update actionConfig title Co-authored-by: Sima-BES <87400368+Sima-BES@users.noreply.github.com> --- packages/admin-panel/src/editor/EditModal.js | 1 + packages/admin-panel/src/editor/actions.js | 6 +- packages/admin-panel/src/editor/reducer.js | 1 + .../src/pages/resources/AccessRequestsPage.js | 5 +- .../src/pages/resources/CountriesPage.js | 2 +- .../src/pages/resources/DashboardItemsPage.js | 4 +- .../pages/resources/DashboardRelationsPage.js | 6 +- .../src/pages/resources/DashboardsPage.js | 4 +- .../src/pages/resources/DataSourcesPage.js | 4 +- .../src/pages/resources/EntitiesPage.js | 28 ++++++--- .../src/pages/resources/IndicatorsPage.js | 6 +- .../src/pages/resources/LegacyReportsPage.js | 4 +- .../resources/MapOverlayGroupRelationsPage.js | 1 + .../pages/resources/MapOverlayGroupsPage.js | 1 + .../src/pages/resources/MapOverlaysPage.js | 4 +- .../src/pages/resources/OptionSetsPage.js | 6 +- .../pages/resources/PermissionGroupsPage.js | 6 +- .../src/pages/resources/PermissionsPage.js | 6 +- .../src/pages/resources/ProjectsPage.js | 13 +--- .../src/pages/resources/QuestionsPage.js | 6 +- .../src/pages/resources/ResourcePage.js | 11 +--- .../src/pages/resources/SocialFeedPage.js | 6 +- .../pages/resources/SurveyResponsesPage.js | 7 +-- .../src/pages/resources/SurveysPage.js | 7 +-- .../src/pages/resources/UsersPage.js | 6 +- .../src/table/DataFetchingTable.js | 15 +---- .../src/apiV2/entities/EditEntity.js | 17 ++++++ .../src/apiV2/entities/index.js | 6 ++ packages/central-server/src/apiV2/index.js | 2 + .../tests/apiV2/entities/EditEntity.test.js | 60 +++++++++++++++++++ 30 files changed, 139 insertions(+), 112 deletions(-) create mode 100644 packages/central-server/src/apiV2/entities/EditEntity.js create mode 100644 packages/central-server/src/apiV2/entities/index.js create mode 100644 packages/central-server/src/tests/apiV2/entities/EditEntity.test.js diff --git a/packages/admin-panel/src/editor/EditModal.js b/packages/admin-panel/src/editor/EditModal.js index 17079776e9..8d7e6a9713 100644 --- a/packages/admin-panel/src/editor/EditModal.js +++ b/packages/admin-panel/src/editor/EditModal.js @@ -152,6 +152,7 @@ const mergeProps = ( } dispatch(saveEdits(endpoint, fieldValuesToSave, isNew)); }, + endpoint, }; }; diff --git a/packages/admin-panel/src/editor/actions.js b/packages/admin-panel/src/editor/actions.js index 527aaa818a..561fe24c61 100644 --- a/packages/admin-panel/src/editor/actions.js +++ b/packages/admin-panel/src/editor/actions.js @@ -18,7 +18,7 @@ import { convertSearchTermToFilter, makeSubstitutionsInString } from '../utiliti const STATIC_FIELD_TYPES = ['link']; export const openBulkEditModal = ( - { bulkGetEndpoint, bulkUpdateEndpoint, fields, baseFilter }, + { bulkGetEndpoint, bulkUpdateEndpoint, fields, title, baseFilter }, recordId, rowData, ) => async (dispatch, getState, { api }) => { @@ -26,6 +26,7 @@ export const openBulkEditModal = ( dispatch({ type: EDITOR_DATA_FETCH_BEGIN, fields, + title, endpoint: bulkUpdateEndpoint, }); // Set up filter @@ -76,7 +77,7 @@ export const openBulkEditModal = ( } }; -export const openEditModal = ({ editEndpoint, fields }, recordId) => async ( +export const openEditModal = ({ editEndpoint, title, fields }, recordId) => async ( dispatch, getState, { api }, @@ -86,6 +87,7 @@ export const openEditModal = ({ editEndpoint, fields }, recordId) => async ( dispatch({ type: EDITOR_DATA_FETCH_BEGIN, fields, + title, endpoint, recordId, }); diff --git a/packages/admin-panel/src/editor/reducer.js b/packages/admin-panel/src/editor/reducer.js index 1f42d5a969..a6598caded 100644 --- a/packages/admin-panel/src/editor/reducer.js +++ b/packages/admin-panel/src/editor/reducer.js @@ -22,6 +22,7 @@ const defaultState = { recordId: null, recordData: null, fields: null, + title: 'Edit', editedFields: {}, }; diff --git a/packages/admin-panel/src/pages/resources/AccessRequestsPage.js b/packages/admin-panel/src/pages/resources/AccessRequestsPage.js index 62dbc733d5..08b15b1edc 100644 --- a/packages/admin-panel/src/pages/resources/AccessRequestsPage.js +++ b/packages/admin-panel/src/pages/resources/AccessRequestsPage.js @@ -84,6 +84,7 @@ const USER_COLUMNS = [ type: 'bulkEdit', width: 150, actionConfig: { + title: 'Edit & Approve Access Requests', bulkGetEndpoint: `users/{user_id}/${ACCESS_REQUESTS_ENDPOINT}`, bulkUpdateEndpoint: `${ACCESS_REQUESTS_ENDPOINT}`, baseFilter: { approved: null }, @@ -147,6 +148,7 @@ const EXPANSION_COLUMNS = [ type: 'edit', source: 'id', actionConfig: { + title: 'Edit & Approve Access Request', editEndpoint: 'accessRequests', fields: [ ...ACCESS_REQUEST_FIELDS, @@ -179,9 +181,6 @@ export const AccessRequestsPage = ({ getHeaderEl }) => ( columns={USER_COLUMNS} expansionTabs={EXPANSION_CONFIG} baseFilter={{ approved: null }} - editConfig={{ - title: 'Edit & Approve Access Request', - }} getHeaderEl={getHeaderEl} onProcessDataForSave={(editedFields, recordData) => { if (!Array.isArray(recordData)) { diff --git a/packages/admin-panel/src/pages/resources/CountriesPage.js b/packages/admin-panel/src/pages/resources/CountriesPage.js index 520ef4dd76..703bfc8400 100644 --- a/packages/admin-panel/src/pages/resources/CountriesPage.js +++ b/packages/admin-panel/src/pages/resources/CountriesPage.js @@ -31,6 +31,7 @@ const EXPANSION_CONFIG = [ const CREATE_CONFIG = { title: 'New Country', actionConfig: { + title: 'Create New Country', editEndpoint: 'countries', fields: FIELDS, }, @@ -42,7 +43,6 @@ export const CountriesPage = ({ getHeaderEl }) => ( endpoint="countries" columns={FIELDS} expansionTabs={EXPANSION_CONFIG} - editConfig={{ title: 'Create New Country' }} createConfig={CREATE_CONFIG} getHeaderEl={getHeaderEl} /> diff --git a/packages/admin-panel/src/pages/resources/DashboardItemsPage.js b/packages/admin-panel/src/pages/resources/DashboardItemsPage.js index 0d65a2584c..b0d43bbadc 100644 --- a/packages/admin-panel/src/pages/resources/DashboardItemsPage.js +++ b/packages/admin-panel/src/pages/resources/DashboardItemsPage.js @@ -116,6 +116,7 @@ export const DashboardItemsPage = ({ getHeaderEl, isBESAdmin, ...props }) => { type: 'edit', source: 'id', actionConfig: { + title: 'Edit Dashboard Item', editEndpoint: DASHBOARD_ITEMS_ENDPOINT, fields: [...FIELDS, ...extraEditFields], }, @@ -136,9 +137,6 @@ export const DashboardItemsPage = ({ getHeaderEl, isBESAdmin, ...props }) => { endpoint={DASHBOARD_ITEMS_ENDPOINT} columns={columns} importConfig={IMPORT_CONFIG} - editConfig={{ - title: 'Edit Dashboard Item', - }} LinksComponent={renderNewDashboardVizButton} getHeaderEl={getHeaderEl} {...props} diff --git a/packages/admin-panel/src/pages/resources/DashboardRelationsPage.js b/packages/admin-panel/src/pages/resources/DashboardRelationsPage.js index 76370d717b..fcbe89d013 100644 --- a/packages/admin-panel/src/pages/resources/DashboardRelationsPage.js +++ b/packages/admin-panel/src/pages/resources/DashboardRelationsPage.js @@ -85,6 +85,7 @@ const FIELDS = [ source: 'id', type: 'edit', actionConfig: { + title: 'Edit Dashboard Relation', editEndpoint: DASHBOARD_RELATION_ENDPOINT, fields: DASHBOARD_RELATION_COLUMNS, }, @@ -99,10 +100,6 @@ const FIELDS = [ }, ]; -const EDIT_CONFIG = { - title: 'Edit Dashboard Relation', -}; - const CREATE_CONFIG = { title: 'Create a new relation between Dashboard and DashboardItem', actionConfig: { @@ -116,7 +113,6 @@ export const DashboardRelationsPage = ({ getHeaderEl }) => ( title="Dashboard Relations" endpoint={DASHBOARD_RELATION_ENDPOINT} columns={FIELDS} - editConfig={EDIT_CONFIG} createConfig={CREATE_CONFIG} getHeaderEl={getHeaderEl} /> diff --git a/packages/admin-panel/src/pages/resources/DashboardsPage.js b/packages/admin-panel/src/pages/resources/DashboardsPage.js index f7e38ece65..0be8f806f2 100644 --- a/packages/admin-panel/src/pages/resources/DashboardsPage.js +++ b/packages/admin-panel/src/pages/resources/DashboardsPage.js @@ -42,6 +42,7 @@ const COLUMNS = [ type: 'edit', source: 'id', actionConfig: { + title: 'Edit Dashboard', editEndpoint: 'dashboards', fields: [...FIELDS], }, @@ -138,9 +139,6 @@ export const DashboardsPage = ({ getHeaderEl }) => ( columns={COLUMNS} expansionTabs={EXPANSION_CONFIG} createConfig={CREATE_CONFIG} - editConfig={{ - title: 'Edit Dashboard', - }} getHeaderEl={getHeaderEl} /> ); diff --git a/packages/admin-panel/src/pages/resources/DataSourcesPage.js b/packages/admin-panel/src/pages/resources/DataSourcesPage.js index 1436c27bac..3c1c9fdba4 100644 --- a/packages/admin-panel/src/pages/resources/DataSourcesPage.js +++ b/packages/admin-panel/src/pages/resources/DataSourcesPage.js @@ -184,10 +184,10 @@ export const DataGroupsPage = ({ getHeaderEl }) => ( columns: [...DATA_ELEMENT_FIELDS, ...getButtonsConfig(DATA_ELEMENT_FIELDS, 'dataElement')], }, ]} - editConfig={{ title: 'Edit Data Group' }} createConfig={{ title: 'New Data Group', actionConfig: { + title: 'Edit Data Group', editEndpoint: 'dataGroups', fields: [...DATA_GROUP_FIELDS], }, @@ -214,11 +214,11 @@ export const DataElementsPage = ({ getHeaderEl }) => ( endpoint="dataElements" reduxId="dataElements" columns={[...DATA_ELEMENT_FIELDS, ...getButtonsConfig(DATA_ELEMENT_FIELDS, 'dataElement')]} - editConfig={{ title: 'Edit Data Element' }} importConfig={IMPORT_CONFIG} createConfig={{ title: 'New Data Element', actionConfig: { + title: 'Edit Data Element', editEndpoint: 'dataElements', fields: [...DATA_ELEMENT_FIELDS], }, diff --git a/packages/admin-panel/src/pages/resources/EntitiesPage.js b/packages/admin-panel/src/pages/resources/EntitiesPage.js index 60aa7c3ecd..7dc19b6089 100644 --- a/packages/admin-panel/src/pages/resources/EntitiesPage.js +++ b/packages/admin-panel/src/pages/resources/EntitiesPage.js @@ -8,6 +8,8 @@ import PropTypes from 'prop-types'; import { ResourcePage } from './ResourcePage'; import { SURVEY_RESPONSE_COLUMNS, ANSWER_COLUMNS } from './SurveyResponsesPage'; +const ENTITY_ENDPOINT = 'entities'; + export const ENTITIES_COLUMNS = [ { source: 'id', show: false }, { @@ -25,12 +27,27 @@ export const ENTITIES_COLUMNS = [ }, ]; -const COLUMNS = [ +const FIELDS = [ ...ENTITIES_COLUMNS, { Header: 'Country', source: 'country_code', }, + { + Header: 'Edit', + source: 'id', + type: 'edit', + actionConfig: { + editEndpoint: ENTITY_ENDPOINT, + title: 'Edit Entity', + fields: [ + { + Header: 'Name', + source: 'name', + }, + ], + }, + }, ]; const EXPANSION_CONFIG = [ @@ -75,16 +92,11 @@ const IMPORT_CONFIG = { ], }; -const EDIT_CONFIG = { - title: 'Edit Answer', -}; - export const EntitiesPage = ({ getHeaderEl }) => ( ( title="Indicators" endpoint="indicators" columns={COLUMNS} - editConfig={EDIT_CONFIG} createConfig={CREATE_CONFIG} getHeaderEl={getHeaderEl} /> diff --git a/packages/admin-panel/src/pages/resources/LegacyReportsPage.js b/packages/admin-panel/src/pages/resources/LegacyReportsPage.js index f85989ff90..36d2faebe5 100644 --- a/packages/admin-panel/src/pages/resources/LegacyReportsPage.js +++ b/packages/admin-panel/src/pages/resources/LegacyReportsPage.js @@ -39,6 +39,7 @@ const COLUMNS = [ type: 'edit', source: 'id', actionConfig: { + title: 'Edit Legacy Report', editEndpoint: 'legacyReports', fields: [...FIELDS], }, @@ -50,9 +51,6 @@ export const LegacyReportsPage = ({ getHeaderEl }) => ( title="Legacy Reports" endpoint="legacyReports" columns={COLUMNS} - editConfig={{ - title: 'Edit Legacy Report', - }} getHeaderEl={getHeaderEl} /> ); diff --git a/packages/admin-panel/src/pages/resources/MapOverlayGroupRelationsPage.js b/packages/admin-panel/src/pages/resources/MapOverlayGroupRelationsPage.js index c89da20771..3e61fafe83 100644 --- a/packages/admin-panel/src/pages/resources/MapOverlayGroupRelationsPage.js +++ b/packages/admin-panel/src/pages/resources/MapOverlayGroupRelationsPage.js @@ -65,6 +65,7 @@ const COLUMNS = [ type: 'edit', source: 'id', actionConfig: { + title: 'Edit Map Overlay Group Relation', editEndpoint: 'mapOverlayGroupRelations', fields: [...FIELDS], }, diff --git a/packages/admin-panel/src/pages/resources/MapOverlayGroupsPage.js b/packages/admin-panel/src/pages/resources/MapOverlayGroupsPage.js index fe570fa1dd..d232f425df 100644 --- a/packages/admin-panel/src/pages/resources/MapOverlayGroupsPage.js +++ b/packages/admin-panel/src/pages/resources/MapOverlayGroupsPage.js @@ -39,6 +39,7 @@ const COLUMNS = [ type: 'edit', source: 'id', actionConfig: { + title: 'Edit Map Overlay Group', editEndpoint: 'mapOverlayGroups', fields: EDIT_FIELDS, }, diff --git a/packages/admin-panel/src/pages/resources/MapOverlaysPage.js b/packages/admin-panel/src/pages/resources/MapOverlaysPage.js index a75e0e4e6b..5a283cee1b 100644 --- a/packages/admin-panel/src/pages/resources/MapOverlaysPage.js +++ b/packages/admin-panel/src/pages/resources/MapOverlaysPage.js @@ -179,6 +179,7 @@ export const MapOverlaysPage = ({ getHeaderEl, isBESAdmin, ...props }) => { type: 'edit', source: 'id', actionConfig: { + title: 'Edit Map Overlay', editEndpoint: MAP_OVERLAYS_ENDPOINT, fields: [...FIELDS, ...extraEditFields], }, @@ -201,9 +202,6 @@ export const MapOverlaysPage = ({ getHeaderEl, isBESAdmin, ...props }) => { importConfig={IMPORT_CONFIG} LinksComponent={renderNewMapOverlayVizButton} getHeaderEl={getHeaderEl} - editConfig={{ - title: 'Edit Map Overlay', - }} {...props} /> ); diff --git a/packages/admin-panel/src/pages/resources/OptionSetsPage.js b/packages/admin-panel/src/pages/resources/OptionSetsPage.js index a4b53d674c..fb231df153 100644 --- a/packages/admin-panel/src/pages/resources/OptionSetsPage.js +++ b/packages/admin-panel/src/pages/resources/OptionSetsPage.js @@ -21,6 +21,7 @@ const OPTION_SET_COLUMNS = [ type: 'edit', source: 'id', actionConfig: { + title: 'Edit Option Set', editEndpoint: 'optionSets', fields: [...OPTION_SET_FIELDS], }, @@ -88,10 +89,6 @@ const IMPORT_CONFIG = { ], }; -const EDIT_CONFIG = { - title: 'Edit Option Set', -}; - export const OptionSetsPage = ({ getHeaderEl }) => ( ( columns={OPTION_SET_COLUMNS} expansionTabs={EXPANSION_CONFIG} importConfig={IMPORT_CONFIG} - editConfig={EDIT_CONFIG} getHeaderEl={getHeaderEl} /> ); diff --git a/packages/admin-panel/src/pages/resources/PermissionGroupsPage.js b/packages/admin-panel/src/pages/resources/PermissionGroupsPage.js index b77a641d1f..e4d1af91af 100644 --- a/packages/admin-panel/src/pages/resources/PermissionGroupsPage.js +++ b/packages/admin-panel/src/pages/resources/PermissionGroupsPage.js @@ -7,10 +7,6 @@ import React from 'react'; import PropTypes from 'prop-types'; import { ResourcePage } from './ResourcePage'; -const EDIT_CONFIG = { - title: 'Edit Permission Groups', -}; - const COLUMNS = [ { Header: 'Name', @@ -21,6 +17,7 @@ const COLUMNS = [ const CREATE_CONFIG = { title: 'Create Permission Group', actionConfig: { + title: 'Edit Permission Group', editEndpoint: 'permissionGroups', fields: [ ...COLUMNS, @@ -40,7 +37,6 @@ export const PermissionGroupsPage = ({ getHeaderEl }) => ( title="Permission Groups" endpoint="permissionGroups" columns={COLUMNS} - editConfig={EDIT_CONFIG} createConfig={CREATE_CONFIG} getHeaderEl={getHeaderEl} defaultSorting={[{ id: 'name', desc: false }]} diff --git a/packages/admin-panel/src/pages/resources/PermissionsPage.js b/packages/admin-panel/src/pages/resources/PermissionsPage.js index b8540cb9bd..363b9e74dd 100644 --- a/packages/admin-panel/src/pages/resources/PermissionsPage.js +++ b/packages/admin-panel/src/pages/resources/PermissionsPage.js @@ -49,6 +49,7 @@ const FIELDS = [ source: 'id', type: 'edit', actionConfig: { + title: "Edit User's Permission", editEndpoint: PERMISSIONS_ENDPOINT, fields: PERMISSIONS_COLUMNS, }, @@ -63,10 +64,6 @@ const FIELDS = [ }, ]; -const EDIT_CONFIG = { - title: "Edit User's Permission", -}; - const CREATE_CONFIG = { title: 'Give User Permission', bulkCreate: true, @@ -142,7 +139,6 @@ export const PermissionsPage = ({ getHeaderEl, ...props }) => ( title="Permissions" endpoint={PERMISSIONS_ENDPOINT} columns={FIELDS} - editConfig={EDIT_CONFIG} createConfig={CREATE_CONFIG} getHeaderEl={getHeaderEl} {...props} diff --git a/packages/admin-panel/src/pages/resources/ProjectsPage.js b/packages/admin-panel/src/pages/resources/ProjectsPage.js index 98ae084820..c35945a65b 100644 --- a/packages/admin-panel/src/pages/resources/ProjectsPage.js +++ b/packages/admin-panel/src/pages/resources/ProjectsPage.js @@ -71,24 +71,15 @@ const COLUMNS = [ type: 'edit', source: 'id', actionConfig: { + title: 'Edit Project', editEndpoint: 'projects', fields: FIELDS, }, }, ]; -const EDIT_CONFIG = { - title: 'Edit Project', -}; - export const ProjectsPage = ({ getHeaderEl }) => ( - + ); ProjectsPage.propTypes = { diff --git a/packages/admin-panel/src/pages/resources/QuestionsPage.js b/packages/admin-panel/src/pages/resources/QuestionsPage.js index b899f80f83..2f98693d58 100644 --- a/packages/admin-panel/src/pages/resources/QuestionsPage.js +++ b/packages/admin-panel/src/pages/resources/QuestionsPage.js @@ -54,6 +54,7 @@ const QUESTION_COLUMNS = [ type: 'edit', source: 'id', actionConfig: { + title: 'Edit Question', editEndpoint: 'questions', fields: QUESTION_FIELDS, displayUsedBy: true, @@ -98,17 +99,12 @@ const EXPANSION_CONFIG = [ }, ]; -const EDIT_CONFIG = { - title: 'Edit Question', -}; - export const QuestionsPage = ({ getHeaderEl }) => ( diff --git a/packages/admin-panel/src/pages/resources/ResourcePage.js b/packages/admin-panel/src/pages/resources/ResourcePage.js index 7908bd62a1..8e0d852373 100644 --- a/packages/admin-panel/src/pages/resources/ResourcePage.js +++ b/packages/admin-panel/src/pages/resources/ResourcePage.js @@ -17,7 +17,6 @@ const Container = styled(PageBody)` export const ResourcePage = ({ columns, - editConfig, createConfig, endpoint, reduxId, @@ -57,11 +56,7 @@ export const ResourcePage = ({ defaultSorting={defaultSorting} /> - + ); }; @@ -69,9 +64,7 @@ export const ResourcePage = ({ ResourcePage.propTypes = { getHeaderEl: PropTypes.func.isRequired, columns: PropTypes.array.isRequired, - ConfirmDeleteModalComponent: PropTypes.oneOfType([PropTypes.func, PropTypes.node]), createConfig: PropTypes.object, - editConfig: PropTypes.object, onProcessDataForSave: PropTypes.func, endpoint: PropTypes.string.isRequired, reduxId: PropTypes.string, @@ -94,9 +87,7 @@ ResourcePage.propTypes = { }; ResourcePage.defaultProps = { - ConfirmDeleteModalComponent: undefined, createConfig: null, - editConfig: null, expansionTabs: null, importConfig: null, ExportModalComponent: null, diff --git a/packages/admin-panel/src/pages/resources/SocialFeedPage.js b/packages/admin-panel/src/pages/resources/SocialFeedPage.js index 1306c75ea5..65f9b401a7 100644 --- a/packages/admin-panel/src/pages/resources/SocialFeedPage.js +++ b/packages/admin-panel/src/pages/resources/SocialFeedPage.js @@ -81,6 +81,7 @@ export const SOCIAL_FEED_COLUMNS = [ source: 'id', type: 'edit', actionConfig: { + title: 'Edit Social Feed item', editEndpoint: 'feedItems', fields: FIELDS, }, @@ -95,10 +96,6 @@ export const SOCIAL_FEED_COLUMNS = [ }, ]; -const EDIT_CONFIG = { - title: 'Edit Social Feed item', -}; - const CREATE_CONFIG = { title: 'Add Social Feed item', actionConfig: { @@ -113,7 +110,6 @@ export const SocialFeedPage = ({ getHeaderEl }) => ( endpoint="feedItems" baseFilter={{ type: 'markdown' }} columns={SOCIAL_FEED_COLUMNS} - editConfig={EDIT_CONFIG} createConfig={CREATE_CONFIG} onProcessDataForSave={data => ({ ...data, type: 'markdown' })} getHeaderEl={getHeaderEl} diff --git a/packages/admin-panel/src/pages/resources/SurveyResponsesPage.js b/packages/admin-panel/src/pages/resources/SurveyResponsesPage.js index c8e6e86e43..d624a7ba53 100644 --- a/packages/admin-panel/src/pages/resources/SurveyResponsesPage.js +++ b/packages/admin-panel/src/pages/resources/SurveyResponsesPage.js @@ -100,6 +100,7 @@ export const SURVEY_RESPONSE_PAGE_COLUMNS = [ type: 'edit', source: 'id', actionConfig: { + title: 'Edit Survey Response', editEndpoint: 'surveyResponses', fields: [ entityName, @@ -127,10 +128,6 @@ export const SURVEY_RESPONSE_PAGE_COLUMNS = [ }, ]; -const EDIT_CONFIG = { - title: 'Edit Survey Response', -}; - const ANSWER_FIELDS = [ { Header: 'Question', @@ -152,6 +149,7 @@ export const ANSWER_COLUMNS = [ type: 'edit', source: 'id', actionConfig: { + title: 'Edit Answer', editEndpoint: 'answers', fields: ANSWER_FIELDS, }, @@ -197,7 +195,6 @@ export const SurveyResponsesPage = ({ getHeaderEl, ...props }) => ( defaultSorting={[{ id: 'data_time', desc: true }]} expansionTabs={EXPANSION_CONFIG} importConfig={IMPORT_CONFIG} - editConfig={EDIT_CONFIG} getHeaderEl={getHeaderEl} ExportModalComponent={SurveyResponsesExportModal} {...props} diff --git a/packages/admin-panel/src/pages/resources/SurveysPage.js b/packages/admin-panel/src/pages/resources/SurveysPage.js index e32b244d50..0db16dd7fa 100644 --- a/packages/admin-panel/src/pages/resources/SurveysPage.js +++ b/packages/admin-panel/src/pages/resources/SurveysPage.js @@ -78,6 +78,7 @@ const SURVEY_COLUMNS = [ type: 'edit', source: 'id', actionConfig: { + title: 'Edit Survey', editEndpoint: 'surveys', fields: [ ...SURVEY_FIELDS, @@ -216,6 +217,7 @@ const QUESTION_COLUMNS = [ type: 'edit', source: 'id', actionConfig: { + title: 'Edit Question', editEndpoint: 'surveyScreenComponents', fields: [ ...QUESTION_FIELDS, @@ -435,10 +437,6 @@ const IMPORT_CONFIG = { ], }; -const EDIT_CONFIG = { - title: 'Edit Survey', -}; - export const SurveysPage = ({ getHeaderEl }) => ( ( columns={SURVEY_COLUMNS} expansionTabs={EXPANSION_CONFIG} importConfig={IMPORT_CONFIG} - editConfig={EDIT_CONFIG} getHeaderEl={getHeaderEl} /> ); diff --git a/packages/admin-panel/src/pages/resources/UsersPage.js b/packages/admin-panel/src/pages/resources/UsersPage.js index 85dcd9259f..fc5e176f63 100644 --- a/packages/admin-panel/src/pages/resources/UsersPage.js +++ b/packages/admin-panel/src/pages/resources/UsersPage.js @@ -104,6 +104,7 @@ const COLUMNS = [ type: 'edit', width: 150, actionConfig: { + title: 'Edit User', editEndpoint: 'users', fields: EDIT_FIELDS, }, @@ -123,10 +124,6 @@ const EXPANSION_CONFIG = [ }, ]; -const EDIT_CONFIG = { - title: 'Edit User', -}; - const IMPORT_CONFIG = { title: 'Import Users', actionConfig: { @@ -159,7 +156,6 @@ export const UsersPage = ({ getHeaderEl, ...props }) => ( columns={COLUMNS} expansionTabs={EXPANSION_CONFIG} importConfig={IMPORT_CONFIG} - editConfig={EDIT_CONFIG} createConfig={CREATE_CONFIG} getHeaderEl={getHeaderEl} {...props} diff --git a/packages/admin-panel/src/table/DataFetchingTable.js b/packages/admin-panel/src/table/DataFetchingTable.js index 8a90e7d149..f1c6c84e4d 100644 --- a/packages/admin-panel/src/table/DataFetchingTable.js +++ b/packages/admin-panel/src/table/DataFetchingTable.js @@ -70,14 +70,9 @@ class DataFetchingTableComponent extends React.Component { } renderConfirmModal() { - const { - confirmActionMessage, - onConfirmAction, - onCancelAction, - ConfirmDeleteModalComponent, - } = this.props; + const { confirmActionMessage, onConfirmAction, onCancelAction } = this.props; return ( - { + const BES_ADMIN_POLICY = { + SB: [BES_ADMIN_PERMISSION_GROUP], + }; + + const TUPAIA_ADMIN_POLICY = { + SB: [TUPAIA_ADMIN_PANEL_PERMISSION_GROUP], + }; + + const app = new TestableApp(); + const { models } = app; + + const ENTITY = { + code: 'test_entity', + id: generateId(), + name: 'original_name', + }; + before(async () => { + await findOrCreateDummyRecord(models.entity, ENTITY); + }); + + afterEach(() => { + app.revokeAccess(); + }); + + describe('PUT /entities/:id', async () => { + it('Successfully changes the entity name', async () => { + await app.grantAccess(BES_ADMIN_POLICY); + await app.put(`entities/${ENTITY.id}`, { + body: { name: 'new_name' }, + }); + + const result = await models.entity.find({ id: ENTITY.id, name: 'new_name' }); + expect(result.length).to.equal(1); + expect(result[0].name).to.equal('new_name'); + }); + + it('Throws an exception if we do not have BES admin access', async () => { + await app.grantAccess(TUPAIA_ADMIN_POLICY); + const { body: result } = await app.put(`entities/${ENTITY.id}`, { + body: { name: 'new_name' }, + }); + + expect(result).to.deep.equal({ error: 'Need BES Admin access' }); + }); + }); +}); From ced2987713a6ff8122088af427d48852dd783a4c Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 29 Aug 2022 16:13:16 +1000 Subject: [PATCH 08/14] Log pm2 dump file on deployment (#4127) --- packages/devops/scripts/deployment/startBackEnds.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/devops/scripts/deployment/startBackEnds.sh b/packages/devops/scripts/deployment/startBackEnds.sh index 81fa4504c3..d262da0c6c 100755 --- a/packages/devops/scripts/deployment/startBackEnds.sh +++ b/packages/devops/scripts/deployment/startBackEnds.sh @@ -41,4 +41,7 @@ setup_startup_command=$(pm2 startup ubuntu -u ubuntu --hp /home/ubuntu | tail -1 eval "$setup_startup_command" pm2 save +# Log dump file +grep status /home/ubuntu/.pm2/dump.pm2 + echo "Finished deploying latest" From ec77baaf9bcfceef734c006b0d0f486b2aed860d Mon Sep 17 00:00:00 2001 From: Andrew Date: Thu, 1 Sep 2022 10:13:39 +1000 Subject: [PATCH 09/14] RN-620 add codegenerator import validation --- .../importSurveys/constructQuestionValidators.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/central-server/src/apiV2/import/importSurveys/constructQuestionValidators.js b/packages/central-server/src/apiV2/import/importSurveys/constructQuestionValidators.js index 464d0e2046..eaf5a845b5 100644 --- a/packages/central-server/src/apiV2/import/importSurveys/constructQuestionValidators.js +++ b/packages/central-server/src/apiV2/import/importSurveys/constructQuestionValidators.js @@ -232,4 +232,14 @@ export const constructQuestionValidators = models => ({ return true; }, ], + config: [ + (cell, row) => { + if (row.type === ANSWER_TYPES.CODE_GENERATOR && isEmpty(cell)) { + throw new Error( + 'CodeGenerator questions must have a configuration defined in the config column', + ); + } + return true; + }, + ], }); From 3be7f485d1ff8ee83077d22cfbd12f9c75e00f23 Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 2 Sep 2022 15:32:04 +1000 Subject: [PATCH 10/14] MAUI-1190 Add TV to public countries list (#4137) --- packages/web-config-server/src/authSession/publicAccess.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/web-config-server/src/authSession/publicAccess.js b/packages/web-config-server/src/authSession/publicAccess.js index 67a81301be..86af63da2e 100644 --- a/packages/web-config-server/src/authSession/publicAccess.js +++ b/packages/web-config-server/src/authSession/publicAccess.js @@ -21,6 +21,7 @@ export const PUBLIC_COUNTRY_CODES = [ 'TK', 'TL', 'TO', + 'TV', 'VE', 'VU', 'WS', From 81f1d0c60025cee835e1411555086e24f5c95129 Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 2 Sep 2022 16:54:44 +1000 Subject: [PATCH 11/14] MAUI-4136 Add facilities to penfaa project (#4136) --- ...42432-AddFacilitiesPenfaa-modifies-data.js | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 packages/database/src/migrations/20220901042432-AddFacilitiesPenfaa-modifies-data.js diff --git a/packages/database/src/migrations/20220901042432-AddFacilitiesPenfaa-modifies-data.js b/packages/database/src/migrations/20220901042432-AddFacilitiesPenfaa-modifies-data.js new file mode 100644 index 0000000000..fdb073b53e --- /dev/null +++ b/packages/database/src/migrations/20220901042432-AddFacilitiesPenfaa-modifies-data.js @@ -0,0 +1,137 @@ +'use strict'; + +const { generateId, nameToId } = require('../utilities'); + +var dbm; +var type; +var seed; + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ +exports.setup = function (options, seedLink) { + dbm = options.dbmigrate; + type = dbm.dataType; + seed = seedLink; +}; + +// Current hierarchy +// country +// |- district +// |- sub_district +// |- ... + +// New hierarchy +// country +// |- district +// |- facility <-- add facilities to alt. hierarchy. All facilities already exist. +// |- sub_district +// |- ... + +const HIERARCHY_CODE = 'penfaa_samoa'; + +// a subset of facilities in Samoa +const FACILITIES = [ + // parent_code code + ['WS_Upolu', 'WS_001'], + ['WS_Savaii', 'WS_002'], + ['WS_Upolu', 'WS_003'], + ['WS_Upolu', 'WS_004'], + ['WS_Upolu', 'WS_005'], + ['WS_Savaii', 'WS_006'], + ['WS_Upolu', 'WS_007'], + ['WS_Upolu', 'WS_008'], + ['WS_Savaii', 'WS_009'], + ['WS_Savaii', 'WS_011'], + ['WS_Savaii', 'WS_012'], + ['WS_Upolu', 'WS_014'], +]; + +const SUB_DISTRICTS = [ + // parent_code code + ['WS_004', 'WS_sd01'], + ['WS_004', 'WS_sd02'], + ['WS_004', 'WS_sd03'], + ['WS_004', 'WS_sd04'], + ['WS_001', 'WS_sd05'], + ['WS_011', 'WS_sd06'], + ['WS_003', 'WS_sd07'], + ['WS_003', 'WS_sd08'], + ['WS_005', 'WS_sd09'], + ['WS_005', 'WS_sd10'], + ['WS_006', 'WS_sd11'], + ['WS_006', 'WS_sd12'], + ['WS_006', 'WS_sd13'], + ['WS_006', 'WS_sd14'], + ['WS_006', 'WS_sd15'], + ['WS_007', 'WS_sd16'], + ['WS_007', 'WS_sd17'], + ['WS_011', 'WS_sd18'], + ['WS_014', 'WS_sd19'], + ['WS_014', 'WS_sd20'], + ['WS_014', 'WS_sd21'], + ['WS_014', 'WS_sd22'], + ['WS_001', 'WS_sd23'], + ['WS_009', 'WS_sd24'], + ['WS_009', 'WS_sd25'], + ['WS_009', 'WS_sd26'], + ['WS_009', 'WS_sd27'], + ['WS_009', 'WS_sd28'], + ['WS_008', 'WS_sd29'], + ['WS_003', 'WS_sd30'], + ['WS_003', 'WS_sd31'], + ['WS_002', 'WS_sd32'], + ['WS_012', 'WS_sd33'], + ['WS_012', 'WS_sd34'], + ['WS_008', 'WS_sd35'], + ['WS_008', 'WS_sd36'], + ['WS_004', 'WS_sd37'], + ['WS_004', 'WS_sd38'], + ['WS_004', 'WS_sd39'], + ['WS_004', 'WS_sd40'], + ['WS_002', 'WS_sd41'], + ['WS_002', 'WS_sd42'], + ['WS_012', 'WS_sd43'], + ['WS_008', 'WS_sd44'], + ['WS_005', 'WS_sd45'], + ['WS_014', 'WS_sd46'], + ['WS_014', 'WS_sd47'], + ['WS_014', 'WS_sd48'], + ['WS_014', 'WS_sd49'], + ['WS_011', 'WS_sd50'], + ['WS_011', 'WS_sd51'], +]; + +exports.up = async function (db) { + const entityHierarchyId = await nameToId(db, 'entity_hierarchy', HIERARCHY_CODE); + + for (const [parentSubDistrictCode, facilityCode] of FACILITIES) { + await db.runSql(` + INSERT INTO entity_relation (id, parent_id, child_id, entity_hierarchy_id) + VALUES ( + '${generateId()}', + (SELECT id FROM entity WHERE code = '${parentSubDistrictCode}'), + (SELECT id FROM entity WHERE code = '${facilityCode}'), + '${entityHierarchyId}' + ) + `); + } + + for (const [parentFacilityCode, subDistrictCode] of SUB_DISTRICTS) { + await db.runSql(` + UPDATE entity_relation + SET parent_id = (SELECT id FROM entity WHERE code = '${parentFacilityCode}') + WHERE child_id = (SELECT id FROM entity WHERE code = '${subDistrictCode}') + AND entity_hierarchy_id = '${entityHierarchyId}' + `); + } +}; + +exports.down = async function (db) { + return null; +}; + +exports._meta = { + version: 1, +}; From 39aa81454bb055b135b2e1ef712e311843106983 Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 2 Sep 2022 17:31:27 +1000 Subject: [PATCH 12/14] MAUI-1076 Add project Tuvalu eHealth (#4138) * Add project Tuvalu eHealth * Fix migration country --- ...8-AddProjectTuvaluEhealth-modifies-data.js | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 packages/database/src/migrations/20220901060328-AddProjectTuvaluEhealth-modifies-data.js diff --git a/packages/database/src/migrations/20220901060328-AddProjectTuvaluEhealth-modifies-data.js b/packages/database/src/migrations/20220901060328-AddProjectTuvaluEhealth-modifies-data.js new file mode 100644 index 0000000000..a174a003d7 --- /dev/null +++ b/packages/database/src/migrations/20220901060328-AddProjectTuvaluEhealth-modifies-data.js @@ -0,0 +1,103 @@ +'use strict'; + +import { insertObject, codeToId, generateId } from '../utilities'; + +var dbm; +var type; +var seed; + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ +exports.setup = function (options, seedLink) { + dbm = options.dbmigrate; + type = dbm.dataType; + seed = seedLink; +}; + +const PROJECT_NAME = 'Tuvalu eHealth'; +const PROJECT_CODE = 'ehealth_tuvalu'; +const DASHBOARD_CODE = 'TV_COVID-19'; +const DASHBOARD_NAME = 'COVID-19 Vaccination'; +const PROJECT = { + code: PROJECT_CODE, + description: 'Aggregate health data for Tuvalu', + sort_order: 8, + image_url: 'https://tupaia.s3.ap-southeast-2.amazonaws.com/uploads/tuvalu-ehealth-image.png', + default_measure: '126', + dashboard_group_name: DASHBOARD_NAME, + permission_groups: '{Tuvalu eHealth Admin,Tuvalu eHealth,Tuvalu COVID-19}', + logo_url: 'https://tupaia.s3.ap-southeast-2.amazonaws.com/uploads/tuvalu-ehealth-logo.png', +}; +const ENTITY = { + code: PROJECT_CODE, + name: PROJECT_NAME, + type: 'project', +}; + +const addDashboard = async db => { + await insertObject(db, 'dashboard', { + id: generateId(), + code: DASHBOARD_CODE, + name: DASHBOARD_NAME, + root_entity_code: PROJECT_CODE, + }); +}; + +const addEntity = async db => { + const id = generateId(); + const parentId = await codeToId(db, 'entity', 'World'); + await insertObject(db, 'entity', { + id, + parent_id: parentId, + ...ENTITY, + }); +}; + +const addEntityHierarchy = async db => { + await insertObject(db, 'entity_hierarchy', { + id: generateId(), + name: PROJECT_CODE, + canonical_types: '{country,district,village}', + }); +}; + +const hierarchyNameToId = async (db, name) => { + const record = await db.runSql(`SELECT id FROM entity_hierarchy WHERE name = '${name}'`); + return record.rows[0] && record.rows[0].id; +}; + +const addEntityRelation = async db => { + await insertObject(db, 'entity_relation', { + id: generateId(), + parent_id: await codeToId(db, 'entity', PROJECT_CODE), + child_id: await codeToId(db, 'entity', 'TV'), + entity_hierarchy_id: await hierarchyNameToId(db, PROJECT_CODE), + }); +}; + +const addProject = async db => { + await insertObject(db, 'project', { + id: generateId(), + entity_id: await codeToId(db, 'entity', PROJECT_CODE), + entity_hierarchy_id: await hierarchyNameToId(db, PROJECT_CODE), + ...PROJECT, + }); +}; + +exports.up = async function (db) { + await addEntity(db); + await addDashboard(db); + await addEntityHierarchy(db); + await addEntityRelation(db); + await addProject(db); +}; + +exports.down = async function (db) { + return null; +}; + +exports._meta = { + version: 1, +}; From 9cfd2d7370ae380429ec7095eaa5d70d2c1e0965 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 5 Sep 2022 09:46:22 +1000 Subject: [PATCH 13/14] MAUI-1059 Lesmis fetch by entity not country (#4121) Co-authored-by: Andrew --- packages/lesmis/src/api/queries/useMapOverlayReportData.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/lesmis/src/api/queries/useMapOverlayReportData.js b/packages/lesmis/src/api/queries/useMapOverlayReportData.js index ecb9f6e370..a33aab358b 100644 --- a/packages/lesmis/src/api/queries/useMapOverlayReportData.js +++ b/packages/lesmis/src/api/queries/useMapOverlayReportData.js @@ -19,7 +19,6 @@ import { useEntitiesData } from './useEntitiesData'; import { yearToApiDates } from './utils'; import { useUrlSearchParam } from '../../utils/useUrlSearchParams'; import { useMapOverlaysData, findOverlay } from './useMapOverlaysData'; -import { COUNTRY_CODE } from '../../constants'; import { get } from '../api'; const getMeasureDataFromResponse = (overlay, measureDataResponse) => { @@ -156,9 +155,9 @@ export const useMapOverlayReportData = ({ entityCode, year }) => { }; const { data: measureDataResponse, isLoading: measureDataLoading } = useQuery( - ['mapOverlay', COUNTRY_CODE, selectedOverlay, params], + ['mapOverlay', entityCode, selectedOverlay, params], () => - get(`report/${COUNTRY_CODE}/${reportCode}`, { + get(`report/${entityCode}/${reportCode}`, { params, }), { From 3565551633b7acf9b54ead7726497512498832d6 Mon Sep 17 00:00:00 2001 From: Rohan Port <59544282+rohan-bes@users.noreply.github.com> Date: Mon, 5 Sep 2022 11:02:46 +1000 Subject: [PATCH 14/14] RN-619: Added ability to perform manual sync and view sync group logs in Admin Panel (#4117) --- packages/admin-panel/package.json | 1 + .../admin-panel/src/logsTable/LogsButton.js | 40 ++++ .../admin-panel/src/logsTable/LogsModal.js | 87 +++++++ .../admin-panel/src/logsTable/LogsTable.js | 46 ++++ packages/admin-panel/src/logsTable/actions.js | 97 ++++++++ .../admin-panel/src/logsTable/constants.js | 10 + packages/admin-panel/src/logsTable/index.js | 8 + packages/admin-panel/src/logsTable/reducer.js | 48 ++++ .../src/pages/resources/ResourcePage.js | 2 + .../src/pages/resources/SyncGroupsPage.js | 28 +++ packages/admin-panel/src/rootReducer.js | 2 + packages/admin-panel/src/sync/SyncStatus.js | 214 ++++++++++++++++++ packages/admin-panel/src/sync/index.js | 6 + .../generateConfigForColumnType.js | 6 +- packages/central-server/src/apiV2/index.js | 15 +- .../src/apiV2/syncGroups/GETSyncGroupLogs.js | 33 +++ .../apiV2/syncGroups/GETSyncGroupLogsCount.js | 28 +++ .../apiV2/syncGroups/ManuallySyncSyncGroup.js | 54 +++++ .../src/apiV2/syncGroups/index.js | 3 + packages/central-server/src/kobo/index.js | 3 +- .../central-server/src/kobo/manualKoBoSync.js | 18 -- .../src/kobo/startSyncWithKoBo.js | 38 ++++ .../src/services/kobo/KoBoService.js | 19 +- .../src/services/kobo/KoBoTranslator.js | 12 +- ...ddSyncStatusToSyncGroup-modifies-schema.js | 33 +++ .../src/modelClasses/DataServiceSyncGroup.js | 51 +++++ packages/kobo-api/src/KoBoApi.js | 55 +++-- .../src/components/Table/Table.js | 4 + .../src/components/Table/TablePaginator.js | 15 +- .../src/components/Table/TableRow.js | 8 +- yarn.lock | 8 + 31 files changed, 937 insertions(+), 55 deletions(-) create mode 100644 packages/admin-panel/src/logsTable/LogsButton.js create mode 100644 packages/admin-panel/src/logsTable/LogsModal.js create mode 100644 packages/admin-panel/src/logsTable/LogsTable.js create mode 100644 packages/admin-panel/src/logsTable/actions.js create mode 100644 packages/admin-panel/src/logsTable/constants.js create mode 100644 packages/admin-panel/src/logsTable/index.js create mode 100644 packages/admin-panel/src/logsTable/reducer.js create mode 100644 packages/admin-panel/src/sync/SyncStatus.js create mode 100644 packages/admin-panel/src/sync/index.js create mode 100644 packages/central-server/src/apiV2/syncGroups/GETSyncGroupLogs.js create mode 100644 packages/central-server/src/apiV2/syncGroups/GETSyncGroupLogsCount.js create mode 100644 packages/central-server/src/apiV2/syncGroups/ManuallySyncSyncGroup.js delete mode 100644 packages/central-server/src/kobo/manualKoBoSync.js create mode 100644 packages/database/src/migrations/20220822035354-AddSyncStatusToSyncGroup-modifies-schema.js diff --git a/packages/admin-panel/package.json b/packages/admin-panel/package.json index 97503a798d..13897c6c3e 100644 --- a/packages/admin-panel/package.json +++ b/packages/admin-panel/package.json @@ -38,6 +38,7 @@ "axios": "^0.21.1", "case": "^1.5.3", "content-disposition-header": "^0.6.0", + "date-fns": "^2.29.2", "file-saver": "^1.3.3", "localforage": "^1.5.0", "lodash.debounce": "^4.0.8", diff --git a/packages/admin-panel/src/logsTable/LogsButton.js b/packages/admin-panel/src/logsTable/LogsButton.js new file mode 100644 index 0000000000..adc4209c4a --- /dev/null +++ b/packages/admin-panel/src/logsTable/LogsButton.js @@ -0,0 +1,40 @@ +/** + * Tupaia MediTrak + * Copyright (c) 2018 Beyond Essential Systems Pty Ltd + */ + +import React from 'react'; +import PropTypes from 'prop-types'; +import { connect } from 'react-redux'; +import DescriptionIcon from '@material-ui/icons/Description'; +import { IconButton } from '../widgets'; +import { openLogsModal } from './actions'; + +export const LogsButtonComponent = props => { + const { openModal } = props; + return ( + + + + ); +}; + +LogsButtonComponent.propTypes = { + openModal: PropTypes.func.isRequired, +}; + +const mapDispatchToProps = (dispatch, { actionConfig, value: recordId, row }) => ({ + openModal: () => { + dispatch(openLogsModal(actionConfig, recordId, row)); + }, +}); + +const mergeProps = ({ ...stateProps }, { ...dispatchProps }, { ...ownProps }) => { + return { + ...ownProps, + ...stateProps, + ...dispatchProps, + }; +}; + +export const LogsButton = connect(null, mapDispatchToProps, mergeProps)(LogsButtonComponent); diff --git a/packages/admin-panel/src/logsTable/LogsModal.js b/packages/admin-panel/src/logsTable/LogsModal.js new file mode 100644 index 0000000000..5e3bda6c93 --- /dev/null +++ b/packages/admin-panel/src/logsTable/LogsModal.js @@ -0,0 +1,87 @@ +/** + * Tupaia MediTrak + * Copyright (c) 2018 Beyond Essential Systems Pty Ltd + */ + +import React from 'react'; +import PropTypes from 'prop-types'; +import { connect } from 'react-redux'; +import { Button, Dialog, DialogFooter, DialogHeader } from '@tupaia/ui-components'; +import { changeLogsTablePage, closeLogsModal } from './actions'; +import { ModalContentProvider } from '../widgets'; +import { LogsTable } from './LogsTable'; + +export const LogsModalComponent = ({ + errorMessage, + logs, + logsCount, + page, + logsPerPage, + onChangeLogsTablePage, + isLoading, + isOpen, + onDismiss, + title, +}) => { + return ( + + + + + + + + + + ); +}; + +LogsModalComponent.propTypes = { + errorMessage: PropTypes.string, + isLoading: PropTypes.object.isRequired, + isOpen: PropTypes.bool.isRequired, + onDismiss: PropTypes.func.isRequired, + title: PropTypes.string, + logs: PropTypes.arrayOf(PropTypes.string).isRequired, + logsCount: PropTypes.number.isRequired, + page: PropTypes.number.isRequired, + logsPerPage: PropTypes.number.isRequired, + onChangeLogsTablePage: PropTypes.func.isRequired, +}; + +LogsModalComponent.defaultProps = { + errorMessage: null, + title: 'Logs', +}; + +const mapStateToProps = state => ({ + ...state.logs, +}); + +const mapDispatchToProps = dispatch => ({ + onDismiss: () => dispatch(closeLogsModal()), + onChangeLogsTablePage: page => dispatch(changeLogsTablePage(page)), + dispatch, +}); + +const mergeProps = ({ ...stateProps }, { dispatch, ...dispatchProps }, { ...ownProps }) => { + return { + ...ownProps, + ...stateProps, + ...dispatchProps, + }; +}; + +export const LogsModal = connect( + mapStateToProps, + mapDispatchToProps, + mergeProps, +)(LogsModalComponent); diff --git a/packages/admin-panel/src/logsTable/LogsTable.js b/packages/admin-panel/src/logsTable/LogsTable.js new file mode 100644 index 0000000000..e0212641ca --- /dev/null +++ b/packages/admin-panel/src/logsTable/LogsTable.js @@ -0,0 +1,46 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd + */ + +import React from 'react'; +import styled from 'styled-components'; +import PropTypes from 'prop-types'; +import { Table, useTableSorting } from '@tupaia/ui-components'; + +const StyledTable = styled(Table)` + .MuiTableCell-root { + height: 30px; + } +`; + +export const LogsTable = ({ logs, logsCount, page, logsPerPage, onChangePage }) => { + const { sortedData, order, orderBy, sortColumn } = useTableSorting(logs); + return ( + + ); +}; + +LogsTable.propTypes = { + logs: PropTypes.arrayOf( + PropTypes.shape({ timestamp: PropTypes.string, message: PropTypes.string }), + ).isRequired, + logsCount: PropTypes.number.isRequired, + page: PropTypes.number.isRequired, + logsPerPage: PropTypes.number.isRequired, + onChangePage: PropTypes.func.isRequired, +}; diff --git a/packages/admin-panel/src/logsTable/actions.js b/packages/admin-panel/src/logsTable/actions.js new file mode 100644 index 0000000000..aadd2d0fe1 --- /dev/null +++ b/packages/admin-panel/src/logsTable/actions.js @@ -0,0 +1,97 @@ +/** + * Tupaia MediTrak + * Copyright (c) 2018 Beyond Essential Systems Pty Ltd + */ + +import { makeSubstitutionsInString } from '../utilities'; +import { + LOGS_DATA_FETCH_BEGIN, + LOGS_DATA_FETCH_SUCCESS, + LOGS_DISMISS, + LOGS_ERROR, + LOGS_OPEN, +} from './constants'; + +const getModalTitle = (titleTemplate, recordData) => + titleTemplate ? makeSubstitutionsInString(titleTemplate, recordData) : null; + +const addQueryParameters = (url, logsPerPage, page) => + `${url}?limit=${logsPerPage}${page !== undefined ? `&offset=${page * logsPerPage}` : ''}`; + +const fetchNewPageOfLogs = async (dispatch, api, logsEndpoint, page, logsPerPage, recordData) => { + const formattedLogsEndpoint = makeSubstitutionsInString(logsEndpoint, recordData); + const finalLogsEndpoint = addQueryParameters(formattedLogsEndpoint, logsPerPage, page); + dispatch({ + type: LOGS_DATA_FETCH_BEGIN, + }); + + try { + const response = await api.get(finalLogsEndpoint); + dispatch({ + type: LOGS_DATA_FETCH_SUCCESS, + data: response.body, + page, + }); + } catch (error) { + dispatch({ + type: LOGS_ERROR, + errorMessage: error.message, + }); + } +}; + +const fetchLogsFirstTime = async ( + dispatch, + api, + logsCountEndpoint, + logsEndpoint, + logsPerPage, + recordData, +) => { + const finalLogsCountEndpoint = makeSubstitutionsInString(logsCountEndpoint, recordData); + const formattedLogsEndpoint = makeSubstitutionsInString(logsEndpoint, recordData); + const finalLogsEndpoint = addQueryParameters(formattedLogsEndpoint, logsPerPage); + dispatch({ + type: LOGS_DATA_FETCH_BEGIN, + }); + + try { + const countResponse = await api.get(finalLogsCountEndpoint); + const logsResponse = await api.get(finalLogsEndpoint); + dispatch({ + type: LOGS_DATA_FETCH_SUCCESS, + data: { ...countResponse.body, ...logsResponse.body }, + }); + } catch (error) { + dispatch({ + type: LOGS_ERROR, + errorMessage: error.message, + }); + } +}; + +export const openLogsModal = ( + { logsEndpoint, logsCountEndpoint, logsPerPage, title }, + recordId, + recordData, +) => async (dispatch, getState, { api }) => { + await fetchLogsFirstTime(dispatch, api, logsCountEndpoint, logsEndpoint, logsPerPage, recordData); + dispatch({ + type: LOGS_OPEN, + recordData, + recordId, + logsEndpoint, + logsCountEndpoint, + logsPerPage, + title: getModalTitle(title, recordData), + }); +}; + +export const changeLogsTablePage = page => async (dispatch, getState, { api }) => { + const { logsEndpoint, logsPerPage, recordData } = getState().logs; + await fetchNewPageOfLogs(dispatch, api, logsEndpoint, page, logsPerPage, recordData); +}; + +export const closeLogsModal = () => ({ + type: LOGS_DISMISS, +}); diff --git a/packages/admin-panel/src/logsTable/constants.js b/packages/admin-panel/src/logsTable/constants.js new file mode 100644 index 0000000000..8fc535399b --- /dev/null +++ b/packages/admin-panel/src/logsTable/constants.js @@ -0,0 +1,10 @@ +/** + * Tupaia MediTrak + * Copyright (c) 2018 Beyond Essential Systems Pty Ltd + */ + +export const LOGS_DATA_FETCH_BEGIN = 'LOGS_DATA_FETCH_BEGIN'; +export const LOGS_DATA_FETCH_SUCCESS = 'LOGS_DATA_FETCH_SUCCESS'; +export const LOGS_DISMISS = 'LOGS_DISMISS'; +export const LOGS_ERROR = 'LOGS_ERROR'; +export const LOGS_OPEN = 'LOGS_OPEN'; diff --git a/packages/admin-panel/src/logsTable/index.js b/packages/admin-panel/src/logsTable/index.js new file mode 100644 index 0000000000..966dbc7359 --- /dev/null +++ b/packages/admin-panel/src/logsTable/index.js @@ -0,0 +1,8 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd + */ + +export { LogsButton } from './LogsButton'; +export { LogsModal } from './LogsModal'; +export { reducer } from './reducer'; diff --git a/packages/admin-panel/src/logsTable/reducer.js b/packages/admin-panel/src/logsTable/reducer.js new file mode 100644 index 0000000000..7e07b2f764 --- /dev/null +++ b/packages/admin-panel/src/logsTable/reducer.js @@ -0,0 +1,48 @@ +/** + * Tupaia MediTrak + * Copyright (c) 2018 Beyond Essential Systems Pty Ltd + */ + +import { createReducer } from '../utilities'; +import { + LOGS_DATA_FETCH_BEGIN, + LOGS_DATA_FETCH_SUCCESS, + LOGS_DISMISS, + LOGS_ERROR, + LOGS_OPEN, +} from './constants'; + +const defaultState = { + errorMessage: '', + isLoading: false, + isOpen: false, + logs: [], + logCount: null, + page: 0, + logsPerPage: 10, + recordId: null, + recordData: null, +}; + +const stateChanges = { + [LOGS_DATA_FETCH_BEGIN]: payload => ({ + isLoading: true, + ...payload, + }), + [LOGS_DATA_FETCH_SUCCESS]: payload => { + const { data, ...restOfPayload } = payload; + return { isLoading: false, ...data, ...restOfPayload }; + }, + [LOGS_DISMISS]: () => ({ + ...defaultState, + }), + [LOGS_ERROR]: (payload, { errorMessage }) => { + if (errorMessage) { + return { errorMessage: defaultState.errorMessage }; // If there is an error, dismiss it + } + return defaultState; // If no error, dismiss the whole modal and clear its state + }, + [LOGS_OPEN]: payload => ({ ...payload, isOpen: true }), +}; + +export const reducer = createReducer(defaultState, stateChanges); diff --git a/packages/admin-panel/src/pages/resources/ResourcePage.js b/packages/admin-panel/src/pages/resources/ResourcePage.js index 8e0d852373..959b7fbd3d 100644 --- a/packages/admin-panel/src/pages/resources/ResourcePage.js +++ b/packages/admin-panel/src/pages/resources/ResourcePage.js @@ -10,6 +10,7 @@ import { DataFetchingTable } from '../../table'; import { EditModal } from '../../editor'; import { Header, PageBody } from '../../widgets'; import { usePortalWithCallback } from '../../utilities'; +import { LogsModal } from '../../logsTable'; const Container = styled(PageBody)` overflow: auto; @@ -57,6 +58,7 @@ export const ResourcePage = ({ /> + ); }; diff --git a/packages/admin-panel/src/pages/resources/SyncGroupsPage.js b/packages/admin-panel/src/pages/resources/SyncGroupsPage.js index 5b6de30d0d..3f2142cf93 100644 --- a/packages/admin-panel/src/pages/resources/SyncGroupsPage.js +++ b/packages/admin-panel/src/pages/resources/SyncGroupsPage.js @@ -10,6 +10,10 @@ import { ResourcePage } from './ResourcePage'; const SERVICE_TYPES = [{ label: 'Kobo', value: 'kobo' }]; const FIELDS = [ + { + Header: 'Code', + source: 'code', + }, { Header: 'Survey Code', source: 'data_group_code', @@ -51,6 +55,30 @@ const COLUMNS = [ endpoint: 'dataServiceSyncGroups', }, }, + { + Header: 'Logs', + type: 'logs', + source: 'id', + actionConfig: { + title: '{code} sync group logs', + logsCountEndpoint: 'dataServiceSyncGroups/{id}/logs/count', + logsEndpoint: 'dataServiceSyncGroups/{id}/logs', + logsPerPage: 100, + }, + }, + { + Header: 'Sync', + type: 'sync', + source: 'sync_status', + filterable: false, + sortable: false, + width: 180, + actionConfig: { + syncStatusEndpoint: 'dataServiceSyncGroups/{id}', + latestSyncLogEndpoint: 'dataServiceSyncGroups/{id}/logs?limit=1', + manualSyncEndpoint: 'dataServiceSyncGroups/{id}/sync', + }, + }, ]; const EDIT_CONFIG = { diff --git a/packages/admin-panel/src/rootReducer.js b/packages/admin-panel/src/rootReducer.js index bf89b8d57e..fadabf7107 100644 --- a/packages/admin-panel/src/rootReducer.js +++ b/packages/admin-panel/src/rootReducer.js @@ -7,6 +7,7 @@ import { reducer as authentication, LOGOUT } from './authentication'; import { reducer as tables } from './table'; import { reducer as autocomplete } from './autocomplete/reducer'; // Needs to be imported from reducer file or console shows autocomplete not found error import { reducer as editor } from './editor'; +import { reducer as logs } from './logsTable'; import { reducer as dataChangeListener } from './dataChangeListener'; import { reducer as usedBy } from './usedBy'; @@ -15,6 +16,7 @@ const appReducer = combineReducers({ tables, autocomplete, editor, + logs, dataChangeListener, usedBy, }); diff --git a/packages/admin-panel/src/sync/SyncStatus.js b/packages/admin-panel/src/sync/SyncStatus.js new file mode 100644 index 0000000000..2af3264b09 --- /dev/null +++ b/packages/admin-panel/src/sync/SyncStatus.js @@ -0,0 +1,214 @@ +/** + * Tupaia MediTrak + * Copyright (c) 2018 Beyond Essential Systems Pty Ltd + */ + +import React, { useState, useEffect } from 'react'; +import PropTypes from 'prop-types'; +import { formatDistance } from 'date-fns'; +import SyncIcon from '@material-ui/icons/Sync'; +import ErrorIcon from '@material-ui/icons/Error'; +import CheckCircleIcon from '@material-ui/icons/CheckCircle'; +import { Tooltip } from '@material-ui/core'; +import styled, { keyframes } from 'styled-components'; +import { useApi } from '../utilities/ApiProvider'; +import { IconButton } from '../widgets'; +import { makeSubstitutionsInString } from '../utilities'; + +const STATUSES = { + IDLE: 'IDLE', + SYNCING: 'SYNCING', + ERROR: 'ERROR', +}; + +const SyncStatusContainer = styled.div` + display: flex; +`; + +const StatusMessageContainer = styled.div` + display: flex; + align-items: center; + padding-left: 5px; +`; + +const spin = keyframes` + 0% { + transform: rotate(360deg); + } + + 100% { + transform: rotate(0deg); + } + `; + +const SyncSuccessIcon = styled(CheckCircleIcon)` + color: ${props => props.theme.palette.success.main}; +`; + +const SyncFailingIcon = styled(ErrorIcon)` + color: ${props => props.theme.palette.error.main}; +`; + +const SpinningSyncIcon = styled(SyncIcon)` + color: white; + + animation: 3s ${spin}; + animation-timing-function: linear; + animation-iteration-count: infinite; +`; + +const SyncingIconButton = styled(IconButton)` + display: flex; + + background-color: ${props => props.theme.palette.blue[100]}; + + &.Mui-disabled { + background-color: ${props => props.theme.palette.primary.main}; + color: white; + } +`; + +// Bit of a hack to work around the fact that the sync button is constantly being recreated +// in the resource page due to parent component re-rendering https://stackoverflow.com/a/33800398 +const externalState = {}; + +const useExternalState = (key, initialState) => { + const [state, setState] = useState(() => { + if (key in externalState) { + return externalState[key]; + } + return initialState; + }); + + const onChange = nextState => { + externalState[key] = nextState; + setState(nextState); + }; + + return [state, onChange]; +}; + +const formatLog = ({ timestamp, message }) => + `${formatDistance(new Date(timestamp.concat(' UTC')), new Date(), { + addSuffix: true, + })}: ${message}`; + +export const SyncStatus = props => { + const { actionConfig, original } = props; + const api = useApi(); + const [status, setStatus] = useExternalState(`${original.id}.status`, original.sync_status); + const [logMessage, setLogMessage] = useExternalState(`${original.id}.logMessage`, ''); + const [errorMessage, setErrorMessage] = useState(null); + + const syncStatusEndpoint = makeSubstitutionsInString(actionConfig.syncStatusEndpoint, original); + const latestSyncLogEndpoint = makeSubstitutionsInString( + actionConfig.latestSyncLogEndpoint, + original, + ); + const manualSyncEndpoint = makeSubstitutionsInString(actionConfig.manualSyncEndpoint, original); + + const pollStatus = async () => { + try { + const statusResponse = await api.get(syncStatusEndpoint); + const latestLogResponse = await api.get(latestSyncLogEndpoint); + const latestLog = latestLogResponse.body.logs[0]; + + setStatus(statusResponse.body.sync_status); + if (latestLog) { + setLogMessage(formatLog(latestLog)); + } + setErrorMessage(null); + } catch (error) { + setErrorMessage(error.message); + } + }; + + // First poll after 0.5 seconds, then poll each 10 seconds + useEffect(() => { + const timeout = setTimeout(pollStatus, 500); + return () => clearTimeout(timeout); + }, []); + + useEffect(() => { + const timer = setInterval(pollStatus, 10000); + return () => clearInterval(timer); + }, []); + + const performManualSync = async () => { + try { + await api.post(manualSyncEndpoint); + setStatus(STATUSES.SYNCING); + setErrorMessage(null); + } catch (error) { + setErrorMessage(error.message); + } + }; + + if (errorMessage) { + return ( + + + + + + + +
Network error
+ + + + ); + } + + if (status === STATUSES.ERROR) { + return ( + + + + + + + +
Sync failing
+ + + + ); + } + + if (status === STATUSES.SYNCING) { + return ( + + + + + +
Sync in progress
+ + + ); + } + + return ( + + + + + + + +
Sync online
+ + + + ); +}; + +SyncStatus.propTypes = { + actionConfig: PropTypes.shape({ + syncStatusEndpoint: PropTypes.string, + latestSyncLogEndpoint: PropTypes.string, + manualSyncEndpoint: PropTypes.string, + }).isRequired, + original: PropTypes.shape({ id: PropTypes.string, sync_status: PropTypes.string }).isRequired, +}; diff --git a/packages/admin-panel/src/sync/index.js b/packages/admin-panel/src/sync/index.js new file mode 100644 index 0000000000..8af0e01f88 --- /dev/null +++ b/packages/admin-panel/src/sync/index.js @@ -0,0 +1,6 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd + */ + +export { SyncStatus } from './SyncStatus'; diff --git a/packages/admin-panel/src/table/columnTypes/generateConfigForColumnType.js b/packages/admin-panel/src/table/columnTypes/generateConfigForColumnType.js index 0ae7375e42..4f84ef2b00 100644 --- a/packages/admin-panel/src/table/columnTypes/generateConfigForColumnType.js +++ b/packages/admin-panel/src/table/columnTypes/generateConfigForColumnType.js @@ -9,6 +9,8 @@ import { DeleteButton } from './DeleteButton'; import { ExportButton } from '../../importExport'; import { BooleanSelectFilter } from './columnFilters'; import { Tooltip, JSONTooltip } from './Tooltip'; +import { LogsButton } from '../../logsTable'; +import { SyncStatus } from '../../sync'; const generateCustomCell = (CustomCell, actionConfig, reduxId) => props => ( @@ -29,9 +31,11 @@ const CUSTOM_CELL_COMPONENTS = { boolean: ({ value }) => (value ? 'Yes' : 'No'), tooltip: Tooltip, jsonTooltip: JSONTooltip, + logs: LogsButton, + sync: SyncStatus, }; -const BUTTON_COLUMN_TYPES = ['edit', 'export', 'delete']; +const BUTTON_COLUMN_TYPES = ['edit', 'export', 'delete', 'logs']; export const generateConfigForColumnType = (type, actionConfig, reduxId) => { const CustomCellComponent = CUSTOM_CELL_COMPONENTS[type]; diff --git a/packages/central-server/src/apiV2/index.js b/packages/central-server/src/apiV2/index.js index 04618b766f..064dee3ac6 100644 --- a/packages/central-server/src/apiV2/index.js +++ b/packages/central-server/src/apiV2/index.js @@ -100,7 +100,6 @@ import { requestPasswordReset } from './requestPasswordReset'; import { getCountryAccessList } from './getCountryAccessList'; import { surveyResponse } from './surveyResponse'; import { verifyEmail, requestResendEmail } from './verifyEmail'; -import { manualKoBoSync } from '../kobo'; import { GETReports } from './reports'; import { GETDataElementDataGroups } from './dataElementDataGroups'; import { @@ -108,7 +107,15 @@ import { EditMapOverlayVisualisation, GETMapOverlayVisualisations, } from './mapOverlayVisualisations'; -import { GETSyncGroups, EditSyncGroups, CreateSyncGroups, DeleteSyncGroups } from './syncGroups'; +import { + GETSyncGroups, + EditSyncGroups, + CreateSyncGroups, + DeleteSyncGroups, + GETSyncGroupLogs, + GETSyncGroupLogsCount, + ManuallySyncSyncGroup, +} from './syncGroups'; // quick and dirty permission wrapper for open endpoints const allowAnyone = routeHandler => (req, res, next) => { @@ -215,6 +222,8 @@ apiV2.get('/geographicalAreas/:recordId?', useRouteHandler(GETGeographicalAreas) apiV2.get('/reports/:recordId?', useRouteHandler(GETReports)); apiV2.get('/dhisInstances/:recordId?', useRouteHandler(BESAdminGETHandler)); apiV2.get('/dataServiceSyncGroups/:recordId?', useRouteHandler(GETSyncGroups)); +apiV2.get('/dataServiceSyncGroups/:recordId/logs', useRouteHandler(GETSyncGroupLogs)); +apiV2.get('/dataServiceSyncGroups/:recordId/logs/count', useRouteHandler(GETSyncGroupLogsCount)); /** * POST routes @@ -245,8 +254,8 @@ apiV2.post('/dashboardRelations', useRouteHandler(CreateDashboardRelation)); apiV2.post('/dashboardVisualisations', useRouteHandler(CreateDashboardVisualisation)); apiV2.post('/mapOverlayVisualisations', useRouteHandler(CreateMapOverlayVisualisation)); apiV2.post('/mapOverlayGroupRelations', useRouteHandler(CreateMapOverlayGroupRelation)); -apiV2.post('/syncFromService', allowAnyone(manualKoBoSync)); apiV2.post('/dataServiceSyncGroups', useRouteHandler(CreateSyncGroups)); +apiV2.post('/dataServiceSyncGroups/:recordId/sync', useRouteHandler(ManuallySyncSyncGroup)); /** * PUT routes diff --git a/packages/central-server/src/apiV2/syncGroups/GETSyncGroupLogs.js b/packages/central-server/src/apiV2/syncGroups/GETSyncGroupLogs.js new file mode 100644 index 0000000000..c06b78e677 --- /dev/null +++ b/packages/central-server/src/apiV2/syncGroups/GETSyncGroupLogs.js @@ -0,0 +1,33 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd + */ + +import { GETHandler } from '../GETHandler'; +import { assertAdminPanelAccess } from '../../permissions'; +import { createSyncGroupDBFilter } from './assertSyncGroupPermissions'; + +export class GETSyncGroupLogs extends GETHandler { + permissionsFilteredInternally = true; + + async buildResponse() { + const { limit, offset } = this.req.query; + const { recordId } = this; + + const dataServiceSyncGroup = await this.models.dataServiceSyncGroup.findById(recordId); + const logs = await dataServiceSyncGroup.getLatestLogs(limit, offset); + const logObjects = logs.map(({ timestamp, log_message: logMessage }) => ({ + timestamp, + message: logMessage, + })); + return { body: { logs: logObjects } }; + } + + async assertUserHasAccess() { + await this.assertPermissions(assertAdminPanelAccess); + } + + async getPermissionsFilter(criteria, options) { + return createSyncGroupDBFilter(this.accessPolicy, this.models, criteria, options); + } +} diff --git a/packages/central-server/src/apiV2/syncGroups/GETSyncGroupLogsCount.js b/packages/central-server/src/apiV2/syncGroups/GETSyncGroupLogsCount.js new file mode 100644 index 0000000000..0846f3654a --- /dev/null +++ b/packages/central-server/src/apiV2/syncGroups/GETSyncGroupLogsCount.js @@ -0,0 +1,28 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd + */ + +import { GETHandler } from '../GETHandler'; +import { assertAdminPanelAccess } from '../../permissions'; +import { createSyncGroupDBFilter } from './assertSyncGroupPermissions'; + +export class GETSyncGroupLogsCount extends GETHandler { + permissionsFilteredInternally = true; + + async buildResponse() { + const { recordId } = this; + + const dataServiceSyncGroup = await this.models.dataServiceSyncGroup.findById(recordId); + const count = await dataServiceSyncGroup.getLogsCount(); + return { body: { logsCount: count } }; + } + + async assertUserHasAccess() { + await this.assertPermissions(assertAdminPanelAccess); + } + + async getPermissionsFilter(criteria, options) { + return createSyncGroupDBFilter(this.accessPolicy, this.models, criteria, options); + } +} diff --git a/packages/central-server/src/apiV2/syncGroups/ManuallySyncSyncGroup.js b/packages/central-server/src/apiV2/syncGroups/ManuallySyncSyncGroup.js new file mode 100644 index 0000000000..ac7ad936c7 --- /dev/null +++ b/packages/central-server/src/apiV2/syncGroups/ManuallySyncSyncGroup.js @@ -0,0 +1,54 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2020 Beyond Essential Systems Pty Ltd + */ + +import { DataBroker } from '@tupaia/data-broker'; +import { respond, ObjectValidator, constructRecordExistsWithId } from '@tupaia/utils'; +import { CRUDHandler } from '../CRUDHandler'; +import { assertAnyPermissions, assertBESAdminAccess } from '../../permissions'; +import { assertSyncGroupEditPermissions } from './assertSyncGroupPermissions'; +import { syncWithKoBo } from '../../kobo'; + +const manualSyncFunctions = { + kobo: (models, syncGroupCode) => { + const dataBroker = new DataBroker(); + return syncWithKoBo(models, dataBroker, syncGroupCode); + }, +}; + +export class ManuallySyncSyncGroup extends CRUDHandler { + async validateRecordExists() { + const validationCriteria = { + id: [constructRecordExistsWithId(this.database, this.recordType)], + }; + + const validator = new ObjectValidator(validationCriteria); + return validator.validate({ id: this.recordId }); // Will throw an error if not valid + } + + async assertUserHasAccess() { + const syncGroupChecker = accessPolicy => + assertSyncGroupEditPermissions(accessPolicy, this.models, this.recordId); + + await this.assertPermissions(assertAnyPermissions([assertBESAdminAccess, syncGroupChecker])); + } + + async handleRequest() { + await this.validateRecordExists(); + await this.assertUserHasAccess(); + + const dataServiceSyncGroup = await this.models.dataServiceSyncGroup.findById(this.recordId); + + const { service_type: serviceType, code: syncGroupCode } = dataServiceSyncGroup; + const syncFunction = manualSyncFunctions[serviceType]; + + if (!syncFunction) { + throw new Error(`Manual sync unsupported for service type: ${serviceType}`); + } + + // Kick off sync and respond immediately (results of sync can be checked in sync group logs) + syncFunction(this.models, syncGroupCode); + respond(this.res, { message: 'Sync triggered' }); + } +} diff --git a/packages/central-server/src/apiV2/syncGroups/index.js b/packages/central-server/src/apiV2/syncGroups/index.js index bc24f83165..f000738ae0 100644 --- a/packages/central-server/src/apiV2/syncGroups/index.js +++ b/packages/central-server/src/apiV2/syncGroups/index.js @@ -6,4 +6,7 @@ export { DeleteSyncGroups } from './DeleteSyncGroups'; export { EditSyncGroups } from './EditSyncGroups'; export { GETSyncGroups } from './GETSyncGroups'; +export { GETSyncGroupLogs } from './GETSyncGroupLogs'; +export { GETSyncGroupLogsCount } from './GETSyncGroupLogsCount'; export { CreateSyncGroups } from './CreateSyncGroups'; +export { ManuallySyncSyncGroup } from './ManuallySyncSyncGroup'; diff --git a/packages/central-server/src/kobo/index.js b/packages/central-server/src/kobo/index.js index ea00237fcc..1fdb9ffb1f 100644 --- a/packages/central-server/src/kobo/index.js +++ b/packages/central-server/src/kobo/index.js @@ -3,5 +3,4 @@ * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd */ -export { startSyncWithKoBo } from './startSyncWithKoBo'; -export { manualKoBoSync } from './manualKoBoSync'; +export { startSyncWithKoBo, syncWithKoBo } from './startSyncWithKoBo'; diff --git a/packages/central-server/src/kobo/manualKoBoSync.js b/packages/central-server/src/kobo/manualKoBoSync.js deleted file mode 100644 index b753c46964..0000000000 --- a/packages/central-server/src/kobo/manualKoBoSync.js +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Tupaia - * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd - */ - -import { DataBroker } from '@tupaia/data-broker'; -import { respond } from '@tupaia/utils'; -import { syncWithKoBo } from './startSyncWithKoBo'; - -export async function manualKoBoSync(req, res) { - const { models } = req; - const { syncGroupCode } = req.query; - - const dataBroker = new DataBroker(); - await syncWithKoBo(models, dataBroker, syncGroupCode); - - respond(res, { message: 'KoBo sync triggered' }); -} diff --git a/packages/central-server/src/kobo/startSyncWithKoBo.js b/packages/central-server/src/kobo/startSyncWithKoBo.js index cb7d4b7755..fbdaa4a4ce 100644 --- a/packages/central-server/src/kobo/startSyncWithKoBo.js +++ b/packages/central-server/src/kobo/startSyncWithKoBo.js @@ -11,6 +11,32 @@ import winston from '../log'; const PERIOD_BETWEEN_SYNCS = 10 * 60 * 1000; // 10 minutes between syncs const SERVICE_TYPE = 'kobo'; +const validateSyncGroup = async (models, dataServiceSyncGroup) => { + const { data_group_code: dataGroupCode, config } = dataServiceSyncGroup; + + const survey = await models.survey.findOne({ code: dataGroupCode }); + if (!survey) { + throw new Error( + `No survey exists in Tupaia with code matching data_group_code: ${dataGroupCode}`, + ); + } + + const questionCodesInQuestionMapping = Object.keys(config.questionMapping || {}); + const questions = await models.question.find({ + code: questionCodesInQuestionMapping, + }); + const questionCodes = questions.map(q => q.code); + const questionsNotDefinedInTupaia = questionCodesInQuestionMapping.filter( + q => !questionCodes.includes(q), + ); + + if (questionsNotDefinedInTupaia.length > 0) { + throw new Error( + `Question codes in sync group questionMapping do not match any existing questions in Tupaia: ${questionsNotDefinedInTupaia}`, + ); + } +}; + const writeKoboDataToTupaia = async (transactingModels, koboData, syncGroupCode) => { const dataServiceSyncGroup = await transactingModels.dataServiceSyncGroup.findOne({ service_type: SERVICE_TYPE, @@ -86,7 +112,16 @@ export async function syncWithKoBo(models, dataBroker, syncGroupCode) { throw new Error(`No KoBo sync group with the code ${syncGroupCode} exists`); } + if (dataServiceSyncGroup.isSyncing()) { + winston.info(`Already syncing ${dataServiceSyncGroup.code}, skipping sync request`); + return; + } + try { + await dataServiceSyncGroup.setSyncStarted(); + + await validateSyncGroup(models, dataServiceSyncGroup); + // Pull data from KoBo const koboData = await dataBroker.pull( { @@ -116,9 +151,12 @@ export async function syncWithKoBo(models, dataBroker, syncGroupCode) { `Sync successful, ${numberOfSurveyResponsesCreated} survey responses created`, ); }); + + await dataServiceSyncGroup.setSyncCompletedSuccessfully(); } catch (e) { // Swallow errors when processing kobo data await dataServiceSyncGroup.log(`ERROR: ${e.message}`); + await dataServiceSyncGroup.setSyncFailed(); winston.error(e.message); } } diff --git a/packages/data-broker/src/services/kobo/KoBoService.js b/packages/data-broker/src/services/kobo/KoBoService.js index a944b2e371..820fd68d00 100644 --- a/packages/data-broker/src/services/kobo/KoBoService.js +++ b/packages/data-broker/src/services/kobo/KoBoService.js @@ -44,12 +44,25 @@ export class KoBoService extends Service { const resultsByDataGroupCode = {}; for (const source of dataSources) { - const results = await this.api.fetchKoBoSubmissions(source.config?.koboSurveyCode, options); + const { koboSurveyCode, questionMapping, entityQuestionCode } = source.config; + if (!koboSurveyCode) { + throw new Error(`Missing 'koboSurveyCode' in sync group config`); + } + + if (!entityQuestionCode) { + throw new Error(`Missing 'entityQuestionCode' in sync group config`); + } + + if (!questionMapping) { + throw new Error(`Missing 'questionMapping' in sync group config`); + } + + const results = await this.api.fetchKoBoSubmissions(koboSurveyCode, options); resultsByDataGroupCode[source.data_group_code] = await this.translator.translateKoBoResults( results, - source.config?.questionMapping, - source.config?.entityQuestionCode, + questionMapping, + entityQuestionCode, ); } diff --git a/packages/data-broker/src/services/kobo/KoBoTranslator.js b/packages/data-broker/src/services/kobo/KoBoTranslator.js index 7825f5e386..eb8be9e483 100644 --- a/packages/data-broker/src/services/kobo/KoBoTranslator.js +++ b/packages/data-broker/src/services/kobo/KoBoTranslator.js @@ -31,13 +31,21 @@ export class KoBoTranslator { ...restOfFields } = result; + if (!koboEntityCode) { + throw new Error( + `Cannot find a question in the Kobo survey response matching the entityQuestionCode: ${entityQuestion}`, + ); + } + const { orgUnit, orgUnitName } = await this.fetchEntityInfoFromKoBoAnswer(koboEntityCode); // Map kobo questions to tupaia question codes const dataValues = {}; - for (const [tupaia, { koboQuestionCode, answerMap }] of Object.entries(questionMapping)) { + for (const [tupaiaQuestionCode, { koboQuestionCode, answerMap }] of Object.entries( + questionMapping, + )) { if (restOfFields[koboQuestionCode] !== undefined) { const koboValue = restOfFields[koboQuestionCode]; - dataValues[tupaia] = + dataValues[tupaiaQuestionCode] = answerMap?.[koboValue] !== undefined ? answerMap[koboValue] : koboValue; } } diff --git a/packages/database/src/migrations/20220822035354-AddSyncStatusToSyncGroup-modifies-schema.js b/packages/database/src/migrations/20220822035354-AddSyncStatusToSyncGroup-modifies-schema.js new file mode 100644 index 0000000000..167e94a53c --- /dev/null +++ b/packages/database/src/migrations/20220822035354-AddSyncStatusToSyncGroup-modifies-schema.js @@ -0,0 +1,33 @@ +'use strict'; + +var dbm; +var type; +var seed; + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ +exports.setup = function (options, seedLink) { + dbm = options.dbmigrate; + type = dbm.dataType; + seed = seedLink; +}; + +exports.up = async function (db) { + return db.runSql(` + CREATE TYPE sync_group_sync_status AS ENUM ('IDLE', 'SYNCING', 'ERROR'); + ALTER TABLE data_service_sync_group ADD COLUMN sync_status sync_group_sync_status DEFAULT 'IDLE'; + `); +}; + +exports.down = function (db) { + return db.runSql(` + ALTER TABLE data_service_sync_group DROP COLUMN sync_status; + DROP TYPE IF EXISTS sync_group_sync_status; + `); +}; + +exports._meta = { + version: 1, +}; diff --git a/packages/database/src/modelClasses/DataServiceSyncGroup.js b/packages/database/src/modelClasses/DataServiceSyncGroup.js index 7b65e7d6cc..f8fb9a8d09 100644 --- a/packages/database/src/modelClasses/DataServiceSyncGroup.js +++ b/packages/database/src/modelClasses/DataServiceSyncGroup.js @@ -13,9 +13,31 @@ const SERVICE_TYPES = { KOBO: 'kobo', }; +const syncStatuses = { + syncing: 'SYNCING', + idle: 'IDLE', + error: 'ERROR', +}; + class DataServiceSyncGroupType extends DatabaseType { static databaseType = TYPES.DATA_SERVICE_SYNC_GROUP; + async setSyncStarted() { + return this.model.update({ id: this.id }, { sync_status: syncStatuses.syncing }); + } + + async setSyncCompletedSuccessfully() { + return this.model.update({ id: this.id }, { sync_status: syncStatuses.idle }); + } + + async setSyncFailed() { + return this.model.update({ id: this.id }, { sync_status: syncStatuses.error }); + } + + isSyncing() { + return this.sync_status === syncStatuses.syncing; + } + async log(message) { winston.info(`${this.code} SYNC_GROUP_LOG: ${message}`); await this.otherModels.syncGroupLog.create({ @@ -25,6 +47,35 @@ class DataServiceSyncGroupType extends DatabaseType { log_message: message, }); } + + async getLogsCount() { + const [{ count }] = await this.database.executeSql( + ` + SELECT count(sgl.*) FROM sync_group_log sgl + JOIN data_service_sync_group dssg ON dssg.code = sgl.sync_group_code + WHERE dssg.id = ? + `, + [this.id], + ); + + return parseInt(count); + } + + async getLatestLogs(limit = 100, offset = 0) { + const logs = await this.database.executeSql( + ` + SELECT sgl.* FROM sync_group_log sgl + JOIN data_service_sync_group dssg ON dssg.code = sgl.sync_group_code + WHERE dssg.id = ? + ORDER BY timestamp DESC + LIMIT ? + OFFSET ? + `, + [this.id, limit, offset], + ); + + return Promise.all(logs.map(this.otherModels.syncGroupLog.generateInstance)); + } } export class DataServiceSyncGroupModel extends DatabaseModel { diff --git a/packages/kobo-api/src/KoBoApi.js b/packages/kobo-api/src/KoBoApi.js index 4aa4d97b58..43f4df218e 100644 --- a/packages/kobo-api/src/KoBoApi.js +++ b/packages/kobo-api/src/KoBoApi.js @@ -2,17 +2,18 @@ * Tupaia * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd */ -import { fetchWithTimeout, stringifyQuery, takesDateForm } from '@tupaia/utils'; +import { + fetchWithTimeout, + requireEnv, + RespondingError, + stringifyQuery, + takesDateForm, +} from '@tupaia/utils'; const MAX_FETCH_WAIT_TIME = 15 * 1000; // 15 seconds const MAX_FETCH_ENTRIES = 50; export class KoBoApi { - constructor() { - this.baseUrl = process.env.KOBO_URL; - this.apiKey = process.env.KOBO_API_KEY; - } - async fetchKoBoSubmissions(koboSurveyCode, optionsInput) { let mongoQuery = {}; if (optionsInput.startSubmissionTime) { @@ -23,35 +24,49 @@ export class KoBoApi { let response; let start = 0; const results = []; - do { - response = await this.fetchFromKoBo(`api/v2/assets/${koboSurveyCode}/data.json`, { - start, - limit: MAX_FETCH_ENTRIES, - query: JSON.stringify(mongoQuery), - }); - start += MAX_FETCH_ENTRIES; - results.push(...response.results); - } while (response.next !== null); + try { + do { + response = await this.fetchFromKoBo(`api/v2/assets/${koboSurveyCode}/data.json`, { + start, + limit: MAX_FETCH_ENTRIES, + query: JSON.stringify(mongoQuery), + }); + start += MAX_FETCH_ENTRIES; + results.push(...response.results); + } while (response.next !== null); + } catch (error) { + if (error.statusCode === 404) { + throw new RespondingError( + `No Kobo survey exists with code: ${koboSurveyCode}`, + error.statusCode, + {}, + error, + ); + } else { + throw error; + } + } return results; } async fetchFromKoBo(endpoint, params) { + const baseUrl = requireEnv('KOBO_URL'); + const apiKey = requireEnv('KOBO_API_KEY'); + const queryParams = { ...params }; - const url = stringifyQuery(this.baseUrl, endpoint, queryParams); + const url = stringifyQuery(baseUrl, endpoint, queryParams); const response = await fetchWithTimeout( url, - { headers: { Authorization: `Token ${this.apiKey}` } }, + { headers: { Authorization: `Token ${apiKey}` } }, MAX_FETCH_WAIT_TIME, ); if (response.status !== 200) { const bodyText = await response.text(); - throw new Error( - `Error response from KoBo API. Status: ${response.status}, body: ${bodyText}`, - ); + throw new RespondingError(`Error response from KoBo API: ${bodyText}`, response.status); } return response.json(); } diff --git a/packages/ui-components/src/components/Table/Table.js b/packages/ui-components/src/components/Table/Table.js index 87c636e65d..645674f450 100644 --- a/packages/ui-components/src/components/Table/Table.js +++ b/packages/ui-components/src/components/Table/Table.js @@ -39,6 +39,7 @@ export const Table = React.memo( order, page, rowsPerPage, + rowsPerPageOptions, rowIdKey, isFetching, className, @@ -74,6 +75,7 @@ export const Table = React.memo( isFetching, count, rowsPerPage, + rowsPerPageOptions, onChangePage, onChangeRowsPerPage, }} @@ -103,6 +105,7 @@ Table.propTypes = { order: PropTypes.string, page: PropTypes.number, rowsPerPage: PropTypes.number, + rowsPerPageOptions: PropTypes.arrayOf(PropTypes.number), rowIdKey: PropTypes.string, className: PropTypes.string, }; @@ -125,6 +128,7 @@ Table.defaultProps = { order: 'asc', page: null, rowsPerPage: 10, + rowsPerPageOptions: null, rowIdKey: 'id', className: null, }; diff --git a/packages/ui-components/src/components/Table/TablePaginator.js b/packages/ui-components/src/components/Table/TablePaginator.js index 60b1e04adb..1d6f399b31 100644 --- a/packages/ui-components/src/components/Table/TablePaginator.js +++ b/packages/ui-components/src/components/Table/TablePaginator.js @@ -60,7 +60,16 @@ const TablePagination = styled(MuiTablePagination)` `; export const TablePaginator = React.memo( - ({ columns, page, count, rowsPerPage, onChangePage, onChangeRowsPerPage, isFetching }) => { + ({ + columns, + page, + count, + rowsPerPage, + rowsPerPageOptions, + onChangePage, + onChangeRowsPerPage, + isFetching, + }) => { const handleChangePage = useCallback( (event, newPage) => { if (onChangePage) onChangePage(newPage); @@ -85,7 +94,7 @@ export const TablePaginator = React.memo( - columns.map( +export const TableRowCells = React.memo(({ columns, rowData, ExpandButton }) => { + return columns.map( ({ key, accessor, CellComponent, width = null, align = 'center', cellColor }, index) => { const value = accessor ? accessor(rowData) : rowData[key]; const displayValue = value === 0 ? '0' : value; @@ -46,8 +46,8 @@ export const TableRowCells = React.memo(({ columns, rowData, ExpandButton }) => ); }, - ), -); + ); +}); TableRowCells.propTypes = { columns: PropTypes.arrayOf(PropTypes.shape(tableColumnShape)).isRequired, diff --git a/yarn.lock b/yarn.lock index 1c751db472..97b3c31166 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5026,6 +5026,7 @@ __metadata: content-disposition-header: ^0.6.0 cross-env: ^7.0.2 cypress-file-upload: ^5.0.8 + date-fns: ^2.29.2 file-saver: ^1.3.3 localforage: ^1.5.0 lodash.debounce: ^4.0.8 @@ -12621,6 +12622,13 @@ __metadata: languageName: node linkType: hard +"date-fns@npm:^2.29.2": + version: 2.29.2 + resolution: "date-fns@npm:2.29.2" + checksum: 08bebcceb0a5dbadae4c55e6592b9d5c07dbd7833433c7e9a1d4a424300db32589b8b48e5979b32863c9b00a48d9bab6663e580c2a4f9f203d46cbf9113b5664 + languageName: node + linkType: hard + "dayjs@npm:^1.8.15": version: 1.9.6 resolution: "dayjs@npm:1.9.6"