RN-603 de dg multi mapping #4158
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,12 +4,13 @@ | |
| */ | ||
|
|
||
| import { lower } from 'case'; | ||
| import isequal from 'lodash.isequal'; | ||
|
|
||
| import { ModelRegistry, TupaiaDatabase } from '@tupaia/database'; | ||
| import { countDistinct, toArray } from '@tupaia/utils'; | ||
| import { createService } from './services'; | ||
| import { DATA_SOURCE_TYPES } from './utils'; | ||
| import { DataServiceResolver } from './services/DataServiceResolver'; | ||
|
|
||
| export const BES_ADMIN_PERMISSION_GROUP = 'BES Admin'; | ||
|
|
||
|
|
@@ -35,6 +36,7 @@ export class DataBroker { | |
| constructor(context = {}) { | ||
| this.context = context; | ||
| this.models = getModelRegistry(); | ||
| this.dataServiceResolver = new DataServiceResolver(this.models); | ||
| this.resultMergers = { | ||
| [this.getDataSourceTypes().DATA_ELEMENT]: this.mergeAnalytics, | ||
| [this.getDataSourceTypes().DATA_GROUP]: this.mergeEvents, | ||
|
|
@@ -153,32 +155,51 @@ export class DataBroker { | |
| return createService(this.models, serviceType, this); | ||
| } | ||
|
|
||
| async push(dataSourceSpec, data, options = {}) { | ||
| const dataSources = await this.fetchDataSources(dataSourceSpec); | ||
| const { type: dataSourceType } = dataSourceSpec; | ||
| const { serviceType, dataServiceMapping } = await this.getSingleServiceAndMapping( | ||
| dataSources, | ||
| options, | ||
| ); | ||
|
|
||
| const service = this.createService(serviceType); | ||
| return service.push(dataSources, data, { type: dataSourceType, dataServiceMapping }); | ||
| } | ||
|
|
||
| async delete(dataSourceSpec, data, options = {}) { | ||
| const dataSources = await this.fetchDataSources(dataSourceSpec); | ||
| const [dataSource] = dataSources; | ||
| const { serviceType, dataServiceMapping } = await this.getSingleServiceAndMapping( | ||
| dataSources, | ||
| options, | ||
| ); | ||
|
|
||
| const service = this.createService(serviceType); | ||
| return service.delete(dataSource, data, { | ||
| type: dataSourceSpec.type, | ||
| dataServiceMapping, | ||
| ...options, | ||
| }); | ||
| } | ||
|
|
||
| async pull(dataSourceSpec, options = {}) { | ||
| const dataSources = await this.fetchDataSources(dataSourceSpec); | ||
| const { type } = dataSourceSpec; | ||
| const { organisationUnitCode, organisationUnitCodes } = options; | ||
| const orgUnitCodes = organisationUnitCodes || [organisationUnitCode]; | ||
|
|
||
| const pulls = await this.getPulls(dataSources, orgUnitCodes); | ||
| const nestedResults = await Promise.all( | ||
| pulls.map(({ dataSources: dataSourcesForThisPull, serviceType, dataServiceMapping }) => { | ||
| return this.pullForServiceAndType( | ||
| dataSourcesForThisPull, | ||
| options, | ||
| type, | ||
| serviceType, | ||
| dataServiceMapping, | ||
| ); | ||
| }), | ||
| ); | ||
| const mergeResults = this.resultMergers[type]; | ||
|
|
||
|
|
@@ -188,13 +209,12 @@ export class DataBroker { | |
| ); | ||
| } | ||
|
|
||
| pullForServiceAndType = async (dataSources, options, type, serviceType, dataServiceMapping) => { | ||
| const permissionChecker = this.permissionCheckers[type]; | ||
| // Permission checkers will throw if they fail | ||
| await permissionChecker(dataSources); | ||
| const service = this.createService(serviceType); | ||
| return service.pull(dataSources, type, { dataServiceMapping, ...options }); | ||
| }; | ||
|
|
||
| mergeAnalytics = (target = { results: [], metadata: {} }, source) => { | ||
|
|
@@ -241,14 +261,108 @@ export class DataBroker { | |
|
|
||
| mergeSyncGroups = (target = [], source) => target.concat(source); | ||
|
|
||
| async pullMetadata(dataSourceSpec, options = {}) { | ||
| const dataSources = await this.fetchDataSources(dataSourceSpec); | ||
| const { serviceType, dataServiceMapping } = await this.getSingleServiceAndMapping( | ||
| dataSources, | ||
| options, | ||
| ); | ||
|
|
||
| const service = this.createService(serviceType); | ||
| // `dataSourceSpec` is defined for a single `type` | ||
| const { type } = dataSourceSpec; | ||
| return service.pullMetadata(dataSources, type, { dataServiceMapping, ...options }); | ||
| } | ||
|
|
||
| /** | ||
| * Given some DataSources, returns a single serviceType or throws an error if multiple found | ||
| * @private | ||
| * @param {DataSource[]} dataSources | ||
| * @param {{organisationUnitCode?: string}} options | ||
| * @return {Promise<{ serviceType: string, mapping: DataServiceMapping }>} | ||
| */ | ||
| async getSingleServiceAndMapping(dataSources, options = {}) { | ||
| const { organisationUnitCode } = options; | ||
|
|
||
| const dataServiceMapping = await this.dataServiceResolver.getMappingByOrgUnitCode( | ||
| dataSources, | ||
| organisationUnitCode, | ||
| ); | ||
| if (dataServiceMapping.uniqueServiceTypes().length > 1) { | ||
| throw new Error('Multiple data service types found, only a single service type expected'); | ||
| } | ||
|
|
||
| const [serviceType] = dataServiceMapping.uniqueServiceTypes(); | ||
| return { | ||
| serviceType, | ||
| dataServiceMapping, | ||
| }; | ||
| } | ||
|
|
||
| async getPulls(dataSources, orgUnitCodes) { | ||
| const orgUnits = await this.models.entity.find({ code: orgUnitCodes }); | ||
|
|
||
| // Note: each service will pull for ALL org units and ALL data sources. | ||
|
There was a problem hiding this comment. Is there a reason we can't fetch for just the countries/data elements in each mapping? We work out which countries and dataElements/groups belong to each along the way, so we'd just need to combine them when building the It feels a little odd to pull for dataElements and countries that we know aren't in the data service (and also could create issues if they actually are in the data service but Tupaia isn't configured to fetch them from there...) There was a problem hiding this comment. Discussed. The issue is that our DataServiceMapping class and mapping resolution doesn't play nice with double ups like DrugB in dhis there. Options are to rework the mapping, or to leave it until we do the full implementation which would drop the need for a DataServiceMapping class at all, make all this trivial. I say we leave it. |
||
| // This will likely lead to problems in the future, for now this is ok because | ||
| // our services happily ignore extra org units, and our vizes do not ask for | ||
| // data elements that don't exist in dhis (dhis throws if it cant find it). | ||
|
|
||
| // First we get the mapping for each country, then if any two countries have the | ||
| // exact same mapping we simply combine them | ||
| const orgUnitCountryCodes = orgUnits | ||
| .map(orgUnit => orgUnit.country_code) | ||
| .filter(countryCode => countryCode !== null && countryCode !== undefined); | ||
| const countryCodes = [...new Set(orgUnitCountryCodes)]; | ||
|
|
||
| if (countryCodes.length === 1) { | ||
| // No special logic needed, exit early | ||
| const [countryCode] = countryCodes; | ||
| const dataServiceMapping = await this.dataServiceResolver.getMappingByCountryCode( | ||
| dataSources, | ||
| countryCode, | ||
| ); | ||
| return Object.entries(dataServiceMapping.dataSourcesByServiceType()).map( | ||
| ([serviceType, dataSourcesForThisServiceType]) => ({ | ||
| dataSources: dataSourcesForThisServiceType, | ||
| serviceType, | ||
| dataServiceMapping, | ||
| }), | ||
| ); | ||
| } | ||
|
|
||
| const mappingsByCountryCode = {}; | ||
| for (const countryCode of countryCodes) { | ||
| mappingsByCountryCode[countryCode] = await this.dataServiceResolver.getMappingByCountryCode( | ||
| dataSources, | ||
| countryCode, | ||
| ); | ||
| } | ||
|
|
||
| const uniqueMappings = []; | ||
| for (const mappingA of Object.values(mappingsByCountryCode)) { | ||
| let alreadyAdded = false; | ||
| for (const mappingB of uniqueMappings) { | ||
| if (mappingA === mappingB) continue; | ||
| if (mappingA.equals(mappingB)) { | ||
| alreadyAdded = true; | ||
| break; | ||
| } | ||
| } | ||
| if (!alreadyAdded) uniqueMappings.push(mappingA); | ||
| } | ||
|
|
||
| // And finally split each by service type | ||
| const pulls = []; | ||
| for (const mapping of uniqueMappings) { | ||
| for (const serviceType of mapping.uniqueServiceTypes()) { | ||
| pulls.push({ | ||
| dataSources, | ||
| serviceType, | ||
| dataServiceMapping: mapping, | ||
| }); | ||
| } | ||
| } | ||
|
|
||
| return pulls; | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We had discussed breaking this out into a separate class right? I'm not too fussed though, feel it's okay being here as well 👍