diff --git a/.gitignore b/.gitignore index 77afa766..0c9b2a5e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,7 @@ regression/output/*-* regression/connectathon regression/ecqm-content-r4-2021 regression/ecqm-content-qicore-2022 +data-requirements/fqm-e-dr/* +data-requirements/jan-2024-connectathon/* +data-requirements/elm-parser-dr/* +data-requirements/elm-parser-for-ecqms \ No newline at end of file diff --git a/README.md b/README.md index 02013183..b0c651d5 100644 --- a/README.md +++ b/README.md @@ -887,39 +887,41 @@ The order of these populations is determined by most inclusive to least inclusiv To disable this behavior, use the `disableHTMLOrdering` calculation option. ### Statement-level HTML -Optionally, `fqm-execution` can generate the stylized HTML markup for each individual statement. To access the statement-level HTML, specify the `buildStatementLevelHTML` in the `CalculationOptions` prior to measure calculation. From the `detailedResults` returned for a given patient, the `statementLevelHTML` will be available as an element on each `statementResult` whose relevance is *not* N/A. + +Optionally, `fqm-execution` can generate the stylized HTML markup for each individual statement. To access the statement-level HTML, specify the `buildStatementLevelHTML` in the `CalculationOptions` prior to measure calculation. From the `detailedResults` returned for a given patient, the `statementLevelHTML` will be available as an element on each `statementResult` whose relevance is _not_ N/A. ```typescript [ { - "patientId": "test-patient", - "detailedResults": [ + patientId: 'test-patient', + detailedResults: [ { - "groupId": "test-group", - "statementResults": [ + groupId: 'test-group', + statementResults: [ // no HTML returned since relevance is NA { - "libraryName": "MATGlobalCommonFunctionsFHIR4", - "statementName": "Patient", - "final": "NA", - "relevance": "NA", - "isFunction": false, - "pretty": "NA" + libraryName: 'MATGlobalCommonFunctionsFHIR4', + statementName: 'Patient', + final: 'NA', + relevance: 'NA', + isFunction: false, + pretty: 'NA' }, { - "libraryName": "CancerScreening", - "statementName": "SDE Sex", - "final": "TRUE", - "relevance": "TRUE", - "isFunction": false, - "pretty": "CODE: http://hl7.org/fhir/v3/AdministrativeGender F, Female", - "statementLevelHTML": "
\n...\n
" - }, + libraryName: 'CancerScreening', + statementName: 'SDE Sex', + final: 'TRUE', + relevance: 'TRUE', + isFunction: false, + pretty: 'CODE: http://hl7.org/fhir/v3/AdministrativeGender F, Female', + statementLevelHTML: + '
\n...\n
' + } ] } ] } -] +]; ``` ## Group Clause Coverage Highlighting @@ -1214,6 +1216,27 @@ const evaluateMeasure = async (args, { req }) => { }; ``` +## Special Testing + +### Regression Testing + +The `./regression` directory is organized for internal calculation testing between branches of `fqm-execution`. The idea is that if changes are made to calculation on a local branch, running regression will compare the calculation output of measures in the following three repositories (`connectathon`, `ecqm-content-qi-2022`, `ecqm-content-r4-2021`) from the local branch to the calculation output of those measures from the master branch (or another user-specified branch). The `./run-regression.sh` script takes the following options: + +``` +-b, --base-branch Base branch to compare results with (default: master) +-v, --verbose Use verbose regression. Will print out diffs of failing JSON files with spacing (default: false) +``` + +To run the regression testing script, in the `./regression` directory run: + +```bash +./run-regression +``` + +### Data Requirements Testing + +The `./data-requirements` directory is organized for internal data-requirements calculation testing between `fqm-execution` and the `fhir_review` branch of [elm-parser-for-ecqms](https://github.com/projecttacoma/elm-parser-for-ecqms/tree/fhir_review). See the [README](https://github.com/projecttacoma/fqm-execution/data-requirements/README.md) in this directory for more information. + # Contributing For suggestions or contributions, please use [GitHub Issues](https://github.com/projecttacoma/fqm-execution/issues) or open a [Pull Request](https://github.com/projecttacoma/fqm-execution/pulls). diff --git a/data-requirements/README.md b/data-requirements/README.md new file mode 100644 index 00000000..ee2b902d --- /dev/null +++ b/data-requirements/README.md @@ -0,0 +1,54 @@ +# fqm-execution Data Requirements Output Testing/Comparison + +This directory includes scripts for comparing the data-requirements output of [fqm-execution](https://github.com/projecttacoma/fqm-execution) to the data-requirements output of the fhir_review branch of the [elm-parser-for-ecqms](https://github.com/projecttacoma/elm-parser-for-ecqms/tree/fhir_review). + +## Getting Data Requirements from the elm-parser-for-ecqms + +The scripts in this directory will get the data requirements output from the elm-parser-for-ecqms fhir_review branch for the January 2024 Connectathon bundles. On the fhir_review branch of elm-parser-for-ecqms, data-requirements are calculated for the measures in [elm-parser-for-ecqms/measures/qicore/measures](https://github.com/projecttacoma/elm-parser-for-ecqms/tree/fhir_review/measures/qicore/measures) by running the command `ruby parse_elm.rb --bundle qicore`. The results are outputted to JSON files per measure to `elm-parser-for-ecqms/data_requirements/library`. This is all done by the script and the results are moved to the `elm-parser-dr` directory. + +## Getting Data Requirements from fqm-execution + +The scripts in this directory will get the data requirements output from fqm-execution for the January 2024 Connectathon bundles. The data requirements are calculated with fqm-execution on every run for ease of testing changes to fqm-execution. Since the January 2024 Connectathon bundles are not in a GitHub repository, they will have to be manually dropped into the `jan-2024-connectathon` directory that is empty. The data requirements JSON output files will be moved to the `fqm-e-dr` directory after calculation. + +## Comparing Data Requirements + +Right now there are two ways to compare data-requirements. `compare.sh` takes a similar approach to the regression tests: data-requirements are calculated using fqm-execution and then using elm-parser-for-ecqms and their outputs are compared. This will likely not be useful with the current state of fqm-execution's data-requirements calculation as it is very different from the elm-parser-for-ecqms, but it may be in the future. + +`summary-compare.sh` also compares the data-requirements outputs from fqm-execution and elm-parser-for-ecqms, but in a way that parses through each of the data-requirements in the data-requirements array. This script may be more useful to look at how many data-requirements of each type are being outputted by either repository and if they match up. By default, this script compares the data requirements outputs of all of the measures, however using the -m|--measure flag, one can specify a single measure to compare. + +## Running the Scripts + +Before running any of the scripts, be sure to populate the `jan-2024-connectathon` directory with the corresponding measure bundles. Also confirm that additional dependencies required for these scripts are installed with `npm install`. + +To run `compare.sh`: + +``` +./compare.sh +``` + +To run `summary-compare.sh` for all measures: + +``` +./summary-compare.sh +``` + +To run `summary-compare.sh` for one measure (example: CMS996): + +``` +./summary-compare.sh -m CMS996 +``` + +## Summary Compare Output + +Right now, the `summary-compare.sh` script compares three things: the types of data requirements in either output, the data requirements of each type in either output, and the mustSupports of each data requirement in either output. + +The summary output is currently structured in the following format: + +``` +-----Data Requirements Comparison for ----- +[PASS/DIFF]: Whether or not either of the outputs has data requirements of a type that the other does not + +[PASS/FAIL (Data Requirement Type)]: Whether or not the data requirements of each output match my codeFilter.valueSet and provides details if they do not +MUST SUPPORTS +[MUST SUPPORTS PASS/FAIL(Data Requirement Type-ValueSet)]: Whether or not the mustSupports of the data requirements of each output match +``` diff --git a/data-requirements/compare.sh b/data-requirements/compare.sh new file mode 100755 index 00000000..08338f38 --- /dev/null +++ b/data-requirements/compare.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' + +VERBOSE=false + +function usage() { + cat < { + if (file.endsWith('.json')) { + fs.rmSync(`./fqm-e-dr/${file}`); + } + }); + } else { + // create new fqm-e-dr directory within the data-requirements directory + fs.mkdirSync('./fqm-e-dr'); + } + + // get all of the file names (short and fullPath) from the jan-2024-connectathon directory + const allBundles = fs + .readdirSync(JAN_2024_CONNECTATHON_BASE_PATH) + .filter(f => !f.startsWith('.')) + .map(f => ({ + shortName: f.split('v')[0], + fullPath: path.join(JAN_2024_CONNECTATHON_BASE_PATH, f) + })); + + for (const bundle of allBundles) { + const measureBundle = JSON.parse(fs.readFileSync(bundle.fullPath, 'utf8')) as fhir4.Bundle; + + // try to calculate the data requirements for the measure bundle + try { + const { results } = await Calculator.calculateDataRequirements(measureBundle, {}); + + // write the data-requirements results to the measure's shortName-dr.json file in the fqm-e-dr directory + fs.writeFileSync(`./fqm-e-dr/${bundle.shortName}-dr.json`, JSON.stringify(results, undefined, 2), 'utf8'); + + console.log(`${FG_GREEN}%s${RESET}: Results written to ./fqm-e-dr/${bundle.shortName}-dr.json`, 'SUCCESS'); + } catch (e) { + if (e instanceof Error) { + fs.writeFileSync( + `./fqm-e-dr/${bundle.shortName}-dr.json`, + JSON.stringify({ error: e.message }, undefined, 2), + 'utf8' + ); + console.log( + `${FG_YELLOW}%s${RESET}: Results written to ./fqm-e-dr/${bundle.shortName}-dr.json`, + 'EXECUTION ERROR' + ); + } + } + } +} + +main().then(() => console.log('fqm-execution data-requirement calculation finished')); diff --git a/data-requirements/fqm-e-dr/.gitkeep b/data-requirements/fqm-e-dr/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/data-requirements/jan-2024-connectathon/.gitkeep b/data-requirements/jan-2024-connectathon/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/data-requirements/summary-compare.sh b/data-requirements/summary-compare.sh new file mode 100755 index 00000000..56d60462 --- /dev/null +++ b/data-requirements/summary-compare.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' + +MUST_SUPPORT=false + +function usage() { + cat <] + + Options: + -m/--measure: Name of the CMS measure to compare data requirements results (default: all) +USAGE + exit 1 +} + +while test $# != 0 +do + case "$1" in + -m | --measure) + shift + MEASURE=$1 + ;; + *) usage ;; + esac + shift +done + + +echo "Gathering data-requirements output from the fhir_review branch of elm-parser-for-ecqms" + +# Clone the elm-parser-for-ecqms in the data-requirements directory if it hasn't been, swtich to the fhir_review branch, +# run parse_elm.rb with --bundle qicore to get data-requirements for measure bundles from the January 2024 Connectathon +if [ ! -d "elm-parser-for-ecqms" ]; then + git clone https://github.com/projecttacoma/elm-parser-for-ecqms.git + git fetch --all + cd elm-parser-for-ecqms + git checkout "fhir_review" + ruby parse_elm.rb --bundle qicore + cd .. + if [ -d "elm-parser-dr" ]; then + rm -rf elm-parser-dr/*.json + fi + mkdir elm-parser-dr + SOURCE_DIR="elm-parser-for-ecqms/data_requirements/library" + TARGET_DIR="elm-parser-dr" + + # Move all files from source to target directory + mv "$SOURCE_DIR"/* "$TARGET_DIR" +fi + +echo "Gathering data-requirements output from fqm-execution using the measure bundles from the January 2024 Connectathon" + +npx ts-node fqm-e-dr.ts + +npx ts-node summary-compare.ts $MEASURE \ No newline at end of file diff --git a/data-requirements/summary-compare.ts b/data-requirements/summary-compare.ts new file mode 100644 index 00000000..01082d73 --- /dev/null +++ b/data-requirements/summary-compare.ts @@ -0,0 +1,207 @@ +import fs from 'fs'; +import _ from 'lodash'; +import path from 'path'; + +const ELM_PARSER_DR_BASE_PATH = path.join(__dirname, './elm-parser-dr'); +const FQM_E_DR_BASE_PATH = path.join(__dirname, './fqm-e-dr'); + +const measure = process.argv[2] ?? 'all'; + +const RESET = '\x1b[0m'; +const FG_YELLOW = '\x1b[33m'; +const FG_GREEN = '\x1b[32m'; +const FG_RED = '\x1b[31m'; + +export type DRFilePath = { + shortName: string; + fullPath: string; +}; + +async function main() { + let files: DRFilePath[] = []; + + // if no measure is specified, go through all the data requirements output for all of the measures + if (measure === 'all') { + // get all of the data-requirements output files from elm-parser-dr + files = fs.readdirSync(ELM_PARSER_DR_BASE_PATH).map(f => ({ + shortName: f.split('.xml')[0], + fullPath: path.join(ELM_PARSER_DR_BASE_PATH, f) + })); + } else { + const file = fs.existsSync(path.join(ELM_PARSER_DR_BASE_PATH, `${measure}.xml.json`)); + if (file) { + files = [{ shortName: measure, fullPath: path.join(ELM_PARSER_DR_BASE_PATH, `${measure}.xml.json`) }]; + } else { + console.log(`No data-requirements output found for measure ${measure}`); + } + } + + for (const elmParserDRFile of files) { + const measureName = elmParserDRFile.shortName; + const fqmEDRFilePath = path.join(FQM_E_DR_BASE_PATH, `${measureName}-dr.json`); + + // Skip measures that do not have a corresponding data-requirements output in fqm-e-dr + if (!fs.existsSync(fqmEDRFilePath)) continue; + + console.log(`\n-----Data Requirements Comparison for ${measureName}-----`); + + const fqmEDRLib = JSON.parse(fs.readFileSync(fqmEDRFilePath, 'utf8')) as fhir4.Library; + const elmParserDRLib = JSON.parse(fs.readFileSync(elmParserDRFile.fullPath, 'utf8')) as fhir4.Library; + + if (!fqmEDRLib.resourceType) { + console.log( + `${FG_RED}%s${RESET}: An error occurred in calculating data-requirements in fqm-execution for ${measureName}`, + 'EXECUTION ERROR' + ); + continue; + } + + const fqmEDR = fqmEDRLib.dataRequirement as fhir4.DataRequirement[]; + const elmParserDR = elmParserDRLib.dataRequirement as fhir4.DataRequirement[]; + + const fqmEData: Record = {}; + fqmEDR.forEach(dr => { + if (fqmEData[dr.type]) { + fqmEData[dr.type].push(dr); + } else { + fqmEData[dr.type] = [dr]; + } + }); + + // Sort the data requirements by type alphabetically + const sortedFqmEData: Record = {}; + Object.keys(fqmEData) + .sort() + .forEach(key => { + sortedFqmEData[key] = fqmEData[key]; + }); + + // Group data requirements by type in Record + const elmParserData: Record = {}; + elmParserDR.forEach(dr => { + if (elmParserData[dr.type]) { + elmParserData[dr.type].push(dr); + } else { + elmParserData[dr.type] = [dr]; + } + }); + + // Sort the data requirements by type alphabetically + const sortedElmParserData: Record = {}; + Object.keys(elmParserData) + .sort() + .forEach(key => { + sortedElmParserData[key] = elmParserData[key]; + }); + + // get the data requirements types that the output do not have in common, if any + const keyDifferences = _.difference(Object.keys(sortedElmParserData), Object.keys(sortedFqmEData)); + + if (keyDifferences.length > 0) { + const missingElmParserKeys: string[] = []; + const missingFqmEKeys: string[] = []; + + keyDifferences.forEach(key => { + if (Object.keys(sortedElmParserData).includes(key)) { + missingFqmEKeys.push(key); + } else { + missingElmParserKeys.push(key); + } + }); + console.log(`${FG_RED}%s${RESET}: Missing DR Types`, 'DIFF'); + if (missingElmParserKeys.length > 0) { + console.log( + `elm-parser: missing data-requirements of the following type(s): ${missingElmParserKeys.toString()}` + ); + } + if (missingFqmEKeys.length > 0) { + console.log(`fqm-execution: missing data-requirements of the following type(s): ${missingFqmEKeys.toString()}`); + } + } else { + console.log(`${FG_GREEN}%s${RESET}: No Missing DR Types`, 'PASS'); + } + + console.log('\n'); + + // get the keys that both outputs have in common to go through + const keys = _.intersection(Object.keys(sortedElmParserData), Object.keys(sortedFqmEData)); + + for (const key of keys) { + const elmParserDRByKey = sortedElmParserData[key]; + const fqmEDRByKey = sortedFqmEData[key]; + + // get the data requirements from the elm-parser output that do not exist in the fqm-execution output + // i.e. is there a data requirement in the fqm-execution output of the same type that has a codeFilter entry with + // a valueSet that is the same + const missingDRs = elmParserDRByKey.filter(dr => + fqmEDRByKey.every(dr2 => dr.codeFilter?.every(cf => dr2.codeFilter?.every(cf2 => cf.valueSet !== cf2.valueSet))) + ); + + // get the data requirements from the fqm-execution output that do not exist in the elm-parser output + const missingELMDRs = fqmEDRByKey.filter(dr => + elmParserDRByKey.every(dr2 => + dr.codeFilter?.every(cf => dr2.codeFilter?.every(cf2 => cf.valueSet !== cf2.valueSet)) + ) + ); + + if (missingDRs.length === 0 && missingELMDRs.length === 0 && elmParserDRByKey.length === fqmEDRByKey.length) { + console.log(`${FG_GREEN}%s${RESET}: data requirements of type ${key} match`, `PASS (${key})`); + } else if (missingDRs.length > 0 || missingELMDRs.length > 0) { + console.log(`${FG_RED}%s${RESET}: `, `FAIL (${key})`); + if (missingDRs.length > 0) { + const missingvValueSets = missingDRs.map(dr => dr.codeFilter?.find(cf => cf.valueSet)?.valueSet); + console.log( + `fqm-execution is missing the data requirement of type ${key} for the following valuesets: ${JSON.stringify( + missingvValueSets + )}` + ); + } + if (missingELMDRs.length > 0) { + const missingELMValueSets = missingELMDRs.map(dr => dr.codeFilter?.find(cf => cf.valueSet)?.valueSet); + console.log( + `elm-parser-for-ecqms is missing the data requirement(s) of type ${key} for the following valueset(s): ${JSON.stringify( + missingELMValueSets + )}` + ); + } + } else { + console.log( + `${FG_RED}%s${RESET}: something else went wrong (fqm-execution: ${fqmEDRByKey.length}, elm-parser: ${elmParserDRByKey.length})`, + `FAIL (${key})` + ); + } + + console.log(`${FG_YELLOW}%s${RESET}`, 'MUST SUPPORTS'); + + // go through all of the data requirements from the elm-parser and if any of them have a codeFilter.valueSet that match a data requirement + // from fqm-execution, then see if the mustSupports match and if they don't, print them out + // TO DO: maybe make this a flag + elmParserDRByKey.forEach(dr => { + const elmParserMustSupports = dr.mustSupport; + const elmParserValueSet = dr.codeFilter?.find(cf => cf.valueSet)?.valueSet; + + if (elmParserValueSet) { + const fqmEMatchMustSupports = fqmEDRByKey.find(dr => + dr.codeFilter?.some(cf => cf.valueSet === elmParserValueSet) + )?.mustSupport; + + const equalMustSupports = _.isEqual(elmParserMustSupports, fqmEMatchMustSupports); + + if (!equalMustSupports) { + console.log(`${FG_RED}%s${RESET}:`, `MUST SUPPORTS FAIL (${key}-${elmParserValueSet})`); + console.log(`fqm-execution has the following mustSupports: ${fqmEMatchMustSupports ?? ''}`); + console.log(`elm-parser-for-ecqms has the following mustSupports: ${elmParserMustSupports}`); + } else { + console.log( + `${FG_GREEN}%s${RESET}: matching mustSupports`, + `MUST SUPPORTS PASS (${key}-${elmParserValueSet})` + ); + } + } + }); + console.log('\n'); + } + } +} + +main().then(() => console.log('done'));