From 87786bd9c2ea405390672d8a23c2434ece16fab6 Mon Sep 17 00:00:00 2001 From: lmd59 Date: Wed, 20 Nov 2024 09:19:51 -0500 Subject: [PATCH] Regression cql (#321) * add npm install * Test install * regression work with coverage bundles * fix new directory structure bugs * Revert cql-execution change * Prettier fix * Make coverage files optional * prettier * coverage-structured folders in bundles dir --- .gitignore | 5 +- regression/bundles/.gitkeep | 1 + regression/default-bundles/.gitkeep | 1 + regression/regression.ts | 138 +++++++++++++++++++++------- regression/run-regression.sh | 31 +++++-- 5 files changed, 132 insertions(+), 44 deletions(-) create mode 100644 regression/bundles/.gitkeep create mode 100644 regression/default-bundles/.gitkeep diff --git a/.gitignore b/.gitignore index abcf1305..a7a3c32d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,9 +9,8 @@ coverage cache test/integration/.start-translator regression/output/*-* -regression/connectathon -regression/ecqm-content-r4-2021 -regression/ecqm-content-qicore-2022 +regression/bundles/* +regression/default-bundles/* data-requirements/fqm-e-dr/* data-requirements/jan-2024-connectathon/* data-requirements/sept-2023-connectathon/* diff --git a/regression/bundles/.gitkeep b/regression/bundles/.gitkeep new file mode 100644 index 00000000..3d81b7ba --- /dev/null +++ b/regression/bundles/.gitkeep @@ -0,0 +1 @@ +# keep this directory \ No newline at end of file diff --git a/regression/default-bundles/.gitkeep b/regression/default-bundles/.gitkeep new file mode 100644 index 00000000..3d81b7ba --- /dev/null +++ b/regression/default-bundles/.gitkeep @@ -0,0 +1 @@ +# keep this directory \ No newline at end of file diff --git a/regression/regression.ts b/regression/regression.ts index 2aad3924..d9af2b62 100644 --- a/regression/regression.ts +++ b/regression/regression.ts @@ -6,14 +6,81 @@ const regressionBaseName = process.argv[2] ?? 'regression-output'; const verbose = process.argv[3] === 'true'; const REGRESSION_OUTPUT_DIR = path.join(__dirname, `./output/${regressionBaseName}`); -const CTHON_BASE_PATH = path.join(__dirname, './connectathon/fhir401/bundles/measure'); -const ECQM_CONTENT_BASE_PATH = path.join(__dirname, './ecqm-content-r4-2021/bundles/measure'); -const ECQM_CONTENT_QICORE_BASE_PATH = path.join(__dirname, './ecqm-content-qicore-2022/bundles/measure'); +const CTHON_BASE_PATH = path.join(__dirname, './default-bundles/connectathon/fhir401/bundles/measure'); +const ECQM_CONTENT_BASE_PATH = path.join(__dirname, './default-bundles/ecqm-content-r4-2021/bundles/measure'); +const ECQM_CONTENT_QICORE_BASE_PATH = path.join( + __dirname, + './default-bundles/ecqm-content-qicore-2022/bundles/measure' +); +const BUNDLES_BASE_PATH = path.join(__dirname, './bundles'); // folders in this directory are assumed to have a coverage script bundles format const RESET = '\x1b[0m'; const FG_YELLOW = '\x1b[33m'; const FG_GREEN = '\x1b[32m'; +function findPatientBundlePathsInDirectory(patientDir: string): string[] { + const paths: string[] = []; + // iterate over the given directory + fs.readdirSync(patientDir, { withFileTypes: true }).forEach(ent => { + // if this item is a directory, look for .json files under it + if (ent.isDirectory()) { + fs.readdirSync(path.join(patientDir, ent.name), { withFileTypes: true }).forEach(subEnt => { + if (!subEnt.isDirectory() && subEnt.name.endsWith('.json')) { + paths.push(path.join(ent.name, subEnt.name)); + } + }); + } else if (ent.name.endsWith('.json')) { + paths.push(ent.name); + } + }); + return paths; +} + +/* + * @public + * @param {string} filesPath - patient file directory path + * @param {string[]} testFilePaths - individual test files within that directory + * @param {fhir4.Bundle} measureBundle - parsed measure bundle + * @param {string} shortName - measure shortname for location of results + */ +async function calculateRegression( + filesPath: string, + testFilePaths: string[], + measureBundle: fhir4.Bundle, + shortName: string +) { + const regressionResultsPath = path.join(REGRESSION_OUTPUT_DIR, shortName); + console.log(`Path: ${regressionResultsPath}`); + fs.mkdirSync(regressionResultsPath); + + for (const tfp of testFilePaths) { + const fullTestFilePath = path.join(filesPath, tfp); + const patientBundle = JSON.parse(fs.readFileSync(fullTestFilePath, 'utf8')) as fhir4.Bundle; + + //turn tfp into un-nested path for results file + const testResultsPath = path.join(regressionResultsPath, `results-${tfp.replace('/', '-')}`); + + try { + const { results } = await Calculator.calculate(measureBundle, [patientBundle], {}); + + fs.writeFileSync(testResultsPath, JSON.stringify(results, undefined, verbose ? 2 : undefined), 'utf8'); + console.log(`${FG_GREEN}%s${RESET}: Results written to ${testResultsPath}`, 'SUCCESS'); + } catch (e) { + if (e instanceof Error) { + // Errors will not halt regression. For the purposes of these tests, what matters is that there aren't any new errors that weren't there before + // or that the behavior related to the error differs from the base branch to the branch in question. + // Errors that occur will be diffed just like normal calculation results + fs.writeFileSync( + testResultsPath, + JSON.stringify({ error: e.message }, undefined, verbose ? 2 : undefined), + 'utf8' + ); + console.log(`${FG_YELLOW}%s${RESET}: Results written to ${testResultsPath}`, 'EXECUTION ERROR'); + } + } + } +} + async function main() { if (fs.existsSync(REGRESSION_OUTPUT_DIR)) { fs.rmSync(REGRESSION_OUTPUT_DIR, { recursive: true }); @@ -47,41 +114,48 @@ async function main() { // Skip measures with no test patients in the `*-files` directory if (testFilePaths.length === 0) continue; - const regressionResultsPath = path.join(REGRESSION_OUTPUT_DIR, dir.shortName); - - fs.mkdirSync(regressionResultsPath); - // It is assumed that the bundle lives under the base directory with `-bundle.json` added to the extension const measureBundle = JSON.parse( fs.readFileSync(path.join(basePath, `${dir.shortName}-bundle.json`), 'utf8') ) as fhir4.Bundle; - for (const tfp of testFilePaths) { - const fullTestFilePath = path.join(filesPath, tfp); - const patientBundle = JSON.parse(fs.readFileSync(fullTestFilePath, 'utf8')) as fhir4.Bundle; - - const testResultsPath = path.join(regressionResultsPath, `results-${tfp}`); - - try { - const { results } = await Calculator.calculate(measureBundle, [patientBundle], {}); - - fs.writeFileSync(testResultsPath, JSON.stringify(results, undefined, verbose ? 2 : undefined), 'utf8'); - console.log(`${FG_GREEN}%s${RESET}: Results written to ${testResultsPath}`, 'SUCCESS'); - } catch (e) { - if (e instanceof Error) { - // Errors will not halt regression. For the purposes of these tests, what matters is that there aren't any new errors that weren't there before - // or that the behavior related to the error differs from the base branch to the branch in question. - // Errors that occur will be diffed just like normal calculation results - fs.writeFileSync( - testResultsPath, - JSON.stringify({ error: e.message }, undefined, verbose ? 2 : undefined), - 'utf8' - ); - console.log(`${FG_YELLOW}%s${RESET}: Results written to ${testResultsPath}`, 'EXECUTION ERROR'); - } - } - } + await calculateRegression(filesPath, testFilePaths, measureBundle, dir.shortName); } + + // Everything in the BUNDLES_BASE_PATH directory is expected in $set_name/measure/$measure_name structure with + // $measure_name-v332.json, $measure_name-v314.json, and $measure_name-TestCases folder within + const baseBundlePaths = fs + .readdirSync(BUNDLES_BASE_PATH) + .filter(f => !f.startsWith('.')) + .map(f => path.join(BUNDLES_BASE_PATH, f, 'measure')); + await baseBundlePaths.forEach(async dirPath => { + // coverage directory organized with multiple measures for each set of test files + const covDirs = fs.readdirSync(dirPath).map(f => ({ + shortName: f, + fullPath: path.join(dirPath, f) + })); + for (const dir of covDirs) { + const basePath = dir.fullPath; + + const patientDirectoryPath = path.join(basePath, `${dir.shortName}-TestCases`); + // Skip measures with no `*-TestCases` directory + if (!fs.existsSync(patientDirectoryPath)) continue; + const testFilePaths = findPatientBundlePathsInDirectory(patientDirectoryPath); + // Skip measures with no test patients in the `*-files` directory + if (testFilePaths.length === 0) continue; + + // Two versions of measure + const measureBundle314 = JSON.parse( + fs.readFileSync(path.join(basePath, `${dir.shortName}-v314.json`), 'utf8') + ) as fhir4.Bundle; + await calculateRegression(patientDirectoryPath, testFilePaths, measureBundle314, `${dir.shortName}-v314`); + + const measureBundle332 = JSON.parse( + fs.readFileSync(path.join(basePath, `${dir.shortName}-v332.json`), 'utf8') + ) as fhir4.Bundle; + await calculateRegression(patientDirectoryPath, testFilePaths, measureBundle332, `${dir.shortName}-v332`); + } + }); } main().then(() => console.log('done')); diff --git a/regression/run-regression.sh b/regression/run-regression.sh index 6498a4eb..62149ba2 100755 --- a/regression/run-regression.sh +++ b/regression/run-regression.sh @@ -10,16 +10,18 @@ GREEN='\033[0;32m' NC='\033[0m' VERBOSE=false +PULL_BUNDLES=false BASE_BRANCH="master" function usage() { cat <] [-v|--verbose] + Usage: $0 [-b|--base-branch ] [-v|--verbose] [-p|--pull-bundles] Options: -b/--base-branch: Base branch to compare results with (default: master) -v/--verbose: Use verbose regression. Will print out diffs of failing JSON files with spacing (default: false) + -p/--pull-bundles: Pull bundles from default repositories into the default-bundles directory USAGE exit 1 } @@ -28,6 +30,7 @@ while test $# != 0 do case "$1" in -v | --verbose) VERBOSE=true ;; + -p | --pull-bundles) PULL_BUNDLES=true ;; -b | --base-branch) shift BASE_BRANCH=$1 @@ -37,16 +40,24 @@ do shift done -if [ ! -d "regression/connectathon" ]; then - git clone https://github.com/dbcg/connectathon.git regression/connectathon -fi +if [ $PULL_BUNDLES = "true" ]; then + if [ ! -d "regression/default-bundles/connectathon" ]; then + git clone https://github.com/dbcg/connectathon.git regression/default-bundles/connectathon + else + git -C regression/default-bundles/connectathon/ pull origin master + fi -if [ ! -d "regression/ecqm-content-r4-2021" ]; then - git clone https://github.com/cqframework/ecqm-content-r4-2021.git regression/ecqm-content-r4-2021 -fi + if [ ! -d "regression/default-bundles/ecqm-content-r4-2021" ]; then + git clone https://github.com/cqframework/ecqm-content-r4-2021.git regression/default-bundles/ecqm-content-r4-2021 + else + git -C regression/default-bundles/ecqm-content-r4-2021/ pull origin master + fi -if [ ! -d "regression/ecqm-content-qicore-2022" ]; then - git clone https://github.com/cqframework/ecqm-content-qicore-2022.git regression/ecqm-content-qicore-2022 + if [ ! -d "regression/default-bundles/ecqm-content-qicore-2022" ]; then + git clone https://github.com/cqframework/ecqm-content-qicore-2022.git regression/default-bundles/ecqm-content-qicore-2022 + else + git -C regression/default-bundles/ecqm-content-qicore-2022/ pull origin master + fi fi git fetch --all @@ -62,11 +73,13 @@ TIMESTAMP=$(date +%s) echo "Gathering results on current branch '$CURRENT_BRANCH'" +npm i npx ts-node --files ./regression/regression.ts "$CURRENT_BRANCH-$TIMESTAMP" $VERBOSE echo "Gathering results on base branch '$BASE_BRANCH'" git checkout $BASE_BRANCH +npm i npx ts-node --files ./regression/regression.ts "$BASE_BRANCH-$TIMESTAMP" $VERBOSE FAILURES=()