Regression cql (#321)

* add npm install * Test install * regression work with coverage bundles * fix new directory structure bugs * Revert cql-execution change * Prettier fix * Make coverage files optional * prettier * coverage-structured folders in bundles dir
projecttacoma · Nov 20, 2024 · 87786bd · 87786bd
1 parent 6f8471b
commit 87786bd
Show file tree

Hide file tree

Showing 5 changed files with 132 additions and 44 deletions.
diff --git a/.gitignore b/.gitignore
@@ -9,9 +9,8 @@ coverage
 cache
 test/integration/.start-translator
 regression/output/*-*
-regression/connectathon
-regression/ecqm-content-r4-2021
-regression/ecqm-content-qicore-2022
+regression/bundles/*
+regression/default-bundles/*
 data-requirements/fqm-e-dr/*
 data-requirements/jan-2024-connectathon/*
 data-requirements/sept-2023-connectathon/*

diff --git a/regression/bundles/.gitkeep b/regression/bundles/.gitkeep
@@ -0,0 +1 @@
+# keep this directory
diff --git a/regression/default-bundles/.gitkeep b/regression/default-bundles/.gitkeep
@@ -0,0 +1 @@
+# keep this directory
diff --git a/regression/regression.ts b/regression/regression.ts
@@ -6,14 +6,81 @@ const regressionBaseName = process.argv[2] ?? 'regression-output';
 const verbose = process.argv[3] === 'true';
 
 const REGRESSION_OUTPUT_DIR = path.join(__dirname, `./output/${regressionBaseName}`);
-const CTHON_BASE_PATH = path.join(__dirname, './connectathon/fhir401/bundles/measure');
-const ECQM_CONTENT_BASE_PATH = path.join(__dirname, './ecqm-content-r4-2021/bundles/measure');
-const ECQM_CONTENT_QICORE_BASE_PATH = path.join(__dirname, './ecqm-content-qicore-2022/bundles/measure');
+const CTHON_BASE_PATH = path.join(__dirname, './default-bundles/connectathon/fhir401/bundles/measure');
+const ECQM_CONTENT_BASE_PATH = path.join(__dirname, './default-bundles/ecqm-content-r4-2021/bundles/measure');
+const ECQM_CONTENT_QICORE_BASE_PATH = path.join(
+  __dirname,
+  './default-bundles/ecqm-content-qicore-2022/bundles/measure'
+);
+const BUNDLES_BASE_PATH = path.join(__dirname, './bundles'); // folders in this directory are assumed to have a coverage script bundles format
 
 const RESET = '\x1b[0m';
 const FG_YELLOW = '\x1b[33m';
 const FG_GREEN = '\x1b[32m';
 
+function findPatientBundlePathsInDirectory(patientDir: string): string[] {
+  const paths: string[] = [];
+  // iterate over the given directory
+  fs.readdirSync(patientDir, { withFileTypes: true }).forEach(ent => {
+    // if this item is a directory, look for .json files under it
+    if (ent.isDirectory()) {
+      fs.readdirSync(path.join(patientDir, ent.name), { withFileTypes: true }).forEach(subEnt => {
+        if (!subEnt.isDirectory() && subEnt.name.endsWith('.json')) {
+          paths.push(path.join(ent.name, subEnt.name));
+        }
+      });
+    } else if (ent.name.endsWith('.json')) {
+      paths.push(ent.name);
+    }
+  });
+  return paths;
+}
+
+/*
+ * @public
+ * @param {string} filesPath - patient file directory path
+ * @param {string[]} testFilePaths - individual test files within that directory
+ * @param {fhir4.Bundle} measureBundle - parsed measure bundle
+ * @param {string} shortName - measure shortname for location of results
+ */
+async function calculateRegression(
+  filesPath: string,
+  testFilePaths: string[],
+  measureBundle: fhir4.Bundle,
+  shortName: string
+) {
+  const regressionResultsPath = path.join(REGRESSION_OUTPUT_DIR, shortName);
+  console.log(`Path: ${regressionResultsPath}`);
+  fs.mkdirSync(regressionResultsPath);
+
+  for (const tfp of testFilePaths) {
+    const fullTestFilePath = path.join(filesPath, tfp);
+    const patientBundle = JSON.parse(fs.readFileSync(fullTestFilePath, 'utf8')) as fhir4.Bundle;
+
+    //turn tfp into un-nested path for results file
+    const testResultsPath = path.join(regressionResultsPath, `results-${tfp.replace('/', '-')}`);
+
+    try {
+      const { results } = await Calculator.calculate(measureBundle, [patientBundle], {});
+
+      fs.writeFileSync(testResultsPath, JSON.stringify(results, undefined, verbose ? 2 : undefined), 'utf8');
+      console.log(`${FG_GREEN}%s${RESET}: Results written to ${testResultsPath}`, 'SUCCESS');
+    } catch (e) {
+      if (e instanceof Error) {
+        // Errors will not halt regression. For the purposes of these tests, what matters is that there aren't any new errors that weren't there before
+        // or that the behavior related to the error differs from the base branch to the branch in question.
+        // Errors that occur will be diffed just like normal calculation results
+        fs.writeFileSync(
+          testResultsPath,
+          JSON.stringify({ error: e.message }, undefined, verbose ? 2 : undefined),
+          'utf8'
+        );
+        console.log(`${FG_YELLOW}%s${RESET}: Results written to ${testResultsPath}`, 'EXECUTION ERROR');
+      }
+    }
+  }
+}
+
 async function main() {
   if (fs.existsSync(REGRESSION_OUTPUT_DIR)) {
     fs.rmSync(REGRESSION_OUTPUT_DIR, { recursive: true });
@@ -47,41 +114,48 @@ async function main() {
     // Skip measures with no test patients in the `*-files` directory
     if (testFilePaths.length === 0) continue;
 
-    const regressionResultsPath = path.join(REGRESSION_OUTPUT_DIR, dir.shortName);
-
-    fs.mkdirSync(regressionResultsPath);
-
     // It is assumed that the bundle lives under the base directory with `-bundle.json` added to the extension
     const measureBundle = JSON.parse(
       fs.readFileSync(path.join(basePath, `${dir.shortName}-bundle.json`), 'utf8')
     ) as fhir4.Bundle;
 
-    for (const tfp of testFilePaths) {
-      const fullTestFilePath = path.join(filesPath, tfp);
-      const patientBundle = JSON.parse(fs.readFileSync(fullTestFilePath, 'utf8')) as fhir4.Bundle;
-
-      const testResultsPath = path.join(regressionResultsPath, `results-${tfp}`);
-
-      try {
-        const { results } = await Calculator.calculate(measureBundle, [patientBundle], {});
-
-        fs.writeFileSync(testResultsPath, JSON.stringify(results, undefined, verbose ? 2 : undefined), 'utf8');
-        console.log(`${FG_GREEN}%s${RESET}: Results written to ${testResultsPath}`, 'SUCCESS');
-      } catch (e) {
-        if (e instanceof Error) {
-          // Errors will not halt regression. For the purposes of these tests, what matters is that there aren't any new errors that weren't there before
-          // or that the behavior related to the error differs from the base branch to the branch in question.
-          // Errors that occur will be diffed just like normal calculation results
-          fs.writeFileSync(
-            testResultsPath,
-            JSON.stringify({ error: e.message }, undefined, verbose ? 2 : undefined),
-            'utf8'
-          );
-          console.log(`${FG_YELLOW}%s${RESET}: Results written to ${testResultsPath}`, 'EXECUTION ERROR');
-        }
-      }
-    }
+    await calculateRegression(filesPath, testFilePaths, measureBundle, dir.shortName);
   }
+
+  // Everything in the BUNDLES_BASE_PATH directory is expected in $set_name/measure/$measure_name structure with
+  // $measure_name-v332.json, $measure_name-v314.json, and $measure_name-TestCases folder within
+  const baseBundlePaths = fs
+    .readdirSync(BUNDLES_BASE_PATH)
+    .filter(f => !f.startsWith('.'))
+    .map(f => path.join(BUNDLES_BASE_PATH, f, 'measure'));
+  await baseBundlePaths.forEach(async dirPath => {
+    // coverage directory organized with multiple measures for each set of test files
+    const covDirs = fs.readdirSync(dirPath).map(f => ({
+      shortName: f,
+      fullPath: path.join(dirPath, f)
+    }));
+    for (const dir of covDirs) {
+      const basePath = dir.fullPath;
+
+      const patientDirectoryPath = path.join(basePath, `${dir.shortName}-TestCases`);
+      // Skip measures with no `*-TestCases` directory
+      if (!fs.existsSync(patientDirectoryPath)) continue;
+      const testFilePaths = findPatientBundlePathsInDirectory(patientDirectoryPath);
+      // Skip measures with no test patients in the `*-files` directory
+      if (testFilePaths.length === 0) continue;
+
+      // Two versions of measure
+      const measureBundle314 = JSON.parse(
+        fs.readFileSync(path.join(basePath, `${dir.shortName}-v314.json`), 'utf8')
+      ) as fhir4.Bundle;
+      await calculateRegression(patientDirectoryPath, testFilePaths, measureBundle314, `${dir.shortName}-v314`);
+
+      const measureBundle332 = JSON.parse(
+        fs.readFileSync(path.join(basePath, `${dir.shortName}-v332.json`), 'utf8')
+      ) as fhir4.Bundle;
+      await calculateRegression(patientDirectoryPath, testFilePaths, measureBundle332, `${dir.shortName}-v332`);
+    }
+  });
 }
 
 main().then(() => console.log('done'));
diff --git a/regression/run-regression.sh b/regression/run-regression.sh
@@ -10,16 +10,18 @@ GREEN='\033[0;32m'
 NC='\033[0m'
 
 VERBOSE=false
+PULL_BUNDLES=false
 BASE_BRANCH="master"
 
 function usage() {
     cat <<USAGE
 
-    Usage: $0 [-b|--base-branch <branch-name>] [-v|--verbose]
+    Usage: $0 [-b|--base-branch <branch-name>] [-v|--verbose] [-p|--pull-bundles]
 
     Options:
         -b/--base-branch:   Base branch to compare results with (default: master)
         -v/--verbose:       Use verbose regression. Will print out diffs of failing JSON files with spacing (default: false)
+        -p/--pull-bundles:  Pull bundles from default repositories into the default-bundles directory
 USAGE
     exit 1
 }
@@ -28,6 +30,7 @@ while test $# != 0
 do
     case "$1" in
     -v | --verbose) VERBOSE=true ;;
+    -p | --pull-bundles) PULL_BUNDLES=true ;;
     -b | --base-branch)
       shift
       BASE_BRANCH=$1
@@ -37,16 +40,24 @@ do
     shift
 done
 
-if [ ! -d "regression/connectathon" ]; then
-  git clone https://github.com/dbcg/connectathon.git regression/connectathon
-fi
+if [ $PULL_BUNDLES = "true" ]; then
+  if [ ! -d "regression/default-bundles/connectathon" ]; then
+    git clone https://github.com/dbcg/connectathon.git regression/default-bundles/connectathon
+  else
+    git -C regression/default-bundles/connectathon/ pull origin master
+  fi
 
-if [ ! -d "regression/ecqm-content-r4-2021" ]; then
-  git clone https://github.com/cqframework/ecqm-content-r4-2021.git regression/ecqm-content-r4-2021
-fi
+  if [ ! -d "regression/default-bundles/ecqm-content-r4-2021" ]; then
+    git clone https://github.com/cqframework/ecqm-content-r4-2021.git regression/default-bundles/ecqm-content-r4-2021
+  else
+    git -C regression/default-bundles/ecqm-content-r4-2021/ pull origin master
+  fi
 
-if [ ! -d "regression/ecqm-content-qicore-2022" ]; then
-  git clone https://github.com/cqframework/ecqm-content-qicore-2022.git regression/ecqm-content-qicore-2022
+  if [ ! -d "regression/default-bundles/ecqm-content-qicore-2022" ]; then
+    git clone https://github.com/cqframework/ecqm-content-qicore-2022.git regression/default-bundles/ecqm-content-qicore-2022
+  else
+    git -C regression/default-bundles/ecqm-content-qicore-2022/ pull origin master
+  fi
 fi
 
 git fetch --all
@@ -62,11 +73,13 @@ TIMESTAMP=$(date +%s)
 
 echo "Gathering results on current branch '$CURRENT_BRANCH'"
 
+npm i
 npx ts-node --files ./regression/regression.ts "$CURRENT_BRANCH-$TIMESTAMP" $VERBOSE
 
 echo "Gathering results on base branch '$BASE_BRANCH'"
 git checkout $BASE_BRANCH
 
+npm i
 npx ts-node --files ./regression/regression.ts "$BASE_BRANCH-$TIMESTAMP" $VERBOSE
 
 FAILURES=()