Uncoverage

README.md

@@ -450,6 +450,8 @@ The options that we support for calculation are as follows:
 
 - `[buildStatementLevelHTML]`<[boolean](#calculation-options)>: Builds and returns HTML at the statement level (default: `false`)
 - `[calculateClauseCoverage]`<[boolean](#calculation-options)>: Include HTML structure with clause coverage highlighting (default: `true`)
+- `[calculateClauseUncoverage]`<[boolean](#calculation-options)>: Include HTML structure with clause uncoverage highlighting (default: `false`)
+- `[calculateCoverageDetails]`<[boolean](#calculation-options)>: Include details on logic clause coverage. (default: `false`)
 - `[calculateHTML]`<[boolean](#calculation-options)>: Include HTML structure for highlighting (default: `true`)
 - `[calculateSDEs]`<[boolean](#calculation-options)>: Include Supplemental Data Elements (SDEs) in calculation (default: `true`)
 - `[clearElmJsonsCache]`<[boolean](#calculation-options)>: If `true`, clears ELM JSON cache before running calculation (default: `false`)
@@ -931,9 +933,11 @@ have a blue background and dashed underline in the highlighted HTML, indicating
 
 The highlighted HTML also provides an approximate percentage for what percentage of the measure logic is covered by the patients that were passed in to execution.
 
+This option, `calculateClauseCoverage`, defaults to enabled. When running the CLI in `--debug` mode this option will be enabled and output to the debug directory.
+
 ![Screenshot of Highlighted Clause Coverage HTML](./static/coverage-highlighting-example.png)
 
-HTML strings are returned for each group defined in the `Measure` resource as a lookup object `groupClauseCoverageHTML` which maps `group ID -> HTML string`
+HTML strings are returned for each group defined in the `Measure` resource in the lookup object, `groupClauseCoverageHTML`, which maps `group ID -> HTML string`
 
 ```typescript
 import { Calculator } from 'fqm-execution';
@@ -953,6 +957,68 @@ const { results, groupClauseCoverageHTML } = await Calculator.calculate(measureB
 */
 ```
 
+### Uncoverage Highlighting
+
+`fqm-execution` can also generate highlighted HTML that indicate which pieces of the measure logic did NOT have a "truthy" value during calculation. This is the complement to coverage. Clauses that are not covered will be highlighted red.
+
+This option, `calculateClauseUncoverage`, defaults to disabled. When running the CLI in `--debug` mode this option will be enabled and output to the debug directory.
+
+HTML strings are returned for each group defined in the `Measure` resource in the lookup object,`groupClauseUncoverageHTML`, which is structured similarly to Group Clause Coverage.
+
+```typescript
+import { Calculator } from 'fqm-execution';
+
+const { results, groupClauseUncoverageHTML } = await Calculator.calculate(measureBundle, patientBundles, {
+  calculateClauseUncoverage: true /* false by default */
+});
+
+// groupClauseUncoverageHTML
+/*      ^?
+        {
+          'population-group-1': '<div><h2> population-group-1 Clause Uncoverage: X of Y clauses</h2> ...'
+          ...
+        }
+
+
+*/
+```
+
+### Coverage Details
+
+Details on clause coverage can also be returned. This includes a count of how many clauses there are, how many are covered and uncovered, and information about which clauses are uncovered.
+
+This option, `calculateCoverageDetails`, defaults to disabled. When running the CLI in `--debug` mode this option will be enabled and output to the debug directory.
+
+This information is returned for each group defined in the `Measure` resource in the lookup object,`groupClauseCoverageDetails`, which maps `group ID -> coverageDetails`. See example below for structure of this object.
+
+```typescript
+import { Calculator } from 'fqm-execution';
+
+const { results, groupClauseCoverageDetails } = await Calculator.calculate(measureBundle, patientBundles, {
+  calculateCoverageDetails: true /* false by default */
+});
+
+// groupClauseCoverageDetails
+/*      ^?
+        {
+          "population-group-1": {
+            "totalClauseCount": 333,
+            "coveredClauseCount": 330,
+            "uncoveredClauseCount": 3,
+            "uncoveredClauses": [
+              {
+                "localId": "97",
+                "libraryName": "MAT6725TestingMeasureCoverage",
+                "statementName": "Has Medical or Patient Reason for Not Ordering ACEI or ARB or ARNI"
+              },
+              ...
+            ]
+          }
+          ...
+        }
+*/
+```
+
 ### Clause Coverage of Null and False Literal Values
 
 Since clause coverage calculation and highlighting are based on whether individual pieces of the measure logic CQL were processed at all during calculation and held "truthy" values, `Null` and false `Literal`s that are processed during calculation would prevent 100% clause coverage and highlighting. In order to handle this special case, these clauses that will never hold "truthy" values will be highlighted and counted as covered if they were simply processed during calculation.

src/calculation/Calculator.ts

@@ -14,7 +14,8 @@ import {
   OneOrManyBundles,
   OneOrMultiPatient,
   PopulationGroupResult,
-  DetailedPopulationGroupResult
+  DetailedPopulationGroupResult,
+  ClauseCoverageDetails
 } from '../types/Calculator';
 import { PopulationType, MeasureScoreType, ImprovementNotation } from '../types/Enums';
 import * as Execution from '../execution/Execution';
@@ -70,6 +71,8 @@ export async function calculate<T extends CalculationOptions>(
   options.calculateHTML = options.calculateHTML ?? true;
   options.calculateSDEs = options.calculateSDEs ?? true;
   options.calculateClauseCoverage = options.calculateClauseCoverage ?? true;
+  options.calculateClauseUncoverage = options.calculateClauseUncoverage ?? false;
+  options.calculateCoverageDetails = options.calculateCoverageDetails ?? false;
   options.disableHTMLOrdering = options.disableHTMLOrdering ?? false;
   options.buildStatementLevelHTML = options.buildStatementLevelHTML ?? false;
 
@@ -87,6 +90,8 @@ export async function calculate<T extends CalculationOptions>(
   const executionResults: ExecutionResult<DetailedPopulationGroupResult>[] = [];
   let overallClauseCoverageHTML: string | undefined;
   let groupClauseCoverageHTML: Record<string, string> | undefined;
+  let groupClauseUncoverageHTML: Record<string, string> | undefined;
+  let groupClauseCoverageDetails: Record<string, ClauseCoverageDetails> | undefined;
 
   let newValueSetCache: fhir4.ValueSet[] | undefined = [...valueSetCache];
   const allELM: ELM[] = [];
@@ -251,12 +256,8 @@ export async function calculate<T extends CalculationOptions>(
     patientSource = resolvePatientSource(patientBundles, options);
 
     if (!isCompositeExecution && options.calculateClauseCoverage) {
-      groupClauseCoverageHTML = generateClauseCoverageHTML(
-        measure,
-        executedELM,
-        executionResults,
-        options.disableHTMLOrdering
-      );
+      const coverage = generateClauseCoverageHTML(measure, executedELM, executionResults, options);
+      groupClauseCoverageHTML = coverage.coverage;
       overallClauseCoverageHTML = '';
       Object.entries(groupClauseCoverageHTML).forEach(([groupId, result]) => {
         overallClauseCoverageHTML += result;
@@ -272,13 +273,40 @@ export async function calculate<T extends CalculationOptions>(
           }
         }
       });
+
+      // pull out uncoverage html
+      if (options.calculateClauseUncoverage && coverage.uncoverage) {
+        groupClauseUncoverageHTML = coverage.uncoverage;
+        if (debugObject && options.enableDebugOutput) {
+          Object.entries(groupClauseUncoverageHTML).forEach(([groupId, result]) => {
+            const debugUncoverageHTML = {
+              name: `clause-uncoverage-${groupId}.html`,
+              html: result
+            };
+            if (Array.isArray(debugObject.html)) {
+              debugObject.html.push(debugUncoverageHTML);
+            } else {
+              debugObject.html = [debugUncoverageHTML];
+            }
+          });
+        }
+      }
+
       // don't necessarily need this file, but adding it for backwards compatibility
       if (debugObject && options.enableDebugOutput) {
         debugObject.html?.push({
           name: 'overall-clause-coverage.html',
           html: overallClauseCoverageHTML
         });
       }
+
+      // grab coverage details
+      if (options.calculateCoverageDetails && coverage.details) {
+        groupClauseCoverageDetails = coverage.details;
+        if (debugObject && options.enableDebugOutput) {
+          debugObject.coverageDetails = groupClauseCoverageDetails;
+        }
+      }
     }
   }
 
@@ -314,7 +342,9 @@ export async function calculate<T extends CalculationOptions>(
         )
       }),
     ...(overallClauseCoverageHTML && { coverageHTML: overallClauseCoverageHTML }),
-    ...(groupClauseCoverageHTML && { groupClauseCoverageHTML: groupClauseCoverageHTML })
+    ...(groupClauseCoverageHTML && { groupClauseCoverageHTML: groupClauseCoverageHTML }),
+    ...(groupClauseUncoverageHTML && { groupClauseUncoverageHTML: groupClauseUncoverageHTML }),
+    ...(groupClauseCoverageDetails && { groupClauseCoverageDetails: groupClauseCoverageDetails })
   };
 }
 

src/calculation/HTMLBuilder.ts

@@ -2,6 +2,7 @@
 import Handlebars from 'handlebars';
 import {
   CalculationOptions,
+  ClauseCoverageDetails,
   ClauseResult,
   DetailedPopulationGroupResult,
   ExecutionResult,
@@ -91,6 +92,29 @@
   return '';
 });
 
+// apply highlighting style to uncovered clauses
+Handlebars.registerHelper('highlightUncoverage', (localId, context) => {
+  const libraryName: string = context.data.root.libraryName;
+
+  if (
+    (context.data.root.uncoveredClauses as ClauseResult[]).some(
+      result => result.libraryName === libraryName && result.localId === localId
+    )
+  ) {
+    // Mark with red styling if clause is found in uncoverage list
+    return objToCSS(cqlLogicClauseFalseStyle);
+  } else if (
+    (context.data.root.coveredClauses as ClauseResult[]).some(
+      result => result.libraryName === libraryName && result.localId === localId
+    )
+  ) {
+    // Mark with white (clear out styling) if the clause is in coverage list
+    return objToCSS(cqlLogicUncoveredClauseStyle);
+  }
+  // If this clause has no results then it should not be styled
+  return '';
+});
+
 /**
  * Sort statements into population, then non-functions, then functions
  */
@@ -212,14 +236,20 @@
  * @returns a lookup object where the key is the groupId and the value is the
  * clause coverage HTML
  */
-export function generateClauseCoverageHTML(
+export function generateClauseCoverageHTML<T extends CalculationOptions>(
   measure: fhir4.Measure,
   elmLibraries: ELM[],
   executionResults: ExecutionResult<DetailedPopulationGroupResult>[],
-  disableHTMLOrdering?: boolean
-): Record<string, string> {
+  options: T
+): {
+  coverage: Record<string, string>;
+  uncoverage?: Record<string, string>;
+  details?: Record<string, ClauseCoverageDetails>;
+} {
   const groupResultLookup: Record<string, DetailedPopulationGroupResult[]> = {};
-  const htmlGroupLookup: Record<string, string> = {};
+  const coverageHtmlGroupLookup: Record<string, string> = {};
+  const uncoverageHtmlGroupLookup: Record<string, string> = {};
+  const coverageDetailsGroupLookup: Record<string, ClauseCoverageDetails> = {};
 
   // get the detailed result for each group within each patient and add it
   // to the key in groupResults that matches the groupId
@@ -244,7 +274,7 @@
     // Filter out any statement results where the statement relevance is NA
     const uniqueRelevantStatements = flattenedStatementResults.filter(s => s.relevance !== Relevance.NA);
 
-    if (!disableHTMLOrdering) {
+    if (!options.disableHTMLOrdering) {
       sortStatements(measure, groupId, uniqueRelevantStatements);
     }
 
@@ -270,28 +300,72 @@
       }
     });
 
-    let htmlString = `<div><h2> ${groupId} Clause Coverage: ${calculateClauseCoverage(
-      uniqueRelevantStatements,
-      flattenedClauseResults
-    )}%</h2>`;
+    const clauseCoverage = calculateClauseCoverage(uniqueRelevantStatements, flattenedClauseResults);
+    const uniqueCoverageClauses = clauseCoverage.coveredClauses.concat(clauseCoverage.uncoveredClauses);
+
+    // setup initial html for coverage
+    let coverageHtmlString = `<div><h2> ${groupId} Clause Coverage: ${clauseCoverage.percentage}%</h2>`;
+
+    // setup initial html for uncoverage
+    let uncoverageHtmlString = '';
+    if (options.calculateClauseUncoverage) {
+      uncoverageHtmlString = `<div><h2> ${groupId} Clause Uncoverage: ${clauseCoverage.uncoveredClauses.length} of ${
+        clauseCoverage.coveredClauses.length + clauseCoverage.uncoveredClauses.length
+      } clauses</h2>`;
+    }
 
     // generate HTML clauses using hbs template for each annotation
     statementAnnotations.forEach(a => {
-      const res = main({
+      coverageHtmlString += main({
         libraryName: a.libraryName,
         statementName: a.statementName,
-        clauseResults: flattenedClauseResults,
+        clauseResults: uniqueCoverageClauses,
         ...a.annotation[0].s,
         highlightCoverage: true
       });
-      htmlString += res;
+
+      // calculate for uncoverage
+      if (options.calculateClauseUncoverage) {
+        uncoverageHtmlString += main({
+          libraryName: a.libraryName,
+          statementName: a.statementName,
+          uncoveredClauses: clauseCoverage.uncoveredClauses,
+          coveredClauses: clauseCoverage.coveredClauses,
+          ...a.annotation[0].s,
+          highlightUncoverage: true
+        });
+      }
     });
-    htmlString += '</div>';
+    coverageHtmlString += '</div>';
+    uncoverageHtmlString += '</div>';
+
+    coverageHtmlGroupLookup[groupId] = coverageHtmlString;
+    if (options.calculateClauseUncoverage) {
+      uncoverageHtmlGroupLookup[groupId] = uncoverageHtmlString;
+    }
 
-    htmlGroupLookup[groupId] = htmlString;
+    // If details on coverage are requested, tally them up and add them to the map.
+    if (options.calculateCoverageDetails) {
+      coverageDetailsGroupLookup[groupId] = {
+        totalClauseCount: clauseCoverage.coveredClauses.length + clauseCoverage.uncoveredClauses.length,
+        coveredClauseCount: clauseCoverage.coveredClauses.length,
+        uncoveredClauseCount: clauseCoverage.uncoveredClauses.length,
+        uncoveredClauses: clauseCoverage.uncoveredClauses.map(uncoveredClause => {
+          return {
+            localId: uncoveredClause.localId,
+            libraryName: uncoveredClause.libraryName,
+            statementName: uncoveredClause.statementName
+          };
+        })
+      };
+    }
   });
 
-  return htmlGroupLookup;
+  return {
+    coverage: coverageHtmlGroupLookup,
+    ...(options.calculateClauseUncoverage && { uncoverage: uncoverageHtmlGroupLookup }),
+    ...(options.calculateCoverageDetails && { details: coverageDetailsGroupLookup })
+  };
 }
 
 /**
@@ -301,7 +375,10 @@
  * @param clauseResults ClauseResult array from calculation
  * @returns percentage out of 100, represented as a string
  */
-export function calculateClauseCoverage(relevantStatements: StatementResult[], clauseResults: ClauseResult[]): string {
+export function calculateClauseCoverage(
+  relevantStatements: StatementResult[],
+  clauseResults: ClauseResult[]
+): { percentage: string; coveredClauses: ClauseResult[]; uncoveredClauses: ClauseResult[] } {
   // find all relevant clauses using statementName and libraryName from relevant statements
   const allRelevantClauses = clauseResults.filter(c =>
     relevantStatements.some(
@@ -317,5 +394,14 @@
     allRelevantClauses.filter(clause => clause.final === FinalResult.TRUE),
     (c1, c2) => c1.libraryName === c2.libraryName && c1.localId === c2.localId
   );
-  return ((coveredClauses.length / allUniqueClauses.length) * 100).toPrecision(3);
+
+  const uncoveredClauses = allUniqueClauses.filter(c => {
+    return !coveredClauses.find(coveredC => c.libraryName === coveredC.libraryName && c.localId === coveredC.localId);
+  });
+
+  return {
+    percentage: ((coveredClauses.length / allUniqueClauses.length) * 100).toPrecision(3),
+    coveredClauses,
+    uncoveredClauses
+  };
 }

src/cli.ts

@@ -114,6 +114,8 @@ async function calc(
   if (program.outputType === 'raw') {
     result = await calculateRaw(measureBundle, patientBundles, calcOptions, valueSetCache);
   } else if (program.outputType === 'detailed') {
+    calcOptions.calculateClauseUncoverage = true;
+    calcOptions.calculateCoverageDetails = true;
     result = await calculate(measureBundle, patientBundles, calcOptions, valueSetCache);
   } else if (program.outputType === 'reports') {
     calcOptions.reportType = program.reportType || 'individual';
@@ -242,6 +244,10 @@ populatePatientBundles().then(async patientBundles => {
         dumpObject(debugOutput.gaps, 'gaps.json');
       }
 
+      if (debugOutput?.coverageDetails) {
+        dumpObject(debugOutput.coverageDetails, 'coverageDetails.json');
+      }
+
       // Dump ELM
       if (debugOutput?.elm) {
         dumpELMJSONs(debugOutput.elm);