Squashed commit of the following:

commit f08e187
Author: csmig <[email protected]>
Date:   Thu Feb 29 18:58:21 2024 +0000

    chore: remove/update dependencies (NUWCDIVNPT#97)

commit a70d38a
Author: csmig <[email protected]>
Date:   Thu Feb 22 19:41:45 2024 +0000

    feat: in scan mode, migrate addToHistory() calls to the queue handlers (NUWCDIVNPT#93)

    Co-authored-by: matte22 <[email protected]>

commit d007792
Author: Mathew <[email protected]>
Date:   Thu Feb 22 13:28:34 2024 -0500

    test: Create Workflow for unit testing.  (NUWCDIVNPT#92)

commit e398da9
Author: Mathew <[email protected]>
Date:   Mon Feb 19 17:57:22 2024 -0500

    feat: scan mode history management (NUWCDIVNPT#90)

commit 00f497f
Author: csmig <[email protected]>
Date:   Fri Feb 16 15:37:43 2024 +0000

    feat: pass filename to parsers as sourceRef  (NUWCDIVNPT#91)

.github/workflows/sonarcloud.yml

@@ -40,7 +40,7 @@ jobs:
             -Dsonar.projectName=NUWCDIVNPT_stigman-watcher
             -Dsonar.organization=nuwcdivnpt
             -Dsonar.inclusions=**/*.js
-            -Dsonar.exclusions=**/node_modules/**, 
+            -Dsonar.exclusions=**/node_modules/**,**/test/**,**/coverage/**
 
 
     #   This will fail the action if Quality Gate fails (leaving out for now )

.github/workflows/unit-testing.yml

@@ -0,0 +1,40 @@
+name: Run Unit Tests and Upload Coverage Artifact
+on:
+  workflow_dispatch:
+  push:
+    branches:
+        - main
+    paths:
+        - "lib/**"
+        - "index.js"
+        - "test/**"
+  pull_request:
+    branches:
+        - main
+    paths:
+        - "lib/**"
+        - "index.js"
+        - "test/**"
+
+jobs:
+  build_test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Setup Node.js 
+      uses: actions/setup-node@v3
+      with:
+        node-version: 18
+    - name: Install app dependencies
+      run: npm ci
+    - name: Create .env file
+      run: echo -e "WATCHER_COLLECTION=1\nWATCHER_API_BASE=url\nWATCHER_AUTHORITY=auth\nWATCHER_CLIENT_ID=clientId\nWATCHER_CLIENT_SECRET=secret" > .env
+    - name: Run tests
+      run: npm test
+    - name: Upload coverage to github
+      uses: actions/upload-artifact@v3
+      if: ${{ always() }}
+      with:
+        name: coverage
+        path: coverage

.gitignore

@@ -4,4 +4,5 @@ watched/
 .env
 *.log
 log.json
-bundle.js
+bundle.js
+coverage/

dist/.gitignore

@@ -9,4 +9,4 @@
 *
 
 # Except this file
-!.gitignore
+!.gitignore

index.js

@@ -10,18 +10,27 @@ import startFsEventWatcher from './lib/events.js'
 import { getOpenIDConfiguration, getToken } from './lib/auth.js'
 import * as api from './lib/api.js'
 import { serializeError } from 'serialize-error'
-import startScanner from './lib/scan.js'
+import { initScanner } from './lib/scan.js'
 import semverGte from 'semver/functions/gte.js'
 
 const minApiVersion = '1.2.7'
 
+process.on('SIGINT', () => {
+  logger.info({
+    component: 'main',
+    message: 'received SIGINT, exiting'
+  })
+  process.exit(0)
+}) 
+
 run()
 
 async function run() {
   try {
     logger.info({
       component: 'main',
       message: 'running',
+      pid: process.pid,
       options: getObfuscatedConfig(options)
     })
 
@@ -30,7 +39,7 @@ async function run() {
       startFsEventWatcher()
     }
     else if (options.mode === 'scan') {
-      startScanner()
+      initScanner()
     }
   }
   catch (e) {

lib/args.js

@@ -14,17 +14,17 @@ const component = 'args'
 
 function getVersion() {
   try {
-    const packageJsonText = readFileSync('./package.json', 'utf8');
-    return JSON.parse(packageJsonText).version;
+    const packageJsonText = readFileSync('./package.json', 'utf8')
+    return JSON.parse(packageJsonText).version
   } 
   catch (error) {
-    console.error('Error reading package.json:', error);
+    console.error('Error reading package.json:', error)
   }
 }
 
 let configValid = true
 
-const version = getVersion();
+const version = getVersion()
 
 // Use .env, if present, to setup the environment
 config()
@@ -54,18 +54,18 @@ const getBoolean = (envvar, defaultState = true) => {
   }
 }
 const parseIntegerArg = (value) => {
-  const parsedValue = parseInt(value, 10);
+  const parsedValue = parseInt(value, 10)
   if (isNaN(parsedValue)) {
-    throw new InvalidOptionArgumentError('Not a number.');
+    throw new InvalidOptionArgumentError('Not a number.')
   }
-  return parsedValue;
+  return parsedValue
 }
 const parseIntegerEnv = (value) => {
-  const parsedValue = parseInt(value, 10);
+  const parsedValue = parseInt(value, 10)
   if (isNaN(parsedValue)) {
     return undefined
   }
-  return parsedValue;
+  return parsedValue
 }
 
 // Build the Command
@@ -92,7 +92,8 @@ program
 .option('--add-existing', 'For `--mode events`, existing files in the path will generate an `add` event (`WATCHER_ADD_EXISTING=1`). Ignored if `--mode scan`, negate with `--no-add-existing`.',  getBoolean('WATCHER_ADD_EXISTING', false))
 .option('--no-add-existing', 'Ignore existing files in the watched path (`WATCHER_ADD_EXISTING=0`).')
 .option('--cargo-delay <ms>', 'Milliseconds to delay processing the queue (`WATCHER_CARGO_DELAY`)',  parseIntegerArg,  parseIntegerEnv(pe.WATCHER_CARGO_DELAY) ?? 2000)
-.option('--cargo-size <number>', 'Maximum queue size that triggers processing (`WATCHER_CARGO_SIZE`)', parseIntegerArg, parseIntegerEnv(pe.WATCHER_CARGO_SIZE) ?? 25)
+.option('--history-write-interval <ms>', 'Interval in milliseconds for when to periodically sync history file(`WATCHER_HISTORY_WRITE_INTERVAL`)',  parseIntegerArg,  parseIntegerEnv(pe.WATCHER_HISTORY_WRITE_INTERVAL) ?? 15000)
+.option('--cargo-size <number>', 'Maximum queue size that triggers processing (`WATCHER_CARGO_SIZE`)', parseIntegerArg, parseIntegerEnv(pe.WATCHER_CARGO_SIZE) ?? 10)
 .option('--create-objects', 'Create Assets or STIG Assignments as needed (`WATCHER_CREATE_OBJECTS=1`). Negate with `--no-create-objects`.', getBoolean('WATCHER_CREATE_OBJECTS', true))
 .option('--no-create-objects', 'Do not create Assets or STIG Assignments (`WATCHER_CREATE_OBJECTS=0`).')
 .option('--ignore-dir [name...]', 'DEPRECATED, use --ignore-glob. Sub-directory name to ignore. Can be invoked multiple times.(`WATCHER_IGNORE_DIRS=<csv>`)', pe.WATCHER_IGNORE_DIRS?.split(','))

lib/cargo.js

@@ -3,7 +3,8 @@ import { logger, getSymbol } from './logger.js'
 import Queue from 'better-queue'
 import * as api from './api.js'
 import { serializeError } from 'serialize-error'
-import { TaskObject } from 'stig-manager-client-modules'
+import { TaskObject } from '@nuwcdivnpt/stig-manager-client-modules'
+import { addToHistory } from './scan.js'
 
 const component = 'cargo'
 let batchId = 0
@@ -64,6 +65,7 @@ async function writer ( taskAsset ) {
       })
 
     }
+    return true
   }  
   catch (error) {
     const errorObj = {
@@ -87,19 +89,23 @@ async function writer ( taskAsset ) {
       errorObj.error = serializeError(error)
     }
     logger.error(errorObj)
+    return false
   }
 }
 
 async function resultsHandler( parsedResults, cb ) {
   const component = 'batch'
   try {
     batchId++
+    const isModeScan = options.mode === 'scan'
     logger.info({component: component, message: `batch started`, batchId: batchId, size: parsedResults.length})
     const apiAssets = await api.getCollectionAssets(options.collectionId)
     const apiStigs = await api.getInstalledStigs()
     const tasks = new TaskObject ({ parsedResults, apiAssets, apiStigs, options:options })
+    isModeScan && tasks.errors.length && addToHistory(tasks.errors.map(e => e.sourceRef))
     for ( const taskAsset of tasks.taskAssets.values() ) {
-      await writer( taskAsset )
+      const success = await writer( taskAsset )
+      isModeScan && success && addToHistory(taskAsset.sourceRefs)
     }
     logger.info({component: component, message: 'batch ended', batchId: batchId})
     cb()

lib/parse.js

@@ -1,15 +1,11 @@
 import { cache } from './api.js'
-import { XMLParser } from 'fast-xml-parser'
 import Queue from 'better-queue'
 import { logger } from './logger.js'
 import { cargoQueue } from './cargo.js'
 import { promises as fs } from 'fs'
-import he from 'he'
-import { reviewsFromCkl, reviewsFromScc, reviewsFromCklb } from 'stig-manager-client-modules'
-
-const valueProcessor = function (tagName, tagValue, jPath, hasAttributes, isLeafNode) {
-  he.decode(tagValue)
-}
+import { reviewsFromCkl, reviewsFromScc, reviewsFromCklb } from '@nuwcdivnpt/stig-manager-client-modules'
+import { addToHistory } from './scan.js'
+import { options } from './args.js'
 
 const defaultImportOptions = {
   autoStatus: 'saved',
@@ -69,11 +65,9 @@ async function parseFileAndEnqueue (file, cb) {
       importOptions,
       fieldSettings,
       allowAccept,
-      valueProcessor,
-      XMLParser,
-      scapBenchmarkMap
+      scapBenchmarkMap,
+      sourceRef: file
     })
-    parseResult.file = file
     logger.debug({component: component, message: `parse results`, results: parseResult})
 
     cargoQueue.push( parseResult )
@@ -85,12 +79,13 @@ async function parseFileAndEnqueue (file, cb) {
         stats: checklist.stats
       })
     }
-    logger.verbose({component: component, message: `results queued`, file: parseResult.file, 
+    logger.verbose({component: component, message: `results queued`, file: parseResult.sourceRef, 
       target: parseResult.target.name, checklists: checklistInfo })
     cb(null, parseResult)
   }
   catch (e) {
     logger.warn({component: component, message: e.message, file: file})
+    options.mode === 'scan' && addToHistory(file)
     cb(e, null)
   }
 }