feat: in scan mode, migrate addToHistory() calls to the queue handlers

.github/workflows/unit-testing.yml

@@ -28,6 +28,8 @@ jobs:
         node-version: 18
     - name: Install app dependencies
       run: npm ci
+    - name: Create .env file
+      run: echo -e "WATCHER_COLLECTION=1\nWATCHER_API_BASE=url\nWATCHER_AUTHORITY=auth\nWATCHER_CLIENT_ID=clientId\nWATCHER_CLIENT_SECRET=secret" > .env
     - name: Run tests
       run: npm test
     - name: Upload coverage to github

lib/cargo.js

@@ -4,6 +4,7 @@ import Queue from 'better-queue'
 import * as api from './api.js'
 import { serializeError } from 'serialize-error'
 import { TaskObject } from 'stig-manager-client-modules'
+import { addToHistory } from './scan.js'
 
 const component = 'cargo'
 let batchId = 0
@@ -64,6 +65,7 @@ async function writer ( taskAsset ) {
       })
 
     }
+    return true
   }  
   catch (error) {
     const errorObj = {
@@ -87,19 +89,23 @@ async function writer ( taskAsset ) {
       errorObj.error = serializeError(error)
     }
     logger.error(errorObj)
+    return false
   }
 }
 
 async function resultsHandler( parsedResults, cb ) {
   const component = 'batch'
   try {
     batchId++
+    const isModeScan = options.mode === 'scan'
     logger.info({component: component, message: `batch started`, batchId: batchId, size: parsedResults.length})
     const apiAssets = await api.getCollectionAssets(options.collectionId)
     const apiStigs = await api.getInstalledStigs()
     const tasks = new TaskObject ({ parsedResults, apiAssets, apiStigs, options:options })
+    isModeScan && tasks.errors.length && addToHistory(tasks.errors.map(e => e.sourceRef))
     for ( const taskAsset of tasks.taskAssets.values() ) {
-      await writer( taskAsset )
+      const success = await writer( taskAsset )
+      isModeScan && success && addToHistory(taskAsset.sourceRefs)
     }
     logger.info({component: component, message: 'batch ended', batchId: batchId})
     cb()

lib/parse.js

@@ -6,6 +6,8 @@ import { cargoQueue } from './cargo.js'
 import { promises as fs } from 'fs'
 import he from 'he'
 import { reviewsFromCkl, reviewsFromScc, reviewsFromCklb } from 'stig-manager-client-modules'
+import { addToHistory } from './scan.js'
+import { options } from './args.js'
 
 const valueProcessor = function (tagName, tagValue, jPath, hasAttributes, isLeafNode) {
   he.decode(tagValue)
@@ -91,6 +93,7 @@ async function parseFileAndEnqueue (file, cb) {
   }
   catch (e) {
     logger.warn({component: component, message: e.message, file: file})
+    options.mode === 'scan' && addToHistory(file)
     cb(e, null)
   }
 }

lib/scan.js

@@ -42,7 +42,6 @@ async function startScanner() {
         }
         // if the file is not in the history, add it to the in memory history set.
         else {
-          addToHistory(entry)
           parseQueue.push(entry)
           logger.info({component, message: `queued for parsing`, file: entry})
         }

package.json

@@ -43,7 +43,6 @@
     "chai": "^5.0.3",
     "esbuild": "^0.20.0",
     "mocha": "^10.2.0",
-    "nodemon": "^3.0.3",
-    "sinon": "^17.0.1"
+    "nodemon": "^3.0.3"
   }
 }