diff --git a/.gitignore b/.gitignore index f6a4897..2706b84 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ watched/ .vscode/ .env *.log -log.json \ No newline at end of file +log.json +bundle.js \ No newline at end of file diff --git a/build.sh b/build.sh index b4348d4..59a48a2 100755 --- a/build.sh +++ b/build.sh @@ -29,6 +29,9 @@ printf "[BUILD_TASK] Fetching node_modules\n" rm -rf ./node_modules npm ci +# bundle +npx esbuild index.js --bundle --platform=node --outfile=bundle.js + # version=$(git describe --tags | sed 's/\(.*\)-.*/\1/') version=$(jq -r .version package.json) printf "\n[BUILD_TASK] Using version string: $version\n" diff --git a/index.js b/index.js index 79f90b3..344f430 100755 --- a/index.js +++ b/index.js @@ -1,16 +1,19 @@ #!/usr/bin/env node - -const minApiVersion = '1.2.7' - -const { logger, getSymbol } = require('./lib/logger') -const config = require('./lib/args') -if (!config) { +import { logger, getSymbol } from './lib/logger.js' +import { options, configValid } from './lib/args.js' +if (!configValid) { + logger.error({ component: 'main', message: 'invalid configuration... Exiting'}) logger.end() - return + process.exit(1) } -const auth = require('./lib/auth') -const api = require('./lib/api') -const {serializeError} = require('serialize-error') +import startFsEventWatcher from './lib/events.js' +import { getOpenIDConfiguration, getToken } from './lib/auth.js' +import * as api from './lib/api.js' +import { serializeError } from 'serialize-error' +import startScanner from './lib/scan.js' +import semverGte from 'semver/functions/gte.js' + +const minApiVersion = '1.2.7' run() @@ -19,22 +22,20 @@ async function run() { logger.info({ component: 'main', message: 'running', - config: getObfuscatedConfig(config) + options: getObfuscatedConfig(options) }) await preflightServices() - if (config.mode === 'events') { - const watcher = require('./lib/events') - watcher.startFsEventWatcher() + if (options.mode === 'events') { + startFsEventWatcher() } - else if (config.mode === 'scan') { - const scanner = require('./lib/scan') - scanner.startScanner() + else if (options.mode === 'scan') { + startScanner() } } catch (e) { logError(e) - await logger.end() + logger.end() } } @@ -63,25 +64,24 @@ function logError(e) { } async function hasMinApiVersion () { - const semverGte = require('semver/functions/gte') const [remoteApiVersion] = await api.getDefinition('$.info.version') logger.info({ component: 'main', message: `preflight API version`, minApiVersion, remoteApiVersion}) if (semverGte(remoteApiVersion, minApiVersion)) { return true } else { - throw( `Remote API version ${remoteApiVersion} is not compatible with this release.` ) + throw new Error(`Remote API version ${remoteApiVersion} is not compatible with this release.`) } } async function preflightServices () { await hasMinApiVersion() - await auth.getOpenIDConfiguration() - await auth.getToken() + await getOpenIDConfiguration() + await getToken() logger.info({ component: 'main', message: `preflight token request suceeded`}) const promises = [ - api.getCollection(config.collectionId), - api.getCollectionAssets(config.collectionId), + api.getCollection(options.collectionId), + api.getCollectionAssets(options.collectionId), api.getInstalledStigs(), api.getScapBenchmarkMap() ] @@ -100,8 +100,8 @@ async function preflightServices () { logger.info({ component: 'main', message: `prefilght api requests suceeded`}) } -function getObfuscatedConfig (config) { - const securedConfig = {...config} +function getObfuscatedConfig (options) { + const securedConfig = {...options} if (securedConfig.clientSecret) { securedConfig.clientSecret = '[hidden]' } @@ -119,7 +119,7 @@ async function refreshUser() { async function refreshCollection() { try { - await api.getCollection(config.collectionId) + await api.getCollection(options.collectionId) } catch (e) { logError(e) diff --git a/lib/api.js b/lib/api.js index c827958..00fd696 100644 --- a/lib/api.js +++ b/lib/api.js @@ -1,9 +1,7 @@ - -const got = require('got') -const config = require('./args') -const auth = require('./auth') -const { logger, getSymbol } = require('./logger') -const { serializeError } = require('serialize-error') +import got from 'got' +import { options } from './args.js' +import { getToken, tokens } from './auth.js' +import { logger, getSymbol } from './logger.js' const cache = { collection: null, @@ -13,21 +11,21 @@ const cache = { scapBenchmarkMap: null, stigs: null } - -module.exports.cache = cache +const _cache = cache +export { _cache as cache } async function apiGet(endpoint, authenticate = true) { try { - const options = { + const requestOptions = { responseType: 'json' } if (authenticate) { - await auth.getToken() - options.headers = { - Authorization: `Bearer ${auth.tokens.access_token}` + await getToken() + requestOptions.headers = { + Authorization: `Bearer ${tokens.access_token}` } } - const response = await got.get(`${config.api}${endpoint}`, options) + const response = await got.get(`${options.api}${endpoint}`, requestOptions) logResponse (response ) return response.body } @@ -38,38 +36,38 @@ async function apiGet(endpoint, authenticate = true) { } } -module.exports.getScapBenchmarkMap = async function () { +export async function getScapBenchmarkMap() { const response = await apiGet('/stigs/scap-maps') cache.scapBenchmarkMap = new Map(response.map(apiScapMap => [apiScapMap.scapBenchmarkId, apiScapMap.benchmarkId])) return cache.scapBenchmarkMap } -module.exports.getDefinition = async function (jsonPath) { +export async function getDefinition(jsonPath) { cache.definition = await apiGet(`/op/definition${jsonPath ? '?jsonpath=' + encodeURIComponent(jsonPath) : ''}`, false) return cache.definition } -module.exports.getCollection = async function (collectionId) { +export async function getCollection(collectionId) { cache.collection = await apiGet(`/collections/${collectionId}`) return cache.collection } -module.exports.getCollectionAssets = async function (collectionId) { +export async function getCollectionAssets(collectionId) { cache.assets = await apiGet(`/assets?collectionId=${collectionId}&projection=stigs`) return cache.assets } -module.exports.getInstalledStigs = async function () { +export async function getInstalledStigs() { cache.stigs = await apiGet('/stigs') return cache.stigs } -module.exports.createOrGetAsset = async function (asset) { +export async function createOrGetAsset(asset) { try { - await auth.getToken() - const response = await got.post(`${config.api}/assets?projection=stigs`, { + await getToken() + const response = await got.post(`${options.api}/assets?projection=stigs`, { headers: { - Authorization: `Bearer ${auth.tokens.access_token}` + Authorization: `Bearer ${tokens.access_token}` }, json: asset, responseType: 'json' @@ -87,12 +85,12 @@ module.exports.createOrGetAsset = async function (asset) { } } -module.exports.patchAsset = async function (assetId, body) { +export async function patchAsset(assetId, body) { try { - await auth.getToken() - const response = await got.patch(`${config.api}/assets/${assetId}?projection=stigs`, { + await getToken() + const response = await got.patch(`${options.api}/assets/${assetId}?projection=stigs`, { headers: { - Authorization: `Bearer ${auth.tokens.access_token}` + Authorization: `Bearer ${tokens.access_token}` }, json: body, responseType: 'json' @@ -106,12 +104,12 @@ module.exports.patchAsset = async function (assetId, body) { } } -module.exports.postReviews = async function (collectionId, assetId, reviews) { +export async function postReviews(collectionId, assetId, reviews) { try { - await auth.getToken() - const response = await got.post(`${config.api}/collections/${collectionId}/reviews/${assetId}`, { + await getToken() + const response = await got.post(`${options.api}/collections/${collectionId}/reviews/${assetId}`, { headers: { - Authorization: `Bearer ${auth.tokens.access_token}` + Authorization: `Bearer ${tokens.access_token}` }, json: reviews, responseType: 'json' @@ -125,12 +123,12 @@ module.exports.postReviews = async function (collectionId, assetId, reviews) { } } -module.exports.getUser = async function () { +export async function getUser() { cache.user = await apiGet('/user') return cache.user } -module.exports.canUserAccept = function () { +export function canUserAccept() { const curUser = cache.user const apiCollection = cache.collection const userGrant = curUser.collectionGrants.find( i => i.collection.collectionId === apiCollection.collectionId )?.accessLevel diff --git a/lib/args.js b/lib/args.js index 383e4aa..c556140 100644 --- a/lib/args.js +++ b/lib/args.js @@ -1,16 +1,33 @@ -const { Command, Option, InvalidOptionArgumentError } = require ('commander') -// set up a custom help for commander -require('./help')() +import help_default from './help.js' +help_default() +import { Command, Option, InvalidOptionArgumentError } from 'commander' +import { readFileSync } from 'fs' +import * as logger from './logger.js' +import { config } from 'dotenv' +import { resolve, sep, posix } from 'path' +import promptSync from 'prompt-sync' +import { createPrivateKey } from 'crypto' + +const prompt = promptSync({ sigint:true }) const component = 'args' -const version = require("../package.json").version -const fs = require('fs') -const { logger, addConsoleTransport, addFileTransport } = require('./logger') -const dotenv = require('dotenv') -const Path = require('path') + +function getVersion() { + try { + const packageJsonText = readFileSync('./package.json', 'utf8'); + return JSON.parse(packageJsonText).version; + } + catch (error) { + console.error('Error reading package.json:', error); + } +} + +let configValid = true + +const version = getVersion(); // Use .env, if present, to setup the environment -dotenv.config() +config() const pe = process.env //shorthand variable @@ -112,8 +129,8 @@ options.version = version // Set path variations options._originalPath = options.path -options._resolvedPath = Path.resolve(options.path) -options.path = options.path.split(Path.sep).join(Path.posix.sep) +options._resolvedPath = resolve(options.path) +options.path = options.path.split(sep).join(posix.sep) // Set dependent options if (options.oneShot) { @@ -125,13 +142,12 @@ if (options.debug) { } // Start logging -addConsoleTransport( options.logLevel, options.logColor, options.silent ) +logger.addConsoleTransport( options.logLevel, options.logColor, options.silent ) if (options.logFile) { - addFileTransport( options.logFileLevel, options.logFile ) + logger.addFileTransport( options.logFileLevel, options.logFile ) } // Validate we can perform the requested client authentication -const prompt = require('prompt-sync')({ sigint:true }) if (options.clientKey) { try { // Transform the path into a crypto private key object @@ -139,16 +155,14 @@ if (options.clientKey) { } catch (e) { // Could not make a private key - logger.log({ + logger.logger.log({ level: 'error', component: component, message: 'private key error', file: options.clientKey, error: e }) - // Bail with no export object - module.exports = false - return + configValid = false } } else { @@ -159,17 +173,15 @@ else { } if (!options.clientSecret) { // Don't know the client secret - logger.error({ + logger.logger.error({ component: component, message: 'Missing client secret' }) - // Bail with no export object - module.exports = false - return + configValid = false } } -logger.log({ +logger.logger.log({ level: 'debug', component: component, message: 'parsed options', @@ -177,22 +189,20 @@ logger.log({ }) function getPrivateKey( pemFile, passphrase, canPrompt) { - const prompt = require('prompt-sync')({ sigint:true }) - const crypto = require('crypto') let pemKey try { - pemKey = fs.readFileSync(pemFile) + pemKey = readFileSync(pemFile) } finally {} try { - return crypto.createPrivateKey({ key: pemKey, passphrase: passphrase }) + return createPrivateKey({ key: pemKey, passphrase: passphrase }) } catch (e) { let clientKeyPassphrase if (e.code === 'ERR_MISSING_PASSPHRASE' && canPrompt) { clientKeyPassphrase = prompt(`Provide passphrase for the client private key: `, { echo: "*" }) try { - return crypto.createPrivateKey({ key: pemKey, passphrase: clientKeyPassphrase }) + return createPrivateKey({ key: pemKey, passphrase: clientKeyPassphrase }) } finally {} } @@ -202,5 +212,6 @@ function getPrivateKey( pemFile, passphrase, canPrompt) { } } -module.exports = options + +export { options, configValid } diff --git a/lib/auth.js b/lib/auth.js index fdaebec..dd932e2 100644 --- a/lib/auth.js +++ b/lib/auth.js @@ -1,20 +1,20 @@ -const { logger } = require('./logger') -const got = require('got') -const atob = require('atob') -const config = require('./args') -const jwt = require('jsonwebtoken') -const crypto = require('crypto') -const { log } = require('console') +import { logger } from './logger.js' +import got from 'got' +import atob from 'atob' +import {options} from './args.js' +import jwt from 'jsonwebtoken' +import { randomBytes } from 'crypto' -let self = this +const self = {} self.url = null self.threshold = 10 self.scope = 'openid stig-manager:collection stig-manager:stig:read stig-manager:user:read' -self.key = config.clientKey -self.authenticateFn = config.clientKey ? authenticateSignedJwt : authenticateClientSecret -self.authentication = config.clientKey ? 'signed-jwt' : 'client-secret' +self.key = options.clientKey +self.authenticateFn = options.clientKey ? authenticateSignedJwt : authenticateClientSecret +self.authentication = options.clientKey ? 'signed-jwt' : 'client-secret' +let tokens, tokenDecoded /** * Fetches OpenID configuration from the specified authority URL. * @async @@ -24,10 +24,10 @@ self.authentication = config.clientKey ? 'signed-jwt' : 'client-secret' */ async function getOpenIDConfiguration () { try { - const wellKnownUrl = `${config.authority}/.well-known/openid-configuration` + const wellKnownUrl = `${options.authority}/.well-known/openid-configuration` logger.debug({ component: 'auth', - message: `sending openId config request`, + message: `sending openId configuration request`, request: { method: 'GET', url: wellKnownUrl @@ -55,24 +55,24 @@ async function getOpenIDConfiguration () { */ async function getToken () { try { - if (self.tokenDecoded) { + if (tokenDecoded) { let expiresIn = - self.tokenDecoded.exp - Math.ceil(new Date().getTime() / 1000) + tokenDecoded.exp - Math.ceil(new Date().getTime() / 1000) expiresIn -= self.threshold if (expiresIn > self.threshold) { - return self.tokenDecoded + return tokenDecoded } } // getting new token - self.tokens = await self.authenticateFn() - self.tokenDecoded = decodeToken(self.tokens.access_token) + tokens = await self.authenticateFn() + tokenDecoded = decodeToken(tokens.access_token) logger.debug({ component: 'auth', message: `received token response`, - tokens: self.tokens, - tokenDecoded: self.tokenDecoded + tokens: tokens, + tokenDecoded: tokenDecoded }) - return self.tokenDecoded + return tokenDecoded } catch (e) { if (e.response) { @@ -94,8 +94,8 @@ async function authenticateClientSecret () { form: { grant_type: 'client_credentials' }, - username: config.clientId, - password: config.clientSecret, + username: options.clientId, + password: options.clientSecret, scope: self.scope, responseType: 'json' } @@ -124,11 +124,11 @@ async function authenticateClientSecret () { */ async function authenticateSignedJwt () { // IAW RFC 7523 - const jti = crypto.randomBytes(16).toString('hex') + const jti = randomBytes(16).toString('hex') const payload = { - aud: config.authority, - iss: config.clientId, - sub: config.clientId, + aud: options.authority, + iss: options.clientId, + sub: options.clientId, jti: jti } logger.debug({ @@ -189,7 +189,7 @@ function decodeToken (str) { str += '=' break default: - throw 'Invalid token' + throw new Error('Invalid token') } str = decodeURIComponent(escape(atob(str))) str = JSON.parse(str) @@ -224,5 +224,4 @@ function logResponse (response) { }) } -module.exports.getToken = getToken -module.exports.getOpenIDConfiguration = getOpenIDConfiguration +export { getToken, getOpenIDConfiguration, tokens } diff --git a/lib/cargo.js b/lib/cargo.js index ef91e33..ef46260 100644 --- a/lib/cargo.js +++ b/lib/cargo.js @@ -1,9 +1,9 @@ -const config = require ('./args') -const { logger, getSymbol } = require('./logger') -const Queue = require('better-queue') -const api = require('./api') -const { serializeError } = require('serialize-error') -const {TaskObject} = require('stig-manager-client-modules') +import { options } from './args.js' +import { logger, getSymbol } from './logger.js' +import Queue from 'better-queue' +import * as api from './api.js' +import { serializeError } from 'serialize-error' +import { TaskObject } from 'stig-manager-client-modules' const component = 'cargo' let batchId = 0 @@ -30,7 +30,7 @@ async function writer ( taskAsset ) { if ( taskAsset.knownAsset && taskAsset.hasNewAssignment ) { const r = await api.patchAsset(taskAsset.assetProps.assetId, { // remove collectionId when https://github.com/NUWCDIVNPT/stig-manager/issues/259 is closed - collectionId: config.collectionId, + collectionId: options.collectionId, stigs: taskAsset.assetProps.stigs }) r.stigs = r.stigs.map( stig => stig.benchmarkId ) @@ -47,7 +47,7 @@ async function writer ( taskAsset ) { reviews = reviews.concat(checklist.reviews) } if (reviews.length > 0) { - const r = await api.postReviews(config.collectionId, taskAsset.assetProps.assetId, reviews) + const r = await api.postReviews(options.collectionId, taskAsset.assetProps.assetId, reviews) logger.info({ component: component, message: `posted reviews`, @@ -95,9 +95,9 @@ async function resultsHandler( parsedResults, cb ) { try { batchId++ logger.info({component: component, message: `batch started`, batchId: batchId, size: parsedResults.length}) - const apiAssets = await api.getCollectionAssets(config.collectionId) + const apiAssets = await api.getCollectionAssets(options.collectionId) const apiStigs = await api.getInstalledStigs() - const tasks = new TaskObject ({ parsedResults, apiAssets, apiStigs, options:config }) + const tasks = new TaskObject ({ parsedResults, apiAssets, apiStigs, options:options }) for ( const taskAsset of tasks.taskAssets.values() ) { await writer( taskAsset ) } @@ -112,9 +112,9 @@ async function resultsHandler( parsedResults, cb ) { const cargoQueue = new Queue(resultsHandler, { id: 'file', - batchSize: config.cargoSize, - batchDelay: config.oneShot ? 0 : config.cargoDelay, - // batchDelayTimeout: config.cargoDelay + batchSize: options.cargoSize, + batchDelay: options.oneShot ? 0 : options.cargoDelay, + // batchDelayTimeout: options.cargoDelay }) cargoQueue .on('batch_failed', (err) => { @@ -127,10 +127,10 @@ cargoQueue // console.log(`waiting ${cargoQueue._store._queue.length}`) }) .on('drain', () => { - if (config.oneShot) { + if (options.oneShot) { logger.info({component: 'cargo', message: 'finished one shot mode'}) process.exit() } }) -module.exports.queue = cargoQueue +export { cargoQueue } diff --git a/lib/events.js b/lib/events.js index 925d06b..d9f1855 100644 --- a/lib/events.js +++ b/lib/events.js @@ -1,26 +1,25 @@ -const config = require('./args') -const { logger } = require('./logger') -const parse = require('./parse') -const { serializeError } = require('serialize-error') -const { resolve } = require('path') -const chokidar = require('chokidar') +import {options} from './args.js' +import { logger } from './logger.js' +import { queue } from './parse.js' +import { serializeError } from 'serialize-error' +import { watch } from 'chokidar' const component = 'events' -module.exports.startFsEventWatcher = () => { - const awaitWriteFinishVal = config.stabilityThreshold ? { stabilityThreshold: config.stabilityThreshold } : false - const ignored = config.ignoreGlob ?? [] - if (config.ignoreDot) ignored.push(/(^|[\/\\])\../) - const watcher = chokidar.watch(config.path, { +export default function startFsEventWatcher () { + const awaitWriteFinishVal = options.stabilityThreshold ? { stabilityThreshold: options.stabilityThreshold } : false + const ignored = options.ignoreGlob ?? [] + if (options.ignoreDot) ignored.push(/(^|[\/\\])\../) + const watcher = watch(options.path, { ignored, - ignoreInitial: !config.addExisting, + ignoreInitial: !options.addExisting, persistent: true, - usePolling: config.usePolling, + usePolling: options.usePolling, awaitWriteFinish: awaitWriteFinishVal }) - logger.info({component: component, message: `watching`, path: config.path}) + logger.info({component: component, message: `watching`, path: options.path}) watcher.on('ready', e => { - if (config.oneShot) { + if (options.oneShot) { watcher.close() } }) @@ -42,7 +41,7 @@ module.exports.startFsEventWatcher = () => { event: 'add', file: file }) - parse.queue.push( file ) + queue.push( file ) } }) } diff --git a/lib/help.js b/lib/help.js index 9096788..5a4ff84 100644 --- a/lib/help.js +++ b/lib/help.js @@ -1,6 +1,6 @@ -const { Help } = require ('commander') +import { Help } from 'commander'; -module.exports = function (style = 'cli') { +export default function (style = 'cli') { if (style === 'md') { Help.prototype.optionDescription = (option) => { if (option.negate) { @@ -23,14 +23,11 @@ module.exports = function (style = 'cli') { return option.description; } Help.prototype.formatHelp = (cmd, helper) => { - const termWidth = helper.padWidth(cmd, helper); - const helpWidth = helper.helpWidth || 80; + const itemIndentWidth = 2; - const itemSeparatorWidth = 2; // between term and description + function formatItem(term, description) { if (description) { - // const fullText = `${term.padEnd(termWidth + itemSeparatorWidth)}${description}`; - // return helper.wrap(fullText, helpWidth - itemIndentWidth, termWidth + itemSeparatorWidth); term = term.replace('<','*') term = term.replace('>','*') return `**${term}**\n\n${description}` diff --git a/lib/logger.js b/lib/logger.js index f736959..148af80 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -1,4 +1,4 @@ -const { createLogger, format, transports } = require('winston') +import { createLogger, format, transports } from 'winston' class JsonFieldOrder { constructor(enabled = true) { @@ -18,9 +18,7 @@ class JsonFieldOrder { } } -// const colorFormat = format.colorize({ all: true, colors: { -// info: 'white' -// } }) + const colorFormat = format.colorize({ all: true, colors: { @@ -82,4 +80,4 @@ function getSymbol( obj, description ) { return null } -module.exports = { logger, addConsoleTransport, addFileTransport, getSymbol } \ No newline at end of file +export { logger, addConsoleTransport, addFileTransport, getSymbol } \ No newline at end of file diff --git a/lib/parse.js b/lib/parse.js index d24bd65..f598f94 100644 --- a/lib/parse.js +++ b/lib/parse.js @@ -1,14 +1,16 @@ -const api = require ('./api') -const { XMLParser } = require('fast-xml-parser') -const {reviewsFromCkl, reviewsFromScc, reviewsFromCklb} = require('stig-manager-client-modules') -const Queue = require('better-queue') -const { logger } = require('./logger') -const cargo = require('./cargo') -const fs = require('fs').promises -const he = require('he') +import { cache } from './api.js' +import { XMLParser } from 'fast-xml-parser' +import Queue from 'better-queue' +import { logger } from './logger.js' +import { cargoQueue } from './cargo.js' +import { promises as fs } from 'fs' +import he from 'he' +import { reviewsFromCkl, reviewsFromScc, reviewsFromCklb } from 'stig-manager-client-modules' + const valueProcessor = function (tagName, tagValue, jPath, hasAttributes, isLeafNode) { he.decode(tagValue) } + const defaultImportOptions = { autoStatus: 'saved', unreviewed: 'commented', @@ -17,6 +19,7 @@ const defaultImportOptions = { emptyComment: 'ignore', allowCustom: true } + function safeJSONParse (value) { try { return JSON.parse(value) @@ -25,12 +28,12 @@ function safeJSONParse (value) { return undefined } } + function canUserAccept () { - if (!api.cache.user) return false + if (!cache.user) return false - const apiCollection = api.cache.collection - const userGrant = api.cache.user.collectionGrants.find( i => i.collection.collectionId === apiCollection.collectionId )?.accessLevel - + const apiCollection = cache.collection + const userGrant = cache.user.collectionGrants.find( i => i.collection.collectionId === apiCollection.collectionId )?.accessLevel return apiCollection.settings.status.canAccept && (userGrant >= apiCollection.settings.status.minAcceptGrant) } @@ -38,31 +41,28 @@ async function parseFileAndEnqueue (file, cb) { const component = 'parser' try { const extension = file.substring(file.lastIndexOf(".") + 1) - let parseFn, type + let parseFn if (extension.toLowerCase() === 'ckl') { parseFn = reviewsFromCkl - type = 'CKL' } else if (extension.toLowerCase() === 'xml') { parseFn = reviewsFromScc - type = "XCCDF" } else if (extension.toLowerCase() === 'cklb') { parseFn = reviewsFromCklb - type = "CKLB" } else { - throw (`Ignored unknown extension`) + throw new Error('Ignored unknown extension') } // ReviewParser params const data = await fs.readFile(file) logger.verbose({component: component, message: `readFile succeeded`, file: file}) - const apiCollection = api.cache.collection + const apiCollection = cache.collection const importOptions = safeJSONParse(apiCollection.metadata?.importOptions) ?? defaultImportOptions const fieldSettings = apiCollection.settings.fields const allowAccept = canUserAccept() - const scapBenchmarkMap = api.cache.scapBenchmarkMap + const scapBenchmarkMap = cache.scapBenchmarkMap let parseResult = parseFn({ data, @@ -76,7 +76,7 @@ async function parseFileAndEnqueue (file, cb) { parseResult.file = file logger.debug({component: component, message: `parse results`, results: parseResult}) - cargo.queue.push( parseResult ) + cargoQueue.push( parseResult ) const checklistInfo = [] for (const checklist of parseResult.checklists) { @@ -95,7 +95,7 @@ async function parseFileAndEnqueue (file, cb) { } } -module.exports.queue = new Queue (parseFileAndEnqueue, { +export const queue = new Queue (parseFileAndEnqueue, { concurrent: 8 }) diff --git a/lib/scan.js b/lib/scan.js index c573f13..ced42ca 100644 --- a/lib/scan.js +++ b/lib/scan.js @@ -1,17 +1,19 @@ -const fg = require('fast-glob') -const config = require('./args') -const { logger } = require('./logger') -const fs = require('fs') -const parse = require('./parse') -const { serializeError } = require('serialize-error') -const { resolve } = require('path') + +import fastGlob from 'fast-glob' +import {options} from './args.js' +import { logger } from './logger.js' +import { existsSync, createWriteStream } from 'fs' +import { queue } from './parse.js' +import { serializeError } from 'serialize-error' +import lineByLine from 'n-readlines' + const component = 'scan' const history = new Set() -if (config.historyFile && fs.existsSync(config.historyFile)) { - const lineByLine = require('n-readlines') - const liner = new lineByLine(config.historyFile) +if (options.historyFile && existsSync(options.historyFile)) { + const liner = new lineByLine(options.historyFile) let lineCount = 0 + let line while (line = liner.next()) { history.add(line.toString('ascii')) lineCount++ @@ -19,26 +21,26 @@ if (config.historyFile && fs.existsSync(config.historyFile)) { logger.verbose({ component: component, message: `history initialized from file`, - file: config.historyFile, + file: options.historyFile, entries: lineCount }) } let historyStream -if (config.historyFile) { - historyStream = fs.createWriteStream(config.historyFile, { flags: 'a' }); +if (options.historyFile) { + historyStream = createWriteStream(options.historyFile, { flags: 'a' }); } -const interval = config.scanInterval +const interval = options.scanInterval -async function startScanner () { +export default async function startScanner () { try { - const stream = fg.stream([`${config.path}/**/*.ckl`, `${config.path}/**/*.xml`,`${config.path}/**/*.cklb` ], { - dot: !config.ignoreDot, + const stream = fastGlob.stream([`${options.path}/**/*.ckl`, `${options.path}/**/*.xml`,`${options.path}/**/*.cklb` ], { + dot: !options.ignoreDot, suppressErrors: true, - ignore: config.ignoreGlob ?? [] + ignore: options.ignoreGlob ?? [] }) - logger.info({component: component, message: `scan started`, path: config.path}) + logger.info({component: component, message: `scan started`, path: options.path}) for await (const entry of stream) { logger.verbose({component: component, message: `discovered file`, file: entry}) @@ -48,18 +50,18 @@ async function startScanner () { else { history.add(entry) logger.verbose({component: component, message: `history add`, file: entry}) - if (config.historyFile) _writeHistory(entry) - parse.queue.push(entry) + if (options.historyFile) _writeHistory(entry) + queue.push(entry) logger.info({component: component, message: `queued for parsing`, file: entry}) } } - logger.info({component: component, message: `scan ended`, path: config.path}) + logger.info({component: component, message: `scan ended`, path: options.path}) } catch(e) { logger.error({component: component, error: serializeError(e)}) } finally { - if (!config.oneShot) scheduleNextScan() + if (!options.oneShot) scheduleNextScan() } } @@ -75,14 +77,18 @@ async function _writeHistory (entry) { } } -function scheduleNextScan ( delay = config.scanInterval ) { - setTimeout(startScanner, delay) +function scheduleNextScan(delay = options.scanInterval) { + setTimeout(() => { + startScanner().catch(e => { + logger.error({ component: component, error: serializeError(e) }); + }); + }, delay); + logger.info({ component: component, message: `scan scheduled`, - path: config.path, - delay: config.scanInterval - }) + path: options.path, + delay: delay + }); } -module.exports = { startScanner, scheduleNextScan } diff --git a/package-lock.json b/package-lock.json index dfbd116..7580ddf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,6 +29,9 @@ "bin": { "stigman-watcher": "index.js" }, + "devDependencies": { + "esbuild": "^0.20.0" + }, "engines": { "node": ">=14" } @@ -51,6 +54,374 @@ "kuler": "^2.0.0" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.20.0.tgz", + "integrity": "sha512-fGFDEctNh0CcSwsiRPxiaqX0P5rq+AqE0SRhYGZ4PX46Lg1FNR6oCxJghf8YgY0WQEgQuh3lErUFE4KxLeRmmw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.20.0.tgz", + "integrity": "sha512-3bMAfInvByLHfJwYPJRlpTeaQA75n8C/QKpEaiS4HrFWFiJlNI0vzq/zCjBrhAYcPyVPG7Eo9dMrcQXuqmNk5g==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.20.0.tgz", + "integrity": "sha512-aVpnM4lURNkp0D3qPoAzSG92VXStYmoVPOgXveAUoQBWRSuQzt51yvSju29J6AHPmwY1BjH49uR29oyfH1ra8Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.20.0.tgz", + "integrity": "sha512-uK7wAnlRvjkCPzh8jJ+QejFyrP8ObKuR5cBIsQZ+qbMunwR8sbd8krmMbxTLSrDhiPZaJYKQAU5Y3iMDcZPhyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.20.0.tgz", + "integrity": "sha512-AjEcivGAlPs3UAcJedMa9qYg9eSfU6FnGHJjT8s346HSKkrcWlYezGE8VaO2xKfvvlZkgAhyvl06OJOxiMgOYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.20.0.tgz", + "integrity": "sha512-bsgTPoyYDnPv8ER0HqnJggXK6RyFy4PH4rtsId0V7Efa90u2+EifxytE9pZnsDgExgkARy24WUQGv9irVbTvIw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.20.0.tgz", + "integrity": "sha512-kQ7jYdlKS335mpGbMW5tEe3IrQFIok9r84EM3PXB8qBFJPSc6dpWfrtsC/y1pyrz82xfUIn5ZrnSHQQsd6jebQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.20.0.tgz", + "integrity": "sha512-uG8B0WSepMRsBNVXAQcHf9+Ko/Tr+XqmK7Ptel9HVmnykupXdS4J7ovSQUIi0tQGIndhbqWLaIL/qO/cWhXKyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.20.0.tgz", + "integrity": "sha512-2ezuhdiZw8vuHf1HKSf4TIk80naTbP9At7sOqZmdVwvvMyuoDiZB49YZKLsLOfKIr77+I40dWpHVeY5JHpIEIg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.20.0.tgz", + "integrity": "sha512-uTtyYAP5veqi2z9b6Gr0NUoNv9F/rOzI8tOD5jKcCvRUn7T60Bb+42NDBCWNhMjkQzI0qqwXkQGo1SY41G52nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.20.0.tgz", + "integrity": "sha512-c88wwtfs8tTffPaoJ+SQn3y+lKtgTzyjkD8NgsyCtCmtoIC8RDL7PrJU05an/e9VuAke6eJqGkoMhJK1RY6z4w==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.20.0.tgz", + "integrity": "sha512-lR2rr/128/6svngnVta6JN4gxSXle/yZEZL3o4XZ6esOqhyR4wsKyfu6qXAL04S4S5CgGfG+GYZnjFd4YiG3Aw==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.20.0.tgz", + "integrity": "sha512-9Sycc+1uUsDnJCelDf6ZNqgZQoK1mJvFtqf2MUz4ujTxGhvCWw+4chYfDLPepMEvVL9PDwn6HrXad5yOrNzIsQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.20.0.tgz", + "integrity": "sha512-CoWSaaAXOZd+CjbUTdXIJE/t7Oz+4g90A3VBCHLbfuc5yUQU/nFDLOzQsN0cdxgXd97lYW/psIIBdjzQIwTBGw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.20.0.tgz", + "integrity": "sha512-mlb1hg/eYRJUpv8h/x+4ShgoNLL8wgZ64SUr26KwglTYnwAWjkhR2GpoKftDbPOCnodA9t4Y/b68H4J9XmmPzA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.20.0.tgz", + "integrity": "sha512-fgf9ubb53xSnOBqyvWEY6ukBNRl1mVX1srPNu06B6mNsNK20JfH6xV6jECzrQ69/VMiTLvHMicQR/PgTOgqJUQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.20.0.tgz", + "integrity": "sha512-H9Eu6MGse++204XZcYsse1yFHmRXEWgadk2N58O/xd50P9EvFMLJTQLg+lB4E1cF2xhLZU5luSWtGTb0l9UeSg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.20.0.tgz", + "integrity": "sha512-lCT675rTN1v8Fo+RGrE5KjSnfY0x9Og4RN7t7lVrN3vMSjy34/+3na0q7RIfWDAj0e0rCh0OL+P88lu3Rt21MQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.20.0.tgz", + "integrity": "sha512-HKoUGXz/TOVXKQ+67NhxyHv+aDSZf44QpWLa3I1lLvAwGq8x1k0T+e2HHSRvxWhfJrFxaaqre1+YyzQ99KixoA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.20.0.tgz", + "integrity": "sha512-GDwAqgHQm1mVoPppGsoq4WJwT3vhnz/2N62CzhvApFD1eJyTroob30FPpOZabN+FgCjhG+AgcZyOPIkR8dfD7g==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.20.0.tgz", + "integrity": "sha512-0vYsP8aC4TvMlOQYozoksiaxjlvUcQrac+muDqj1Fxy6jh9l9CZJzj7zmh8JGfiV49cYLTorFLxg7593pGldwQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.20.0.tgz", + "integrity": "sha512-p98u4rIgfh4gdpV00IqknBD5pC84LCub+4a3MO+zjqvU5MVXOc3hqR2UgT2jI2nh3h8s9EQxmOsVI3tyzv1iFg==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.20.0.tgz", + "integrity": "sha512-NgJnesu1RtWihtTtXGFMU5YSE6JyyHPMxCwBZK7a6/8d31GuSo9l0Ss7w1Jw5QnKUawG6UEehs883kcXf5fYwg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -381,6 +752,44 @@ "once": "^1.4.0" } }, + "node_modules/esbuild": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.20.0.tgz", + "integrity": "sha512-6iwE3Y2RVYCME1jLpBqq7LQWK3MW6vjV2bZy6gt/WrqkY+WE74Spyc0ThAOYpMtITvnjX09CrC6ym7A/m9mebA==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.20.0", + "@esbuild/android-arm": "0.20.0", + "@esbuild/android-arm64": "0.20.0", + "@esbuild/android-x64": "0.20.0", + "@esbuild/darwin-arm64": "0.20.0", + "@esbuild/darwin-x64": "0.20.0", + "@esbuild/freebsd-arm64": "0.20.0", + "@esbuild/freebsd-x64": "0.20.0", + "@esbuild/linux-arm": "0.20.0", + "@esbuild/linux-arm64": "0.20.0", + "@esbuild/linux-ia32": "0.20.0", + "@esbuild/linux-loong64": "0.20.0", + "@esbuild/linux-mips64el": "0.20.0", + "@esbuild/linux-ppc64": "0.20.0", + "@esbuild/linux-riscv64": "0.20.0", + "@esbuild/linux-s390x": "0.20.0", + "@esbuild/linux-x64": "0.20.0", + "@esbuild/netbsd-x64": "0.20.0", + "@esbuild/openbsd-x64": "0.20.0", + "@esbuild/sunos-x64": "0.20.0", + "@esbuild/win32-arm64": "0.20.0", + "@esbuild/win32-ia32": "0.20.0", + "@esbuild/win32-x64": "0.20.0" + } + }, "node_modules/fast-glob": { "version": "3.2.12", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", diff --git a/package.json b/package.json index 2f24e48..97f7979 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "bin": { "stigman-watcher": "index.js" }, + "type": "module", "author": "carl.a.smigielski@saic.com", "license": "MIT", "repository": { @@ -32,5 +33,8 @@ "serialize-error": "^8.0.1", "stig-manager-client-modules": "github:nuwcdivnpt/stig-manager-client-modules#semver:^1.0.0", "winston": "^3.3.3" + }, + "devDependencies": { + "esbuild": "^0.20.0" } } diff --git a/pkg.config.json b/pkg.config.json index f184371..d1e4f55 100644 --- a/pkg.config.json +++ b/pkg.config.json @@ -1,9 +1,9 @@ { "name": "stigman-watcher", - "bin": "./index.js", + "bin": "./bundle.js", "pkg": { "targets": [ "node18-win", "node18-linuxstatic" ], - "assets": ["./node_modules/stig-manager-client-modules/index.cjs"], + "assets": ["./node_modules/better-queue-memory/**"], "outputPath": "./bin" } }