diff --git a/index.js b/index.js index 79f90b3..282e281 100755 --- a/index.js +++ b/index.js @@ -1,16 +1,18 @@ #!/usr/bin/env node - -const minApiVersion = '1.2.7' - -const { logger, getSymbol } = require('./lib/logger') -const config = require('./lib/args') -if (!config) { +import { logger, getSymbol } from './lib/logger.js' +import { options, configValid } from './lib/args.js' +if (!configValid) { logger.end() - return + process.exit(1) } -const auth = require('./lib/auth') -const api = require('./lib/api') -const {serializeError} = require('serialize-error') +import { startFsEventWatcher } from './lib/events.js' +import { getOpenIDConfiguration, getToken } from './lib/auth.js' +import { getDefinition, getCollection, getCollectionAssets, getInstalledStigs, getScapBenchmarkMap, getUser } from './lib/api.js' +import { serializeError } from 'serialize-error' +import { startScanner } from './lib/scan.js' +import semverGte from 'semver/functions/gte.js' + +const minApiVersion = '1.2.7' run() @@ -19,17 +21,15 @@ async function run() { logger.info({ component: 'main', message: 'running', - config: getObfuscatedConfig(config) + options: getObfuscatedConfig(options) }) await preflightServices() - if (config.mode === 'events') { - const watcher = require('./lib/events') - watcher.startFsEventWatcher() + if (options.mode === 'events') { + startFsEventWatcher() } - else if (config.mode === 'scan') { - const scanner = require('./lib/scan') - scanner.startScanner() + else if (options.mode === 'scan') { + startScanner() } } catch (e) { @@ -63,8 +63,7 @@ function logError(e) { } async function hasMinApiVersion () { - const semverGte = require('semver/functions/gte') - const [remoteApiVersion] = await api.getDefinition('$.info.version') + const [remoteApiVersion] = await getDefinition('$.info.version') logger.info({ component: 'main', message: `preflight API version`, minApiVersion, remoteApiVersion}) if (semverGte(remoteApiVersion, minApiVersion)) { return true @@ -76,14 +75,14 @@ async function hasMinApiVersion () { async function preflightServices () { await hasMinApiVersion() - await auth.getOpenIDConfiguration() - await auth.getToken() + await getOpenIDConfiguration() + await getToken() logger.info({ component: 'main', message: `preflight token request suceeded`}) const promises = [ - api.getCollection(config.collectionId), - api.getCollectionAssets(config.collectionId), - api.getInstalledStigs(), - api.getScapBenchmarkMap() + getCollection(options.collectionId), + getCollectionAssets(options.collectionId), + getInstalledStigs(), + getScapBenchmarkMap() ] await Promise.all(promises) setInterval(refreshCollection, 10 * 60000) @@ -91,7 +90,7 @@ async function preflightServices () { // OAuth scope 'stig-manager:user:read' was not required for early versions of Watcher // For now, fail gracefully if we are blocked from calling /user try { - await api.getUser() + await getUser() setInterval(refreshUser, 10 * 60000) } catch (e) { @@ -100,8 +99,8 @@ async function preflightServices () { logger.info({ component: 'main', message: `prefilght api requests suceeded`}) } -function getObfuscatedConfig (config) { - const securedConfig = {...config} +function getObfuscatedConfig (options) { + const securedConfig = {...options} if (securedConfig.clientSecret) { securedConfig.clientSecret = '[hidden]' } @@ -110,7 +109,7 @@ function getObfuscatedConfig (config) { async function refreshUser() { try { - await api.getUser() + await getUser() } catch (e) { logError(e) @@ -119,7 +118,7 @@ async function refreshUser() { async function refreshCollection() { try { - await api.getCollection(config.collectionId) + await getCollection(options.collectionId) } catch (e) { logError(e) diff --git a/lib/api.js b/lib/api.js index c827958..00fd696 100644 --- a/lib/api.js +++ b/lib/api.js @@ -1,9 +1,7 @@ - -const got = require('got') -const config = require('./args') -const auth = require('./auth') -const { logger, getSymbol } = require('./logger') -const { serializeError } = require('serialize-error') +import got from 'got' +import { options } from './args.js' +import { getToken, tokens } from './auth.js' +import { logger, getSymbol } from './logger.js' const cache = { collection: null, @@ -13,21 +11,21 @@ const cache = { scapBenchmarkMap: null, stigs: null } - -module.exports.cache = cache +const _cache = cache +export { _cache as cache } async function apiGet(endpoint, authenticate = true) { try { - const options = { + const requestOptions = { responseType: 'json' } if (authenticate) { - await auth.getToken() - options.headers = { - Authorization: `Bearer ${auth.tokens.access_token}` + await getToken() + requestOptions.headers = { + Authorization: `Bearer ${tokens.access_token}` } } - const response = await got.get(`${config.api}${endpoint}`, options) + const response = await got.get(`${options.api}${endpoint}`, requestOptions) logResponse (response ) return response.body } @@ -38,38 +36,38 @@ async function apiGet(endpoint, authenticate = true) { } } -module.exports.getScapBenchmarkMap = async function () { +export async function getScapBenchmarkMap() { const response = await apiGet('/stigs/scap-maps') cache.scapBenchmarkMap = new Map(response.map(apiScapMap => [apiScapMap.scapBenchmarkId, apiScapMap.benchmarkId])) return cache.scapBenchmarkMap } -module.exports.getDefinition = async function (jsonPath) { +export async function getDefinition(jsonPath) { cache.definition = await apiGet(`/op/definition${jsonPath ? '?jsonpath=' + encodeURIComponent(jsonPath) : ''}`, false) return cache.definition } -module.exports.getCollection = async function (collectionId) { +export async function getCollection(collectionId) { cache.collection = await apiGet(`/collections/${collectionId}`) return cache.collection } -module.exports.getCollectionAssets = async function (collectionId) { +export async function getCollectionAssets(collectionId) { cache.assets = await apiGet(`/assets?collectionId=${collectionId}&projection=stigs`) return cache.assets } -module.exports.getInstalledStigs = async function () { +export async function getInstalledStigs() { cache.stigs = await apiGet('/stigs') return cache.stigs } -module.exports.createOrGetAsset = async function (asset) { +export async function createOrGetAsset(asset) { try { - await auth.getToken() - const response = await got.post(`${config.api}/assets?projection=stigs`, { + await getToken() + const response = await got.post(`${options.api}/assets?projection=stigs`, { headers: { - Authorization: `Bearer ${auth.tokens.access_token}` + Authorization: `Bearer ${tokens.access_token}` }, json: asset, responseType: 'json' @@ -87,12 +85,12 @@ module.exports.createOrGetAsset = async function (asset) { } } -module.exports.patchAsset = async function (assetId, body) { +export async function patchAsset(assetId, body) { try { - await auth.getToken() - const response = await got.patch(`${config.api}/assets/${assetId}?projection=stigs`, { + await getToken() + const response = await got.patch(`${options.api}/assets/${assetId}?projection=stigs`, { headers: { - Authorization: `Bearer ${auth.tokens.access_token}` + Authorization: `Bearer ${tokens.access_token}` }, json: body, responseType: 'json' @@ -106,12 +104,12 @@ module.exports.patchAsset = async function (assetId, body) { } } -module.exports.postReviews = async function (collectionId, assetId, reviews) { +export async function postReviews(collectionId, assetId, reviews) { try { - await auth.getToken() - const response = await got.post(`${config.api}/collections/${collectionId}/reviews/${assetId}`, { + await getToken() + const response = await got.post(`${options.api}/collections/${collectionId}/reviews/${assetId}`, { headers: { - Authorization: `Bearer ${auth.tokens.access_token}` + Authorization: `Bearer ${tokens.access_token}` }, json: reviews, responseType: 'json' @@ -125,12 +123,12 @@ module.exports.postReviews = async function (collectionId, assetId, reviews) { } } -module.exports.getUser = async function () { +export async function getUser() { cache.user = await apiGet('/user') return cache.user } -module.exports.canUserAccept = function () { +export function canUserAccept() { const curUser = cache.user const apiCollection = cache.collection const userGrant = curUser.collectionGrants.find( i => i.collection.collectionId === apiCollection.collectionId )?.accessLevel diff --git a/lib/args.js b/lib/args.js index 383e4aa..c989d7e 100644 --- a/lib/args.js +++ b/lib/args.js @@ -1,16 +1,35 @@ -const { Command, Option, InvalidOptionArgumentError } = require ('commander') -// set up a custom help for commander -require('./help')() +import help_default from './help.js' +help_default() + +import { Command, Option, InvalidOptionArgumentError } from 'commander' +import { readFileSync } from 'fs' +import { logger, addConsoleTransport, addFileTransport } from './logger.js' +import { config } from 'dotenv' +import { resolve, sep, posix } from 'path' +import promptSync from 'prompt-sync' +import { createPrivateKey } from 'crypto' +import { fileURLToPath } from 'url'; + +const prompt = promptSync({ sigint:true }) const component = 'args' -const version = require("../package.json").version -const fs = require('fs') -const { logger, addConsoleTransport, addFileTransport } = require('./logger') -const dotenv = require('dotenv') -const Path = require('path') + +function getVersion() { + try { + const packageJsonPath = fileURLToPath(new URL('../package.json', import.meta.url)); + const packageJsonText = readFileSync(packageJsonPath, 'utf8'); + return JSON.parse(packageJsonText).version; + } catch (error) { + console.error('Error reading package.json:', error); + } +} + +let configValid = true + +const version = getVersion(); // Use .env, if present, to setup the environment -dotenv.config() +config() const pe = process.env //shorthand variable @@ -112,8 +131,8 @@ options.version = version // Set path variations options._originalPath = options.path -options._resolvedPath = Path.resolve(options.path) -options.path = options.path.split(Path.sep).join(Path.posix.sep) +options._resolvedPath = resolve(options.path) +options.path = options.path.split(sep).join(posix.sep) // Set dependent options if (options.oneShot) { @@ -131,7 +150,6 @@ if (options.logFile) { } // Validate we can perform the requested client authentication -const prompt = require('prompt-sync')({ sigint:true }) if (options.clientKey) { try { // Transform the path into a crypto private key object @@ -146,9 +164,7 @@ if (options.clientKey) { file: options.clientKey, error: e }) - // Bail with no export object - module.exports = false - return + configValid = false } } else { @@ -163,9 +179,7 @@ else { component: component, message: 'Missing client secret' }) - // Bail with no export object - module.exports = false - return + configValid = false } } @@ -177,22 +191,20 @@ logger.log({ }) function getPrivateKey( pemFile, passphrase, canPrompt) { - const prompt = require('prompt-sync')({ sigint:true }) - const crypto = require('crypto') let pemKey try { - pemKey = fs.readFileSync(pemFile) + pemKey = readFileSync(pemFile) } finally {} try { - return crypto.createPrivateKey({ key: pemKey, passphrase: passphrase }) + return createPrivateKey({ key: pemKey, passphrase: passphrase }) } catch (e) { let clientKeyPassphrase if (e.code === 'ERR_MISSING_PASSPHRASE' && canPrompt) { clientKeyPassphrase = prompt(`Provide passphrase for the client private key: `, { echo: "*" }) try { - return crypto.createPrivateKey({ key: pemKey, passphrase: clientKeyPassphrase }) + return createPrivateKey({ key: pemKey, passphrase: clientKeyPassphrase }) } finally {} } @@ -202,5 +214,5 @@ function getPrivateKey( pemFile, passphrase, canPrompt) { } } -module.exports = options +export { options, configValid } diff --git a/lib/auth.js b/lib/auth.js index fdaebec..667bebb 100644 --- a/lib/auth.js +++ b/lib/auth.js @@ -1,20 +1,20 @@ -const { logger } = require('./logger') -const got = require('got') -const atob = require('atob') -const config = require('./args') -const jwt = require('jsonwebtoken') -const crypto = require('crypto') -const { log } = require('console') +import { logger } from './logger.js' +import got from 'got' +import atob from 'atob' +import {options} from './args.js' +import jwt from 'jsonwebtoken' +import { randomBytes } from 'crypto' -let self = this +const self = {} self.url = null self.threshold = 10 self.scope = 'openid stig-manager:collection stig-manager:stig:read stig-manager:user:read' -self.key = config.clientKey -self.authenticateFn = config.clientKey ? authenticateSignedJwt : authenticateClientSecret -self.authentication = config.clientKey ? 'signed-jwt' : 'client-secret' +self.key = options.clientKey +self.authenticateFn = options.clientKey ? authenticateSignedJwt : authenticateClientSecret +self.authentication = options.clientKey ? 'signed-jwt' : 'client-secret' +let tokens, tokenDecoded /** * Fetches OpenID configuration from the specified authority URL. * @async @@ -24,10 +24,10 @@ self.authentication = config.clientKey ? 'signed-jwt' : 'client-secret' */ async function getOpenIDConfiguration () { try { - const wellKnownUrl = `${config.authority}/.well-known/openid-configuration` + const wellKnownUrl = `${options.authority}/.well-known/openid-configuration` logger.debug({ component: 'auth', - message: `sending openId config request`, + message: `sending openId options request`, request: { method: 'GET', url: wellKnownUrl @@ -55,24 +55,24 @@ async function getOpenIDConfiguration () { */ async function getToken () { try { - if (self.tokenDecoded) { + if (tokenDecoded) { let expiresIn = - self.tokenDecoded.exp - Math.ceil(new Date().getTime() / 1000) + tokenDecoded.exp - Math.ceil(new Date().getTime() / 1000) expiresIn -= self.threshold if (expiresIn > self.threshold) { - return self.tokenDecoded + return tokenDecoded } } // getting new token - self.tokens = await self.authenticateFn() - self.tokenDecoded = decodeToken(self.tokens.access_token) + tokens = await self.authenticateFn() + tokenDecoded = decodeToken(tokens.access_token) logger.debug({ component: 'auth', message: `received token response`, - tokens: self.tokens, - tokenDecoded: self.tokenDecoded + tokens: tokens, + tokenDecoded: tokenDecoded }) - return self.tokenDecoded + return tokenDecoded } catch (e) { if (e.response) { @@ -94,8 +94,8 @@ async function authenticateClientSecret () { form: { grant_type: 'client_credentials' }, - username: config.clientId, - password: config.clientSecret, + username: options.clientId, + password: options.clientSecret, scope: self.scope, responseType: 'json' } @@ -124,11 +124,11 @@ async function authenticateClientSecret () { */ async function authenticateSignedJwt () { // IAW RFC 7523 - const jti = crypto.randomBytes(16).toString('hex') + const jti = randomBytes(16).toString('hex') const payload = { - aud: config.authority, - iss: config.clientId, - sub: config.clientId, + aud: options.authority, + iss: options.clientId, + sub: options.clientId, jti: jti } logger.debug({ @@ -189,7 +189,7 @@ function decodeToken (str) { str += '=' break default: - throw 'Invalid token' + throw new Error('Invalid token') } str = decodeURIComponent(escape(atob(str))) str = JSON.parse(str) @@ -224,5 +224,4 @@ function logResponse (response) { }) } -module.exports.getToken = getToken -module.exports.getOpenIDConfiguration = getOpenIDConfiguration +export { getToken, getOpenIDConfiguration, tokens } diff --git a/lib/cargo.js b/lib/cargo.js index ef91e33..66f6bfd 100644 --- a/lib/cargo.js +++ b/lib/cargo.js @@ -1,9 +1,9 @@ -const config = require ('./args') -const { logger, getSymbol } = require('./logger') -const Queue = require('better-queue') -const api = require('./api') -const { serializeError } = require('serialize-error') -const {TaskObject} = require('stig-manager-client-modules') +import { options } from './args.js' +import { logger, getSymbol } from './logger.js' +import Queue from 'better-queue' +import { createOrGetAsset, patchAsset, postReviews, getCollectionAssets, getInstalledStigs } from './api.js' +import { serializeError } from 'serialize-error' +import { TaskObject } from 'stig-manager-client-modules' const component = 'cargo' let batchId = 0 @@ -18,7 +18,7 @@ async function writer ( taskAsset ) { // Create new asset if necessary if ( !taskAsset.knownAsset ) { - const r = await api.createOrGetAsset( taskAsset.assetProps ) + const r = await createOrGetAsset( taskAsset.assetProps ) // GET projection=stigs is an object, we just need the benchmarkIds r.apiAsset.stigs = r.apiAsset.stigs.map ( stig => stig.benchmarkId ) logger.info({ component: component, message: `asset ${r.created ? 'created' : 'found'}`, asset: r.apiAsset }) @@ -28,9 +28,9 @@ async function writer ( taskAsset ) { // Assign new STIGs, if necessary if ( taskAsset.knownAsset && taskAsset.hasNewAssignment ) { - const r = await api.patchAsset(taskAsset.assetProps.assetId, { + const r = await patchAsset(taskAsset.assetProps.assetId, { // remove collectionId when https://github.com/NUWCDIVNPT/stig-manager/issues/259 is closed - collectionId: config.collectionId, + collectionId: options.collectionId, stigs: taskAsset.assetProps.stigs }) r.stigs = r.stigs.map( stig => stig.benchmarkId ) @@ -47,7 +47,7 @@ async function writer ( taskAsset ) { reviews = reviews.concat(checklist.reviews) } if (reviews.length > 0) { - const r = await api.postReviews(config.collectionId, taskAsset.assetProps.assetId, reviews) + const r = await postReviews(options.collectionId, taskAsset.assetProps.assetId, reviews) logger.info({ component: component, message: `posted reviews`, @@ -95,9 +95,9 @@ async function resultsHandler( parsedResults, cb ) { try { batchId++ logger.info({component: component, message: `batch started`, batchId: batchId, size: parsedResults.length}) - const apiAssets = await api.getCollectionAssets(config.collectionId) - const apiStigs = await api.getInstalledStigs() - const tasks = new TaskObject ({ parsedResults, apiAssets, apiStigs, options:config }) + const apiAssets = await getCollectionAssets(options.collectionId) + const apiStigs = await getInstalledStigs() + const tasks = new TaskObject ({ parsedResults, apiAssets, apiStigs, options:options }) for ( const taskAsset of tasks.taskAssets.values() ) { await writer( taskAsset ) } @@ -112,9 +112,9 @@ async function resultsHandler( parsedResults, cb ) { const cargoQueue = new Queue(resultsHandler, { id: 'file', - batchSize: config.cargoSize, - batchDelay: config.oneShot ? 0 : config.cargoDelay, - // batchDelayTimeout: config.cargoDelay + batchSize: options.cargoSize, + batchDelay: options.oneShot ? 0 : options.cargoDelay, + // batchDelayTimeout: options.cargoDelay }) cargoQueue .on('batch_failed', (err) => { @@ -127,10 +127,10 @@ cargoQueue // console.log(`waiting ${cargoQueue._store._queue.length}`) }) .on('drain', () => { - if (config.oneShot) { + if (options.oneShot) { logger.info({component: 'cargo', message: 'finished one shot mode'}) process.exit() } }) -module.exports.queue = cargoQueue +export { cargoQueue } diff --git a/lib/events.js b/lib/events.js index 925d06b..34c8b27 100644 --- a/lib/events.js +++ b/lib/events.js @@ -1,26 +1,25 @@ -const config = require('./args') -const { logger } = require('./logger') -const parse = require('./parse') -const { serializeError } = require('serialize-error') -const { resolve } = require('path') -const chokidar = require('chokidar') +import {options} from './args.js' +import { logger } from './logger.js' +import { queue } from './parse.js' +import { serializeError } from 'serialize-error' +import { watch } from 'chokidar' const component = 'events' -module.exports.startFsEventWatcher = () => { - const awaitWriteFinishVal = config.stabilityThreshold ? { stabilityThreshold: config.stabilityThreshold } : false - const ignored = config.ignoreGlob ?? [] - if (config.ignoreDot) ignored.push(/(^|[\/\\])\../) - const watcher = chokidar.watch(config.path, { +export function startFsEventWatcher () { + const awaitWriteFinishVal = options.stabilityThreshold ? { stabilityThreshold: options.stabilityThreshold } : false + const ignored = options.ignoreGlob ?? [] + if (options.ignoreDot) ignored.push(/(^|[\/\\])\../) + const watcher = watch(options.path, { ignored, - ignoreInitial: !config.addExisting, + ignoreInitial: !options.addExisting, persistent: true, - usePolling: config.usePolling, + usePolling: options.usePolling, awaitWriteFinish: awaitWriteFinishVal }) - logger.info({component: component, message: `watching`, path: config.path}) + logger.info({component: component, message: `watching`, path: options.path}) watcher.on('ready', e => { - if (config.oneShot) { + if (options.oneShot) { watcher.close() } }) @@ -42,7 +41,7 @@ module.exports.startFsEventWatcher = () => { event: 'add', file: file }) - parse.queue.push( file ) + queue.push( file ) } }) } diff --git a/lib/help.js b/lib/help.js index 9096788..11a86f1 100644 --- a/lib/help.js +++ b/lib/help.js @@ -1,6 +1,6 @@ -const { Help } = require ('commander') +import { Help } from 'commander'; -module.exports = function (style = 'cli') { +export default function (style = 'cli') { if (style === 'md') { Help.prototype.optionDescription = (option) => { if (option.negate) { diff --git a/lib/logger.js b/lib/logger.js index f736959..f5f3db6 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -1,4 +1,4 @@ -const { createLogger, format, transports } = require('winston') +import { createLogger, format, transports } from 'winston' class JsonFieldOrder { constructor(enabled = true) { @@ -82,4 +82,4 @@ function getSymbol( obj, description ) { return null } -module.exports = { logger, addConsoleTransport, addFileTransport, getSymbol } \ No newline at end of file +export { logger, addConsoleTransport, addFileTransport, getSymbol } \ No newline at end of file diff --git a/lib/parse.js b/lib/parse.js index d24bd65..f598f94 100644 --- a/lib/parse.js +++ b/lib/parse.js @@ -1,14 +1,16 @@ -const api = require ('./api') -const { XMLParser } = require('fast-xml-parser') -const {reviewsFromCkl, reviewsFromScc, reviewsFromCklb} = require('stig-manager-client-modules') -const Queue = require('better-queue') -const { logger } = require('./logger') -const cargo = require('./cargo') -const fs = require('fs').promises -const he = require('he') +import { cache } from './api.js' +import { XMLParser } from 'fast-xml-parser' +import Queue from 'better-queue' +import { logger } from './logger.js' +import { cargoQueue } from './cargo.js' +import { promises as fs } from 'fs' +import he from 'he' +import { reviewsFromCkl, reviewsFromScc, reviewsFromCklb } from 'stig-manager-client-modules' + const valueProcessor = function (tagName, tagValue, jPath, hasAttributes, isLeafNode) { he.decode(tagValue) } + const defaultImportOptions = { autoStatus: 'saved', unreviewed: 'commented', @@ -17,6 +19,7 @@ const defaultImportOptions = { emptyComment: 'ignore', allowCustom: true } + function safeJSONParse (value) { try { return JSON.parse(value) @@ -25,12 +28,12 @@ function safeJSONParse (value) { return undefined } } + function canUserAccept () { - if (!api.cache.user) return false + if (!cache.user) return false - const apiCollection = api.cache.collection - const userGrant = api.cache.user.collectionGrants.find( i => i.collection.collectionId === apiCollection.collectionId )?.accessLevel - + const apiCollection = cache.collection + const userGrant = cache.user.collectionGrants.find( i => i.collection.collectionId === apiCollection.collectionId )?.accessLevel return apiCollection.settings.status.canAccept && (userGrant >= apiCollection.settings.status.minAcceptGrant) } @@ -38,31 +41,28 @@ async function parseFileAndEnqueue (file, cb) { const component = 'parser' try { const extension = file.substring(file.lastIndexOf(".") + 1) - let parseFn, type + let parseFn if (extension.toLowerCase() === 'ckl') { parseFn = reviewsFromCkl - type = 'CKL' } else if (extension.toLowerCase() === 'xml') { parseFn = reviewsFromScc - type = "XCCDF" } else if (extension.toLowerCase() === 'cklb') { parseFn = reviewsFromCklb - type = "CKLB" } else { - throw (`Ignored unknown extension`) + throw new Error('Ignored unknown extension') } // ReviewParser params const data = await fs.readFile(file) logger.verbose({component: component, message: `readFile succeeded`, file: file}) - const apiCollection = api.cache.collection + const apiCollection = cache.collection const importOptions = safeJSONParse(apiCollection.metadata?.importOptions) ?? defaultImportOptions const fieldSettings = apiCollection.settings.fields const allowAccept = canUserAccept() - const scapBenchmarkMap = api.cache.scapBenchmarkMap + const scapBenchmarkMap = cache.scapBenchmarkMap let parseResult = parseFn({ data, @@ -76,7 +76,7 @@ async function parseFileAndEnqueue (file, cb) { parseResult.file = file logger.debug({component: component, message: `parse results`, results: parseResult}) - cargo.queue.push( parseResult ) + cargoQueue.push( parseResult ) const checklistInfo = [] for (const checklist of parseResult.checklists) { @@ -95,7 +95,7 @@ async function parseFileAndEnqueue (file, cb) { } } -module.exports.queue = new Queue (parseFileAndEnqueue, { +export const queue = new Queue (parseFileAndEnqueue, { concurrent: 8 }) diff --git a/lib/scan.js b/lib/scan.js index c573f13..6eb03b7 100644 --- a/lib/scan.js +++ b/lib/scan.js @@ -1,17 +1,19 @@ -const fg = require('fast-glob') -const config = require('./args') -const { logger } = require('./logger') -const fs = require('fs') -const parse = require('./parse') -const { serializeError } = require('serialize-error') -const { resolve } = require('path') + +import fastGlob from 'fast-glob' +import {options} from './args.js' +import { logger } from './logger.js' +import { existsSync, createWriteStream } from 'fs' +import { queue } from './parse.js' +import { serializeError } from 'serialize-error' +import lineByLine from 'n-readlines' + const component = 'scan' const history = new Set() -if (config.historyFile && fs.existsSync(config.historyFile)) { - const lineByLine = require('n-readlines') - const liner = new lineByLine(config.historyFile) +if (options.historyFile && existsSync(options.historyFile)) { + const liner = new lineByLine(options.historyFile) let lineCount = 0 + let line while (line = liner.next()) { history.add(line.toString('ascii')) lineCount++ @@ -19,26 +21,26 @@ if (config.historyFile && fs.existsSync(config.historyFile)) { logger.verbose({ component: component, message: `history initialized from file`, - file: config.historyFile, + file: options.historyFile, entries: lineCount }) } let historyStream -if (config.historyFile) { - historyStream = fs.createWriteStream(config.historyFile, { flags: 'a' }); +if (options.historyFile) { + historyStream = createWriteStream(options.historyFile, { flags: 'a' }); } -const interval = config.scanInterval +const interval = options.scanInterval async function startScanner () { try { - const stream = fg.stream([`${config.path}/**/*.ckl`, `${config.path}/**/*.xml`,`${config.path}/**/*.cklb` ], { - dot: !config.ignoreDot, + const stream = fastGlob.stream([`${options.path}/**/*.ckl`, `${options.path}/**/*.xml`,`${options.path}/**/*.cklb` ], { + dot: !options.ignoreDot, suppressErrors: true, - ignore: config.ignoreGlob ?? [] + ignore: options.ignoreGlob ?? [] }) - logger.info({component: component, message: `scan started`, path: config.path}) + logger.info({component: component, message: `scan started`, path: options.path}) for await (const entry of stream) { logger.verbose({component: component, message: `discovered file`, file: entry}) @@ -48,18 +50,18 @@ async function startScanner () { else { history.add(entry) logger.verbose({component: component, message: `history add`, file: entry}) - if (config.historyFile) _writeHistory(entry) - parse.queue.push(entry) + if (options.historyFile) _writeHistory(entry) + queue.push(entry) logger.info({component: component, message: `queued for parsing`, file: entry}) } } - logger.info({component: component, message: `scan ended`, path: config.path}) + logger.info({component: component, message: `scan ended`, path: options.path}) } catch(e) { logger.error({component: component, error: serializeError(e)}) } finally { - if (!config.oneShot) scheduleNextScan() + if (!options.oneShot) scheduleNextScan() } } @@ -75,14 +77,14 @@ async function _writeHistory (entry) { } } -function scheduleNextScan ( delay = config.scanInterval ) { +function scheduleNextScan ( delay = options.scanInterval ) { setTimeout(startScanner, delay) logger.info({ component: component, message: `scan scheduled`, - path: config.path, - delay: config.scanInterval + path: options.path, + delay: options.scanInterval }) } -module.exports = { startScanner, scheduleNextScan } +export { startScanner, scheduleNextScan } diff --git a/package.json b/package.json index 2f24e48..abed507 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "bin": { "stigman-watcher": "index.js" }, + "type": "module", "author": "carl.a.smigielski@saic.com", "license": "MIT", "repository": {