From 6981161edd4f8a31fa214357c2fc243aea267ee1 Mon Sep 17 00:00:00 2001 From: csmig Date: Mon, 10 Jun 2024 11:24:45 -0400 Subject: [PATCH 1/5] feat: concurrent asset fetches --- index.js | 1 - lib/api.js | 65 ++++++++++++++++++++++++++++++++++++++++++++++++---- lib/cargo.js | 16 +++++++++---- 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/index.js b/index.js index 0d5cda2..038514f 100755 --- a/index.js +++ b/index.js @@ -129,7 +129,6 @@ async function preflightServices () { logger.info({ component, message: `preflight token request suceeded`}) const promises = [ api.getCollection(options.collectionId), - api.getCollectionAssets(options.collectionId), api.getInstalledStigs(), api.getScapBenchmarkMap() ] diff --git a/lib/api.js b/lib/api.js index f663bdc..3759d46 100644 --- a/lib/api.js +++ b/lib/api.js @@ -61,7 +61,7 @@ async function apiRequest({method = 'GET', endpoint, json, authorize = true, ful } catch (e) { // accept a client error for POST /assets if it reports a duplicate name - if (e.response?.statusCode === 400 && e.response?.body?.message === 'Duplicate name') { + if (e.response?.statusCode === 422 && e.response?.body?.message === 'Duplicate name exists') { logResponse(e.response) return fullResponse ? e?.response : e?.response?.body } @@ -71,13 +71,40 @@ async function apiRequest({method = 'GET', endpoint, json, authorize = true, ful if (e.response?.statusCode === 403) { Alarm.noGrant(true) } - else { + else if (e.code === 'ETIMEDOUT' || e.response?.statusCode === 503) { Alarm.apiOffline(true) } throw (e) } } +async function apiGetRequestsParallel({endpoints, authorize = true}) { + const requestOptions = { + responseType: 'json', + timeout: { + request: CONSTANTS.REQUEST_TIMEOUT + } + } + if (authorize) { + try { + await getToken() + } + catch (e) { + e.component = 'api' + logError(e) + throw(e) + } + requestOptions.headers = { + Authorization: `Bearer ${tokens.access_token}` + } + } + const requests = [] + for (const endpoint of endpoints) { + requests.push(got.get(endpoint, requestOptions)) + } + return Promise.all(requests) +} + export async function getScapBenchmarkMap() { const body = await apiRequest({endpoint: '/stigs/scap-maps'}) cache.scapBenchmarkMap = new Map(body.map(apiScapMap => [apiScapMap.scapBenchmarkId, apiScapMap.benchmarkId])) @@ -98,9 +125,37 @@ export async function getCollection(collectionId) { return cache.collection } -export async function getCollectionAssets(collectionId) { - cache.assets = await apiRequest({endpoint: `/assets?collectionId=${collectionId}&projection=stigs`}) - return cache.assets +export async function getCollectionAssets({collectionId, targets}) { + + let namedEndpoints = [] + let metadataEndpoints = [] + + if (targets) { + const names = targets.filter(t => !t.metadata.cklHostName).map(t => t.name) + const cklMetadatas = targets.filter(t => t.metadata.cklHostName).map(t => (({cklHostName, cklWebDbSite, cklWebDbInstance}) => ({cklHostName, cklWebDbSite, cklWebDbInstance}))(t.metadata)) + + namedEndpoints = names.map(name => `${options.api}/assets?collectionId=${collectionId}&name=${name}&projection=stigs`) + metadataEndpoints = cklMetadatas.map(md => { + let metadataParams = [] + for (const metadataKey of Object.keys(md).filter(k=>md[k])) { + metadataParams.push(`metadata=${metadataKey}%3A${md[metadataKey]}`) + } + return `${options.api}/assets?collectionId=${collectionId}&${metadataParams.join('&')}&projection=stigs` + }) + const responses = await apiGetRequestsParallel({ + endpoints: new Set([...namedEndpoints, ...metadataEndpoints]) + }) + const assets = [] + for (const response of responses) { + logResponse (response) + if (response.body?.[0]) assets.push(response.body[0]) + } + return assets + } + else { + cache.assets = await apiRequest({endpoint: `/assets?collectionId=${collectionId}&projection=stigs`}) + return cache.assets + } } export async function getInstalledStigs() { diff --git a/lib/cargo.js b/lib/cargo.js index ffa5814..3c71fec 100644 --- a/lib/cargo.js +++ b/lib/cargo.js @@ -23,7 +23,7 @@ async function writer ( taskAsset ) { // GET projection=stigs is an object, we just need the benchmarkIds r.apiAsset.stigs = r.apiAsset.stigs.map ( stig => stig.benchmarkId ) logger.info({ component: component, message: `asset ${r.created ? 'created' : 'found'}`, asset: r.apiAsset }) - // TODO: If created === false, then STIG assignments should be vetted again + // Iterate: If created === false, then STIG assignments should be vetted again taskAsset.assetProps = r.apiAsset } @@ -99,19 +99,26 @@ async function resultsHandler( parsedResults, cb ) { batchId++ const isModeScan = options.mode === 'scan' logger.info({component: component, message: `batch started`, batchId: batchId, size: parsedResults.length}) - const apiAssets = await api.getCollectionAssets(options.collectionId) + const apiAssets = await api.getCollectionAssets( + { + collectionId: options.collectionId, + targets: parsedResults.map(pr=>pr.target) + } + ) + logger.info({component: component, message: `asset data received`, batchId: batchId, size: apiAssets.length}) const apiStigs = await api.getInstalledStigs() + logger.info({component: component, message: `stig data received`, batchId: batchId, size: apiStigs.length}) const tasks = new TaskObject ({ parsedResults, apiAssets, apiStigs, options:options }) isModeScan && tasks.errors.length && addToHistory(tasks.errors.map(e => e.sourceRef)) for ( const taskAsset of tasks.taskAssets.values() ) { const success = await writer( taskAsset ) isModeScan && success && addToHistory(taskAsset.sourceRefs) } - logger.info({component: component, message: 'batch ended', batchId: batchId}) + logger.info({component, message: 'batch ended with success', batchId: batchId}) cb() } catch (e) { - logger.error({component: component, message: e.message, error: serializeError(e)}) + logger.error({component, message: 'batch ended with error', error: serializeError(e), batchId: batchId}) cb( e, undefined) } } @@ -120,7 +127,6 @@ const cargoQueue = new Queue(resultsHandler, { id: 'file', batchSize: options.cargoSize, batchDelay: options.oneShot ? 0 : options.cargoDelay, - // batchDelayTimeout: options.cargoDelay }) cargoQueue .on('batch_failed', (err) => { From 2cb6d5a937318558a2bab7912faf6e1108beb6e5 Mon Sep 17 00:00:00 2001 From: csmig Date: Mon, 10 Jun 2024 12:43:26 -0400 Subject: [PATCH 2/5] Merge branch 'main' into concurrent-asset-fetches --- lib/api.js | 2 +- lib/args.js | 1 + lib/auth.js | 6 +++--- lib/consts.js | 3 +-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/api.js b/lib/api.js index 3759d46..d068bb3 100644 --- a/lib/api.js +++ b/lib/api.js @@ -34,7 +34,7 @@ async function apiRequest({method = 'GET', endpoint, json, authorize = true, ful url: `${options.api}${endpoint}`, responseType: 'json', timeout: { - request: CONSTANTS.REQUEST_TIMEOUT + response: options.responseTimeout } } diff --git a/lib/args.js b/lib/args.js index 9ef150c..cdf8c5e 100644 --- a/lib/args.js +++ b/lib/args.js @@ -112,6 +112,7 @@ program .option('--no-ignore-dot', 'Do not ignore dotfiles in the path (`WATCHER_IGNORE_DOT=0`).') .option('--strict-revision-check', 'For CKL, ignore checklist of uninstalled STIG revision (`WATCHER_STRICT_REVISION_CHECK=1`). Negate with `--no-strict-revision-check`.', getBoolean('WATCHER_STRICT_REVISION_CHECK', false)) .option('--no-strict-revision-check', 'For CKL, allow checklist of uninstalled STIG revision (`WATCHER_STRICT_REVISION_CHECK=0`). This is the default behavior.') +.option('--response-timeout ', 'Specify the timeout duration in milliseconds for an API response to begin. If a response takes longer than this time, an error will be thrown.', parseIntegerArg, parseIntegerEnv(pe.WATCHER_RESPONSE_TIMEOUT) ?? 20000) // 20 secs // Parse ARGV and get the parsed options object // Options properties are created as camelCase versions of the long option name diff --git a/lib/auth.js b/lib/auth.js index f3c039d..01be618 100644 --- a/lib/auth.js +++ b/lib/auth.js @@ -51,7 +51,7 @@ async function getOpenIDConfiguration () { const requestOptions = { responseType: 'json', timeout: { - request: CONSTANTS.REQUEST_TIMEOUT + response: options.responseTimeout } } let response @@ -122,7 +122,7 @@ async function authenticateClientSecret () { password: options.clientSecret, responseType: 'json', timeout: { - request: CONSTANTS.REQUEST_TIMEOUT + response: options.responseTimeout } } @@ -180,7 +180,7 @@ async function authenticateSignedJwt () { }, responseType: 'json', timeout: { - request: CONSTANTS.REQUEST_TIMEOUT + response: options.responseTimeout } } logger.debug({ diff --git a/lib/consts.js b/lib/consts.js index a584ca2..ee4e86a 100644 --- a/lib/consts.js +++ b/lib/consts.js @@ -3,5 +3,4 @@ export const ERR_AUTHOFFLINE = 2 export const ERR_NOTOKEN = 3 export const ERR_NOGRANT = 4 export const ERR_UNKNOWN = 5 -export const ERR_FAILINIT = 6 -export const REQUEST_TIMEOUT = 5000 +export const ERR_FAILINIT = 6 \ No newline at end of file From 043a7498388a737f5ecf803173402d61c1078587 Mon Sep 17 00:00:00 2001 From: csmig Date: Mon, 10 Jun 2024 12:45:58 -0400 Subject: [PATCH 3/5] update to timeout response --- lib/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api.js b/lib/api.js index d068bb3..91bed57 100644 --- a/lib/api.js +++ b/lib/api.js @@ -82,7 +82,7 @@ async function apiGetRequestsParallel({endpoints, authorize = true}) { const requestOptions = { responseType: 'json', timeout: { - request: CONSTANTS.REQUEST_TIMEOUT + response: options.responseTimeout } } if (authorize) { From 78b7a04bfc039b7fa06765e1400db04fd2bcb625 Mon Sep 17 00:00:00 2001 From: csmig Date: Mon, 10 Jun 2024 16:19:04 -0400 Subject: [PATCH 4/5] escape metadata and names with encodeURIComponent --- lib/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/api.js b/lib/api.js index 91bed57..99a611b 100644 --- a/lib/api.js +++ b/lib/api.js @@ -134,11 +134,11 @@ export async function getCollectionAssets({collectionId, targets}) { const names = targets.filter(t => !t.metadata.cklHostName).map(t => t.name) const cklMetadatas = targets.filter(t => t.metadata.cklHostName).map(t => (({cklHostName, cklWebDbSite, cklWebDbInstance}) => ({cklHostName, cklWebDbSite, cklWebDbInstance}))(t.metadata)) - namedEndpoints = names.map(name => `${options.api}/assets?collectionId=${collectionId}&name=${name}&projection=stigs`) + namedEndpoints = names.map(name => `${options.api}/assets?collectionId=${collectionId}&name=${encodeURIComponent(name)}&projection=stigs`) metadataEndpoints = cklMetadatas.map(md => { let metadataParams = [] for (const metadataKey of Object.keys(md).filter(k=>md[k])) { - metadataParams.push(`metadata=${metadataKey}%3A${md[metadataKey]}`) + metadataParams.push(`metadata=${metadataKey}%3A${encodeURIComponent(md[metadataKey])}`) } return `${options.api}/assets?collectionId=${collectionId}&${metadataParams.join('&')}&projection=stigs` }) From 866c882ba641c3aed8cfa2abd55e7c6020def13b Mon Sep 17 00:00:00 2001 From: csmig Date: Mon, 10 Jun 2024 18:59:50 -0400 Subject: [PATCH 5/5] refactor to use shorthand properties --- lib/cargo.js | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/cargo.js b/lib/cargo.js index 3c71fec..6c51eb1 100644 --- a/lib/cargo.js +++ b/lib/cargo.js @@ -13,7 +13,7 @@ async function writer ( taskAsset ) { const component = 'writer' try { logger.debug({ - component: component, + component, message: `${taskAsset.assetProps.name} started` }) @@ -50,7 +50,7 @@ async function writer ( taskAsset ) { if (reviews.length > 0) { const r = await api.postReviews(options.collectionId, taskAsset.assetProps.assetId, reviews) logger.info({ - component: component, + component, message: `posted reviews`, asset: { name: taskAsset.assetProps.name, id: taskAsset.assetProps.assetId }, rejected: r.rejected, @@ -59,7 +59,7 @@ async function writer ( taskAsset ) { } else { logger.warn({ - component: component, + component, message: `no reviews to post`, asset: { name: taskAsset.assetProps.name, id: taskAsset.assetProps.assetId }, }) @@ -94,31 +94,30 @@ async function writer ( taskAsset ) { } async function resultsHandler( parsedResults, cb ) { - const component = 'batch' try { batchId++ const isModeScan = options.mode === 'scan' - logger.info({component: component, message: `batch started`, batchId: batchId, size: parsedResults.length}) + logger.info({component, message: `batch started`, batchId, size: parsedResults.length}) const apiAssets = await api.getCollectionAssets( { collectionId: options.collectionId, targets: parsedResults.map(pr=>pr.target) } ) - logger.info({component: component, message: `asset data received`, batchId: batchId, size: apiAssets.length}) + logger.info({component, message: `asset data received`, batchId, size: apiAssets.length}) const apiStigs = await api.getInstalledStigs() - logger.info({component: component, message: `stig data received`, batchId: batchId, size: apiStigs.length}) - const tasks = new TaskObject ({ parsedResults, apiAssets, apiStigs, options:options }) + logger.info({component, message: `stig data received`, batchId, size: apiStigs.length}) + const tasks = new TaskObject ({ parsedResults, apiAssets, apiStigs, options }) isModeScan && tasks.errors.length && addToHistory(tasks.errors.map(e => e.sourceRef)) for ( const taskAsset of tasks.taskAssets.values() ) { const success = await writer( taskAsset ) isModeScan && success && addToHistory(taskAsset.sourceRefs) } - logger.info({component, message: 'batch ended with success', batchId: batchId}) + logger.info({component, message: 'batch ended', batchId}) cb() } catch (e) { - logger.error({component, message: 'batch ended with error', error: serializeError(e), batchId: batchId}) + logger.error({component, message: 'batch ended', error: serializeError(e), batchId}) cb( e, undefined) } }