From 78832f8f8d2ca869ae09f9d726ef4d9a7b1a5493 Mon Sep 17 00:00:00 2001 From: Karl Stoney Date: Thu, 2 Jun 2022 18:15:04 +0100 Subject: [PATCH] Support parallel and fail fast --- README.md | 14 +++++++++++ bin/helm-test.ts | 7 ++++-- lib/app.ts | 13 ++++++++++- lib/helm.ts | 36 +++++++++++++++++++++-------- lib/resultsParsers/helm.ts | 6 ++--- lib/resultsParsers/index.ts | 10 ++++++-- lib/resultsParsers/istioctl.ts | 9 ++++---- lib/resultsParsers/kubeval.ts | 9 ++++---- lib/resultsParsers/tmpFileWriter.ts | 23 ------------------ package.json | 8 ++++--- 10 files changed, 81 insertions(+), 54 deletions(-) delete mode 100644 lib/resultsParsers/tmpFileWriter.ts diff --git a/README.md b/README.md index c397ed2..dafa584 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,20 @@ Is a simple as doing `helm-test`: You can have helm-test run every time it detects a change in your chart by simply doing `helm-test --watch` +### Running in parallel + +You can get significant improvements in performance by using mochas `--parallel` by doing `helm-test --parallel`. Please note that `.only` will not work. + +Please also note that this will use `NCPU-1` for threads, if you're also using `istioctl` and `kubeval`, that can spawn a lot of sub processes! + +### Failing fast + +By default, all tests will run and then report back. You can fail on the first test failure by doing `helm-test --bail`. + +### Debugging + +Set `export LOG_LEVEL=debug` to see more info about what `helm-test` is doing. + ## License Copyright (c) 2022 Karl Stoney diff --git a/bin/helm-test.ts b/bin/helm-test.ts index 82b3863..404bf22 100755 --- a/bin/helm-test.ts +++ b/bin/helm-test.ts @@ -7,7 +7,6 @@ import { App } from '../lib/app'; import { Logger } from '../lib/logger'; import { IstioCtlResultsParser } from '../lib/resultsParsers/istioctl'; import { KubeValResultsParser } from '../lib/resultsParsers/kubeval'; -import { TmpFileWriter } from '../lib/resultsParsers/tmpFileWriter'; const version = JSON.parse( fs.readFileSync(path.join(__dirname, '../package.json')).toString() @@ -23,7 +22,6 @@ const kubevalSchemaLocation = process.env.KUBEVAL_SCHEMA_LOCATION; logger.info( `kubeval enabled: ${kubevalEnabled}, kubevalVersion: ${kubevalVersion}, kubevalSchemaLocation: ${kubevalSchemaLocation}` ); -logger.info(`tmp file location: ${TmpFileWriter.LOCATION}`); const istioctlEnabled = IstioCtlResultsParser.ENABLED; logger.info(`istioctl enabled: ${istioctlEnabled}`); @@ -31,6 +29,11 @@ logger.info(`istioctl enabled: ${istioctlEnabled}`); program .version(version) .option('-w, --watch', 'Watch for file changes and re-run tests') + .option('-b, --bail', 'Bail out on the first failure') + .option( + '-p, --parallel', + 'Run tests in parallel, be aware that .only is not supported' + ) .option('-h, --helm-binary ', 'location of the helm binary') .parse(process.argv); diff --git a/lib/app.ts b/lib/app.ts index 2757be7..4173ef1 100644 --- a/lib/app.ts +++ b/lib/app.ts @@ -15,6 +15,8 @@ export class App { public async test(options: { helmBinary: string; watch: boolean; + bail: boolean; + parallel: boolean; }): Promise { const execOptions = { output: true, cwd: process.cwd() }; const mocha = path.join(__dirname, '../node_modules/.bin/mocha'); @@ -28,8 +30,17 @@ export class App { this.logger.info('Watching for file changes enabled.'); watch = ' --watch --watch-extensions yaml,tpl'; } - const command = `${mocha}${watch} -r should -r ${globals} --recursive tests`; + const extraFlags: string[] = []; + if (options.bail) { + extraFlags.push('--bail'); + } + if (options.parallel) { + extraFlags.push('--parallel'); + } + const flags = extraFlags.length > 0 ? `${extraFlags.join(' ')} ` : ''; + const command = `${mocha}${watch} -r should -r ${globals} ${flags}--recursive tests`; + console.log(command); await this.exec.command(command, execOptions); } } diff --git a/lib/helm.ts b/lib/helm.ts index d4325bb..dfe6252 100644 --- a/lib/helm.ts +++ b/lib/helm.ts @@ -1,11 +1,13 @@ +import * as fs from 'fs'; +import * as os from 'os'; import * as path from 'path'; +import * as randomstring from 'randomstring'; import { Exec } from './exec'; import { Logger } from './logger'; -import { type IResultsParser } from './resultsParsers'; +import type { IPhaseOneParser, IPhaseTwoParser } from './resultsParsers'; import { HelmResultParser } from './resultsParsers/helm'; import { IstioCtlResultsParser } from './resultsParsers/istioctl'; import { KubeValResultsParser } from './resultsParsers/kubeval'; -import { TmpFileWriter } from './resultsParsers/tmpFileWriter'; export class Helm { private readonly helmBinary = process.env.HELM_BINARY @@ -17,18 +19,14 @@ export class Helm { private readonly exec: Exec; private readonly logger: Logger; - private readonly phase1: IResultsParser[] = []; - private readonly phase2: IResultsParser[] = []; + private readonly phase1: IPhaseOneParser[] = []; + private readonly phase2: IPhaseTwoParser[] = []; constructor() { this.exec = new Exec(); this.logger = new Logger({ namespace: 'helm' }); this.phase1.push(new HelmResultParser()); - if (KubeValResultsParser.ENABLED || IstioCtlResultsParser.ENABLED) { - this.phase1.push(new TmpFileWriter()); - } - if (KubeValResultsParser.ENABLED) { this.phase2.push(new KubeValResultsParser()); } @@ -50,6 +48,8 @@ export class Helm { } public async go(done?: (err?: Error) => void): Promise { + let filename: string | null = null; + try { let command = this.command; if (this.files.length > 0) { @@ -64,12 +64,23 @@ export class Helm { const result = await this.exec.command(command, { throw: true }); + if (IstioCtlResultsParser.ENABLED || KubeValResultsParser.ENABLED) { + filename = path.join( + os.tmpdir(), + randomstring.generate({ + length: 20, + charset: 'alphabetic' + }) + ); + fs.writeFileSync(filename, result.stdout); + } + await Promise.all( this.phase1.map(async (parser) => { this.logger.debug( `running results parser: ${parser.constructor.name}` ); - await parser.parse(result); + await parser.parse({ result }); }) ); @@ -78,9 +89,10 @@ export class Helm { this.logger.debug( `running results parser: ${parser.constructor.name}` ); - await parser.parse(result); + await parser.parse({ result, onDisk: filename as string }); }) ); + if (done) { done(); } @@ -89,6 +101,10 @@ export class Helm { if (done) { done(ex); } + } finally { + if (filename) { + fs.unlinkSync(filename); + } } } } diff --git a/lib/resultsParsers/helm.ts b/lib/resultsParsers/helm.ts index 1bfb4b9..88fb104 100644 --- a/lib/resultsParsers/helm.ts +++ b/lib/resultsParsers/helm.ts @@ -1,14 +1,14 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import * as async from 'async'; import * as YAML from 'js-yaml'; -import { type IResultsParser } from '.'; +import type { IPhaseOneParser, PhaseOneOptions } from '.'; import { Logger } from '../logger'; declare var global: { results: { length: number; byType: any; ofType: (type: string) => any[] }; }; -export class HelmResultParser implements IResultsParser { +export class HelmResultParser implements IPhaseOneParser { private logger: Logger; constructor() { this.logger = new Logger({ namespace: 'helm-parser' }); @@ -24,7 +24,7 @@ export class HelmResultParser implements IResultsParser { }; } - public async parse(result: { stdout: string }): Promise { + public async parse({ result }: PhaseOneOptions): Promise { global.results.byType = []; global.results.length = 0; const removeNonManifests = (manifest: string): boolean => { diff --git a/lib/resultsParsers/index.ts b/lib/resultsParsers/index.ts index 154a3c5..dd164a9 100644 --- a/lib/resultsParsers/index.ts +++ b/lib/resultsParsers/index.ts @@ -1,3 +1,9 @@ -export interface IResultsParser { - parse(result: { stdout: string }): Promise; +export type PhaseOneOptions = { result: { stdout: string } }; +export interface IPhaseOneParser { + parse(options: PhaseOneOptions): Promise; +} + +export type PhaseTwoOptions = PhaseOneOptions & { onDisk: string }; +export interface IPhaseTwoParser { + parse(options: PhaseTwoOptions): Promise; } diff --git a/lib/resultsParsers/istioctl.ts b/lib/resultsParsers/istioctl.ts index 282d09f..8e84922 100644 --- a/lib/resultsParsers/istioctl.ts +++ b/lib/resultsParsers/istioctl.ts @@ -1,9 +1,8 @@ -import { type IResultsParser } from '.'; +import type { IPhaseOneParser, PhaseTwoOptions } from '.'; import { Exec } from '../exec'; import { Logger } from '../logger'; -import { TmpFileWriter } from './tmpFileWriter'; -export class IstioCtlResultsParser implements IResultsParser { +export class IstioCtlResultsParser implements IPhaseOneParser { public static readonly ENABLED = process.env.HELM_TEST_ISTIOCTL_ENABLED === 'true'; private logger: Logger; @@ -20,9 +19,9 @@ export class IstioCtlResultsParser implements IResultsParser { this.exec = new Exec(); } - public async parse(): Promise { + public async parse({ onDisk }: PhaseTwoOptions): Promise { this.logger.debug('running istioctl validate'); - const command = `${this.istioctlBinary} validate -f ${TmpFileWriter.LOCATION}`; + const command = `${this.istioctlBinary} validate -f ${onDisk}`; await this.exec.command(command, { throw: true }); } } diff --git a/lib/resultsParsers/kubeval.ts b/lib/resultsParsers/kubeval.ts index fb10c22..d7b730c 100644 --- a/lib/resultsParsers/kubeval.ts +++ b/lib/resultsParsers/kubeval.ts @@ -1,11 +1,10 @@ import * as fs from 'fs'; import * as path from 'path'; -import { type IResultsParser } from '.'; +import type { IPhaseOneParser, PhaseTwoOptions } from '.'; import { Exec } from '../exec'; import { Logger } from '../logger'; -import { TmpFileWriter } from './tmpFileWriter'; -export class KubeValResultsParser implements IResultsParser { +export class KubeValResultsParser implements IPhaseOneParser { public static readonly ENABLED = process.env.HELM_TEST_KUBEVAL_ENABLED === 'true'; private logger: Logger; @@ -53,8 +52,8 @@ export class KubeValResultsParser implements IResultsParser { this.exec = new Exec(); } - public async parse(): Promise { - let command = `${this.kubevalBinary} --ignore-missing-schemas --strict -o json --kubernetes-version=${this.kubeVersion} --quiet ${TmpFileWriter.LOCATION}`; + public async parse({ onDisk }: PhaseTwoOptions): Promise { + let command = `${this.kubevalBinary} --ignore-missing-schemas --strict -o json --kubernetes-version=${this.kubeVersion} --quiet ${onDisk}`; if (this.schemaLocation) { if (!fs.existsSync(this.schemaLocation)) { throw new Error( diff --git a/lib/resultsParsers/tmpFileWriter.ts b/lib/resultsParsers/tmpFileWriter.ts deleted file mode 100644 index 217ba63..0000000 --- a/lib/resultsParsers/tmpFileWriter.ts +++ /dev/null @@ -1,23 +0,0 @@ -import * as fs from 'fs'; -import * as os from 'os'; -import * as path from 'path'; -import { type IResultsParser } from '.'; -import { Logger } from '../logger'; - -export class TmpFileWriter implements IResultsParser { - public static readonly LOCATION: string = path.join( - os.tmpdir(), - 'helm-test-manifests' - ); - private logger: Logger; - constructor() { - this.logger = new Logger({ namespace: 'tmp-file' }); - } - - public async parse(result: { stdout: string }): Promise { - this.logger.debug( - `writing manifests to temp file: ${TmpFileWriter.LOCATION}` - ); - fs.writeFileSync(TmpFileWriter.LOCATION, result.stdout); - } -} diff --git a/package.json b/package.json index bfd1a6d..4255b93 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "helm-test", "description": "A CLI to test Helm charts", - "version": "1.1.4", + "version": "1.2.0", "homepage": "https://github.com/Stono/helm-test", "author": { "name": "Karl Stoney", @@ -35,8 +35,9 @@ "commander": "9.3.0", "debug": "4.3.4", "js-yaml": "4.1.0", - "should": "13.2.3", - "mocha": "10.0.0" + "mocha": "10.0.0", + "randomstring": "1.2.2", + "should": "13.2.3" }, "devDependencies": { "@types/async": "3.2.13", @@ -46,6 +47,7 @@ "@types/js-yaml": "4.0.5", "@types/mocha": "9.1.1", "@types/node": "17.0.38", + "@types/randomstring": "1.1.8", "@typescript-eslint/eslint-plugin": "5.27.0", "@typescript-eslint/parser": "5.27.0", "eslint": "8.16.0",