Support parallel and fail fast

README.md

@@ -156,6 +156,20 @@ Is a simple as doing `helm-test`:
 
 You can have helm-test run every time it detects a change in your chart by simply doing `helm-test --watch`
 
+### Running in parallel
+
+You can get significant improvements in performance by using mochas `--parallel` by doing `helm-test --parallel`. Please note that `.only` will not work.
+
+Please also note that this will use `NCPU-1` for threads, if you're also using `istioctl` and `kubeval`, that can spawn a lot of sub processes!
+
+### Failing fast
+
+By default, all tests will run and then report back. You can fail on the first test failure by doing `helm-test --bail`.
+
+### Debugging
+
+Set `export LOG_LEVEL=debug` to see more info about what `helm-test` is doing.
+
 ## License
 
 Copyright (c) 2022 Karl Stoney

bin/helm-test.ts

@@ -7,7 +7,6 @@ import { App } from '../lib/app';
 import { Logger } from '../lib/logger';
 import { IstioCtlResultsParser } from '../lib/resultsParsers/istioctl';
 import { KubeValResultsParser } from '../lib/resultsParsers/kubeval';
-import { TmpFileWriter } from '../lib/resultsParsers/tmpFileWriter';
 
 const version = JSON.parse(
   fs.readFileSync(path.join(__dirname, '../package.json')).toString()
@@ -23,14 +22,18 @@ const kubevalSchemaLocation = process.env.KUBEVAL_SCHEMA_LOCATION;
 logger.info(
   `kubeval enabled: ${kubevalEnabled}, kubevalVersion: ${kubevalVersion}, kubevalSchemaLocation: ${kubevalSchemaLocation}`
 );
-logger.info(`tmp file location: ${TmpFileWriter.LOCATION}`);
 
 const istioctlEnabled = IstioCtlResultsParser.ENABLED;
 logger.info(`istioctl enabled: ${istioctlEnabled}`);
 
 program
   .version(version)
   .option('-w, --watch', 'Watch for file changes and re-run tests')
+  .option('-b, --bail', 'Bail out on the first failure')
+  .option(
+    '-p, --parallel',
+    'Run tests in parallel, be aware that .only is not supported'
+  )
   .option('-h, --helm-binary <location>', 'location of the helm binary')
   .parse(process.argv);
 

lib/app.ts

@@ -15,6 +15,8 @@ export class App {
   public async test(options: {
     helmBinary: string;
     watch: boolean;
+    bail: boolean;
+    parallel: boolean;
   }): Promise<void> {
     const execOptions = { output: true, cwd: process.cwd() };
     const mocha = path.join(__dirname, '../node_modules/.bin/mocha');
@@ -28,8 +30,17 @@ export class App {
       this.logger.info('Watching for file changes enabled.');
       watch = ' --watch --watch-extensions yaml,tpl';
     }
-    const command = `${mocha}${watch} -r should -r ${globals} --recursive tests`;
+    const extraFlags: string[] = [];
+    if (options.bail) {
+      extraFlags.push('--bail');
+    }
+    if (options.parallel) {
+      extraFlags.push('--parallel');
+    }
 
+    const flags = extraFlags.length > 0 ? `${extraFlags.join(' ')} ` : '';
+    const command = `${mocha}${watch} -r should -r ${globals} ${flags}--recursive tests`;
+    console.log(command);
     await this.exec.command(command, execOptions);
   }
 }

lib/helm.ts

@@ -1,11 +1,13 @@
+import * as fs from 'fs';
+import * as os from 'os';
 import * as path from 'path';
+import * as randomstring from 'randomstring';
 import { Exec } from './exec';
 import { Logger } from './logger';
-import { type IResultsParser } from './resultsParsers';
+import type { IPhaseOneParser, IPhaseTwoParser } from './resultsParsers';
 import { HelmResultParser } from './resultsParsers/helm';
 import { IstioCtlResultsParser } from './resultsParsers/istioctl';
 import { KubeValResultsParser } from './resultsParsers/kubeval';
-import { TmpFileWriter } from './resultsParsers/tmpFileWriter';
 
 export class Helm {
   private readonly helmBinary = process.env.HELM_BINARY
@@ -17,18 +19,14 @@ export class Helm {
   private readonly exec: Exec;
 
   private readonly logger: Logger;
-  private readonly phase1: IResultsParser[] = [];
-  private readonly phase2: IResultsParser[] = [];
+  private readonly phase1: IPhaseOneParser[] = [];
+  private readonly phase2: IPhaseTwoParser[] = [];
 
   constructor() {
     this.exec = new Exec();
     this.logger = new Logger({ namespace: 'helm' });
     this.phase1.push(new HelmResultParser());
 
-    if (KubeValResultsParser.ENABLED || IstioCtlResultsParser.ENABLED) {
-      this.phase1.push(new TmpFileWriter());
-    }
-
     if (KubeValResultsParser.ENABLED) {
       this.phase2.push(new KubeValResultsParser());
     }
@@ -50,6 +48,8 @@ export class Helm {
   }
 
   public async go(done?: (err?: Error) => void): Promise<void> {
+    let filename: string | null = null;
+
     try {
       let command = this.command;
       if (this.files.length > 0) {
@@ -64,12 +64,23 @@ export class Helm {
 
       const result = await this.exec.command(command, { throw: true });
 
+      if (IstioCtlResultsParser.ENABLED || KubeValResultsParser.ENABLED) {
+        filename = path.join(
+          os.tmpdir(),
+          randomstring.generate({
+            length: 20,
+            charset: 'alphabetic'
+          })
+        );
+        fs.writeFileSync(filename, result.stdout);
+      }
+
       await Promise.all(
         this.phase1.map(async (parser) => {
           this.logger.debug(
             `running results parser: ${parser.constructor.name}`
           );
-          await parser.parse(result);
+          await parser.parse({ result });
         })
       );
 
@@ -78,9 +89,10 @@ export class Helm {
           this.logger.debug(
             `running results parser: ${parser.constructor.name}`
           );
-          await parser.parse(result);
+          await parser.parse({ result, onDisk: filename as string });
         })
       );
+
       if (done) {
         done();
       }
@@ -89,6 +101,10 @@ export class Helm {
       if (done) {
         done(ex);
       }
+    } finally {
+      if (filename) {
+        fs.unlinkSync(filename);
+      }
     }
   }
 }

lib/resultsParsers/helm.ts

@@ -1,14 +1,14 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import * as async from 'async';
 import * as YAML from 'js-yaml';
-import { type IResultsParser } from '.';
+import type { IPhaseOneParser, PhaseOneOptions } from '.';
 import { Logger } from '../logger';
 
 declare var global: {
   results: { length: number; byType: any; ofType: (type: string) => any[] };
 };
 
-export class HelmResultParser implements IResultsParser {
+export class HelmResultParser implements IPhaseOneParser {
   private logger: Logger;
   constructor() {
     this.logger = new Logger({ namespace: 'helm-parser' });
@@ -24,7 +24,7 @@ export class HelmResultParser implements IResultsParser {
     };
   }
 
-  public async parse(result: { stdout: string }): Promise<void> {
+  public async parse({ result }: PhaseOneOptions): Promise<void> {
     global.results.byType = [];
     global.results.length = 0;
     const removeNonManifests = (manifest: string): boolean => {

lib/resultsParsers/index.ts

@@ -1,3 +1,9 @@
-export interface IResultsParser {
-  parse(result: { stdout: string }): Promise<void>;
+export type PhaseOneOptions = { result: { stdout: string } };
+export interface IPhaseOneParser {
+  parse(options: PhaseOneOptions): Promise<void>;
+}
+
+export type PhaseTwoOptions = PhaseOneOptions & { onDisk: string };
+export interface IPhaseTwoParser {
+  parse(options: PhaseTwoOptions): Promise<void>;
 }

lib/resultsParsers/istioctl.ts

@@ -1,9 +1,8 @@
-import { type IResultsParser } from '.';
+import type { IPhaseOneParser, PhaseTwoOptions } from '.';
 import { Exec } from '../exec';
 import { Logger } from '../logger';
-import { TmpFileWriter } from './tmpFileWriter';
 
-export class IstioCtlResultsParser implements IResultsParser {
+export class IstioCtlResultsParser implements IPhaseOneParser {
   public static readonly ENABLED =
     process.env.HELM_TEST_ISTIOCTL_ENABLED === 'true';
   private logger: Logger;
@@ -20,9 +19,9 @@ export class IstioCtlResultsParser implements IResultsParser {
     this.exec = new Exec();
   }
 
-  public async parse(): Promise<void> {
+  public async parse({ onDisk }: PhaseTwoOptions): Promise<void> {
     this.logger.debug('running istioctl validate');
-    const command = `${this.istioctlBinary} validate -f ${TmpFileWriter.LOCATION}`;
+    const command = `${this.istioctlBinary} validate -f ${onDisk}`;
     await this.exec.command(command, { throw: true });
   }
 }

lib/resultsParsers/kubeval.ts

@@ -1,11 +1,10 @@
 import * as fs from 'fs';
 import * as path from 'path';
-import { type IResultsParser } from '.';
+import type { IPhaseOneParser, PhaseTwoOptions } from '.';
 import { Exec } from '../exec';
 import { Logger } from '../logger';
-import { TmpFileWriter } from './tmpFileWriter';
 
-export class KubeValResultsParser implements IResultsParser {
+export class KubeValResultsParser implements IPhaseOneParser {
   public static readonly ENABLED =
     process.env.HELM_TEST_KUBEVAL_ENABLED === 'true';
   private logger: Logger;
@@ -53,8 +52,8 @@ export class KubeValResultsParser implements IResultsParser {
     this.exec = new Exec();
   }
 
-  public async parse(): Promise<void> {
-    let command = `${this.kubevalBinary} --ignore-missing-schemas --strict -o json --kubernetes-version=${this.kubeVersion} --quiet ${TmpFileWriter.LOCATION}`;
+  public async parse({ onDisk }: PhaseTwoOptions): Promise<void> {
+    let command = `${this.kubevalBinary} --ignore-missing-schemas --strict -o json --kubernetes-version=${this.kubeVersion} --quiet ${onDisk}`;
     if (this.schemaLocation) {
       if (!fs.existsSync(this.schemaLocation)) {
         throw new Error(

package.json

@@ -1,7 +1,7 @@
 {
   "name": "helm-test",
   "description": "A CLI to test Helm charts",
-  "version": "1.1.4",
+  "version": "1.2.0",
   "homepage": "https://github.com/Stono/helm-test",
   "author": {
     "name": "Karl Stoney",
@@ -35,8 +35,9 @@
     "commander": "9.3.0",
     "debug": "4.3.4",
     "js-yaml": "4.1.0",
-    "should": "13.2.3",
-    "mocha": "10.0.0"
+    "mocha": "10.0.0",
+    "randomstring": "1.2.2",
+    "should": "13.2.3"
   },
   "devDependencies": {
     "@types/async": "3.2.13",
@@ -46,6 +47,7 @@
     "@types/js-yaml": "4.0.5",
     "@types/mocha": "9.1.1",
     "@types/node": "17.0.38",
+    "@types/randomstring": "1.1.8",
     "@typescript-eslint/eslint-plugin": "5.27.0",
     "@typescript-eslint/parser": "5.27.0",
     "eslint": "8.16.0",