diff --git a/README.md b/README.md index 906eee1..81c8f75 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ import { ```js // adapt import to the targeted JS runtime -import { baseline, bench, group, run } from 'tatami-ng' +import { baseline, bench, group, run, bmf } from 'tatami-ng' bench('noop', () => {}) bench('noop2', () => {}) @@ -130,7 +130,8 @@ group({ name: 'group2', summary: false }, () => { await run({ units: false, // print units cheatsheet (default: false) silent: false, // enable/disable stdout output (default: false) - json: false, // enable/disable json output or set json output format (default: false) + json: false, // enable/disable json output or set json output indentation (default: false) + reporter: bmf // custom reporter function (default: undefined) file: 'results.json', // write json output to file (default: undefined) colors: true, // enable/disable colors (default: true) now: () => 1e6 * performance.now.bind(performance)(), // custom nanoseconds timestamp function to replace default one (default: undefined) diff --git a/cli.js b/cli.js index 3200423..2615ca5 100755 --- a/cli.js +++ b/cli.js @@ -62,7 +62,7 @@ const { }, silent: { listGroup: 'Output options', - description: 'No stdout output', + description: 'No standard output', type: 'boolean', }, json: { diff --git a/src/benchmark.js b/src/benchmark.js index e302603..05835a8 100644 --- a/src/benchmark.js +++ b/src/benchmark.js @@ -2,7 +2,6 @@ import { defaultSamples, defaultTime, emptyFunction, - jsonOutputFormat, tatamiNgGroup, } from './constants.js' import { @@ -17,7 +16,6 @@ import { writeFileSync, } from './lib.js' import { logger } from './logger.js' -import { bmf } from './reporter/json/index.js' import { benchmarkError, benchmarkReport, @@ -279,7 +277,8 @@ const executeBenchmarks = async ( * @param {Object} [opts={}] options object * @param {Boolean} [opts.units=false] print units cheatsheet * @param {Boolean} [opts.silent=false] enable/disable stdout output - * @param {Boolean|Number|'bmf'} [opts.json=false] enable/disable json output or set json output format + * @param {Boolean|Number} [opts.json=false] enable/disable json output or set json output indentation + * @param {Function} [opts.reporter=undefined] custom reporter function * @param {String} [opts.file=undefined] write json output to file * @param {NowType} [opts.now=undefined] custom nanoseconds timestamp function to replace default one * @param {Boolean} [opts.colors=true] enable/disable colors @@ -314,20 +313,14 @@ export async function run(opts = {}) { if ( opts.json != null && 'number' !== typeof opts.json && - 'boolean' !== typeof opts.json && - 'string' !== typeof opts.json + 'boolean' !== typeof opts.json ) throw new TypeError( - `expected number or boolean or string as 'json' option, got ${opts.json.constructor.name}` + `expected number or boolean as 'json' option, got ${opts.json.constructor.name}` ) - if ( - 'string' === typeof opts.json && - !Object.values(jsonOutputFormat).includes(opts.json) - ) + if (opts.reporter != null && !isFunction(opts.reporter)) throw new TypeError( - `expected one of ${Object.values(jsonOutputFormat).join( - ', ' - )} as 'json' option, got ${opts.json}` + `expected function as 'reporter' option, got ${opts.reporter.constructor.name}` ) if (opts.file != null && 'string' !== typeof opts.file) throw new TypeError( @@ -335,7 +328,7 @@ export async function run(opts = {}) { ) if ('string' === typeof opts.file && opts.file.trim().length === 0) throw new TypeError(`expected non-empty string as 'file' option`) - if (opts.now != null && Function !== opts.now.constructor) + if (opts.now != null && !isFunction(opts.now)) throw new TypeError( `expected function as 'now' option, got ${opts.now.constructor.name}` ) @@ -347,7 +340,7 @@ export async function run(opts = {}) { const log = opts.silent === true ? emptyFunction : logger - const report = { + let report = { benchmarks, cpu: `${cpuModel}`, runtime: `${runtime} ${version} (${os})`, @@ -386,20 +379,15 @@ export async function run(opts = {}) { : groupOpts.after() } + report = isFunction(opts.reporter) ? opts.reporter(report) : report + if (!opts.json && opts.units) log(units(opts)) if (opts.json || opts.file) { - let jsonReport - switch (opts.json) { - case jsonOutputFormat.bmf: - jsonReport = JSON.stringify(bmf(report)) - break - default: - jsonReport = JSON.stringify( - report, - undefined, - 'number' !== typeof opts.json ? 0 : opts.json - ) - } + const jsonReport = JSON.stringify( + report, + undefined, + 'number' !== typeof opts.json ? 0 : opts.json + ) if (opts.json) log(jsonReport) if (opts.file) writeFileSync(opts.file, jsonReport) } diff --git a/src/constants.js b/src/constants.js index bf1a90b..949fe26 100644 --- a/src/constants.js +++ b/src/constants.js @@ -1052,8 +1052,4 @@ export const defaultTime = 1e9 // ns export const defaultWarmupRuns = 12 -export const jsonOutputFormat = { - bmf: 'bmf', -} - export const highRelativeMarginOfError = 8 diff --git a/src/index.d.ts b/src/index.d.ts index 67eb33d..8559aad 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -45,7 +45,7 @@ export function baseline( options?: BenchmarkOptions ): void -export function run(options?: { +export function run(options?: { now?: () => number silent?: boolean colors?: boolean @@ -56,10 +56,11 @@ export function run(options?: { throughput?: boolean latencyMinMax?: boolean latencyPercentiles?: boolean - json?: number | boolean | 'bmf' + json?: number | boolean file?: string + reporter?: (report: Report) => T // custom reporter units?: boolean -}): Promise +}): Promise export type Stats = { min: number @@ -71,9 +72,10 @@ export type Stats = { avg: number // average vr: number // variance sd: number // standard deviation + moe: number // margin of error rmoe: number // relative margin of error - aad: number // average time absolute deviation - mad: number // median time absolute deviation + aad: number // average absolute deviation + mad: number // median absolute deviation } export type BenchmarkReport = { diff --git a/src/lib.js b/src/lib.js index 3fc805d..f1c4995 100644 --- a/src/lib.js +++ b/src/lib.js @@ -161,7 +161,7 @@ export const checkBenchmarkArgs = (fn, opts = {}) => { throw new TypeError( `expected number or boolean as 'warmup' option, got ${opts.warmup.constructor.name}` ) - if (opts.now != null && Function !== opts.now.constructor) + if (opts.now != null && !isFunction(opts.now)) throw new TypeError( `expected function as 'now' option, got ${opts.now.constructor.name}` ) @@ -311,6 +311,7 @@ const buildMeasurementStats = latencySamples => { avg: latencyAvg, vr: latencyVr, sd: latencySd, + moe: latencyMoe, rmoe: latencyRmoe, aad: absoluteDeviation(latencySamples, average), mad: absoluteDeviation(latencySamples, medianSorted), @@ -325,6 +326,7 @@ const buildMeasurementStats = latencySamples => { avg: throughputAvg, vr: throughputVr, sd: throughputSd, + moe: throughputMoe, rmoe: throughputRmoe, aad: absoluteDeviation(throughputSamples, average), mad: absoluteDeviation(throughputSamples, medianSorted), diff --git a/src/reporter/terminal/table.js b/src/reporter/terminal/table.js index fee81cb..20adb60 100644 --- a/src/reporter/terminal/table.js +++ b/src/reporter/terminal/table.js @@ -71,7 +71,7 @@ export function header( latencyPercentiles = true, } ) { - return `${dim(colors, white(colors, `cpu: ${report.cpu}`))}\n${dim(colors, white(colors, `runtime: ${report.runtime}`))}\n\n${'benchmark'.padEnd(size, ' ')}${!latency ? '' : 'time/iter'.padStart(20, ' ')}${!throughput ? '' : 'iters/s'.padStart(20, ' ')}${!latencyMinMax ? '' : '(min … max)'.padStart(24, ' ')}${ + return `${dim(colors, white(colors, `cpu: ${report.cpu}`))}\n${dim(colors, white(colors, `runtime: ${report.runtime}`))}\n\n${'benchmark'.padEnd(size, ' ')}${!latency ? '' : 'time/iter'.padStart(20, ' ')}${!throughput ? '' : 'iters/s'.padStart(20, ' ')}${!latencyMinMax ? '' : '(min … max time/iter)'.padStart(24, ' ')}${ !latencyPercentiles ? '' : ` ${'p50/median'.padStart(20, ' ')} ${'p75'.padStart(9, ' ')} ${'p99'.padStart(9, ' ')} ${'p995'.padStart(9, ' ')}` @@ -144,7 +144,10 @@ export function benchmarkReport( }` } -export function warning(benchmarks, { colors = true }) { +export function warning( + benchmarks, + { latency = true, throughput = true, colors = true } +) { if (benchmarks.some(benchmark => benchmark.error != null)) { throw new Error('Cannot display warning on benchmarks with error') } @@ -155,22 +158,25 @@ export function warning(benchmarks, { colors = true }) { `${bold(colors, yellow(colors, 'Warning'))}: ${bold(colors, cyan(colors, benchmark.name))} has a sample size below statistical significance: ${red(colors, benchmark.samples)}` ) } - if (benchmark.stats.latency.rmoe > highRelativeMarginOfError) { + if (latency && benchmark.stats.latency.rmoe > highRelativeMarginOfError) { warnings.push( `${bold(colors, yellow(colors, 'Warning'))}: ${bold(colors, cyan(colors, benchmark.name))} has a high latency relative margin of error: ${red(colors, errorMargin(benchmark.stats.latency.rmoe))}` ) } - if (benchmark.stats.throughput.rmoe > highRelativeMarginOfError) { + if ( + throughput && + benchmark.stats.throughput.rmoe > highRelativeMarginOfError + ) { warnings.push( `${bold(colors, yellow(colors, 'Warning'))}: ${bold(colors, cyan(colors, benchmark.name))} has a high latency throughput margin of error: ${red(colors, errorMargin(benchmark.stats.throughput.rmoe))}` ) } - if (benchmark.stats.latency.mad > 0) { + if (latency && benchmark.stats.latency.mad > 0) { warnings.push( `${bold(colors, yellow(colors, 'Warning'))}: ${bold(colors, cyan(colors, benchmark.name))} has a non zero latency median absolute deviation: ${red(colors, duration(benchmark.stats.latency.mad))}` ) } - if (benchmark.stats.throughput.mad > 0) { + if (throughput && benchmark.stats.throughput.mad > 0) { warnings.push( `${bold(colors, yellow(colors, 'Warning'))}: ${bold(colors, cyan(colors, benchmark.name))} has a non zero throughput median absolute deviation: ${red(colors, duration(benchmark.stats.throughput.mad))}` ) diff --git a/src/runtime.js b/src/runtime.js index 2436b16..6fc8af3 100644 --- a/src/runtime.js +++ b/src/runtime.js @@ -3,8 +3,8 @@ import { runtimes } from './constants.js' const isBun = !!globalThis.Bun || !!globalThis.process?.versions?.bun const isDeno = !!globalThis.Deno const isNode = globalThis.process?.release?.name === 'node' -const isHermes = !!globalThis.HermesInternal -const isWorkerd = globalThis.navigator?.userAgent === 'Cloudflare-Workers' +// const isHermes = !!globalThis.HermesInternal +// const isWorkerd = globalThis.navigator?.userAgent === 'Cloudflare-Workers' const isBrowser = !!globalThis.navigator export const runtime = (() => { diff --git a/tests/formatting.js b/tests/formatting.js index ba3f173..c4cc154 100644 --- a/tests/formatting.js +++ b/tests/formatting.js @@ -103,7 +103,8 @@ group({ summary: true }, () => { await run({ units: true, // print units cheatsheet (default: false) latency: true, // enable/disable time/iter column (default: true) - json: false, // enable/disable json output or set json output format (default: false) + throughput: true, // enable/disable iters/s column (default: true) + json: false, // enable/disable json output or set json output indentation (default: false) colors: true, // enable/disable colors (default: true) latencyMinMax: true, // enable/disable latency (min...max) column (default: true) latencyPercentiles: true, // enable/disable latency percentile columns (default: true) diff --git a/tests/test.js b/tests/test.js index 9be7de2..68f1d4b 100644 --- a/tests/test.js +++ b/tests/test.js @@ -49,7 +49,7 @@ group({ name: 'group2', summary: false }, () => { const report = await run({ latency: true, // enable/disable time/iter column (default: true) throughput: true, // enable/disable iters/s column (default: true) - json: false, // enable/disable json output or set json output format (default: false) + json: false, // enable/disable json output or set json output indentation (default: false) colors: true, // enable/disable colors (default: true) latencyMinMax: true, // enable/disable latency (min...max) column (default: true) latencyPercentiles: true, // enable/disable latency percentile columns (default: true) @@ -60,7 +60,7 @@ console.log(report) await run({ latency: true, // enable/disable time/iter column (default: true) throughput: true, // enable/disable iters/s column (default: true) - json: false, // enable/disable json output or set json output format (default: false) + json: false, // enable/disable json output or set json output indentation (default: false) colors: true, // enable/disable colors (default: true) latencyMinMax: true, // enable/disable latency (min...max) column (default: true) latencyPercentiles: true, // enable/disable latency percentile columns (default: true) diff --git a/tests/test.ts b/tests/test.ts index 9be7de2..24d70c2 100644 --- a/tests/test.ts +++ b/tests/test.ts @@ -48,8 +48,8 @@ group({ name: 'group2', summary: false }, () => { const report = await run({ latency: true, // enable/disable time/iter column (default: true) - throughput: true, // enable/disable iters/s column (default: true) - json: false, // enable/disable json output or set json output format (default: false) + throughput: false, // enable/disable iters/s column (default: true) + json: false, // enable/disable json output or set json output indentation (default: false) colors: true, // enable/disable colors (default: true) latencyMinMax: true, // enable/disable latency (min...max) column (default: true) latencyPercentiles: true, // enable/disable latency percentile columns (default: true) @@ -60,7 +60,7 @@ console.log(report) await run({ latency: true, // enable/disable time/iter column (default: true) throughput: true, // enable/disable iters/s column (default: true) - json: false, // enable/disable json output or set json output format (default: false) + json: false, // enable/disable json output or set json output indentation (default: false) colors: true, // enable/disable colors (default: true) latencyMinMax: true, // enable/disable latency (min...max) column (default: true) latencyPercentiles: true, // enable/disable latency percentile columns (default: true)