From 25ac40973a9b769d9052e1da19039a3c11662c76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Benoit?= Date: Wed, 2 Oct 2024 23:17:31 +0200 Subject: [PATCH] refactor: major restructuration to ease pluggable features MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jérôme Benoit --- src/benchmark.js | 50 ++++++----- src/index.d.ts | 6 +- src/index.js | 3 +- src/lib.js | 121 +++------------------------ src/reporter/json/bmf.js | 24 ++++++ src/reporter/json/index.js | 1 + src/reporter/{ => terminal}/clr.js | 0 src/reporter/{ => terminal}/fmt.js | 0 src/reporter/terminal/index.js | 15 ++++ src/reporter/{ => terminal}/table.js | 63 +++++++------- src/stats-utils.js | 69 +++++++++++++++ src/utils.js | 11 +++ 12 files changed, 203 insertions(+), 160 deletions(-) create mode 100644 src/reporter/json/bmf.js create mode 100644 src/reporter/json/index.js rename src/reporter/{ => terminal}/clr.js (100%) rename src/reporter/{ => terminal}/fmt.js (100%) create mode 100644 src/reporter/terminal/index.js rename src/reporter/{ => terminal}/table.js (65%) create mode 100644 src/stats-utils.js create mode 100644 src/utils.js diff --git a/src/benchmark.js b/src/benchmark.js index 4e1525f..00e5cfe 100644 --- a/src/benchmark.js +++ b/src/benchmark.js @@ -10,20 +10,31 @@ import { AsyncFunction, checkBenchmarkArgs, colors, - convertReportToBmf, cpuModel, gc, - isObject, measure, overrideBenchmarkOptions, version, writeFileSync, } from './lib.js' import { logger } from './logger.js' -import * as clr from './reporter/clr.js' -import * as table from './reporter/table.js' +import { bmf } from './reporter/json/index.js' +import { + benchmarkError, + benchmarkReport, + bold, + br, + dim, + header, + size, + summary, + units, + warning, + white, +} from './reporter/terminal/index.js' import { runtime } from './runtime.js' import { now } from './time.js' +import { isObject } from './utils.js' let groupName = null const groups = new Map() @@ -247,18 +258,18 @@ const executeBenchmarks = async ( after: benchmark.after, }) if (!opts.json) - logFn(table.benchmark(benchmark.name, benchmark.stats, opts)) + logFn(benchmarkReport(benchmark.name, benchmark.stats, opts)) } catch (err) { benchmark.error = err if (!opts.json) - logFn(table.benchmarkError(benchmark.name, benchmark.error, opts)) + logFn(benchmarkError(benchmark.name, benchmark.error, opts)) } } // biome-ignore lint/style/noParameterAssign: benchmarks = benchmarks.filter(benchmark => benchmark.error == null) - if (!opts.json && table.warning(benchmarks, opts)) { + if (!opts.json && warning(benchmarks, opts)) { logFn('') - logFn(table.warning(benchmarks, opts)) + logFn(warning(benchmarks, opts)) } if ( (Object.keys(groupOpts).length === 0 || groupOpts.summary === true) && @@ -266,7 +277,7 @@ const executeBenchmarks = async ( benchmarks.length > 1 ) { logFn('') - logFn(table.summary(benchmarks, opts)) + logFn(summary(benchmarks, opts)) } return once } @@ -341,7 +352,7 @@ export async function run(opts = {}) { opts.silent = opts.silent ?? false opts.units = opts.units ?? false opts.colors = opts.colors ?? colors - opts.size = table.size(benchmarks.map(benchmark => benchmark.name)) + opts.size = size(benchmarks.map(benchmark => benchmark.name)) const log = opts.silent === true ? emptyFunction : logger @@ -352,14 +363,12 @@ export async function run(opts = {}) { } if (!opts.json && benchmarks.length > 0) { - log(clr.dim(opts.colors, clr.white(opts.colors, `cpu: ${report.cpu}`))) - log( - clr.dim(opts.colors, clr.white(opts.colors, `runtime: ${report.runtime}`)) - ) + log(dim(opts.colors, white(opts.colors, `cpu: ${report.cpu}`))) + log(dim(opts.colors, white(opts.colors, `runtime: ${report.runtime}`))) log('') - log(table.header(opts)) - log(table.br(opts)) + log(header(opts)) + log(br(opts)) } let once = await executeBenchmarks( @@ -371,10 +380,9 @@ export async function run(opts = {}) { for (const [group, groupOpts] of groups) { if (!opts.json) { if (once) log('') - if (!group.startsWith(tatamiNgGroup)) - log(`• ${clr.bold(opts.colors, group)}`) + if (!group.startsWith(tatamiNgGroup)) log(`• ${bold(opts.colors, group)}`) if (once || !group.startsWith(tatamiNgGroup)) - log(clr.dim(opts.colors, clr.white(opts.colors, table.br(opts)))) + log(dim(opts.colors, white(opts.colors, br(opts)))) } AsyncFunction === groupOpts.before.constructor @@ -393,12 +401,12 @@ export async function run(opts = {}) { : groupOpts.after() } - if (!opts.json && opts.units) log(table.units(opts)) + if (!opts.json && opts.units) log(units(opts)) if (opts.json || opts.file) { let jsonReport switch (opts.json) { case jsonOutputFormat.bmf: - jsonReport = JSON.stringify(convertReportToBmf(report)) + jsonReport = JSON.stringify(bmf(report)) break default: jsonReport = JSON.stringify( diff --git a/src/index.d.ts b/src/index.d.ts index 59e0497..cf27318 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -62,7 +62,7 @@ export function run(options?: { units?: boolean }): Promise -export interface Report { +export type BenchmarkReport = { cpu: string runtime: string @@ -97,5 +97,7 @@ export interface Report { mad: number // median time absolute deviation ss: boolean // statistical significance } - }[] + } } + +export type Report = BenchmarkReport[] diff --git a/src/index.js b/src/index.js index 2d47fea..6bc4db4 100644 --- a/src/index.js +++ b/src/index.js @@ -1 +1,2 @@ -export { group, bench, baseline, run } from './benchmark.js' +export { baseline, bench, group, run } from './benchmark.js' +export { bmf } from './reporter/json/index.js' diff --git a/src/lib.js b/src/lib.js index 1f24e06..a5f3771 100644 --- a/src/lib.js +++ b/src/lib.js @@ -1,3 +1,6 @@ +import { spawnSync as nodeSpawnSync } from 'node:child_process' +import { setFlagsFromString } from 'node:v8' +import { runInNewContext } from 'node:vm' import { defaultSamples, defaultTime, @@ -7,11 +10,15 @@ import { tTable, } from './constants.js' import { runtime } from './runtime.js' +import { + absoluteDeviation, + average, + medianSorted, + quantileSorted, + variance, +} from './stats-utils.js' import { now } from './time.js' - -import { spawnSync as nodeSpawnSync } from 'node:child_process' -import { setFlagsFromString } from 'node:v8' -import { runInNewContext } from 'node:vm' +import { checkDividend, isObject } from './utils.js' export const AsyncFunction = (async () => {}).constructor @@ -133,28 +140,6 @@ export const gc = (() => { }[runtime]() })() -export const convertReportToBmf = report => { - return report.benchmarks - .map(({ name, stats }) => { - const throughputSd = ratioStandardDeviation(1e9, 0, stats.avg, stats.sd) - return { - [name]: { - latency: { - value: stats?.avg, - lower_value: stats?.avg - stats?.sd, - upper_value: stats?.avg + stats?.sd, - }, - throughput: { - value: stats?.iters, - lower_value: stats?.iters - throughputSd, - upper_value: stats?.iters + throughputSd, - }, - }, - } - }) - .reduce((obj, item) => Object.assign(obj, item), {}) -} - export const checkBenchmarkArgs = (fn, opts = {}) => { if (![Function, AsyncFunction].includes(fn.constructor)) throw new TypeError(`expected function, got ${fn.constructor.name}`) @@ -203,18 +188,6 @@ export const overrideBenchmarkOptions = (benchmark, opts) => { benchmark.warmup = opts.warmup ?? benchmark.warmup } -export const isObject = value => { - return Object.prototype.toString.call(value).slice(8, -1) === 'Object' -} - -export const checkDividend = n => { - if (n == null) throw new TypeError(`Invalid dividend: ${n}`) - if ('number' !== typeof n) - throw new TypeError(`expected number, got ${n.constructor.name}`) - if (n === 0 || Number.isNaN(n)) throw new RangeError(`Invalid dividend: ${n}`) - return n -} - /** * Measure a function runtime. * @@ -297,68 +270,10 @@ export async function measure(fn, opts = {}) { ? await benchmark(fn, opts.before, opts.after, opts.now) : benchmark(fn, opts.before, opts.after, opts.now) - return buildStats(samples) -} - -const variance = (samples, avg = average(samples)) => { - return ( - samples.reduce((a, b) => a + (b - avg) ** 2, 0) / - checkDividend(samples.length - 1) // Bessel's correction - ) -} - -const quantileSorted = (samples, q) => { - if (!Array.isArray(samples)) { - throw new TypeError(`expected array, got ${samples.constructor.name}`) - } - if (samples.length === 0) { - throw new Error('expected non-empty array, got empty array') - } - if (q < 0 || q > 1) { - throw new Error('q must be between 0 and 1') - } - if (q === 0) { - return samples[0] - } - if (q === 1) { - return samples[samples.length - 1] - } - const base = (samples.length - 1) * q - const baseIndex = Math.floor(base) - if (samples[baseIndex + 1] != null) { - return ( - samples[baseIndex] + - (base - baseIndex) * (samples[baseIndex + 1] - samples[baseIndex]) - ) - } - return samples[baseIndex] -} - -const medianSorted = samples => quantileSorted(samples, 0.5) - -const average = samples => { - if (!Array.isArray(samples)) { - throw new TypeError(`expected array, got ${samples.constructor.name}`) - } - if (samples.length === 0) { - throw new Error('expected non-empty array, got empty array') - } - - return samples.reduce((a, b) => a + b, 0) / samples.length + return buildMeasurementStats(samples) } -const absoluteDeviation = (samples, aggFn) => { - const value = aggFn(samples) - const absoluteDeviations = [] - - for (const sample of samples) { - absoluteDeviations.push(Math.abs(sample - value)) - } - - return aggFn(absoluteDeviations) -} - -const buildStats = samples => { +const buildMeasurementStats = samples => { if (!Array.isArray(samples)) throw new TypeError(`expected array, got ${samples.constructor.name}`) if (samples.length === 0) @@ -394,13 +309,3 @@ const buildStats = samples => { ss: samples.length >= minimumSamples, } } - -// https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulae -export const ratioStandardDeviation = (avgA, sdA, avgB, sdB) => { - return ( - (avgA / checkDividend(avgB)) * - Math.sqrt( - (sdA / checkDividend(avgA)) ** 2 + (sdB / checkDividend(avgB)) ** 2 - ) - ) -} diff --git a/src/reporter/json/bmf.js b/src/reporter/json/bmf.js new file mode 100644 index 0000000..b03eac7 --- /dev/null +++ b/src/reporter/json/bmf.js @@ -0,0 +1,24 @@ +import { ratioStandardDeviation } from '../../stats-utils.js' + +export const bmf = report => { + return report.benchmarks + .filter(benchmark => benchmark.error == null) + .map(({ name, stats }) => { + const throughputSd = ratioStandardDeviation(1e9, 0, stats?.avg, stats?.sd) + return { + [name]: { + latency: { + value: stats?.avg, + lower_value: stats?.avg - stats?.sd, + upper_value: stats?.avg + stats?.sd, + }, + throughput: { + value: stats?.iters, + lower_value: stats?.iters - throughputSd, + upper_value: stats?.iters + throughputSd, + }, + }, + } + }) + .reduce((obj, item) => Object.assign(obj, item), {}) +} diff --git a/src/reporter/json/index.js b/src/reporter/json/index.js new file mode 100644 index 0000000..4ae2ebd --- /dev/null +++ b/src/reporter/json/index.js @@ -0,0 +1 @@ +export { bmf } from './bmf.js' diff --git a/src/reporter/clr.js b/src/reporter/terminal/clr.js similarity index 100% rename from src/reporter/clr.js rename to src/reporter/terminal/clr.js diff --git a/src/reporter/fmt.js b/src/reporter/terminal/fmt.js similarity index 100% rename from src/reporter/fmt.js rename to src/reporter/terminal/fmt.js diff --git a/src/reporter/terminal/index.js b/src/reporter/terminal/index.js new file mode 100644 index 0000000..ccb544b --- /dev/null +++ b/src/reporter/terminal/index.js @@ -0,0 +1,15 @@ +export { + bold, + dim, + white, +} from './clr.js' +export { + benchmarkError, + benchmarkReport, + br, + header, + size, + summary, + units, + warning, +} from './table.js' diff --git a/src/reporter/table.js b/src/reporter/terminal/table.js similarity index 65% rename from src/reporter/table.js rename to src/reporter/terminal/table.js index ea796c0..a73db69 100644 --- a/src/reporter/table.js +++ b/src/reporter/terminal/table.js @@ -2,9 +2,21 @@ import { highRelativeMarginOfError, tTable, tatamiNgGroup, -} from '../constants.js' -import { checkDividend, ratioStandardDeviation } from '../lib.js' -import * as clr from './clr.js' +} from '../../constants.js' +import { ratioStandardDeviation } from '../../stats-utils.js' +import { checkDividend } from '../../utils.js' +import { + blue, + bold, + cyan, + dim, + gray, + green, + magenta, + red, + white, + yellow, +} from './clr.js' import { duration, errorMargin, itersPerSecond, speedRatio } from './fmt.js' export function size(names) { @@ -28,15 +40,15 @@ export function br({ } export function benchmarkError(name, error, { size, colors = true }) { - return `${name.padEnd(size, ' ')}${clr.red(colors, 'error')}: ${ + return `${name.padEnd(size, ' ')}${red(colors, 'error')}: ${ error.message - }${error.stack ? `\n${clr.gray(colors, error.stack)}` : ''}` + }${error.stack ? `\n${gray(colors, error.stack)}` : ''}` } export function units({ colors = true } = {}) { - return clr.dim( + return dim( colors, - clr.white( + white( colors, ` 1 ps = 1 picosecond = 1e-12s @@ -67,7 +79,7 @@ export function header({ }` } -export function benchmark( +export function benchmarkReport( name, stats, { @@ -83,28 +95,25 @@ export function benchmark( return `${name.padEnd(size, ' ')}${ !avg ? '' - : `${clr.yellow(colors, duration(stats.avg))}`.padStart( - 14 + 10 * colors, - ' ' - ) + : `${yellow(colors, duration(stats.avg))}`.padStart(14 + 10 * colors, ' ') }${ !iters ? '' - : `${clr.yellow(colors, itersPerSecond(stats.iters))}`.padStart( + : `${yellow(colors, itersPerSecond(stats.iters))}`.padStart( 14 + 10 * colors, ' ' ) }${ !rmoe ? '' - : `± ${clr[stats.rmoe > highRelativeMarginOfError ? 'red' : 'blue'](colors, errorMargin(stats.rmoe))}`.padStart( + : `± ${(stats.rmoe > highRelativeMarginOfError ? red : blue)(colors, errorMargin(stats.rmoe))}`.padStart( 14 + 10 * colors, ' ' ) }${ !min_max ? '' - : `(${clr.cyan(colors, duration(stats.min))} … ${clr.magenta( + : `(${cyan(colors, duration(stats.min))} … ${magenta( colors, duration(stats.max) )})`.padStart(24 + 2 * 10 * colors, ' ') @@ -113,14 +122,12 @@ export function benchmark( ? '' : ` ${ stats.mad > 0 - ? `${clr.green(colors, duration(stats.p50))} ± ${clr.red(colors, duration(stats.mad))}`.padStart( + ? `${green(colors, duration(stats.p50))} ± ${red(colors, duration(stats.mad))}`.padStart( 20 + 2 * 10 * colors, ' ' ) - : clr - .green(colors, duration(stats.p50)) - .padStart(20 + 10 * colors, ' ') - } ${clr.green(colors, duration(stats.p75)).padStart(9 + 10 * colors, ' ')} ${clr.green(colors, duration(stats.p99)).padStart(9 + 10 * colors, ' ')} ${clr.green(colors, duration(stats.p995)).padStart(9 + 10 * colors, ' ')}` + : green(colors, duration(stats.p50)).padStart(20 + 10 * colors, ' ') + } ${green(colors, duration(stats.p75)).padStart(9 + 10 * colors, ' ')} ${green(colors, duration(stats.p99)).padStart(9 + 10 * colors, ' ')} ${green(colors, duration(stats.p995)).padStart(9 + 10 * colors, ' ')}` }` } @@ -132,17 +139,17 @@ export function warning(benchmarks, { colors = true }) { for (const benchmark of benchmarks) { if (benchmark.stats.ss === false) { warnings.push( - `${clr.bold(colors, clr.yellow(colors, 'Warning'))}: ${clr.bold(colors, clr.cyan(colors, benchmark.name))} has a sample size below statistical significance: ${clr.red(colors, benchmark.samples)}` + `${bold(colors, yellow(colors, 'Warning'))}: ${bold(colors, cyan(colors, benchmark.name))} has a sample size below statistical significance: ${red(colors, benchmark.samples)}` ) } if (benchmark.stats.rmoe > highRelativeMarginOfError) { warnings.push( - `${clr.bold(colors, clr.yellow(colors, 'Warning'))}: ${clr.bold(colors, clr.cyan(colors, benchmark.name))} has a high relative margin of error: ${clr.red(colors, errorMargin(benchmark.stats.rmoe))}` + `${bold(colors, yellow(colors, 'Warning'))}: ${bold(colors, cyan(colors, benchmark.name))} has a high relative margin of error: ${red(colors, errorMargin(benchmark.stats.rmoe))}` ) } if (benchmark.stats.mad > 0) { warnings.push( - `${clr.bold(colors, clr.yellow(colors, 'Warning'))}: ${clr.bold(colors, clr.cyan(colors, benchmark.name))} has a non zero median absolute deviation: ${clr.red(colors, duration(benchmark.stats.mad))}` + `${bold(colors, yellow(colors, 'Warning'))}: ${bold(colors, cyan(colors, benchmark.name))} has a non zero median absolute deviation: ${red(colors, duration(benchmark.stats.mad))}` ) } } @@ -170,8 +177,8 @@ export function summary(benchmarks, { colors = true }) { return `${`${ baseline.group == null || baseline.group.startsWith(tatamiNgGroup) ? '' - : `${clr.bold(colors, clr.white(colors, baseline.group.trim().split(/\s+/).length > 1 ? `'${baseline.group}'` : `${baseline.group}`))} ` - }${clr.bold(colors, clr.white(colors, 'summary'))}`}\n ${clr.bold(colors, clr.cyan(colors, baseline.name))}${benchmarks + : `${bold(colors, white(colors, baseline.group.trim().split(/\s+/).length > 1 ? `'${baseline.group}'` : `${baseline.group}`))} ` + }${bold(colors, white(colors, 'summary'))}`}\n ${bold(colors, cyan(colors, baseline.name))}${benchmarks .filter(benchmark => benchmark !== baseline) .map(benchmark => { const ratio = benchmark.stats.avg / checkDividend(baseline.stats.avg) @@ -192,12 +199,12 @@ export function summary(benchmarks, { colors = true }) { ] || tTable.infinity const ratioMoe = ratioSem * critical const ratioRmoe = (ratioMoe / checkDividend(ratio)) * 100 - return `\n ${clr[1 > ratio ? 'red' : 'green']( + return `\n ${(1 > ratio ? red : green)( colors, 1 > ratio ? speedRatio(1 / checkDividend(ratio)) : speedRatio(ratio) - )} ± ${clr.blue(colors, errorMargin(ratioRmoe))} times ${ + )} ± ${blue(colors, errorMargin(ratioRmoe))} times ${ 1 > ratio ? 'slower' : 'faster' - } than ${clr.bold(colors, clr.cyan(colors, benchmark.name))}` + } than ${bold(colors, cyan(colors, benchmark.name))}` }) .join('')}` } diff --git a/src/stats-utils.js b/src/stats-utils.js new file mode 100644 index 0000000..22fe0c5 --- /dev/null +++ b/src/stats-utils.js @@ -0,0 +1,69 @@ +import { checkDividend } from './utils.js' + +export const variance = (samples, avg = average(samples)) => { + return ( + samples.reduce((a, b) => a + (b - avg) ** 2, 0) / + checkDividend(samples.length - 1) // Bessel's correction + ) +} + +export const quantileSorted = (samples, q) => { + if (!Array.isArray(samples)) { + throw new TypeError(`expected array, got ${samples.constructor.name}`) + } + if (samples.length === 0) { + throw new Error('expected non-empty array, got empty array') + } + if (q < 0 || q > 1) { + throw new Error('q must be between 0 and 1') + } + if (q === 0) { + return samples[0] + } + if (q === 1) { + return samples[samples.length - 1] + } + const base = (samples.length - 1) * q + const baseIndex = Math.floor(base) + if (samples[baseIndex + 1] != null) { + return ( + samples[baseIndex] + + (base - baseIndex) * (samples[baseIndex + 1] - samples[baseIndex]) + ) + } + return samples[baseIndex] +} + +export const medianSorted = samples => quantileSorted(samples, 0.5) + +export const average = samples => { + if (!Array.isArray(samples)) { + throw new TypeError(`expected array, got ${samples.constructor.name}`) + } + if (samples.length === 0) { + throw new Error('expected non-empty array, got empty array') + } + + return samples.reduce((a, b) => a + b, 0) / samples.length +} + +export const absoluteDeviation = (samples, aggFn) => { + const value = aggFn(samples) + const absoluteDeviations = [] + + for (const sample of samples) { + absoluteDeviations.push(Math.abs(sample - value)) + } + + return aggFn(absoluteDeviations) +} + +// https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulae +export const ratioStandardDeviation = (avgA, sdA, avgB, sdB) => { + return ( + (avgA / checkDividend(avgB)) * + Math.sqrt( + (sdA / checkDividend(avgA)) ** 2 + (sdB / checkDividend(avgB)) ** 2 + ) + ) +} diff --git a/src/utils.js b/src/utils.js new file mode 100644 index 0000000..2f4e726 --- /dev/null +++ b/src/utils.js @@ -0,0 +1,11 @@ +export const isObject = value => { + return Object.prototype.toString.call(value).slice(8, -1) === 'Object' +} + +export const checkDividend = n => { + if (n == null) throw new TypeError(`Invalid dividend: ${n}`) + if ('number' !== typeof n) + throw new TypeError(`expected number, got ${n.constructor.name}`) + if (n === 0 || Number.isNaN(n)) throw new RangeError(`Invalid dividend: ${n}`) + return n +}