feat!: add custom benchmark reporter support

reference #29

Signed-off-by: Jérôme Benoit <[email protected]>

README.md

@@ -103,7 +103,7 @@ import {
 
 ```js
 // adapt import to the targeted JS runtime
-import { baseline, bench, group, run } from 'tatami-ng'
+import { baseline, bench, group, run, bmf } from 'tatami-ng'
 
 bench('noop', () => {})
 bench('noop2', () => {})
@@ -130,7 +130,8 @@ group({ name: 'group2', summary: false }, () => {
 await run({
   units: false, // print units cheatsheet (default: false)
   silent: false, // enable/disable stdout output (default: false)
-  json: false, // enable/disable json output or set json output format (default: false)
+  json: false, // enable/disable json output or set json output indentation (default: false)
+  reporter: bmf // custom reporter function (default: undefined)
   file: 'results.json', // write json output to file (default: undefined)
   colors: true, // enable/disable colors (default: true)
   now: () => 1e6 * performance.now.bind(performance)(), // custom nanoseconds timestamp function to replace default one (default: undefined)

cli.js

@@ -62,7 +62,7 @@ const {
     },
     silent: {
       listGroup: 'Output options',
-      description: 'No stdout output',
+      description: 'No standard output',
       type: 'boolean',
     },
     json: {

src/benchmark.js

@@ -2,7 +2,6 @@ import {
   defaultSamples,
   defaultTime,
   emptyFunction,
-  jsonOutputFormat,
   tatamiNgGroup,
 } from './constants.js'
 import {
@@ -17,7 +16,6 @@ import {
   writeFileSync,
 } from './lib.js'
 import { logger } from './logger.js'
-import { bmf } from './reporter/json/index.js'
 import {
   benchmarkError,
   benchmarkReport,
@@ -279,7 +277,8 @@ const executeBenchmarks = async (
  * @param {Object} [opts={}] options object
  * @param {Boolean} [opts.units=false] print units cheatsheet
  * @param {Boolean} [opts.silent=false] enable/disable stdout output
- * @param {Boolean|Number|'bmf'} [opts.json=false] enable/disable json output or set json output format
+ * @param {Boolean|Number} [opts.json=false] enable/disable json output or set json output indentation
+ * @param {Function} [opts.reporter=undefined] custom reporter function
  * @param {String} [opts.file=undefined] write json output to file
  * @param {NowType} [opts.now=undefined] custom nanoseconds timestamp function to replace default one
  * @param {Boolean} [opts.colors=true] enable/disable colors
@@ -314,28 +313,22 @@ export async function run(opts = {}) {
   if (
     opts.json != null &&
     'number' !== typeof opts.json &&
-    'boolean' !== typeof opts.json &&
-    'string' !== typeof opts.json
+    'boolean' !== typeof opts.json
   )
     throw new TypeError(
-      `expected number or boolean or string as 'json' option, got ${opts.json.constructor.name}`
+      `expected number or boolean as 'json' option, got ${opts.json.constructor.name}`
     )
-  if (
-    'string' === typeof opts.json &&
-    !Object.values(jsonOutputFormat).includes(opts.json)
-  )
+  if (opts.reporter != null && !isFunction(opts.reporter))
     throw new TypeError(
-      `expected one of ${Object.values(jsonOutputFormat).join(
-        ', '
-      )} as 'json' option, got ${opts.json}`
+      `expected function as 'reporter' option, got ${opts.reporter.constructor.name}`
     )
   if (opts.file != null && 'string' !== typeof opts.file)
     throw new TypeError(
       `expected string as 'file' option, got ${opts.file.constructor.name}`
     )
   if ('string' === typeof opts.file && opts.file.trim().length === 0)
     throw new TypeError(`expected non-empty string as 'file' option`)
-  if (opts.now != null && Function !== opts.now.constructor)
+  if (opts.now != null && !isFunction(opts.now))
     throw new TypeError(
       `expected function as 'now' option, got ${opts.now.constructor.name}`
     )
@@ -347,7 +340,7 @@ export async function run(opts = {}) {
 
   const log = opts.silent === true ? emptyFunction : logger
 
-  const report = {
+  let report = {
     benchmarks,
     cpu: `${cpuModel}`,
     runtime: `${runtime} ${version} (${os})`,
@@ -386,20 +379,15 @@ export async function run(opts = {}) {
       : groupOpts.after()
   }
 
+  report = isFunction(opts.reporter) ? opts.reporter(report) : report
+
   if (!opts.json && opts.units) log(units(opts))
   if (opts.json || opts.file) {
-    let jsonReport
-    switch (opts.json) {
-      case jsonOutputFormat.bmf:
-        jsonReport = JSON.stringify(bmf(report))
-        break
-      default:
-        jsonReport = JSON.stringify(
-          report,
-          undefined,
-          'number' !== typeof opts.json ? 0 : opts.json
-        )
-    }
+    const jsonReport = JSON.stringify(
+      report,
+      undefined,
+      'number' !== typeof opts.json ? 0 : opts.json
+    )
     if (opts.json) log(jsonReport)
     if (opts.file) writeFileSync(opts.file, jsonReport)
   }

src/constants.js

@@ -1052,8 +1052,4 @@ export const defaultTime = 1e9 // ns
 
 export const defaultWarmupRuns = 12
 
-export const jsonOutputFormat = {
-  bmf: 'bmf',
-}
-
 export const highRelativeMarginOfError = 8

src/index.d.ts

@@ -45,7 +45,7 @@ export function baseline(
   options?: BenchmarkOptions
 ): void
 
-export function run(options?: {
+export function run<T = Report>(options?: {
   now?: () => number
   silent?: boolean
   colors?: boolean
@@ -56,10 +56,11 @@ export function run(options?: {
   throughput?: boolean
   latencyMinMax?: boolean
   latencyPercentiles?: boolean
-  json?: number | boolean | 'bmf'
+  json?: number | boolean
   file?: string
+  reporter?: <T>(report: Report) => T // custom reporter
   units?: boolean
-}): Promise<Report>
+}): Promise<T>
 
 export type Stats = {
   min: number
@@ -71,9 +72,10 @@ export type Stats = {
   avg: number // average
   vr: number // variance
   sd: number // standard deviation
+  moe: number // margin of error
   rmoe: number // relative margin of error
-  aad: number // average time absolute deviation
-  mad: number // median time absolute deviation
+  aad: number // average absolute deviation
+  mad: number // median absolute deviation
 }
 
 export type BenchmarkReport = {

src/lib.js

@@ -161,7 +161,7 @@ export const checkBenchmarkArgs = (fn, opts = {}) => {
     throw new TypeError(
       `expected number or boolean as 'warmup' option, got ${opts.warmup.constructor.name}`
     )
-  if (opts.now != null && Function !== opts.now.constructor)
+  if (opts.now != null && !isFunction(opts.now))
     throw new TypeError(
       `expected function as 'now' option, got ${opts.now.constructor.name}`
     )
@@ -311,6 +311,7 @@ const buildMeasurementStats = latencySamples => {
       avg: latencyAvg,
       vr: latencyVr,
       sd: latencySd,
+      moe: latencyMoe,
       rmoe: latencyRmoe,
       aad: absoluteDeviation(latencySamples, average),
       mad: absoluteDeviation(latencySamples, medianSorted),
@@ -325,6 +326,7 @@ const buildMeasurementStats = latencySamples => {
       avg: throughputAvg,
       vr: throughputVr,
       sd: throughputSd,
+      moe: throughputMoe,
       rmoe: throughputRmoe,
       aad: absoluteDeviation(throughputSamples, average),
       mad: absoluteDeviation(throughputSamples, medianSorted),

src/reporter/terminal/table.js

@@ -71,7 +71,7 @@ export function header(
     latencyPercentiles = true,
   }
 ) {
-  return `${dim(colors, white(colors, `cpu: ${report.cpu}`))}\n${dim(colors, white(colors, `runtime: ${report.runtime}`))}\n\n${'benchmark'.padEnd(size, ' ')}${!latency ? '' : 'time/iter'.padStart(20, ' ')}${!throughput ? '' : 'iters/s'.padStart(20, ' ')}${!latencyMinMax ? '' : '(min … max)'.padStart(24, ' ')}${
+  return `${dim(colors, white(colors, `cpu: ${report.cpu}`))}\n${dim(colors, white(colors, `runtime: ${report.runtime}`))}\n\n${'benchmark'.padEnd(size, ' ')}${!latency ? '' : 'time/iter'.padStart(20, ' ')}${!throughput ? '' : 'iters/s'.padStart(20, ' ')}${!latencyMinMax ? '' : '(min … max time/iter)'.padStart(24, ' ')}${
     !latencyPercentiles
       ? ''
       : ` ${'p50/median'.padStart(20, ' ')} ${'p75'.padStart(9, ' ')} ${'p99'.padStart(9, ' ')} ${'p995'.padStart(9, ' ')}`
@@ -144,7 +144,10 @@ export function benchmarkReport(
   }`
 }
 
-export function warning(benchmarks, { colors = true }) {
+export function warning(
+  benchmarks,
+  { latency = true, throughput = true, colors = true }
+) {
   if (benchmarks.some(benchmark => benchmark.error != null)) {
     throw new Error('Cannot display warning on benchmarks with error')
   }
@@ -155,22 +158,25 @@ export function warning(benchmarks, { colors = true }) {
         `${bold(colors, yellow(colors, 'Warning'))}: ${bold(colors, cyan(colors, benchmark.name))} has a sample size below statistical significance: ${red(colors, benchmark.samples)}`
       )
     }
-    if (benchmark.stats.latency.rmoe > highRelativeMarginOfError) {
+    if (latency && benchmark.stats.latency.rmoe > highRelativeMarginOfError) {
       warnings.push(
         `${bold(colors, yellow(colors, 'Warning'))}: ${bold(colors, cyan(colors, benchmark.name))} has a high latency relative margin of error: ${red(colors, errorMargin(benchmark.stats.latency.rmoe))}`
       )
     }
-    if (benchmark.stats.throughput.rmoe > highRelativeMarginOfError) {
+    if (
+      throughput &&
+      benchmark.stats.throughput.rmoe > highRelativeMarginOfError
+    ) {
       warnings.push(
         `${bold(colors, yellow(colors, 'Warning'))}: ${bold(colors, cyan(colors, benchmark.name))} has a high latency throughput margin of error: ${red(colors, errorMargin(benchmark.stats.throughput.rmoe))}`
       )
     }
-    if (benchmark.stats.latency.mad > 0) {
+    if (latency && benchmark.stats.latency.mad > 0) {
       warnings.push(
         `${bold(colors, yellow(colors, 'Warning'))}: ${bold(colors, cyan(colors, benchmark.name))} has a non zero latency median absolute deviation: ${red(colors, duration(benchmark.stats.latency.mad))}`
       )
     }
-    if (benchmark.stats.throughput.mad > 0) {
+    if (throughput && benchmark.stats.throughput.mad > 0) {
       warnings.push(
         `${bold(colors, yellow(colors, 'Warning'))}: ${bold(colors, cyan(colors, benchmark.name))} has a non zero throughput median absolute deviation: ${red(colors, duration(benchmark.stats.throughput.mad))}`
       )

src/runtime.js

@@ -3,8 +3,8 @@ import { runtimes } from './constants.js'
 const isBun = !!globalThis.Bun || !!globalThis.process?.versions?.bun
 const isDeno = !!globalThis.Deno
 const isNode = globalThis.process?.release?.name === 'node'
-const isHermes = !!globalThis.HermesInternal
-const isWorkerd = globalThis.navigator?.userAgent === 'Cloudflare-Workers'
+// const isHermes = !!globalThis.HermesInternal
+// const isWorkerd = globalThis.navigator?.userAgent === 'Cloudflare-Workers'
 const isBrowser = !!globalThis.navigator
 
 export const runtime = (() => {

tests/formatting.js

@@ -103,7 +103,8 @@ group({ summary: true }, () => {
 await run({
   units: true, // print units cheatsheet (default: false)
   latency: true, // enable/disable time/iter column (default: true)
-  json: false, // enable/disable json output or set json output format (default: false)
+  throughput: true, // enable/disable iters/s column (default: true)
+  json: false, // enable/disable json output or set json output indentation (default: false)
   colors: true, // enable/disable colors (default: true)
   latencyMinMax: true, // enable/disable latency (min...max) column (default: true)
   latencyPercentiles: true, // enable/disable latency percentile columns (default: true)

tests/test.js

@@ -49,7 +49,7 @@ group({ name: 'group2', summary: false }, () => {
 const report = await run({
   latency: true, // enable/disable time/iter column (default: true)
   throughput: true, // enable/disable iters/s column (default: true)
-  json: false, // enable/disable json output or set json output format (default: false)
+  json: false, // enable/disable json output or set json output indentation (default: false)
   colors: true, // enable/disable colors (default: true)
   latencyMinMax: true, // enable/disable latency (min...max) column (default: true)
   latencyPercentiles: true, // enable/disable latency percentile columns (default: true)
@@ -60,7 +60,7 @@ console.log(report)
 await run({
   latency: true, // enable/disable time/iter column (default: true)
   throughput: true, // enable/disable iters/s column (default: true)
-  json: false, // enable/disable json output or set json output format (default: false)
+  json: false, // enable/disable json output or set json output indentation (default: false)
   colors: true, // enable/disable colors (default: true)
   latencyMinMax: true, // enable/disable latency (min...max) column (default: true)
   latencyPercentiles: true, // enable/disable latency percentile columns (default: true)

tests/test.ts

@@ -48,8 +48,8 @@ group({ name: 'group2', summary: false }, () => {
 
 const report = await run({
   latency: true, // enable/disable time/iter column (default: true)
-  throughput: true, // enable/disable iters/s column (default: true)
-  json: false, // enable/disable json output or set json output format (default: false)
+  throughput: false, // enable/disable iters/s column (default: true)
+  json: false, // enable/disable json output or set json output indentation (default: false)
   colors: true, // enable/disable colors (default: true)
   latencyMinMax: true, // enable/disable latency (min...max) column (default: true)
   latencyPercentiles: true, // enable/disable latency percentile columns (default: true)
@@ -60,7 +60,7 @@ console.log(report)
 await run({
   latency: true, // enable/disable time/iter column (default: true)
   throughput: true, // enable/disable iters/s column (default: true)
-  json: false, // enable/disable json output or set json output format (default: false)
+  json: false, // enable/disable json output or set json output indentation (default: false)
   colors: true, // enable/disable colors (default: true)
   latencyMinMax: true, // enable/disable latency (min...max) column (default: true)
   latencyPercentiles: true, // enable/disable latency percentile columns (default: true)