refactor!: untangle latency and throughput (stats, display, ...)

Signed-off-by: Jérôme Benoit <[email protected]>

README.md

@@ -137,11 +137,10 @@ await run({
   samples: 128, // minimum number of benchmark samples (default: 128)
   time: 1_000_000_000, // minimum benchmark execution time in nanoseconds (default: 1_000_000_000)
   warmup: true, // enable/disable benchmark warmup or set benchmark warmup run(s) (default: true)
-  avg: true, // enable/disable time/iter average column (default: true)
-  iters: true, // enable/disable iters/s column (default: true)
-  rmoe: true, // enable/disable error margin column (default: true)
-  min_max: true, // enable/disable (min...max) column (default: true)
-  percentiles: false, // enable/disable percentile columns (default: true)
+  latency: true, // enable/disable time/iter column (default: true)
+  throughput: true, // enable/disable iters/s column (default: true)
+  latencyMinMax: true, // enable/disable latency (min...max) column (default: true)
+  latencyPercentiles: false, // enable/disable latency percentile columns (default: true)
 })
 ```
 

cli.js

@@ -82,29 +82,24 @@ const {
       description: 'No colors in output',
       type: 'boolean',
     },
-    'no-avg': {
+    'no-latency': {
       listGroup: 'Output options',
-      description: 'No time/iter average column',
+      description: 'No time/iter column',
       type: 'boolean',
     },
-    'no-iters': {
+    'no-throughput': {
       listGroup: 'Output options',
       description: 'No iters/s column',
       type: 'boolean',
     },
-    'no-min_max': {
+    'no-latency-min_max': {
       listGroup: 'Output options',
-      description: 'No (min...max) column',
+      description: 'No latency (min...max) column',
       type: 'boolean',
     },
-    'no-rmoe': {
+    'no-latency-percentiles': {
       listGroup: 'Output options',
-      description: 'No error margin column',
-      type: 'boolean',
-    },
-    'no-percentiles': {
-      listGroup: 'Output options',
-      description: 'No percentile columns',
+      description: 'No latency percentile columns',
       type: 'boolean',
     },
     units: {
@@ -166,12 +161,15 @@ await run({
   ...(flags.json != null && { json: flags.json }),
   ...(flags.file != null && { file: flags.file }),
   ...(flags['no-colors'] != null && { colors: !flags['no-colors'] }),
-  ...(flags['no-avg'] != null && { avg: !flags['no-avg'] }),
-  ...(flags['no-rmoe'] != null && { rmoe: !flags['no-rmoe'] }),
-  ...(flags['no-iters'] != null && { iters: !flags['no-iters'] }),
-  ...(flags['no-min_max'] != null && { min_max: !flags['no-min_max'] }),
-  ...(flags['no-percentiles'] != null && {
-    percentiles: !flags['no-percentiles'],
+  ...(flags['no-latency'] != null && { latency: !flags['no-latency'] }),
+  ...(flags['no-throughput'] != null && {
+    throughput: !flags['no-throughput'],
+  }),
+  ...(flags['no-latency-min_max'] != null && {
+    latencyMinMax: !flags['no-latency-min_max'],
+  }),
+  ...(flags['no-latency-percentiles'] != null && {
+    latencyPercentiles: !flags['no-latency-percentiles'],
   }),
   ...(flags.units != null && { units: flags.units }),
 })

src/benchmark.js

@@ -286,11 +286,10 @@ const executeBenchmarks = async (
  * @param {Number} [opts.samples=128] minimum number of benchmark samples
  * @param {Number} [opts.time=1_000_000_000] minimum benchmark execution time in nanoseconds
  * @param {Boolean|Number} [opts.warmup=true] enable/disable benchmark warmup or set benchmark warmup run(s)
- * @param {Boolean} [opts.avg=true] enable/disable time/iter average column
- * @param {Boolean} [opts.iters=true] enable/disable iters/s column
- * @param {Boolean} [opts.rmoe=true] enable/disable error margin column
- * @param {Boolean} [opts.min_max=true] enable/disable (min...max) column
- * @param {Boolean} [opts.percentiles=true] enable/disable percentile columns
+ * @param {Boolean} [opts.latency=true] enable/disable time/iter column
+ * @param {Boolean} [opts.throughput=true] enable/disable iters/s column
+ * @param {Boolean} [opts.latencyMinMax=true] enable/disable latency (min...max) column
+ * @param {Boolean} [opts.latencyPercentiles=true] enable/disable latency percentile columns
  * @returns {Promise<Object>} defined benchmarks report
  */
 export async function run(opts = {}) {

src/index.d.ts

@@ -52,16 +52,30 @@ export function run(options?: {
   samples?: number // minimum number of samples
   time?: number // minimum execution time
   warmup?: number | boolean
-  avg?: boolean
-  iters?: boolean
-  min_max?: boolean
-  rmoe?: boolean
-  percentiles?: boolean
+  latency?: boolean
+  throughput?: boolean
+  latencyMinMax?: boolean
+  latencyPercentiles?: boolean
   json?: number | boolean | 'bmf'
   file?: string
   units?: boolean
 }): Promise<Report>
 
+export type Stats = {
+  min: number
+  max: number
+  p50: number // median
+  p75: number
+  p99: number
+  p995: number
+  avg: number // average
+  vr: number // variance
+  sd: number // standard deviation
+  rmoe: number // relative margin of error
+  aad: number // average time absolute deviation
+  mad: number // median time absolute deviation
+}
+
 export type BenchmarkReport = {
   cpu: string
   runtime: string
@@ -82,20 +96,9 @@ export type BenchmarkReport = {
 
     stats?: {
       samples: number // number of samples
-      min: number
-      max: number
-      p50: number // median
-      p75: number
-      p99: number
-      p995: number
-      avg: number // average time per iteration
-      iters: number // iterations per second
-      vr: number // variance
-      sd: number // standard deviation
-      rmoe: number // relative margin of error
-      aad: number // average time absolute deviation
-      mad: number // median time absolute deviation
       ss: boolean // statistical significance
+      latency: Stats
+      throughput: Stats
     }
   }
 }

src/lib.js

@@ -267,39 +267,67 @@ export async function measure(fn, opts = {}) {
   return buildMeasurementStats(samples)
 }
 
-const buildMeasurementStats = samples => {
-  if (!Array.isArray(samples))
-    throw new TypeError(`expected array, got ${samples.constructor.name}`)
-  if (samples.length === 0)
+const buildMeasurementStats = latencySamples => {
+  if (!Array.isArray(latencySamples))
+    throw new TypeError(
+      `expected array, got ${latencySamples.constructor.name}`
+    )
+  if (latencySamples.length === 0)
     throw new Error('expected non-empty array, got empty array')
 
-  samples.sort((a, b) => a - b)
+  // Latency
+  latencySamples.sort((a, b) => a - b)
+  const latencyAvg = average(latencySamples)
+  const latencyVr = variance(latencySamples, latencyAvg)
+  const latencySd = Math.sqrt(latencyVr)
+  const latencySem = latencySd / Math.sqrt(latencySamples.length)
+  const latencyCritical =
+    tTable[(latencySamples.length - 1 || 1).toString()] || tTable.infinity
+  const latencyMoe = latencySem * latencyCritical
+  const latencyRmoe = (latencyMoe / checkDividend(latencyAvg)) * 100
 
-  const time = samples.reduce((a, b) => a + b, 0)
-  const avg = time / samples.length
-  const vr = variance(samples, avg)
-  const sd = Math.sqrt(vr)
-  const sem = sd / Math.sqrt(samples.length)
-  const critical =
-    tTable[(samples.length - 1 || 1).toString()] || tTable.infinity
-  const moe = sem * critical
-  const rmoe = (moe / checkDividend(avg)) * 100
+  // Throughput
+  const throughputSamples = latencySamples.map(sample => 1e9 / sample)
+  throughputSamples.sort((a, b) => a - b)
+  const throughputAvg = average(throughputSamples)
+  const throughputVr = variance(throughputSamples, throughputAvg)
+  const throughputSd = Math.sqrt(throughputVr)
+  const throughputSem = throughputSd / Math.sqrt(throughputSamples.length)
+  const throughputCritical =
+    tTable[(throughputSamples.length - 1 || 1).toString()] || tTable.infinity
+  const throughputMoe = throughputSem * throughputCritical
+  const throughputRmoe = (throughputMoe / checkDividend(throughputAvg)) * 100
 
   return {
-    samples: samples.length,
-    min: samples[0],
-    max: samples[samples.length - 1],
-    p50: medianSorted(samples),
-    p75: quantileSorted(samples, 0.75),
-    p99: quantileSorted(samples, 0.99),
-    p995: quantileSorted(samples, 0.995),
-    avg,
-    iters: (1e9 * samples.length) / checkDividend(time),
-    vr,
-    sd,
-    rmoe,
-    aad: absoluteDeviation(samples, average),
-    mad: absoluteDeviation(samples, medianSorted),
-    ss: samples.length >= minimumSamples,
+    samples: latencySamples.length,
+    ss: latencySamples.length >= minimumSamples,
+    latency: {
+      min: latencySamples[0],
+      max: latencySamples[latencySamples.length - 1],
+      p50: medianSorted(latencySamples),
+      p75: quantileSorted(latencySamples, 0.75),
+      p99: quantileSorted(latencySamples, 0.99),
+      p995: quantileSorted(latencySamples, 0.995),
+      avg: latencyAvg,
+      vr: latencyVr,
+      sd: latencySd,
+      rmoe: latencyRmoe,
+      aad: absoluteDeviation(latencySamples, average),
+      mad: absoluteDeviation(latencySamples, medianSorted),
+    },
+    throughput: {
+      min: throughputSamples[0],
+      max: throughputSamples[latencySamples.length - 1],
+      p50: medianSorted(throughputSamples),
+      p75: quantileSorted(throughputSamples, 0.75),
+      p99: quantileSorted(throughputSamples, 0.99),
+      p995: quantileSorted(throughputSamples, 0.995),
+      avg: throughputAvg,
+      vr: throughputVr,
+      sd: throughputSd,
+      rmoe: throughputRmoe,
+      aad: absoluteDeviation(throughputSamples, average),
+      mad: absoluteDeviation(throughputSamples, medianSorted),
+    },
   }
 }

src/reporter/json/bmf.js

@@ -2,19 +2,17 @@ export const bmf = report => {
   return report.benchmarks
     .filter(benchmark => benchmark.error == null)
     .map(({ name, stats }) => {
-      // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulae
-      const throughputSd = (1e9 * stats?.sd) / stats?.avg ** 2
       return {
         [name]: {
           latency: {
-            value: stats?.avg,
-            lower_value: stats?.avg - stats?.sd,
-            upper_value: stats?.avg + stats?.sd,
+            value: stats?.latency?.avg,
+            lower_value: stats?.latency?.avg - stats?.latency?.sd,
+            upper_value: stats?.latency?.avg + stats?.latency?.sd,
           },
           throughput: {
-            value: stats?.iters,
-            lower_value: stats?.iters - throughputSd,
-            upper_value: stats?.iters + throughputSd,
+            value: stats?.throughput?.avg,
+            lower_value: stats?.throughput?.avg - stats?.throughput?.sd,
+            upper_value: stats?.throughput?.avg + stats?.throughput?.sd,
           },
         },
       }