chore: update benchmark framework and result (#939)

Brooooooklyn · Nov 15, 2024 · 6645e30 · 6645e30
1 parent f1a7d7d
commit 6645e30
Show file tree

Hide file tree

Showing 8 changed files with 123 additions and 388 deletions.
diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml
@@ -626,7 +626,6 @@ jobs:
         uses: ./.github/actions/setup-rust
         with:
           targets: x86_64-unknown-linux-gnu
-          toolchain: nightly-2023-08-09
 
       - name: Install dependencies
         run: yarn install --immutable
@@ -642,25 +641,6 @@ jobs:
       - name: 'Run benchmark'
         run: yarn bench
 
-      - name: Store benchmark result
-        uses: rhysd/github-action-benchmark@v1
-        if: github.ref == 'refs/heads/main'
-        with:
-          tool: 'benchmarkjs'
-          output-file-path: bench.txt
-          github-token: ${{ secrets.GH_TOKEN }}
-          auto-push: true
-
-      - name: Store benchmark result
-        uses: rhysd/github-action-benchmark@v1
-        continue-on-error: true
-        if: github.ref != 'refs/heads/main'
-        with:
-          tool: 'benchmarkjs'
-          output-file-path: bench.txt
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          comment-always: true
-
   publish:
     name: Publish
     runs-on: ubuntu-latest

diff --git a/README-zh.md b/README-zh.md
@@ -114,50 +114,44 @@ writeFileSync(join(__dirname, 'draw-emoji.png'), b)
 硬件信息:
 
 ```
-OS: Windows 10 x86_64
-Host: Micro-Star International Co., Ltd. MS-7C35
-Kernel: 10.0.19043
-Terminal: Windows Terminal
-CPU: AMD Ryzen 9 5950X (32) @ 3.400GHz
-Memory: 32688MiB
+                  ,MMMM.           Host        -  xxxxxxxxxxxxxxxxxxxxxxx
+                .MMMMMM            Machine     -  Mac15,9
+                MMMMM,             Kernel      -  24.0.0
+      .;MMMMM:' MMMMMMMMMM;.       OS          -  macOS 15.0.1 Sequoia
+    MMMMMMMMMMMMNWMMMMMMMMMMM:     DE          -  Aqua
+  .MMMMMMMMMMMMMMMMMMMMMMMMWM.     WM          -  Quartz Compositor
+  MMMMMMMMMMMMMMMMMMMMMMMMM.       Packages    -  194 (Homebrew), 32 (cargo)
+ ;MMMMMMMMMMMMMMMMMMMMMMMM:        Shell       -  zsh
+ :MMMMMMMMMMMMMMMMMMMMMMMM:        Terminal    -  warpterminal (Version v0.2024.10.23.14.49.stable_00)
+ .MMMMMMMMMMMMMMMMMMMMMMMMM.       Resolution  -  5120x2880@160fps (as 2560x1440)
+  MMMMMMMMMMMMMMMMMMMMMMMMMMM.                    2992x1934@120fps (as 1496x967)
+   .MMMMMMMMMMMMMMMMMMMMMMMMMM.                   2232x1512@60fps (as 1116x756)
+     MMMMMMMMMMMMMMMMMMMMMMMM      Uptime      -  1d 2h 32m
+      ;MMMMMMMMMMMMMMMMMMMM.       CPU         -  Apple M3 Max (16)
+        .MMMM,.    .MMMM,.         CPU Load    -  16%
+                                   Memory      -  50.1 GB / 134.2 GB
+                                   Battery     -  78% & Discharging
+                                   Disk Space  -  624.0 GB / 994.7 GB
 ```
 
 ```
 ❯ yarn bench
-
-> @napi-rs/[email protected] bench D:\workspace\skia-rs
-> node -r @swc-node/register benchmark/bench.ts
-
-Running "Draw house" suite...
-Progress: 100%
-
-  skia-canvas:
-    26 ops/s, ±0.70%   | slowest, 29.73% slower
-
-  node-canvas:
-    30 ops/s, ±6.95%   | 18.92% slower
-
-  @napi-rs/canvas:
-    37 ops/s, ±6.30%   | fastest
-
-Finished 3 cases!
-  Fastest: @napi-rs/canvas
-  Slowest: skia-canvas
-Running "Draw gradient" suite...
-Progress: 100%
-
-  skia-canvas:
-    36 ops/s, ±6.12%   | 14.29% slower
-
-  node-canvas:
-    34 ops/s, ±5.60%   | slowest, 19.05% slower
-
-  @napi-rs/canvas:
-    42 ops/s, ±0.53%   | fastest
-
-Finished 3 cases!
-  Fastest: @napi-rs/canvas
-  Slowest: node-canvas
+Draw a House and export to PNG
+┌─────────┬─────────────────┬───────────────────────┬──────────────────────────┬────────────────────────────┬───────────────────────────┬─────────┐
+│ (index) │ Task name       │ Latency average (ns)  │ Latency median (ns)      │ Throughput average (ops/s) │ Throughput median (ops/s) │ Samples │
+├─────────┼─────────────────┼───────────────────────┼──────────────────────────┼────────────────────────────┼───────────────────────────┼─────────┤
+│ 0       │ '@napi-rs/skia' │ '14676992.14 ± 0.68%' │ '14602333.00'            │ '68 ± 0.59%'               │ '68'                      │ 69      │
+│ 1       │ 'skia-canvas'   │ '21167809.17 ± 2.05%' │ '20960021.00 ± 13646.00' │ '47 ± 1.31%'               │ '48'                      │ 64      │
+│ 2       │ 'node-canvas'   │ '16552027.42 ± 0.70%' │ '16451291.50 ± 2208.50'  │ '60 ± 0.62%'               │ '61'                      │ 64      │
+└─────────┴─────────────────┴───────────────────────┴──────────────────────────┴────────────────────────────┴───────────────────────────┴─────────┘
+Draw Gradient and export to PNG
+┌─────────┬─────────────────┬───────────────────────┬─────────────────────────┬────────────────────────────┬───────────────────────────┬─────────┐
+│ (index) │ Task name       │ Latency average (ns)  │ Latency median (ns)     │ Throughput average (ops/s) │ Throughput median (ops/s) │ Samples │
+├─────────┼─────────────────┼───────────────────────┼─────────────────────────┼────────────────────────────┼───────────────────────────┼─────────┤
+│ 0       │ '@napi-rs/skia' │ '15228495.58 ± 0.53%' │ '15146312.50 ± 1187.50' │ '66 ± 0.48%'               │ '66'                      │ 66      │
+│ 1       │ 'skia-canvas'   │ '21725564.41 ± 2.20%' │ '21412520.50 ± 2104.50' │ '46 ± 1.39%'               │ '47'                      │ 64      │
+│ 2       │ 'node-canvas'   │ '17976022.14 ± 1.53%' │ '17563479.50 ± 5104.50' │ '56 ± 1.38%'               │ '57'                      │ 64      │
+└─────────┴─────────────────┴───────────────────────┴─────────────────────────┴────────────────────────────┴───────────────────────────┴─────────┘
 ```
 
 # 功能

diff --git a/README.md b/README.md
@@ -131,50 +131,44 @@ See [benchmark](./benchmark) for benchmark code.
 Hardware info:
 
 ```
-OS: Windows 10 x86_64
-Host: Micro-Star International Co., Ltd. MS-7C35
-Kernel: 10.0.19043
-Terminal: Windows Terminal
-CPU: AMD Ryzen 9 5950X (32) @ 3.400GHz
-Memory: 32688MiB
+                  ,MMMM.           Host        -  xxxxxxxxxxxxxxxxxxxxxxx
+                .MMMMMM            Machine     -  Mac15,9
+                MMMMM,             Kernel      -  24.0.0
+      .;MMMMM:' MMMMMMMMMM;.       OS          -  macOS 15.0.1 Sequoia
+    MMMMMMMMMMMMNWMMMMMMMMMMM:     DE          -  Aqua
+  .MMMMMMMMMMMMMMMMMMMMMMMMWM.     WM          -  Quartz Compositor
+  MMMMMMMMMMMMMMMMMMMMMMMMM.       Packages    -  194 (Homebrew), 32 (cargo)
+ ;MMMMMMMMMMMMMMMMMMMMMMMM:        Shell       -  zsh
+ :MMMMMMMMMMMMMMMMMMMMMMMM:        Terminal    -  warpterminal (Version v0.2024.10.23.14.49.stable_00)
+ .MMMMMMMMMMMMMMMMMMMMMMMMM.       Resolution  -  5120x2880@160fps (as 2560x1440)
+  MMMMMMMMMMMMMMMMMMMMMMMMMMM.                    2992x1934@120fps (as 1496x967)
+   .MMMMMMMMMMMMMMMMMMMMMMMMMM.                   2232x1512@60fps (as 1116x756)
+     MMMMMMMMMMMMMMMMMMMMMMMM      Uptime      -  1d 2h 32m
+      ;MMMMMMMMMMMMMMMMMMMM.       CPU         -  Apple M3 Max (16)
+        .MMMM,.    .MMMM,.         CPU Load    -  16%
+                                   Memory      -  50.1 GB / 134.2 GB
+                                   Battery     -  78% & Discharging
+                                   Disk Space  -  624.0 GB / 994.7 GB
 ```
 
 ```
 ❯ yarn bench
-
-> @napi-rs/[email protected] bench D:\workspace\skia-rs
-> node -r @swc-node/register benchmark/bench.ts
-
-Running "Draw house" suite...
-Progress: 100%
-
-  skia-canvas:
-    26 ops/s, ±0.70%   | slowest, 29.73% slower
-
-  node-canvas:
-    30 ops/s, ±6.95%   | 18.92% slower
-
-  @napi-rs/canvas:
-    37 ops/s, ±6.30%   | fastest
-
-Finished 3 cases!
-  Fastest: @napi-rs/canvas
-  Slowest: skia-canvas
-Running "Draw gradient" suite...
-Progress: 100%
-
-  skia-canvas:
-    36 ops/s, ±6.12%   | 14.29% slower
-
-  node-canvas:
-    34 ops/s, ±5.60%   | slowest, 19.05% slower
-
-  @napi-rs/canvas:
-    42 ops/s, ±0.53%   | fastest
-
-Finished 3 cases!
-  Fastest: @napi-rs/canvas
-  Slowest: node-canvas
+Draw a House and export to PNG
+┌─────────┬─────────────────┬───────────────────────┬──────────────────────────┬────────────────────────────┬───────────────────────────┬─────────┐
+│ (index) │ Task name       │ Latency average (ns)  │ Latency median (ns)      │ Throughput average (ops/s) │ Throughput median (ops/s) │ Samples │
+├─────────┼─────────────────┼───────────────────────┼──────────────────────────┼────────────────────────────┼───────────────────────────┼─────────┤
+│ 0       │ '@napi-rs/skia' │ '14676992.14 ± 0.68%' │ '14602333.00'            │ '68 ± 0.59%'               │ '68'                      │ 69      │
+│ 1       │ 'skia-canvas'   │ '21167809.17 ± 2.05%' │ '20960021.00 ± 13646.00' │ '47 ± 1.31%'               │ '48'                      │ 64      │
+│ 2       │ 'node-canvas'   │ '16552027.42 ± 0.70%' │ '16451291.50 ± 2208.50'  │ '60 ± 0.62%'               │ '61'                      │ 64      │
+└─────────┴─────────────────┴───────────────────────┴──────────────────────────┴────────────────────────────┴───────────────────────────┴─────────┘
+Draw Gradient and export to PNG
+┌─────────┬─────────────────┬───────────────────────┬─────────────────────────┬────────────────────────────┬───────────────────────────┬─────────┐
+│ (index) │ Task name       │ Latency average (ns)  │ Latency median (ns)     │ Throughput average (ops/s) │ Throughput median (ops/s) │ Samples │
+├─────────┼─────────────────┼───────────────────────┼─────────────────────────┼────────────────────────────┼───────────────────────────┼─────────┤
+│ 0       │ '@napi-rs/skia' │ '15228495.58 ± 0.53%' │ '15146312.50 ± 1187.50' │ '66 ± 0.48%'               │ '66'                      │ 66      │
+│ 1       │ 'skia-canvas'   │ '21725564.41 ± 2.20%' │ '21412520.50 ± 2104.50' │ '46 ± 1.39%'               │ '47'                      │ 64      │
+│ 2       │ 'node-canvas'   │ '17976022.14 ± 1.53%' │ '17563479.50 ± 5104.50' │ '56 ± 1.38%'               │ '57'                      │ 64      │
+└─────────┴─────────────────┴───────────────────────┴─────────────────────────┴────────────────────────────┴───────────────────────────┴─────────┘
 ```
 
 # Features

diff --git a/benchmark/bench.ts b/benchmark/bench.ts
@@ -1,25 +1,11 @@
-import { promises as fs } from 'fs'
-import { join } from 'path'
-
-import { Summary } from 'benny/lib/internal/common-types'
-
 import { house } from './house'
 import { gradient } from './gradient'
 
 async function run() {
-  const output = [await house(), await gradient()].map(formatSummary).join('\n')
-  await fs.writeFile(join(process.cwd(), 'bench.txt'), output, 'utf8')
+  await house()
+  await gradient()
 }
 
 run().catch((e) => {
   console.error(e)
 })
-
-function formatSummary(summary: Summary): string {
-  return summary.results
-    .map(
-      (result) =>
-        `${summary.name}#${result.name} x ${result.ops} ops/sec ±${result.margin}% (${result.samples} runs sampled)`,
-    )
-    .join('\n')
-}
diff --git a/benchmark/gradient.ts b/benchmark/gradient.ts
@@ -1,6 +1,6 @@
-import b from 'benny'
-
 import { createCanvas, Canvas } from 'canvas'
+import { cyan } from 'colorette'
+import { Bench } from 'tinybench'
 import { Canvas as SkiaCanvas } from 'skia-canvas'
 
 import { createCanvas as skiaCreateCanvas } from '../index'
@@ -30,25 +30,27 @@ function drawGradient(factory: (width: number, height: number) => Canvas) {
   }
 }
 
-export function gradient() {
-  return b.suite(
-    'Draw gradient',
+export async function gradient() {
+  const bench = new Bench({
+    name: 'gradient',
+  })
 
-    b.add('skia-canvas', () => {
+  bench
+    .add('@napi-rs/skia', () => {
+      // @ts-expect-error
+      drawGradient(skiaCreateCanvas)
+    })
+    .add('skia-canvas', () => {
       // @ts-expect-error
       drawGradient((w, h) => new SkiaCanvas(w, h))
-    }),
-
-    b.add('node-canvas', () => {
+    })
+    .add('node-canvas', () => {
       drawGradient(createCanvas)
-    }),
+    })
 
-    b.add('@napi-rs/skia', () => {
-      // @ts-expect-error
-      drawGradient(skiaCreateCanvas)
-    }),
+  await bench.run()
+  console.info(cyan('Draw Gradient and export to PNG'))
+  console.table(bench.table())
 
-    b.cycle(),
-    b.complete(),
-  )
+  return bench
 }
diff --git a/benchmark/house.ts b/benchmark/house.ts
@@ -1,7 +1,6 @@
-import b from 'benny'
-
 import { createCanvas, Canvas } from 'canvas'
-// @ts-expect-error
+import { cyan } from 'colorette'
+import { Bench } from 'tinybench'
 import { Canvas as SkiaCanvas } from 'skia-canvas'
 
 import { createCanvas as skiaCreateCanvas } from '../index'
@@ -30,32 +29,36 @@ function drawHouse(factory: (width: number, height: number) => Canvas) {
   ctx.stroke()
 
   if (canvas instanceof SkiaCanvas) {
-    canvas.toBufferSync('image/png')
+    canvas.toBufferSync('png')
   } else {
     // @ts-expect-error
     canvas.async = false
     canvas.toBuffer('image/png')
   }
 }
 
-export function house() {
-  return b.suite(
-    'Draw house',
+export async function house() {
+  const bench = new Bench({
+    name: 'house',
+  })
 
-    b.add('skia-canvas', () => {
+  bench
+    .add('@napi-rs/skia', () => {
+      // @ts-expect-error
+      drawHouse(skiaCreateCanvas)
+    })
+    .add('skia-canvas', () => {
+      // @ts-expect-error
       drawHouse((w, h) => new SkiaCanvas(w, h))
-    }),
-
-    b.add('node-canvas', () => {
+    })
+    .add('node-canvas', () => {
       drawHouse(createCanvas)
-    }),
+    })
 
-    b.add('@napi-rs/skia', () => {
-      // @ts-expect-error
-      drawHouse(skiaCreateCanvas)
-    }),
+  await bench.run()
+
+  console.info(cyan('Draw a House and export to PNG'))
+  console.table(bench.table())
 
-    b.cycle(),
-    b.complete(),
-  )
+  return bench
 }
diff --git a/package.json b/package.json
@@ -79,7 +79,6 @@
     "@types/node": "^20.11.16",
     "@types/semver": "^7",
     "ava": "^6.1.1",
-    "benny": "^3.7.1",
     "canvas": "^2.11.2",
     "canvaskit-wasm": "^0.39.1",
     "colorette": "^2.0.20",
@@ -98,6 +97,7 @@
     "semver": "^7.5.4",
     "skia-canvas": "^1.0.1",
     "table": "^6.8.1",
+    "tinybench": "^3.0.6",
     "typescript": "^5.3.3"
   },
   "lint-staged": {