feat: bench js api by codspeed

.github/workflows/ci.yml

@@ -21,6 +21,11 @@ on:
     tags-ignore:
       - "**"
 
+env:
+  # Since CI builds are more akin to from-scratch builds, incremental compilation adds unnecessary dependency-tracking and IO overhead, reducing caching effectiveness.
+  # https://github.com/rust-lang/rust-analyzer/blob/25368d24308d6a94ffe8b99f0122bcf5a2175322/.github/workflows/ci.yaml#L11
+  CARGO_INCREMENTAL: 0
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
   cancel-in-progress: ${{ github.ref_name != 'main' }}
@@ -61,10 +66,10 @@ jobs:
     uses: ./.github/workflows/reusable-build.yml
     with:
       target: x86_64-unknown-linux-gnu
-      profile: "debug"
       native: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS == '"ubuntu-22.04"' }}
       runner: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS }}
       skipable: ${{ needs.check-changed.outputs.changed != 'true' }}
+      bench: true
 
   test-release-linux:
     name: Test Release Linux
@@ -352,36 +357,6 @@ jobs:
         # reason for excluding https://github.com/napi-rs/napi-rs/issues/2200
         run: cargo miri test --workspace --exclude rspack_node -- --nocapture
 
-  run_rust_benchmark:
-    name: Run rust benchmark
-    runs-on: ubuntu-latest
-    needs: [rust_changes, rust_check, rust_test]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Pnpm Cache # Required by some tests
-        uses: ./.github/actions/pnpm-cache
-
-      - name: Install Rust Toolchain
-        uses: ./.github/actions/rustup
-        with:
-          save-cache: ${{ github.ref_name == 'main' }}
-          shared-key: check
-
-      - name: Install Tools
-        uses: taiki-e/install-action@v2
-        with:
-          tool: cargo-codspeed
-
-      - name: Build Benchmark
-        run: cargo codspeed build -p rspack_benchmark --features codspeed
-
-      - name: Run benchmark
-        uses: CodSpeedHQ/action@v3
-        timeout-minutes: 30
-        with:
-          run: cargo codspeed run
-          token: ${{ secrets.CODSPEED_TOKEN }}
-
   run_benchmark:
     name: Run benchmark
     runs-on: ubuntu-latest

.github/workflows/reusable-build.yml

@@ -54,6 +54,10 @@ on:
         type: boolean
         required: false
         default: true
+      bench: # Run benchmarks?
+        type: boolean
+        required: false
+        default: false
       ref: # Git reference to checkout
         required: false
         type: string
@@ -95,8 +99,7 @@ jobs:
         if: ${{ !inputs.skipable }}
         uses: ./.github/actions/rustup
         with:
-          save-cache: ${{ github.ref_name == 'main' }} # This should be safe because we have nightly building the cache every day
-          shared-key: build-${{ inputs.target }}
+          shared-key: build-${{ inputs.target }}-${{ inputs.profile }}
 
       # Compile dependencies with optimization to make tests run faster
       - name: Add optimization to debug profile
@@ -389,3 +392,59 @@ jobs:
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           sha: ${{ github.sha }}
+
+  bench:
+    name: Bench
+    needs: build
+    if: ${{ inputs.bench && !inputs.skipable }}
+    runs-on: ${{ fromJSON(needs.build.outputs.runner-labels) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          clean: ${{ startsWith(runner.name, 'GitHub Actions') }}
+
+      - name: Clean
+        if: ${{ !startsWith(runner.name, 'GitHub Actions') }}
+        uses: ./.github/actions/clean
+        with:
+          target: ${{ inputs.target }}
+
+      - name: Download bindings
+        uses: ./.github/actions/download-artifact
+        with:
+          name: bindings-${{ inputs.target }}
+          path: crates/node_binding/
+          try-local-cache: ${{ inputs.profile == 'debug' }}
+          link-when-local: true
+
+      - name: Show restored binding
+        shell: bash
+        run: ls -lah crates/node_binding/*.node
+
+      - name: Install Rust Toolchain
+        uses: ./.github/actions/rustup
+        with:
+          shared-key: build-${{ inputs.target }}-${{ inputs.profile }}
+
+      - name: Pnpm Cache
+        uses: ./.github/actions/pnpm-cache
+
+      - name: Build JS
+        run: pnpm run build:js
+
+      - name: Install cargo-codspeed binary
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-codspeed
+
+      - name: Build Benchmark
+        run: cargo codspeed build -p rspack_benchmark --features codspeed
+
+      - name: Run benchmark
+        uses: CodSpeedHQ/action@v3
+        timeout-minutes: 30
+        with:
+          run: pnpm run bench:ci
+          token: ${{ secrets.CODSPEED_TOKEN }}

package.json

@@ -41,7 +41,8 @@
     "test:webpack": "pnpm --filter \"webpack-test\" test:metric",
     "doc-coverage": "pnpm --filter \"@rspack/core\" doc-coverage",
     "api-extractor:local": "pnpm --filter \"@rspack/*\" api-extractor --local",
-    "api-extractor:ci": "pnpm --filter \"@rspack/*\" api-extractor:ci"
+    "api-extractor:ci": "pnpm --filter \"@rspack/*\" api-extractor:ci",
+    "bench:ci": "cargo codspeed run && pnpm --filter bench run bench"
   },
   "homepage": "https://rspack.dev",
   "bugs": "https://github.com/web-infra-dev/rspack/issues",

packages/bench/README.md

@@ -0,0 +1,9 @@
+# Rspack JavaScript API benchmarks
+
+The primary objective of this project is to track the performance of the Rspack JavaScript API and prevent performance degradation.
+
+## Notes for Writing Benchmark Cases
+
+CPU instrument is better suited for micro-benchmarks (taking less than a second) focused on CPU-bound tasks, not system calls.
+
+System calls introduce variability in execution time. This variability is influenced by several factors, including system load, network latency, and disk I/O performance. As a result, the execution time of system calls can fluctuate significantly, making them the most inconsistent part of a program's execution time.

packages/bench/fixtures/ts-react/index.html

@@ -0,0 +1,12 @@
+<!doctype html>
+<html>
+	<head>
+		<meta charset="UTF-8" />
+		<link rel="icon" type="image/svg+xml" href="/react.svg" />
+		<meta name="viewport" content="width=device-width, initial-scale=1.0" />
+		<title>Rspack + React + TS</title>
+	</head>
+	<body>
+		<div id="root"></div>
+	</body>
+</html>

packages/bench/fixtures/ts-react/rspack.config.ts

@@ -0,0 +1,67 @@
+import { defineConfig } from "@rspack/cli";
+import { rspack } from "@rspack/core";
+import * as RefreshPlugin from "@rspack/plugin-react-refresh";
+
+const isDev = process.env.NODE_ENV === "development";
+
+// Target browsers, see: https://github.com/browserslist/browserslist
+const targets = ["chrome >= 87", "edge >= 88", "firefox >= 78", "safari >= 14"];
+
+export default defineConfig({
+	context: __dirname,
+	entry: {
+		main: "./src/index.tsx"
+	},
+	resolve: {
+		extensions: ["...", ".ts", ".tsx", ".jsx"]
+	},
+	module: {
+		rules: [
+			{
+				test: /\.svg$/,
+				type: "asset"
+			},
+			{
+				test: /\.(jsx?|tsx?)$/,
+				use: [
+					{
+						loader: "builtin:swc-loader",
+						options: {
+							jsc: {
+								parser: {
+									syntax: "typescript",
+									tsx: true
+								},
+								transform: {
+									react: {
+										runtime: "automatic",
+										development: isDev,
+										refresh: isDev
+									}
+								}
+							},
+							env: { targets }
+						}
+					}
+				]
+			}
+		]
+	},
+	plugins: [
+		new rspack.HtmlRspackPlugin({
+			template: "./index.html"
+		}),
+		isDev ? new RefreshPlugin() : null
+	].filter(Boolean),
+	optimization: {
+		minimizer: [
+			new rspack.SwcJsMinimizerRspackPlugin(),
+			new rspack.LightningCssMinimizerRspackPlugin({
+				minimizerOptions: { targets }
+			})
+		]
+	},
+	experiments: {
+		css: true
+	}
+});

packages/bench/fixtures/ts-react/src/App.css

@@ -0,0 +1,41 @@
+#root {
+	max-width: 1280px;
+	margin: 0 auto;
+	padding: 2rem;
+	text-align: center;
+}
+
+.logo {
+	height: 6em;
+	padding: 1.5em;
+	will-change: filter;
+}
+.logo:hover {
+	filter: drop-shadow(0 0 2em #646cffaa);
+}
+.logo.react:hover {
+	filter: drop-shadow(0 0 2em #61dafbaa);
+}
+
+@keyframes logo-spin {
+	from {
+		transform: rotate(0deg);
+	}
+	to {
+		transform: rotate(360deg);
+	}
+}
+
+@media (prefers-reduced-motion: no-preference) {
+	a > .logo {
+		animation: logo-spin infinite 20s linear;
+	}
+}
+
+.card {
+	padding: 2em;
+}
+
+.read-the-docs {
+	color: #888;
+}

packages/bench/fixtures/ts-react/src/App.tsx

@@ -0,0 +1,31 @@
+import { useState } from "react";
+import reactLogo from "./assets/react.svg";
+import "./App.css";
+
+function App() {
+	const [count, setCount] = useState(0);
+
+	return (
+		<div className="App">
+			<div>
+				<a href="https://reactjs.org" target="_blank" rel="noreferrer">
+					<img src={reactLogo} className="logo react" alt="React logo" />
+				</a>
+			</div>
+			<h1>Rspack + React + TypeScript</h1>
+			<div className="card">
+				<button type="button" onClick={() => setCount(count => count + 1)}>
+					count is {count}
+				</button>
+				<p>
+					Edit <code>src/App.tsx</code> and save to test HMR
+				</p>
+			</div>
+			<p className="read-the-docs">
+				Click on the Rspack and React logos to learn more
+			</p>
+		</div>
+	);
+}
+
+export default App;

packages/bench/fixtures/ts-react/src/index.css

@@ -0,0 +1,70 @@
+:root {
+	font-family: Inter, Avenir, Helvetica, Arial, sans-serif;
+	font-size: 16px;
+	line-height: 24px;
+	font-weight: 400;
+
+	color-scheme: light dark;
+	color: rgba(255, 255, 255, 0.87);
+	background-color: #242424;
+
+	font-synthesis: none;
+	text-rendering: optimizeLegibility;
+	-webkit-font-smoothing: antialiased;
+	-moz-osx-font-smoothing: grayscale;
+	-webkit-text-size-adjust: 100%;
+}
+
+a {
+	font-weight: 500;
+	color: #646cff;
+	text-decoration: inherit;
+}
+a:hover {
+	color: #535bf2;
+}
+
+body {
+	margin: 0;
+	display: flex;
+	place-items: center;
+	min-width: 320px;
+	min-height: 100vh;
+}
+
+h1 {
+	font-size: 3.2em;
+	line-height: 1.1;
+}
+
+button {
+	border-radius: 8px;
+	border: 1px solid transparent;
+	padding: 0.6em 1.2em;
+	font-size: 1em;
+	font-weight: 500;
+	font-family: inherit;
+	background-color: #1a1a1a;
+	cursor: pointer;
+	transition: border-color 0.25s;
+}
+button:hover {
+	border-color: #646cff;
+}
+button:focus,
+button:focus-visible {
+	outline: 4px auto -webkit-focus-ring-color;
+}
+
+@media (prefers-color-scheme: light) {
+	:root {
+		color: #213547;
+		background-color: #ffffff;
+	}
+	a:hover {
+		color: #747bff;
+	}
+	button {
+		background-color: #f9f9f9;
+	}
+}