perf: new sample size

.github/workflows/scheduled_bench.yml

@@ -11,6 +11,9 @@ jobs:
       - uses: actions/checkout@v4
       - name: Init env
         uses: ./.github/actions/env
+      - name: Set Scheduled env
+        shell: bash
+        run: echo "SCHEDULED_JOB=true" >> $GITHUB_ENV
       - name: Build rspack
         run: node bin/cli.js build
       - name: Run benchmark

bench.config.js

@@ -1,17 +1,71 @@
+const isScheduled = !!process.env.SCHEDULED_JOB;
+
+// HMR will run 10 times in build plugin, so we should not start multiple instances of Rspack.
+// However, we still need to run multiple instances of Rspack when executing scheduled tasks for longer runtimes.
+const hmrRuns = isScheduled ? 10 : 1;
+
 export default {
 	jobs: [
-		"10000_development-mode",
-		"10000_development-mode_hmr",
-		"10000_production-mode",
-		"arco-pro_development-mode",
-		"arco-pro_development-mode_intercept-plugin",
-		"arco-pro_development-mode_hmr",
-		"arco-pro_development-mode_hmr_intercept-plugin",
-		"arco-pro_production-mode",
-		"arco-pro_production-mode_intercept-plugin",
-		"threejs_development-mode_10x",
-		"threejs_development-mode_10x_hmr",
-		"threejs_production-mode_10x"
+		{
+			name: "10000_development-mode",
+			runs: 10,
+			compareMetrics: ["exec"]
+		},
+		{
+			name: "10000_development-mode_hmr",
+			runs: hmrRuns,
+			compareMetrics: ["stats"]
+		},
+		{
+			name: "10000_production-mode",
+			runs: 10,
+			compareMetrics: ["exec"]
+		},
+		{
+			name: "arco-pro_development-mode",
+			runs: 10,
+			compareMetrics: ["exec"]
+		},
+		{
+			name: "arco-pro_development-mode_intercept-plugin",
+			runs: 10,
+			compareMetrics: ["exec"]
+		},
+		{
+			name: "arco-pro_development-mode_hmr",
+			runs: hmrRuns,
+			compareMetrics: ["stats"]
+		},
+		{
+			name: "arco-pro_development-mode_hmr_intercept-plugin",
+			runs: hmrRuns,
+			compareMetrics: ["stats"]
+		},
+		{
+			name: "arco-pro_production-mode",
+			runs: 10,
+			compareMetrics: ["exec"]
+		},
+		{
+			name: "arco-pro_production-mode_intercept-plugin",
+			runs: 10,
+			compareMetrics: ["exec"]
+		},
+		{
+			name: "threejs_development-mode_10x",
+			runs: 10,
+			compareMetrics: ["exec"]
+		},
+		{
+			name: "threejs_development-mode_10x_hmr",
+			runs: hmrRuns,
+			compareMetrics: ["stats"]
+		},
+		{
+			name: "threejs_production-mode_10x",
+			runs: 10,
+			compareMetrics: ["exec"]
+		}
 	],
 	rspackDirectory: process.env.RSPACK_DIR
 };

bin/cli.js

@@ -68,12 +68,13 @@ const {
 
 const cwd = process.cwd();
 
-const configPath = join(process.cwd(), "bench.config.js");
-const config = (await import(configPath)).default;
+const benchConfigPath = join(process.cwd(), "bench.config.js");
+const benchConfig = (await import(benchConfigPath)).default;
 
-const jobs = config.jobs ?? [];
-const rspackDirectory = config.rspackDirectory ?? join(cwd, ".rspack");
-const benchmarkDirectory = config.benchmarkDirectory ?? join(cwd, "output");
+const jobs = benchConfig.jobs ?? [];
+const rspackDirectory = benchConfig.rspackDirectory ?? join(cwd, ".rspack");
+const benchmarkDirectory =
+	benchConfig.benchmarkDirectory ?? join(cwd, "output");
 
 if (!command || command === "build") {
 	const fetchUrl = `https://github.com/${repository}`;
@@ -125,29 +126,30 @@ if (!command || command === "bench") {
 		console.log(
 			[
 				`Running jobs for shard ${currentIndex}/${totalShards}:`,
-				...shardJobs
+				...shardJobs.map(job => job.name)
 			].join("\n  * ")
 		);
 
 		for (const job of shardJobs) {
 			const start = Date.now();
-			const result = await run(job);
+			const result = await run(job.name, job.runs);
+			const message = `${job.name} was run ${job.runs} times, with the following results:`;
 			if (isGitHubActions) {
-				actionsCore.startGroup(`${job} result is`);
+				actionsCore.startGroup(message);
 			} else {
-				console.log(`${job} result is`);
+				console.log(message);
 			}
 
 			console.log(formatResultTable(result, { verbose: true }));
 
 			if (isGitHubActions) {
 				actionsCore.endGroup();
 				const cost = Math.ceil((Date.now() - start) / 1000);
-				console.log(`Cost for \`${job}\`: ${cost} s`);
+				console.log(`Cost for \`${job.name}\`: ${cost} s`);
 			}
 
 			await writeFile(
-				join(benchmarkDirectory, `${job}.json`),
+				join(benchmarkDirectory, `${job.name}.json`),
 				JSON.stringify(result, null, 2)
 			);
 		}
@@ -157,5 +159,5 @@ if (!command || command === "bench") {
 }
 
 if (!command || command === "compare") {
-	compare(base, current, benchmarkDirectory);
+	compare(base, current, benchmarkDirectory, jobs);
 }

docs/index.html

@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!doctype html>
 <html>
 	<head>
 		<title>rspack/benchmark</title>

docs/index.js

@@ -329,8 +329,8 @@ class BenchmarkChart {
 									context.dataset.yAxisID === "size"
 										? formatSize(value, value)
 										: context.dataset.yAxisID === "ratio"
-										? formatRatio(value, value)
-										: formatTime(value, value);
+											? formatRatio(value, value)
+											: formatTime(value, value);
 								return `${context.dataset.label}: ${text}`;
 							}
 						}

lib/addons/hmr.js

@@ -3,7 +3,6 @@ import { Addon } from "./common.js";
 export default class extends Addon {
 	async afterSetup(ctx) {
 		ctx.rspackArgs.push("--watch");
-		ctx.runTimes = 5;
 		ctx.config =
 			ctx.config +
 			`

lib/bench.js

@@ -5,7 +5,7 @@ import { useAddons, dirExist } from "./utils.js";
 
 const dirname = path.resolve(fileURLToPath(import.meta.url), "..");
 
-export async function run(benchmarkName) {
+export async function run(benchmarkName, runs) {
 	const [caseName, ...addonNames] = benchmarkName.split("_");
 	const scenario = getScenario(caseName);
 	const addons = await Promise.all(
@@ -20,6 +20,8 @@ export async function run(benchmarkName) {
 
 	await useAddons(addons, "beforeSetup");
 	const ctx = await scenario.setup();
+	ctx.runTimes = runs;
+
 	await useAddons(addons, "afterSetup", ctx);
 
 	try {

lib/compare.js

@@ -71,9 +71,7 @@ async function getResults(date, index, benchmarkDirectory) {
 	);
 }
 
-export async function compare(base, current, benchmarkDirectory) {
-	const compareMetric = ["exec"];
-
+export async function compare(base, current, benchmarkDirectory, jobs) {
 	const index = await fetchIndex();
 	if (base === "latest") {
 		base = index[index.length - 1].date;
@@ -86,13 +84,20 @@ export async function compare(base, current, benchmarkDirectory) {
 	]);
 	const baseData = {};
 	const currentData = {};
-	for (const metric of compareMetric) {
-		for (const { name, result } of baseResults) {
+
+	for (const { name, result } of baseResults) {
+		const job = jobs.find(job => job.name === name);
+		const compareMetrics = job ? job.compareMetrics : [];
+		for (const metric of compareMetrics) {
 			const tag = `${name} + ${metric}`;
 			baseData[tag] = result[metric];
 		}
+	}
 
-		for (const { name, result } of currentResults) {
+	for (const { name, result } of currentResults) {
+		const job = jobs.find(job => job.name === name);
+		const compareMetrics = job ? job.compareMetrics : [];
+		for (const metric of compareMetrics) {
 			const tag = `${name} + ${metric}`;
 			currentData[tag] = result[metric];
 		}

lib/display.js

@@ -135,7 +135,7 @@ export function formatResultTable(result, { verbose, limit, threshold }) {
 					con: l => f(l.name, l.confidence),
 					con: l => f(l.name, l.confidence),
 					n: l => `${l.count}`
-			  }
+				}
 			: undefined)
 	};
 	return formatTable(entries, columns);

lib/scenarios/build-plugin.cjs

@@ -8,22 +8,16 @@ module.exports = class BuildPlugin {
 		compiler.hooks.watchRun.tap("BuildPlugin", () => {
 			isWatching = true;
 		});
-		(compiler.hooks.afterDone || compiler.hooks.done).tap("BuildPlugin", () => {
-			setTimeout(() => {
-				if (counter === WARMUP_BUILDS) console.log("#!# start");
-				if (isWatching && counter <= TOTAL_BUILDS) console.log("#!# next");
-			}, 10);
+		compiler.hooks.afterDone.tap("BuildPlugin", () => {
+			if (counter === WARMUP_BUILDS) console.log("#!# start");
+			if (isWatching && counter <= TOTAL_BUILDS) console.log("#!# next");
 		});
 		compiler.hooks.done.tap("BuildPlugin", stats => {
 			if (isWatching) {
 				counter++;
 				if (counter <= WARMUP_BUILDS) return;
 				if (counter > TOTAL_BUILDS) {
-					if (compiler.watching) {
-						compiler.watching.close();
-					} else {
-						process.nextTick(() => process.exit(0));
-					}
+					process.nextTick(() => process.exit(0));
 				}
 			}
 			const { logging, time } = stats.toJson({
@@ -69,10 +63,7 @@ module.exports = class BuildPlugin {
 			if (type !== "time") return;
 			const ms = args[1] * 1000 + args[2] / 1000000;
 			console.log(
-				`#!# ${name}.${args[0].replace(
-					/restore cache content \d.+$/,
-					"restore cache content"
-				)} = ${ms}`
+				`#!# ${name}.${args[0].replace(/restore cache content \d.+$/, "restore cache content")} = ${ms}`
 			);
 		};
 	}

lib/scenarios/index.js

@@ -18,15 +18,20 @@ async function runRspack(ctx) {
 	let counter = 0;
 	let dataSetCounter = -1;
 	let promise = Promise.resolve();
-	const data = {};
+	let data = {};
+	const dataSet = [];
 	const processLine = line => {
 		if (line === "#!# start") {
 			start = Date.now();
 			dataSetCounter = 0;
 		} else if (line === "#!# next") {
 			promise = promise.then(async () => {
 				counter++;
-				if (dataSetCounter >= 0) dataSetCounter++;
+				if (dataSetCounter >= 0) {
+					dataSetCounter++;
+					data = {};
+					dataSet.push(data);
+				}
 				await new Promise(r => setTimeout(r, Math.max(300, 1000 / counter)));
 				for (const item of ctx.hmrConfig) {
 					const content = item.generateContent(
@@ -38,7 +43,7 @@ async function runRspack(ctx) {
 			});
 		} else if (line.startsWith("#!#")) {
 			const [, name, valueStr] = /^#!# (.+) = ((\d|\.|e|-)+)$/.exec(line);
-			data[name] = (data[name] || 0) + +valueStr;
+			data[name] = (data[name] || 0) + Number(valueStr);
 		}
 	};
 	let remainingLine = "";
@@ -63,7 +68,7 @@ async function runRspack(ctx) {
 		}
 	}
 	data["dist size"] = await getDirSizes("dist");
-	return data;
+	return dataSet;
 }
 
 export function getScenario(caseName) {
@@ -137,7 +142,8 @@ module.exports.plugins.push(new (require("../../lib/scenarios/build-plugin.cjs")
 			for (let i = 0; i < ctx.runTimes; i++) {
 				await clearCaches(ctx.caseDir);
 
-				ctx.runData.push(await runRspack(ctx));
+				const result = await runRspack(ctx);
+				ctx.runData.push(...result);
 
 				const runtime = Date.now() - start;
 				if (runtime > ctx.timeout) break;

lib/scenarios/utils.js

@@ -68,8 +68,14 @@ const tDist95Two = n => {
 };
 export function calcStatistics(data) {
 	const stats = {};
-	for (const key of Object.keys(data[0])) {
-		const values = data.map(r => r[key] || 0);
+	const keys = Array.from(data.reduce((set, item) => {
+		for (const key of Object.keys(item)) {
+			set.add(key);
+		}
+		return set;
+	}, new Set()));
+	for (const key of keys) {
+		const values = data.map(r => r[key]).filter(d => typeof d === 'number');
 		if (typeof values[0] === "object") {
 			stats[key] = calcStatistics(values);
 		} else {

lib/utils.js

@@ -21,7 +21,7 @@ export async function runCommand(
 			? {
 					...process.env,
 					...env
-			  }
+				}
 			: undefined
 	});
 	if (hasOnData) {