From 140b6210140c43e232bf039047084788cb9b039a Mon Sep 17 00:00:00 2001 From: Cong-Cong Date: Wed, 29 May 2024 12:58:01 +0800 Subject: [PATCH 1/3] Revert "Revert "feat: only run 1 time when not in scheduled job (#28)" (#29)" This reverts commit 843fe0e9caf3a11a6ddc683657570216946f1c70. --- .github/workflows/scheduled_bench.yml | 3 ++ bench.config.js | 78 ++++++++++++++++++++++----- bin/cli.js | 26 ++++----- docs/index.html | 2 +- docs/index.js | 4 +- lib/addons/hmr.js | 1 - lib/bench.js | 4 +- lib/compare.js | 17 +++--- lib/display.js | 2 +- lib/utils.js | 2 +- 10 files changed, 102 insertions(+), 37 deletions(-) diff --git a/.github/workflows/scheduled_bench.yml b/.github/workflows/scheduled_bench.yml index b35a5f27..e2c3b87e 100644 --- a/.github/workflows/scheduled_bench.yml +++ b/.github/workflows/scheduled_bench.yml @@ -11,6 +11,9 @@ jobs: - uses: actions/checkout@v4 - name: Init env uses: ./.github/actions/env + - name: Set Scheduled env + shell: bash + run: echo "SCHEDULED_JOB=true" >> $GITHUB_ENV - name: Build rspack run: node bin/cli.js build - name: Run benchmark diff --git a/bench.config.js b/bench.config.js index 7fb6c93e..7fcd434b 100644 --- a/bench.config.js +++ b/bench.config.js @@ -1,17 +1,71 @@ +const isScheduled = !!process.env.SCHEDULED_JOB; + +// HMR will run 10 times in build plugin, so we should not start multiple instances of Rspack. +// However, we still need to run multiple instances of Rspack when executing scheduled tasks for longer runtimes. +const hmrRuns = isScheduled ? 10 : 1; + export default { jobs: [ - "10000_development-mode", - "10000_development-mode_hmr", - "10000_production-mode", - "arco-pro_development-mode", - "arco-pro_development-mode_intercept-plugin", - "arco-pro_development-mode_hmr", - "arco-pro_development-mode_hmr_intercept-plugin", - "arco-pro_production-mode", - "arco-pro_production-mode_intercept-plugin", - "threejs_development-mode_10x", - "threejs_development-mode_10x_hmr", - "threejs_production-mode_10x" + { + name: "10000_development-mode", + runs: 10, + compareMetrics: ["exec"] + }, + { + name: "10000_development-mode_hmr", + runs: hmrRuns, + compareMetrics: ["stats"] + }, + { + name: "10000_production-mode", + runs: 10, + compareMetrics: ["exec"] + }, + { + name: "arco-pro_development-mode", + runs: 10, + compareMetrics: ["exec"] + }, + { + name: "arco-pro_development-mode_intercept-plugin", + runs: 10, + compareMetrics: ["exec"] + }, + { + name: "arco-pro_development-mode_hmr", + runs: hmrRuns, + compareMetrics: ["stats"] + }, + { + name: "arco-pro_development-mode_hmr_intercept-plugin", + runs: hmrRuns, + compareMetrics: ["stats"] + }, + { + name: "arco-pro_production-mode", + runs: 10, + compareMetrics: ["exec"] + }, + { + name: "arco-pro_production-mode_intercept-plugin", + runs: 10, + compareMetrics: ["exec"] + }, + { + name: "threejs_development-mode_10x", + runs: 10, + compareMetrics: ["exec"] + }, + { + name: "threejs_development-mode_10x_hmr", + runs: hmrRuns, + compareMetrics: ["stats"] + }, + { + name: "threejs_production-mode_10x", + runs: 10, + compareMetrics: ["exec"] + } ], rspackDirectory: process.env.RSPACK_DIR }; diff --git a/bin/cli.js b/bin/cli.js index 78a5effd..ac75e21f 100644 --- a/bin/cli.js +++ b/bin/cli.js @@ -68,12 +68,13 @@ const { const cwd = process.cwd(); -const configPath = join(process.cwd(), "bench.config.js"); -const config = (await import(configPath)).default; +const benchConfigPath = join(process.cwd(), "bench.config.js"); +const benchConfig = (await import(benchConfigPath)).default; -const jobs = config.jobs ?? []; -const rspackDirectory = config.rspackDirectory ?? join(cwd, ".rspack"); -const benchmarkDirectory = config.benchmarkDirectory ?? join(cwd, "output"); +const jobs = benchConfig.jobs ?? []; +const rspackDirectory = benchConfig.rspackDirectory ?? join(cwd, ".rspack"); +const benchmarkDirectory = + benchConfig.benchmarkDirectory ?? join(cwd, "output"); if (!command || command === "build") { const fetchUrl = `https://github.com/${repository}`; @@ -125,17 +126,18 @@ if (!command || command === "bench") { console.log( [ `Running jobs for shard ${currentIndex}/${totalShards}:`, - ...shardJobs + ...shardJobs.map(job => job.name) ].join("\n * ") ); for (const job of shardJobs) { const start = Date.now(); - const result = await run(job); + const result = await run(job.name, job.runs); + const message = `${job.name} was run ${job.runs} times, with the following results:`; if (isGitHubActions) { - actionsCore.startGroup(`${job} result is`); + actionsCore.startGroup(message); } else { - console.log(`${job} result is`); + console.log(message); } console.log(formatResultTable(result, { verbose: true })); @@ -143,11 +145,11 @@ if (!command || command === "bench") { if (isGitHubActions) { actionsCore.endGroup(); const cost = Math.ceil((Date.now() - start) / 1000); - console.log(`Cost for \`${job}\`: ${cost} s`); + console.log(`Cost for \`${job.name}\`: ${cost} s`); } await writeFile( - join(benchmarkDirectory, `${job}.json`), + join(benchmarkDirectory, `${job.name}.json`), JSON.stringify(result, null, 2) ); } @@ -157,5 +159,5 @@ if (!command || command === "bench") { } if (!command || command === "compare") { - compare(base, current, benchmarkDirectory); + compare(base, current, benchmarkDirectory, jobs); } diff --git a/docs/index.html b/docs/index.html index 010628b6..9ffb16bd 100644 --- a/docs/index.html +++ b/docs/index.html @@ -1,4 +1,4 @@ - + rspack/benchmark diff --git a/docs/index.js b/docs/index.js index c8450459..220d6c6c 100644 --- a/docs/index.js +++ b/docs/index.js @@ -329,8 +329,8 @@ class BenchmarkChart { context.dataset.yAxisID === "size" ? formatSize(value, value) : context.dataset.yAxisID === "ratio" - ? formatRatio(value, value) - : formatTime(value, value); + ? formatRatio(value, value) + : formatTime(value, value); return `${context.dataset.label}: ${text}`; } } diff --git a/lib/addons/hmr.js b/lib/addons/hmr.js index 46ea675d..037f4dec 100644 --- a/lib/addons/hmr.js +++ b/lib/addons/hmr.js @@ -3,7 +3,6 @@ import { Addon } from "./common.js"; export default class extends Addon { async afterSetup(ctx) { ctx.rspackArgs.push("--watch"); - ctx.runTimes = 5; ctx.config = ctx.config + ` diff --git a/lib/bench.js b/lib/bench.js index 2b96fb2f..bfa38f70 100644 --- a/lib/bench.js +++ b/lib/bench.js @@ -5,7 +5,7 @@ import { useAddons, dirExist } from "./utils.js"; const dirname = path.resolve(fileURLToPath(import.meta.url), ".."); -export async function run(benchmarkName) { +export async function run(benchmarkName, runs) { const [caseName, ...addonNames] = benchmarkName.split("_"); const scenario = getScenario(caseName); const addons = await Promise.all( @@ -20,6 +20,8 @@ export async function run(benchmarkName) { await useAddons(addons, "beforeSetup"); const ctx = await scenario.setup(); + ctx.runTimes = runs; + await useAddons(addons, "afterSetup", ctx); try { diff --git a/lib/compare.js b/lib/compare.js index 37aa6a4e..16949f00 100644 --- a/lib/compare.js +++ b/lib/compare.js @@ -71,9 +71,7 @@ async function getResults(date, index, benchmarkDirectory) { ); } -export async function compare(base, current, benchmarkDirectory) { - const compareMetric = ["exec"]; - +export async function compare(base, current, benchmarkDirectory, jobs) { const index = await fetchIndex(); if (base === "latest") { base = index[index.length - 1].date; @@ -86,13 +84,20 @@ export async function compare(base, current, benchmarkDirectory) { ]); const baseData = {}; const currentData = {}; - for (const metric of compareMetric) { - for (const { name, result } of baseResults) { + + for (const { name, result } of baseResults) { + const job = jobs.find(job => job.name === name); + const compareMetrics = job ? job.compareMetrics : []; + for (const metric of compareMetrics) { const tag = `${name} + ${metric}`; baseData[tag] = result[metric]; } + } - for (const { name, result } of currentResults) { + for (const { name, result } of currentResults) { + const job = jobs.find(job => job.name === name); + const compareMetrics = job ? job.compareMetrics : []; + for (const metric of compareMetrics) { const tag = `${name} + ${metric}`; currentData[tag] = result[metric]; } diff --git a/lib/display.js b/lib/display.js index 5b050b1d..e37f74fd 100644 --- a/lib/display.js +++ b/lib/display.js @@ -135,7 +135,7 @@ export function formatResultTable(result, { verbose, limit, threshold }) { con: l => f(l.name, l.confidence), con: l => f(l.name, l.confidence), n: l => `${l.count}` - } + } : undefined) }; return formatTable(entries, columns); diff --git a/lib/utils.js b/lib/utils.js index dec4d930..12b99cbe 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -21,7 +21,7 @@ export async function runCommand( ? { ...process.env, ...env - } + } : undefined }); if (hasOnData) { From 50e436e70c769a523045d39b327eba864b86c2d3 Mon Sep 17 00:00:00 2001 From: Cong-Cong Date: Thu, 30 May 2024 10:52:43 +0800 Subject: [PATCH 2/3] fix: watching.close() is slow --- lib/scenarios/build-plugin.cjs | 21 ++++++--------------- pnpm-lock.yaml | 4 ++++ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/lib/scenarios/build-plugin.cjs b/lib/scenarios/build-plugin.cjs index c2d08c53..48a944fc 100644 --- a/lib/scenarios/build-plugin.cjs +++ b/lib/scenarios/build-plugin.cjs @@ -1,5 +1,5 @@ const WARMUP_BUILDS = 10; -const TOTAL_BUILDS = 20; +const TOTAL_BUILDS = 110; module.exports = class BuildPlugin { apply(compiler) { @@ -8,22 +8,16 @@ module.exports = class BuildPlugin { compiler.hooks.watchRun.tap("BuildPlugin", () => { isWatching = true; }); - (compiler.hooks.afterDone || compiler.hooks.done).tap("BuildPlugin", () => { - setTimeout(() => { - if (counter === WARMUP_BUILDS) console.log("#!# start"); - if (isWatching && counter <= TOTAL_BUILDS) console.log("#!# next"); - }, 10); + compiler.hooks.afterDone.tap("BuildPlugin", () => { + if (counter === WARMUP_BUILDS) console.log("#!# start"); + if (isWatching && counter <= TOTAL_BUILDS) console.log("#!# next"); }); compiler.hooks.done.tap("BuildPlugin", stats => { if (isWatching) { counter++; if (counter <= WARMUP_BUILDS) return; if (counter > TOTAL_BUILDS) { - if (compiler.watching) { - compiler.watching.close(); - } else { - process.nextTick(() => process.exit(0)); - } + process.nextTick(() => process.exit(0)); } } const { logging, time } = stats.toJson({ @@ -69,10 +63,7 @@ module.exports = class BuildPlugin { if (type !== "time") return; const ms = args[1] * 1000 + args[2] / 1000000; console.log( - `#!# ${name}.${args[0].replace( - /restore cache content \d.+$/, - "restore cache content" - )} = ${ms}` + `#!# ${name}.${args[0].replace(/restore cache content \d.+$/, "restore cache content")} = ${ms}` ); }; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ed37658d..40120934 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32,6 +32,10 @@ importers: zx: specifier: ^8.1.1 version: 8.1.1 + devDependencies: + prettier: + specifier: ^2.6.2 + version: 2.8.8 cases/10000: {} From 78ddb95d3328f37fdad4e20d81d34033fd3363b7 Mon Sep 17 00:00:00 2001 From: Cong-Cong Date: Thu, 30 May 2024 11:29:28 +0800 Subject: [PATCH 3/3] fix: return data set when run rspack --- lib/scenarios/build-plugin.cjs | 2 +- lib/scenarios/index.js | 16 +++++++++++----- lib/scenarios/utils.js | 10 ++++++++-- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/lib/scenarios/build-plugin.cjs b/lib/scenarios/build-plugin.cjs index 48a944fc..226e73f5 100644 --- a/lib/scenarios/build-plugin.cjs +++ b/lib/scenarios/build-plugin.cjs @@ -1,5 +1,5 @@ const WARMUP_BUILDS = 10; -const TOTAL_BUILDS = 110; +const TOTAL_BUILDS = 20; module.exports = class BuildPlugin { apply(compiler) { diff --git a/lib/scenarios/index.js b/lib/scenarios/index.js index e95b8809..db4964c5 100644 --- a/lib/scenarios/index.js +++ b/lib/scenarios/index.js @@ -18,7 +18,8 @@ async function runRspack(ctx) { let counter = 0; let dataSetCounter = -1; let promise = Promise.resolve(); - const data = {}; + let data = {}; + const dataSet = []; const processLine = line => { if (line === "#!# start") { start = Date.now(); @@ -26,7 +27,11 @@ async function runRspack(ctx) { } else if (line === "#!# next") { promise = promise.then(async () => { counter++; - if (dataSetCounter >= 0) dataSetCounter++; + if (dataSetCounter >= 0) { + dataSetCounter++; + data = {}; + dataSet.push(data); + } await new Promise(r => setTimeout(r, Math.max(300, 1000 / counter))); for (const item of ctx.hmrConfig) { const content = item.generateContent( @@ -38,7 +43,7 @@ async function runRspack(ctx) { }); } else if (line.startsWith("#!#")) { const [, name, valueStr] = /^#!# (.+) = ((\d|\.|e|-)+)$/.exec(line); - data[name] = (data[name] || 0) + +valueStr; + data[name] = (data[name] || 0) + Number(valueStr); } }; let remainingLine = ""; @@ -63,7 +68,7 @@ async function runRspack(ctx) { } } data["dist size"] = await getDirSizes("dist"); - return data; + return dataSet; } export function getScenario(caseName) { @@ -137,7 +142,8 @@ module.exports.plugins.push(new (require("../../lib/scenarios/build-plugin.cjs") for (let i = 0; i < ctx.runTimes; i++) { await clearCaches(ctx.caseDir); - ctx.runData.push(await runRspack(ctx)); + const result = await runRspack(ctx); + ctx.runData.push(...result); const runtime = Date.now() - start; if (runtime > ctx.timeout) break; diff --git a/lib/scenarios/utils.js b/lib/scenarios/utils.js index 6fcae086..94c1b031 100644 --- a/lib/scenarios/utils.js +++ b/lib/scenarios/utils.js @@ -68,8 +68,14 @@ const tDist95Two = n => { }; export function calcStatistics(data) { const stats = {}; - for (const key of Object.keys(data[0])) { - const values = data.map(r => r[key] || 0); + const keys = Array.from(data.reduce((set, item) => { + for (const key of Object.keys(item)) { + set.add(key); + } + return set; + }, new Set())); + for (const key of keys) { + const values = data.map(r => r[key]).filter(d => typeof d === 'number'); if (typeof values[0] === "object") { stats[key] = calcStatistics(values); } else {