From 948253f5d282eccff71c84e226b159203c62a7a3 Mon Sep 17 00:00:00 2001 From: kirillzyusko Date: Tue, 11 Jun 2024 16:26:55 +0200 Subject: [PATCH 1/9] e2e: new common metrics (FPS, CPU, RAM) --- package-lock.json | 122 ++++++++++++++++++++ package.json | 3 + patches/@perf-profiler+reporter+0.8.1.patch | 25 ++++ tests/e2e/server/index.ts | 11 +- tests/e2e/testRunner.ts | 64 +++++++--- tests/e2e/utils/measure.ts | 48 ++++++++ 6 files changed, 254 insertions(+), 19 deletions(-) create mode 100644 patches/@perf-profiler+reporter+0.8.1.patch create mode 100644 tests/e2e/utils/measure.ts diff --git a/package-lock.json b/package-lock.json index 9af174f0846b..4fd7d999268f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -160,6 +160,9 @@ "@octokit/core": "4.0.4", "@octokit/plugin-paginate-rest": "3.1.0", "@octokit/plugin-throttling": "4.1.0", + "@perf-profiler/profiler": "^0.10.9", + "@perf-profiler/reporter": "^0.8.1", + "@perf-profiler/types": "^0.8.0", "@react-native-community/eslint-config": "3.2.0", "@react-native/babel-preset": "^0.73.21", "@react-native/metro-config": "^0.73.5", @@ -7668,6 +7671,116 @@ "react-native": ">=0.70.0 <1.0.x" } }, + "node_modules/@perf-profiler/android": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@perf-profiler/android/-/android-0.12.0.tgz", + "integrity": "sha512-wLI3D63drtqw3p7aKci+LCtN/ZipLJQvcw8cfmhwxqqRxTraFa8lDz5CNvNsqtCI7Zl0N9VRtnDMOj4e1W1yMQ==", + "dev": true, + "dependencies": { + "@perf-profiler/logger": "^0.3.3", + "@perf-profiler/profiler": "^0.10.9", + "@perf-profiler/types": "^0.8.0", + "commander": "^12.0.0", + "lodash": "^4.17.21" + }, + "bin": { + "perf-profiler-commands": "dist/src/commands.js" + } + }, + "node_modules/@perf-profiler/android/node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "dev": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/@perf-profiler/ios": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@perf-profiler/ios/-/ios-0.3.1.tgz", + "integrity": "sha512-zRAgxLuCHzo47SYynljf+Aplh2K4DMwJ4dqIU30P8uPHiV5yHjE83eH+sTD6I7jUnUvZ8qAO1dhvp6ATJEpP/Q==", + "dev": true, + "dependencies": { + "@perf-profiler/ios-instruments": "^0.3.1", + "@perf-profiler/logger": "^0.3.3", + "@perf-profiler/types": "^0.8.0" + } + }, + "node_modules/@perf-profiler/ios-instruments": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@perf-profiler/ios-instruments/-/ios-instruments-0.3.1.tgz", + "integrity": "sha512-6ZiN9QTmIT8N37SslzjYNk+4+FX0X4IVuM/KiJF/DVgs056CT3MRDF8FFKF17BHsDJBi2a25QkegU8+AQdh+Qg==", + "dev": true, + "dependencies": { + "@perf-profiler/logger": "^0.3.3", + "@perf-profiler/profiler": "^0.10.9", + "@perf-profiler/types": "^0.8.0", + "commander": "^12.0.0", + "fast-xml-parser": "^4.2.7" + }, + "bin": { + "flashlight-ios-poc": "dist/launchIOS.js" + } + }, + "node_modules/@perf-profiler/ios-instruments/node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "dev": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/@perf-profiler/logger": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@perf-profiler/logger/-/logger-0.3.3.tgz", + "integrity": "sha512-iAJJ5gWhJ3zEpdMT7M2+HX0Q0UjSuCOZiEs5g8UKKPFYQjmPWwC6otHoZz6ZzRRddjiA065iD2PTytVFkpFTeQ==", + "dev": true, + "dependencies": { + "kleur": "^4.1.5", + "luxon": "^3.4.4" + }, + "bin": { + "perf-profiler-logger": "dist/bin.js" + } + }, + "node_modules/@perf-profiler/logger/node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/@perf-profiler/profiler": { + "version": "0.10.9", + "resolved": "https://registry.npmjs.org/@perf-profiler/profiler/-/profiler-0.10.9.tgz", + "integrity": "sha512-jhkFyqsrmkI9gCYmK7+R1e+vjWGw2a2YnqruRAUk71saOkLLvRSWKnT0MiGMqzi0aQj//ojeW9viDJgxQB86zg==", + "dev": true, + "dependencies": { + "@perf-profiler/android": "^0.12.0", + "@perf-profiler/ios": "^0.3.1", + "@perf-profiler/types": "^0.8.0" + } + }, + "node_modules/@perf-profiler/reporter": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@perf-profiler/reporter/-/reporter-0.8.1.tgz", + "integrity": "sha512-lZp17uMMLAV4nuDO0JbajbPCyOoD4/ugnZVxsOEEueRo8mxB26TS3R7ANtMZYjHrpQbJry0CgfTIPxflBgtq4A==", + "dev": true, + "dependencies": { + "@perf-profiler/types": "^0.8.0", + "lodash": "^4.17.21" + } + }, + "node_modules/@perf-profiler/types": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@perf-profiler/types/-/types-0.8.0.tgz", + "integrity": "sha512-TFiktv00SzLjjPp1hFYYjT9O36iGIUaF6yPLd7x/UT4CuLd0YYDUj+gvX0fbXtVtV7141tTvWbXFL5HiXGx0kw==", + "dev": true + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "license": "MIT", @@ -27660,6 +27773,15 @@ "node": ">=10" } }, + "node_modules/luxon": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", + "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==", + "dev": true, + "engines": { + "node": ">=12" + } + }, "node_modules/magic-string": { "version": "0.30.9", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.9.tgz", diff --git a/package.json b/package.json index 3d679bd79557..73be247cf568 100644 --- a/package.json +++ b/package.json @@ -213,6 +213,9 @@ "@octokit/core": "4.0.4", "@octokit/plugin-paginate-rest": "3.1.0", "@octokit/plugin-throttling": "4.1.0", + "@perf-profiler/profiler": "^0.10.9", + "@perf-profiler/reporter": "^0.8.1", + "@perf-profiler/types": "^0.8.0", "@react-native-community/eslint-config": "3.2.0", "@react-native/babel-preset": "^0.73.21", "@react-native/metro-config": "^0.73.5", diff --git a/patches/@perf-profiler+reporter+0.8.1.patch b/patches/@perf-profiler+reporter+0.8.1.patch new file mode 100644 index 000000000000..2c918b4049c2 --- /dev/null +++ b/patches/@perf-profiler+reporter+0.8.1.patch @@ -0,0 +1,25 @@ +diff --git a/node_modules/@perf-profiler/reporter/dist/src/index.d.ts b/node_modules/@perf-profiler/reporter/dist/src/index.d.ts +index 2f84d84..14ae688 100644 +--- a/node_modules/@perf-profiler/reporter/dist/src/index.d.ts ++++ b/node_modules/@perf-profiler/reporter/dist/src/index.d.ts +@@ -4,4 +4,6 @@ export * from "./reporting/Report"; + export * from "./utils/sanitizeProcessName"; + export * from "./utils/round"; + export * from "./reporting/cpu"; ++export * from "./reporting/ram"; ++export * from "./reporting/fps"; + export { canComputeHighCpuUsage } from "./reporting/highCpu"; +diff --git a/node_modules/@perf-profiler/reporter/dist/src/index.js b/node_modules/@perf-profiler/reporter/dist/src/index.js +index 4b50e3a..780963a 100644 +--- a/node_modules/@perf-profiler/reporter/dist/src/index.js ++++ b/node_modules/@perf-profiler/reporter/dist/src/index.js +@@ -21,6 +21,8 @@ __exportStar(require("./reporting/Report"), exports); + __exportStar(require("./utils/sanitizeProcessName"), exports); + __exportStar(require("./utils/round"), exports); + __exportStar(require("./reporting/cpu"), exports); ++__exportStar(require("./reporting/fps"), exports); ++__exportStar(require("./reporting/ram"), exports); + var highCpu_1 = require("./reporting/highCpu"); + Object.defineProperty(exports, "canComputeHighCpuUsage", { enumerable: true, get: function () { return highCpu_1.canComputeHighCpuUsage; } }); + //# sourceMappingURL=index.js.map +\ No newline at end of file diff --git a/tests/e2e/server/index.ts b/tests/e2e/server/index.ts index 68367aa29c2a..228a69626e48 100644 --- a/tests/e2e/server/index.ts +++ b/tests/e2e/server/index.ts @@ -20,10 +20,11 @@ type TestDoneListener = () => void; type TestResultListener = (testResult: TestResult) => void; -type AddListener = (listener: TListener) => void; +type AddListener = (listener: TListener) => () => void; type ServerInstance = { setTestConfig: (testConfig: TestConfig) => void; + getTestConfig: () => TestConfig; addTestStartedListener: AddListener; addTestResultListener: AddListener; addTestDoneListener: AddListener; @@ -115,6 +116,13 @@ const createServerInstance = (): ServerInstance => { const setTestConfig = (testConfig: TestConfig) => { activeTestConfig = testConfig; }; + const getTestConfig = (): TestConfig => { + if (!activeTestConfig) { + throw new Error('No test config set'); + } + + return activeTestConfig; + }; const server = createServer((req, res): ServerResponse | void => { res.statusCode = 200; @@ -213,6 +221,7 @@ const createServerInstance = (): ServerInstance => { return { setReadyToAcceptTestResults, setTestConfig, + getTestConfig, addTestStartedListener, addTestResultListener, addTestDoneListener, diff --git a/tests/e2e/testRunner.ts b/tests/e2e/testRunner.ts index a8295f6ddf5c..01249e836dd2 100644 --- a/tests/e2e/testRunner.ts +++ b/tests/e2e/testRunner.ts @@ -16,6 +16,7 @@ /* eslint-disable @lwc/lwc/no-async-await,no-restricted-syntax,no-await-in-loop */ import {execSync} from 'child_process'; import fs from 'fs'; +import type {TestResult} from '@libs/E2E/client'; import type {TestConfig} from '@libs/E2E/types'; import compare from './compare/compare'; import defaultConfig from './config'; @@ -25,6 +26,7 @@ import installApp from './utils/installApp'; import killApp from './utils/killApp'; import launchApp from './utils/launchApp'; import * as Logger from './utils/logger'; +import {start, stop} from './utils/measure'; import sleep from './utils/sleep'; import withFailTimeout from './utils/withFailTimeout'; @@ -91,18 +93,7 @@ const runTests = async (): Promise => { // Create a dict in which we will store the run durations for all tests const results: Record = {}; - // Collect results while tests are being executed - server.addTestResultListener((testResult) => { - const {isCritical = true} = testResult; - - if (testResult?.error != null && isCritical) { - throw new Error(`Test '${testResult.name}' failed with error: ${testResult.error}`); - } - if (testResult?.error != null && !isCritical) { - // force test completion, since we don't want to have timeout error for non being execute test - server.forceTestCompletion(); - Logger.warn(`Test '${testResult.name}' failed with error: ${testResult.error}`); - } + const attachTestResult = (testResult: TestResult) => { let result = 0; if (testResult?.duration !== undefined) { @@ -124,10 +115,26 @@ const runTests = async (): Promise => { if (testResult?.branch && testResult?.name) { results[testResult.branch][testResult.name] = (results[testResult.branch][testResult.name] ?? []).concat(result); } + }; + + // Collect results while tests are being executed + server.addTestResultListener((testResult) => { + const {isCritical = true} = testResult; + + if (testResult?.error != null && isCritical) { + throw new Error(`Test '${testResult.name}' failed with error: ${testResult.error}`); + } + if (testResult?.error != null && !isCritical) { + // force test completion, since we don't want to have timeout error for non being execute test + server.forceTestCompletion(); + Logger.warn(`Test '${testResult.name}' failed with error: ${testResult.error}`); + } + + attachTestResult(testResult); }); // Function to run a single test iteration - async function runTestIteration(appPackage: string, iterationText: string, launchArgs: Record = {}): Promise { + async function runTestIteration(appPackage: string, iterationText: string, branch: 'main' | 'delta', launchArgs: Record = {}): Promise { Logger.info(iterationText); // Making sure the app is really killed (e.g. if a prior test run crashed) @@ -137,10 +144,31 @@ const runTests = async (): Promise => { Logger.log('Launching', appPackage); await launchApp('android', appPackage, config.ACTIVITY_PATH, launchArgs); + start(appPackage); await withFailTimeout( new Promise((resolve) => { - server.addTestDoneListener(() => { + const subscription = server.addTestDoneListener(() => { Logger.success(iterationText); + + const metrics = stop(); + const test = server.getTestConfig(); + + attachTestResult({ + name: `${test.name} (CPU)`, + branch, + duration: metrics.cpu, + }); + attachTestResult({ + name: `${test.name} (FPS)`, + branch, + duration: metrics.fps, + }); + attachTestResult({ + name: `${test.name} (RAM)`, + branch, + duration: metrics.ram, + }); + subscription(); resolve(); }); }), @@ -188,10 +216,10 @@ const runTests = async (): Promise => { const iterations = 2; for (let i = 0; i < iterations; i++) { // Warmup the main app: - await runTestIteration(config.MAIN_APP_PACKAGE, `[MAIN] ${warmupText}. Iteration ${i + 1}/${iterations}`); + await runTestIteration(config.MAIN_APP_PACKAGE, `[MAIN] ${warmupText}. Iteration ${i + 1}/${iterations}`, 'main'); // Warmup the delta app: - await runTestIteration(config.DELTA_APP_PACKAGE, `[DELTA] ${warmupText}. Iteration ${i + 1}/${iterations}`); + await runTestIteration(config.DELTA_APP_PACKAGE, `[DELTA] ${warmupText}. Iteration ${i + 1}/${iterations}`, 'delta'); } server.setReadyToAcceptTestResults(true); @@ -226,10 +254,10 @@ const runTests = async (): Promise => { const deltaIterationText = `[DELTA] ${iterationText}`; try { // Run the test on the main app: - await runTestIteration(config.MAIN_APP_PACKAGE, mainIterationText, launchArgs); + await runTestIteration(config.MAIN_APP_PACKAGE, mainIterationText, 'main', launchArgs); // Run the test on the delta app: - await runTestIteration(config.DELTA_APP_PACKAGE, deltaIterationText, launchArgs); + await runTestIteration(config.DELTA_APP_PACKAGE, deltaIterationText, 'delta', launchArgs); } catch (e) { onError(e as Error); } diff --git a/tests/e2e/utils/measure.ts b/tests/e2e/utils/measure.ts new file mode 100644 index 000000000000..830dc5c8dd55 --- /dev/null +++ b/tests/e2e/utils/measure.ts @@ -0,0 +1,48 @@ +import {profiler} from '@perf-profiler/profiler'; +import {getAverageCpuUsage, getAverageFPSUsage, getAverageRAMUsage} from '@perf-profiler/reporter'; +import type {Measure} from '@perf-profiler/types'; +import noop from 'lodash/noop'; + +let measures: Measure[] = []; +let polling = { + stop: noop, +}; + +const start = (bundleId: string) => { + console.log(12121212, 'start'); + + // clear our measurements results + measures = []; + + console.log('11111!! clean'); + + polling = profiler.pollPerformanceMeasures(bundleId, { + onMeasure: (measure: Measure) => { + measures.push(measure); + + console.log(989898, measures.length); + // console.log(`JS Thread CPU Usage: ${measure.cpu.perName[ThreadNames.RN.JS_ANDROID]}%`); + // console.log(`RAM Usage: ${measure.ram}MB`); + }, + }); +}; + +const stop = () => { + polling.stop(); + + const cpu = getAverageCpuUsage(measures); + const fps = getAverageFPSUsage(measures); + const ram = getAverageRAMUsage(measures); + + console.log(`Average CPU Usage: ${cpu}%`); + console.log(`Average FPS: ${fps}`); + console.log(`Average RAM Usage: ${ram}MB`); + + return { + cpu, + fps, + ram, + }; +}; + +export {start, stop}; From 523ad7af849d895b819cf73a38c4688ec9f7163c Mon Sep 17 00:00:00 2001 From: kirillzyusko Date: Tue, 11 Jun 2024 16:58:00 +0200 Subject: [PATCH 2/9] fix: don't include warmup results into final report --- tests/e2e/testRunner.ts | 32 +++++++++++++++++--------------- tests/e2e/utils/measure.ts | 12 ------------ 2 files changed, 17 insertions(+), 27 deletions(-) diff --git a/tests/e2e/testRunner.ts b/tests/e2e/testRunner.ts index 01249e836dd2..6ac43ad69e9c 100644 --- a/tests/e2e/testRunner.ts +++ b/tests/e2e/testRunner.ts @@ -153,21 +153,23 @@ const runTests = async (): Promise => { const metrics = stop(); const test = server.getTestConfig(); - attachTestResult({ - name: `${test.name} (CPU)`, - branch, - duration: metrics.cpu, - }); - attachTestResult({ - name: `${test.name} (FPS)`, - branch, - duration: metrics.fps, - }); - attachTestResult({ - name: `${test.name} (RAM)`, - branch, - duration: metrics.ram, - }); + if (server.isReadyToAcceptTestResults) { + attachTestResult({ + name: `${test.name} (CPU)`, + branch, + duration: metrics.cpu, + }); + attachTestResult({ + name: `${test.name} (FPS)`, + branch, + duration: metrics.fps, + }); + attachTestResult({ + name: `${test.name} (RAM)`, + branch, + duration: metrics.ram, + }); + } subscription(); resolve(); }); diff --git a/tests/e2e/utils/measure.ts b/tests/e2e/utils/measure.ts index 830dc5c8dd55..9b072879743e 100644 --- a/tests/e2e/utils/measure.ts +++ b/tests/e2e/utils/measure.ts @@ -9,20 +9,12 @@ let polling = { }; const start = (bundleId: string) => { - console.log(12121212, 'start'); - // clear our measurements results measures = []; - console.log('11111!! clean'); - polling = profiler.pollPerformanceMeasures(bundleId, { onMeasure: (measure: Measure) => { measures.push(measure); - - console.log(989898, measures.length); - // console.log(`JS Thread CPU Usage: ${measure.cpu.perName[ThreadNames.RN.JS_ANDROID]}%`); - // console.log(`RAM Usage: ${measure.ram}MB`); }, }); }; @@ -34,10 +26,6 @@ const stop = () => { const fps = getAverageFPSUsage(measures); const ram = getAverageRAMUsage(measures); - console.log(`Average CPU Usage: ${cpu}%`); - console.log(`Average FPS: ${fps}`); - console.log(`Average RAM Usage: ${ram}MB`); - return { cpu, fps, From 4ee79b7ce562881881f87c7e7f424d32caec3e93 Mon Sep 17 00:00:00 2001 From: kirillzyusko Date: Wed, 12 Jun 2024 09:40:20 +0200 Subject: [PATCH 3/9] fix: crash if can not parse a frame --- patches/@perf-profiler+android+0.12.0.patch | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 patches/@perf-profiler+android+0.12.0.patch diff --git a/patches/@perf-profiler+android+0.12.0.patch b/patches/@perf-profiler+android+0.12.0.patch new file mode 100644 index 000000000000..1ff7c2d98b7a --- /dev/null +++ b/patches/@perf-profiler+android+0.12.0.patch @@ -0,0 +1,25 @@ +diff --git a/node_modules/@perf-profiler/android/src/commands/platforms/UnixProfiler.ts b/node_modules/@perf-profiler/android/src/commands/platforms/UnixProfiler.ts +index d6983c1..ccacf09 100644 +--- a/node_modules/@perf-profiler/android/src/commands/platforms/UnixProfiler.ts ++++ b/node_modules/@perf-profiler/android/src/commands/platforms/UnixProfiler.ts +@@ -136,7 +136,19 @@ export abstract class UnixProfiler implements Profiler { + const subProcessesStats = processOutput(cpu, pid); + + const ram = processRamOutput(ramStr, this.getRAMPageSize()); +- const { frameTimes, interval: atraceInterval } = frameTimeParser.getFrameTimes(atrace, pid); ++ ++ let output; ++ try { ++ output = frameTimeParser.getFrameTimes(atrace, pid); ++ } catch (e) { ++ console.error(e); ++ } ++ ++ if (!output) { ++ return; ++ } ++ ++ const { frameTimes, interval: atraceInterval } = output; + + if (!initialTime) { + initialTime = timestamp; From ca8e0d1bc0a9ee646cc5026c8ac0edbc4fb4268f Mon Sep 17 00:00:00 2001 From: kirillzyusko Date: Wed, 12 Jun 2024 18:16:11 +0200 Subject: [PATCH 4/9] fix: port changes to compiled js file --- patches/@perf-profiler+android+0.12.0.patch | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/patches/@perf-profiler+android+0.12.0.patch b/patches/@perf-profiler+android+0.12.0.patch index 1ff7c2d98b7a..f6ecbce9b481 100644 --- a/patches/@perf-profiler+android+0.12.0.patch +++ b/patches/@perf-profiler+android+0.12.0.patch @@ -1,3 +1,32 @@ +diff --git a/node_modules/@perf-profiler/android/dist/src/commands.js b/node_modules/@perf-profiler/android/dist/src/commands.js +old mode 100755 +new mode 100644 +diff --git a/node_modules/@perf-profiler/android/dist/src/commands/platforms/UnixProfiler.js b/node_modules/@perf-profiler/android/dist/src/commands/platforms/UnixProfiler.js +index 77b9ee0..59aeed9 100644 +--- a/node_modules/@perf-profiler/android/dist/src/commands/platforms/UnixProfiler.js ++++ b/node_modules/@perf-profiler/android/dist/src/commands/platforms/UnixProfiler.js +@@ -134,7 +134,20 @@ class UnixProfiler { + } + const subProcessesStats = (0, getCpuStatsByProcess_1.processOutput)(cpu, pid); + const ram = (0, pollRamUsage_1.processOutput)(ramStr, this.getRAMPageSize()); +- const { frameTimes, interval: atraceInterval } = frameTimeParser.getFrameTimes(atrace, pid); ++ ++ let output; ++ try { ++ output = frameTimeParser.getFrameTimes(atrace, pid); ++ } catch (e) { ++ console.error(e); ++ } ++ ++ if (!output) { ++ return; ++ } ++ ++ const { frameTimes, interval: atraceInterval } = output; ++ + if (!initialTime) { + initialTime = timestamp; + } diff --git a/node_modules/@perf-profiler/android/src/commands/platforms/UnixProfiler.ts b/node_modules/@perf-profiler/android/src/commands/platforms/UnixProfiler.ts index d6983c1..ccacf09 100644 --- a/node_modules/@perf-profiler/android/src/commands/platforms/UnixProfiler.ts From e053fa64bea4c39082727900397eb9b8a2ccfa30 Mon Sep 17 00:00:00 2001 From: kirillzyusko Date: Wed, 12 Jun 2024 18:16:40 +0200 Subject: [PATCH 5/9] fix: TS compilation --- tests/e2e/server/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/e2e/server/index.ts b/tests/e2e/server/index.ts index 228a69626e48..cd1bbeca2ba9 100644 --- a/tests/e2e/server/index.ts +++ b/tests/e2e/server/index.ts @@ -30,6 +30,7 @@ type ServerInstance = { addTestDoneListener: AddListener; forceTestCompletion: () => void; setReadyToAcceptTestResults: (isReady: boolean) => void; + isReadyToAcceptTestResults: boolean; start: () => Promise; stop: () => Promise; }; @@ -220,6 +221,9 @@ const createServerInstance = (): ServerInstance => { return { setReadyToAcceptTestResults, + get isReadyToAcceptTestResults() { + return isReadyToAcceptTestResults; + }, setTestConfig, getTestConfig, addTestStartedListener, From eda73caa11b2157c3409fb874c6944ab56cd3f6f Mon Sep 17 00:00:00 2001 From: kirillzyusko Date: Mon, 17 Jun 2024 10:59:01 +0300 Subject: [PATCH 6/9] fix: don't add results to delta if we run dev build --- tests/e2e/config.dev.ts | 2 ++ tests/e2e/config.ts | 3 +++ tests/e2e/testRunner.ts | 10 +++++----- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/e2e/config.dev.ts b/tests/e2e/config.dev.ts index cdd7bce756c8..a400cbf4787e 100644 --- a/tests/e2e/config.dev.ts +++ b/tests/e2e/config.dev.ts @@ -6,6 +6,8 @@ const appPath = './android/app/build/outputs/apk/development/debug/app-developme const config: Config = { MAIN_APP_PACKAGE: packageName, DELTA_APP_PACKAGE: packageName, + BRANCH_MAIN: 'main', + BRANCH_DELTA: 'main', MAIN_APP_PATH: appPath, DELTA_APP_PATH: appPath, RUNS: 8, diff --git a/tests/e2e/config.ts b/tests/e2e/config.ts index 6eb6bb839ae2..8963e07c31c8 100644 --- a/tests/e2e/config.ts +++ b/tests/e2e/config.ts @@ -26,6 +26,9 @@ export default { MAIN_APP_PATH: './app-e2eRelease.apk', DELTA_APP_PATH: './app-e2edeltaRelease.apk', + BRANCH_MAIN: 'main', + BRANCH_DELTA: 'delta', + ENTRY_FILE: 'src/libs/E2E/reactNativeLaunchingTest.ts', // The path to the activity within the app that we want to launch. diff --git a/tests/e2e/testRunner.ts b/tests/e2e/testRunner.ts index 6ac43ad69e9c..cc5f494b9154 100644 --- a/tests/e2e/testRunner.ts +++ b/tests/e2e/testRunner.ts @@ -134,7 +134,7 @@ const runTests = async (): Promise => { }); // Function to run a single test iteration - async function runTestIteration(appPackage: string, iterationText: string, branch: 'main' | 'delta', launchArgs: Record = {}): Promise { + async function runTestIteration(appPackage: string, iterationText: string, branch: string, launchArgs: Record = {}): Promise { Logger.info(iterationText); // Making sure the app is really killed (e.g. if a prior test run crashed) @@ -218,10 +218,10 @@ const runTests = async (): Promise => { const iterations = 2; for (let i = 0; i < iterations; i++) { // Warmup the main app: - await runTestIteration(config.MAIN_APP_PACKAGE, `[MAIN] ${warmupText}. Iteration ${i + 1}/${iterations}`, 'main'); + await runTestIteration(config.MAIN_APP_PACKAGE, `[MAIN] ${warmupText}. Iteration ${i + 1}/${iterations}`, config.BRANCH_MAIN); // Warmup the delta app: - await runTestIteration(config.DELTA_APP_PACKAGE, `[DELTA] ${warmupText}. Iteration ${i + 1}/${iterations}`, 'delta'); + await runTestIteration(config.DELTA_APP_PACKAGE, `[DELTA] ${warmupText}. Iteration ${i + 1}/${iterations}`, config.BRANCH_DELTA); } server.setReadyToAcceptTestResults(true); @@ -256,10 +256,10 @@ const runTests = async (): Promise => { const deltaIterationText = `[DELTA] ${iterationText}`; try { // Run the test on the main app: - await runTestIteration(config.MAIN_APP_PACKAGE, mainIterationText, 'main', launchArgs); + await runTestIteration(config.MAIN_APP_PACKAGE, mainIterationText, config.BRANCH_MAIN, launchArgs); // Run the test on the delta app: - await runTestIteration(config.DELTA_APP_PACKAGE, deltaIterationText, 'delta', launchArgs); + await runTestIteration(config.DELTA_APP_PACKAGE, deltaIterationText, config.BRANCH_DELTA, launchArgs); } catch (e) { onError(e as Error); } From f1788433da211048eecf836795f51849ed464f6a Mon Sep 17 00:00:00 2001 From: kirillzyusko Date: Tue, 18 Jun 2024 12:05:03 +0300 Subject: [PATCH 7/9] chore: fixes after code review --- tests/e2e/testRunner.ts | 10 +++++----- tests/e2e/utils/measure.ts | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/e2e/testRunner.ts b/tests/e2e/testRunner.ts index cc5f494b9154..b674ac418f3e 100644 --- a/tests/e2e/testRunner.ts +++ b/tests/e2e/testRunner.ts @@ -26,7 +26,7 @@ import installApp from './utils/installApp'; import killApp from './utils/killApp'; import launchApp from './utils/launchApp'; import * as Logger from './utils/logger'; -import {start, stop} from './utils/measure'; +import * as MeasureUtils from './utils/measure'; import sleep from './utils/sleep'; import withFailTimeout from './utils/withFailTimeout'; @@ -144,13 +144,13 @@ const runTests = async (): Promise => { Logger.log('Launching', appPackage); await launchApp('android', appPackage, config.ACTIVITY_PATH, launchArgs); - start(appPackage); + MeasureUtils.start(appPackage); await withFailTimeout( new Promise((resolve) => { - const subscription = server.addTestDoneListener(() => { + const removeListener = server.addTestDoneListener(() => { Logger.success(iterationText); - const metrics = stop(); + const metrics = MeasureUtils.stop(); const test = server.getTestConfig(); if (server.isReadyToAcceptTestResults) { @@ -170,7 +170,7 @@ const runTests = async (): Promise => { duration: metrics.ram, }); } - subscription(); + removeListener(); resolve(); }); }), diff --git a/tests/e2e/utils/measure.ts b/tests/e2e/utils/measure.ts index 9b072879743e..c7e7e1837e14 100644 --- a/tests/e2e/utils/measure.ts +++ b/tests/e2e/utils/measure.ts @@ -1,11 +1,12 @@ import {profiler} from '@perf-profiler/profiler'; import {getAverageCpuUsage, getAverageFPSUsage, getAverageRAMUsage} from '@perf-profiler/reporter'; import type {Measure} from '@perf-profiler/types'; -import noop from 'lodash/noop'; let measures: Measure[] = []; let polling = { - stop: noop, + stop: (): void => { + throw new Error('Cannot stop polling on a stopped profiler'); + }, }; const start = (bundleId: string) => { From eeaa4643911029ffc9964cedd48046bc408c707e Mon Sep 17 00:00:00 2001 From: kirillzyusko Date: Thu, 20 Jun 2024 12:25:19 +0300 Subject: [PATCH 8/9] feat: JS and UI thread metrics added --- tests/e2e/testRunner.ts | 10 ++++++++++ tests/e2e/utils/measure.ts | 9 ++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/e2e/testRunner.ts b/tests/e2e/testRunner.ts index b674ac418f3e..297cbbfcce57 100644 --- a/tests/e2e/testRunner.ts +++ b/tests/e2e/testRunner.ts @@ -169,6 +169,16 @@ const runTests = async (): Promise => { branch, duration: metrics.ram, }); + attachTestResult({ + name: `${test.name} (CPU/JS)`, + branch, + duration: metrics.jsThread, + }); + attachTestResult({ + name: `${test.name} (CPU/UI)`, + branch, + duration: metrics.uiThread, + }); } removeListener(); resolve(); diff --git a/tests/e2e/utils/measure.ts b/tests/e2e/utils/measure.ts index c7e7e1837e14..0a5aface56f2 100644 --- a/tests/e2e/utils/measure.ts +++ b/tests/e2e/utils/measure.ts @@ -1,5 +1,6 @@ import {profiler} from '@perf-profiler/profiler'; -import {getAverageCpuUsage, getAverageFPSUsage, getAverageRAMUsage} from '@perf-profiler/reporter'; +import {getAverageCpuUsage, getAverageCpuUsagePerProcess, getAverageFPSUsage, getAverageRAMUsage} from '@perf-profiler/reporter'; +import {ThreadNames} from '@perf-profiler/types'; import type {Measure} from '@perf-profiler/types'; let measures: Measure[] = []; @@ -23,11 +24,17 @@ const start = (bundleId: string) => { const stop = () => { polling.stop(); + const average = getAverageCpuUsagePerProcess(measures); + const uiThread = average.find(({processName}) => processName === ThreadNames.ANDROID.UI)?.cpuUsage; + // most likely this line needs to be updated when we migrate to RN 0.74 with bridgeless mode + const jsThread = average.find(({processName}) => processName === ThreadNames.RN.JS_ANDROID)?.cpuUsage; const cpu = getAverageCpuUsage(measures); const fps = getAverageFPSUsage(measures); const ram = getAverageRAMUsage(measures); return { + uiThread, + jsThread, cpu, fps, ram, From 7b1994bc6745ae6703b322c30b1f6f334dd494cc Mon Sep 17 00:00:00 2001 From: kirillzyusko Date: Mon, 24 Jun 2024 10:57:31 +0300 Subject: [PATCH 9/9] chore: changes after review --- tests/e2e/utils/measure.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e/utils/measure.ts b/tests/e2e/utils/measure.ts index 0a5aface56f2..96ac1bb4541e 100644 --- a/tests/e2e/utils/measure.ts +++ b/tests/e2e/utils/measure.ts @@ -4,11 +4,12 @@ import {ThreadNames} from '@perf-profiler/types'; import type {Measure} from '@perf-profiler/types'; let measures: Measure[] = []; -let polling = { +const POLLING_STOPPED = { stop: (): void => { throw new Error('Cannot stop polling on a stopped profiler'); }, }; +let polling = POLLING_STOPPED; const start = (bundleId: string) => { // clear our measurements results @@ -23,6 +24,7 @@ const start = (bundleId: string) => { const stop = () => { polling.stop(); + polling = POLLING_STOPPED; const average = getAverageCpuUsagePerProcess(measures); const uiThread = average.find(({processName}) => processName === ThreadNames.ANDROID.UI)?.cpuUsage;