From 0f6db110e60edca6da167fb37408b4e0905d99e0 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Mon, 4 Mar 2024 16:36:50 -0800 Subject: [PATCH] chore: migrate to the tele reporter --- src/debugHighlight.ts | 2 +- src/extension.ts | 100 +++--- src/oopReporter.ts | 186 ++--------- src/playwrightTest.ts | 68 ++-- src/reporter.d.ts | 188 ++++++++--- src/reporterServer.ts | 66 ++-- src/testModel.ts | 60 ++-- src/testServerController.ts | 29 +- src/testServerInterface.ts | 1 + src/testTree.ts | 28 +- src/upstream/teleEmitter.ts | 278 ++++++++++++++++ src/upstream/teleReceiver.ts | 630 +++++++++++++++++++++++++++++++++++ 12 files changed, 1245 insertions(+), 391 deletions(-) create mode 100644 src/upstream/teleEmitter.ts create mode 100644 src/upstream/teleReceiver.ts diff --git a/src/debugHighlight.ts b/src/debugHighlight.ts index db867cf97..52ec94f5c 100644 --- a/src/debugHighlight.ts +++ b/src/debugHighlight.ts @@ -17,7 +17,7 @@ import { locatorForSourcePosition, pruneAstCaches } from './babelHighlightUtil'; import { debugSessionName } from './debugSessionName'; import { replaceActionWithLocator, locatorMethodRegex } from './methodNames'; -import type { Location } from './oopReporter'; +import type { Location } from './reporter'; import { ReusedBrowser } from './reusedBrowser'; import * as vscodeTypes from './vscodeTypes'; diff --git a/src/extension.ts b/src/extension.ts index 78464278d..a5c2395b5 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -19,8 +19,8 @@ import StackUtils from 'stack-utils'; import { DebugHighlight } from './debugHighlight'; import { installBrowsers, installPlaywright } from './installer'; import { MultiMap } from './multimap'; -import { PlaywrightTest, RunHooks, TestConfig, TestListener } from './playwrightTest'; -import type { Location, TestError, Entry } from './oopReporter'; +import { PlaywrightTest, RunHooks, TestConfig } from './playwrightTest'; +import * as reporterTypes from './reporter'; import { ReusedBrowser } from './reusedBrowser'; import { SettingsModel } from './settingsModel'; import { SettingsView } from './settingsView'; @@ -67,8 +67,8 @@ export class Extension implements RunHooks { private _workspaceObserver: WorkspaceObserver; private _testItemUnderDebug: vscodeTypes.TestItem | undefined; - private _activeSteps = new Map(); - private _completedSteps = new Map(); + private _activeSteps = new Map(); + private _completedSteps = new Map(); private _testRun: vscodeTypes.TestRun | undefined; private _models: TestModel[] = []; private _activeStepDecorationType: vscodeTypes.TextEditorDecorationType; @@ -257,7 +257,7 @@ export class Extension implements RunHooks { const configFiles = await this._vscode.workspace.findFiles('**/*playwright*.config.{ts,js,mjs}', '**/node_modules/**'); - const configErrors = new MultiMap(); + const configErrors = new MultiMap(); for (const configFileUri of configFiles) { const configFilePath = configFileUri.fsPath; // TODO: parse .gitignore @@ -329,7 +329,7 @@ export class Extension implements RunHooks { return configFiles; } - private _reportConfigErrorsToUser(configErrors: MultiMap) { + private _reportConfigErrorsToUser(configErrors: MultiMap) { this._updateDiagnostics('configErrors', configErrors); if (!configErrors.size) return; @@ -570,25 +570,23 @@ located next to Run / Debug Tests toolbar buttons.`); locations: string[] | null, parametrizedTestTitle: string | undefined, enqueuedSingleTest: boolean) { - const testListener: TestListener = { - onBegin: ({ projects }) => { - model.updateFromRunningProjects(projects); - const visit = (entry: Entry) => { - if (entry.type === 'test') { - const testItem = this._testTree.testItemForLocation(entry.location, entry.titlePath); - if (testItem) - testRun.enqueued(testItem); - } - (entry.children || []).forEach(visit); + const testListener: reporterTypes.ReporterV2 = { + onBegin: (rootSuite: reporterTypes.Suite) => { + model.updateFromRunningProjects(rootSuite.suites); + const visitTest = (test: reporterTypes.TestCase) => { + const testItem = this._testTree.testItemForLocation(test.location, test.titlePath().slice(3)); + if (testItem) + testRun.enqueued(testItem); }; - projects.forEach(visit); + rootSuite.allTests().forEach(visitTest); }, - onTestBegin: params => { - const testItem = this._testTree.testItemForLocation(params.location, params.titlePath); + onTestBegin: (test: reporterTypes.TestCase, result: reporterTypes.TestResult) => { + const testItem = this._testTree.testItemForLocation(test.location, test.titlePath().slice(3)); if (testItem) { testRun.started(testItem); - const traceUrl = `${params.outputDir}/.playwright-artifacts-${params.workerIndex}/traces/${params.testId}.json`; + const fullProject = ancestorProject(test); + const traceUrl = `${fullProject.outputDir}/.playwright-artifacts-${result.workerIndex}/traces/${test.id}.json`; (testItem as any)[traceUrlSymbol] = traceUrl; } @@ -600,59 +598,62 @@ located next to Run / Debug Tests toolbar buttons.`); } }, - onTestEnd: params => { + onTestEnd: (test: reporterTypes.TestCase, result: reporterTypes.TestResult) => { this._testItemUnderDebug = undefined; this._activeSteps.clear(); this._executionLinesChanged(); - const testItem = this._testTree.testItemForLocation(params.location, params.titlePath); + const testItem = this._testTree.testItemForLocation(test.location, test.titlePath().slice(3)); if (!testItem) return; - (testItem as any)[traceUrlSymbol] = params.trace || ''; + const trace = result.attachments.find(a => a.name === 'trace')?.path || ''; + (testItem as any)[traceUrlSymbol] = trace; if (enqueuedSingleTest) this._showTrace(testItem); - if (params.status === params.expectedStatus) { + if (result.status === test.expectedStatus) { if (!testFailures.has(testItem)) { - if (params.status === 'skipped') + if (result.status === 'skipped') testRun.skipped(testItem); else - testRun.passed(testItem, params.duration); + testRun.passed(testItem, result.duration); } return; } testFailures.add(testItem); - testRun.failed(testItem, params.errors.map(error => this._testMessageForTestError(error, testItem)), params.duration); + testRun.failed(testItem, result.errors.map(error => this._testMessageForTestError(error, testItem)), result.duration); }, - onStepBegin: params => { - const stepId = params.location.file + ':' + params.location.line; - let step = this._activeSteps.get(stepId); + onStepBegin: (test: reporterTypes.TestCase, result: reporterTypes.TestResult, testStep: reporterTypes.TestStep) => { + if (!testStep.location) + return; + let step = this._activeSteps.get(testStep); if (!step) { step = { location: new this._vscode.Location( - this._vscode.Uri.file(params.location.file), - new this._vscode.Position(params.location.line - 1, params.location.column - 1)), + this._vscode.Uri.file(testStep.location.file), + new this._vscode.Position(testStep.location.line - 1, testStep.location?.column - 1)), activeCount: 0, duration: 0, }; - this._activeSteps.set(stepId, step); + this._activeSteps.set(testStep, step); } ++step.activeCount; this._executionLinesChanged(); }, - onStepEnd: params => { - const stepId = params.location.file + ':' + params.location.line; - const step = this._activeSteps.get(stepId)!; + onStepEnd: (test: reporterTypes.TestCase, result: reporterTypes.TestResult, testStep: reporterTypes.TestStep) => { + if (!testStep.location) + return; + const step = this._activeSteps.get(testStep); if (!step) return; --step.activeCount; - step.duration = params.duration; - this._completedSteps.set(stepId, step); + step.duration = testStep.duration; + this._completedSteps.set(testStep, step); if (step.activeCount === 0) - this._activeSteps.delete(stepId); + this._activeSteps.delete(testStep); this._executionLinesChanged(); }, @@ -664,13 +665,13 @@ located next to Run / Debug Tests toolbar buttons.`); testRun.appendOutput(data.toString().replace(/\n/g, '\r\n')); }, - onError: data => { + onError: (error: reporterTypes.TestError) => { // Global errors don't have associated tests, so we'll be allocating them // to the first item / current. if (testItemForGlobalErrors) { // Force UI to reveal the item if that is a file that has never been started. testRun.started(testItemForGlobalErrors); - testRun.failed(testItemForGlobalErrors, this._testMessageForTestError(data.error), 0); + testRun.failed(testItemForGlobalErrors, this._testMessageForTestError(error), 0); } } }; @@ -702,7 +703,7 @@ located next to Run / Debug Tests toolbar buttons.`); const timer = setTimeout(async () => { delete this._filesPendingListTests; const allErrors = new Set(); - const errorsByFile = new MultiMap(); + const errorsByFile = new MultiMap(); for (const model of this._models.slice()) { const filteredFiles = model.narrowDownFilesToEnabledProjects(files); if (!filteredFiles.size) @@ -736,7 +737,7 @@ located next to Run / Debug Tests toolbar buttons.`); return this._filesPendingListTests.promise; } - private _updateDiagnostics(diagnosticsType: 'configErrors' | 'testErrors' , errorsByFile: MultiMap) { + private _updateDiagnostics(diagnosticsType: 'configErrors' | 'testErrors' , errorsByFile: MultiMap) { const diagnosticsCollection = this._diagnostics[diagnosticsType]!; diagnosticsCollection.clear(); for (const [file, errors] of errorsByFile) { @@ -753,7 +754,7 @@ located next to Run / Debug Tests toolbar buttons.`); } } - private _errorInDebugger(errorStack: string, location: Location) { + private _errorInDebugger(errorStack: string, location: reporterTypes.Location) { if (!this._testRun || !this._testItemUnderDebug) return; const testMessage = this._testMessageFromText(errorStack); @@ -843,7 +844,7 @@ located next to Run / Debug Tests toolbar buttons.`); return new this._vscode.TestMessage(markdownString); } - private _testMessageForTestError(error: TestError, testItem?: vscodeTypes.TestItem): vscodeTypes.TestMessage { + private _testMessageForTestError(error: reporterTypes.TestError, testItem?: vscodeTypes.TestItem): vscodeTypes.TestMessage { const text = error.stack || error.message || error.value!; let testMessage: vscodeTypes.TestMessage; if (text.includes('Looks like Playwright Test or Playwright')) { @@ -890,7 +891,7 @@ located next to Run / Debug Tests toolbar buttons.`); } } -function parseLocationFromStack(stack: string | undefined, testItem?: vscodeTypes.TestItem): Location | undefined { +function parseLocationFromStack(stack: string | undefined, testItem?: vscodeTypes.TestItem): reporterTypes.Location | undefined { const lines = stack?.split('\n') || []; for (const line of lines) { const frame = stackUtils.parseLine(line); @@ -941,4 +942,11 @@ class TreeItemObserver implements vscodeTypes.Disposable{ } } +function ancestorProject(test: reporterTypes.TestCase): reporterTypes.FullProject { + let suite: reporterTypes.Suite = test.parent; + while (!suite.project()) + suite = suite.parent!; + return suite.project()!; +} + const traceUrlSymbol = Symbol('traceUrl'); diff --git a/src/oopReporter.ts b/src/oopReporter.ts index 501c6625d..9d454e467 100644 --- a/src/oopReporter.ts +++ b/src/oopReporter.ts @@ -14,186 +14,44 @@ * limitations under the License. */ -import type { FullConfig, FullProject, FullResult, Location, Reporter, Suite, TestCase, TestError, TestResult, TestStatus, TestStep } from './reporter'; -import { ConnectionTransport, WebSocketTransport } from './transport'; +import { TeleReporterEmitter } from './upstream/teleEmitter'; +import { WebSocketTransport } from './transport'; +import { FullResult } from './reporter'; -export type { TestError, Location } from './reporter'; -export type EntryType = 'project' | 'file' | 'suite' | 'test'; - -export type Entry = { - type: EntryType; - title: string; - titlePath: string[]; - location: Location; - children?: Entry[]; - testId?: string; -}; - -export type TestBeginParams = { - testId: string; - title: string; - titlePath: string[]; - location: Location; - outputDir: string; - workerIndex: number; -}; - -export type TestEndParams = { - testId: string; - title: string; - titlePath: string[]; - location: Location; - duration: number; - errors: TestError[]; - expectedStatus: TestStatus; - status: TestStatus; - trace?: string; -}; - -export type StepBeginParams = { - title: string; - location: Location; -}; - -export type StepEndParams = { - duration: number; - location: Location; - error: TestError | undefined; -}; - -class OopReporter implements Reporter { - private _transport: Promise; +class TeleReporter extends TeleReporterEmitter { private _hasSender: boolean; - constructor(sender: (message: any) => void) { - this._hasSender = !!sender; - if (sender) { - this._transport = Promise.resolve({ - send: message => sender(message), - close: () => {}, - isClosed: () => false, - }); - } else { - this._transport = WebSocketTransport.connect(process.env.PW_TEST_REPORTER_WS_ENDPOINT!); - this._transport.then(t => { + constructor(options: any) { + let messageSink: (message: any) => void; + if (options?._send) { + messageSink = options._send; + } else if (process.env.PW_TEST_REPORTER_WS_ENDPOINT) { + const transport = WebSocketTransport.connect(process.env.PW_TEST_REPORTER_WS_ENDPOINT!); + transport.then(t => { t.onmessage = message => { if (message.method === 'stop') process.emit('SIGINT' as any); }; t.onclose = () => process.exit(0); }); + messageSink = (message => { + transport.then(t => t.send(message)); + }); + } else { + messageSink = message => { + console.log(message); + }; } - } - - printsToStdio() { - return false; - } - - onBegin(config: FullConfig, rootSuite: Suite) { - const visit = (suite: Suite, collection: Entry[]) => { - // Don't produce entries for file suits. - for (const child of suite.suites) { - let type: 'project' | 'file' | 'suite' | 'test'; - if (!child.location) - type = 'project'; - else if (child.location.line === 0) - type = 'file'; - else - type = 'suite'; - const entry: Entry = { - type, - title: child.title, - titlePath: child.titlePath().slice(3), - location: child.location || { file: '', line: 0, column: 0 }, - children: [], - }; - collection.push(entry); - visit(child, entry.children!); - } - - for (const test of suite.tests) { - const entry: Entry = { - type: 'test', - title: test.title, - titlePath: test.titlePath().slice(3), - location: test.location, - testId: test.id - }; - collection.push(entry); - } - }; - - const projects: Entry[] = []; - visit(rootSuite, projects); - this._emit('onBegin', { projects }); - } - - onTestBegin?(test: TestCase, result: TestResult): void { - let project: FullProject | undefined; - let suite: Suite | undefined = test.parent; - while (!project && suite) { - project = suite.project(); - suite = suite.parent; - } - - const params: TestBeginParams = { - testId: test.id, - title: test.title, - titlePath: test.titlePath().slice(3), - location: test.location, - workerIndex: result.workerIndex, - outputDir: project?.outputDir || '', - }; - this._emit('onTestBegin', params); - } - - onTestEnd(test: TestCase, result: TestResult): void { - const params: TestEndParams = { - testId: test.id, - title: test.title, - titlePath: test.titlePath().slice(3), - location: test.location, - duration: result.duration, - errors: result.errors, - expectedStatus: test.expectedStatus, - status: result.status, - trace: result.attachments.find(a => a.name === 'trace')?.path, - }; - this._emit('onTestEnd', params); - } - - onStepBegin(test: TestCase, result: TestResult, step: TestStep) { - if (!step.location) - return; - const params: StepBeginParams = { title: step.title, location: step.location }; - this._emit('onStepBegin', params); - } - - onStepEnd(test: TestCase, result: TestResult, step: TestStep) { - if (!step.location) - return; - const params: StepEndParams = { - location: step.location, - duration: step.duration, - error: step.error, - }; - this._emit('onStepEnd', params); - } - - onError(error: TestError): void { - this._emit('onError', { error }); + super(messageSink, { omitBuffers: true, omitOutput: true }); + this._hasSender = !!options?._send; } async onEnd(result: FullResult) { - this._emit('onEnd', {}); + super.onEnd(result); // Embedder is responsible for terminating the connection. if (!this._hasSender) await new Promise(() => {}); } - - private _emit(method: string, params: Object) { - this._transport.then(t => t.send({ id: 0, method, params })); - } } -export default OopReporter; +export default TeleReporter; diff --git a/src/playwrightTest.ts b/src/playwrightTest.ts index c3e22ee92..0e7b8e8e2 100644 --- a/src/playwrightTest.ts +++ b/src/playwrightTest.ts @@ -18,12 +18,12 @@ import { spawn } from 'child_process'; import path from 'path'; import { debugSessionName } from './debugSessionName'; import { ConfigFindRelatedTestFilesReport, ConfigListFilesReport } from './listTests'; -import type { TestError, Entry, StepBeginParams, StepEndParams, TestBeginParams, TestEndParams } from './oopReporter'; import { ReporterServer } from './reporterServer'; import { findNode, spawnAsync } from './utils'; import * as vscodeTypes from './vscodeTypes'; import { SettingsModel } from './settingsModel'; import { TestServerController } from './testServerController'; +import * as reporterTypes from './reporter'; export type TestConfig = { workspaceFolder: string; @@ -33,18 +33,6 @@ export type TestConfig = { testIdAttributeName?: string; }; -export interface TestListener { - onBegin?(params: { projects: Entry[] }): void; - onTestBegin?(params: TestBeginParams): void; - onTestEnd?(params: TestEndParams): void; - onStepBegin?(params: StepBeginParams): void; - onStepEnd?(params: StepEndParams): void; - onError?(params: { error: TestError }): void; - onEnd?(): void; - onStdOut?(data: Buffer | string): void; - onStdErr?(data: Buffer | string): void; -} - const pathSeparator = process.platform === 'win32' ? ';' : ':'; export type PlaywrightTestOptions = { @@ -136,7 +124,7 @@ export class PlaywrightTest { return await testServer.listFiles({ configFile: config.configFile }); } - async runTests(config: TestConfig, projectNames: string[], locations: string[] | null, listener: TestListener, parametrizedTestTitle: string | undefined, token: vscodeTypes.CancellationToken) { + async runTests(config: TestConfig, projectNames: string[], locations: string[] | null, listener: reporterTypes.ReporterV2, parametrizedTestTitle: string | undefined, token: vscodeTypes.CancellationToken) { const locationArg = locations ? locations : []; if (token?.isCancellationRequested) return; @@ -166,38 +154,38 @@ export class PlaywrightTest { try { if (token?.isCancellationRequested) return; - await this._test(config, locationArg, 'run', options, listener, token); + await this._test(config, locationArg, 'test', options, listener, token); } finally { await this._runHooks.onDidRunTests(false); } } - async listTests(config: TestConfig, files: string[]): Promise<{ entries: Entry[], errors: TestError[] }> { - let entries: Entry[] = []; - const errors: TestError[] = []; + async listTests(config: TestConfig, files: string[]): Promise<{ rootSuite: reporterTypes.Suite, errors: reporterTypes.TestError[] }> { + const errors: reporterTypes.TestError[] = []; + let rootSuite: reporterTypes.Suite | undefined; await this._test(config, files, 'list', {}, { - onBegin: params => { - entries = params.projects as Entry[]; + onBegin: (suite: reporterTypes.Suite) => { + rootSuite = suite; }, - onError: params => { - errors.push(params.error); + onError: (error: reporterTypes.TestError) => { + errors.push(error); }, }, new this._vscode.CancellationTokenSource().token); - return { entries, errors }; + return { rootSuite: rootSuite!, errors }; } private _useTestServer(config: TestConfig) { return this._settingsModel.useTestServer.get(); } - private async _test(config: TestConfig, locations: string[], mode: 'list' | 'run', options: PlaywrightTestOptions, listener: TestListener, token: vscodeTypes.CancellationToken): Promise { + private async _test(config: TestConfig, locations: string[], mode: 'list' | 'test', options: PlaywrightTestOptions, reporter: reporterTypes.ReporterV2, token: vscodeTypes.CancellationToken): Promise { if (this._useTestServer(config)) - await this._testWithServer(config, locations, mode, options, listener, token); + await this._testWithServer(config, locations, mode, options, reporter, token); else - await this._testWithCLI(config, locations, mode, options, listener, token); + await this._testWithCLI(config, locations, mode, options, reporter, token); } - private async _testWithCLI(config: TestConfig, locations: string[], mode: 'list' | 'run', options: PlaywrightTestOptions, listener: TestListener, token: vscodeTypes.CancellationToken): Promise { + private async _testWithCLI(config: TestConfig, locations: string[], mode: 'list' | 'test', options: PlaywrightTestOptions, reporter: reporterTypes.ReporterV2, token: vscodeTypes.CancellationToken): Promise { // Playwright will restart itself as child process in the ESM mode and won't inherit the 3/4 pipes. // Always use ws transport to mitigate it. const reporterServer = new ReporterServer(this._vscode); @@ -257,34 +245,34 @@ export class PlaywrightTest { }); const stdio = childProcess.stdio; - stdio[1].on('data', data => listener.onStdOut?.(data)); - stdio[2].on('data', data => listener.onStdErr?.(data)); - await reporterServer.wireTestListener(listener, token); + stdio[1].on('data', data => reporter.onStdOut?.(data)); + stdio[2].on('data', data => reporter.onStdErr?.(data)); + await reporterServer.wireTestListener(mode, reporter, token); } - private async _testWithServer(config: TestConfig, locations: string[], mode: 'list' | 'run', options: PlaywrightTestOptions, listener: TestListener, token: vscodeTypes.CancellationToken): Promise { + private async _testWithServer(config: TestConfig, locations: string[], mode: 'list' | 'test', options: PlaywrightTestOptions, reporter: reporterTypes.ReporterV2, token: vscodeTypes.CancellationToken): Promise { const testServer = await this._testServerController.testServerFor(config); if (token?.isCancellationRequested) return; if (!testServer) return; - const reporter = require.resolve('./oopReporter'); + const oopReporter = require.resolve('./oopReporter'); if (mode === 'list') - testServer.listTests({ configFile: config.configFile, locations, reporter }); - if (mode === 'run') { - testServer.test({ configFile: config.configFile, locations, reporter, ...options }); + testServer.listTests({ configFile: config.configFile, locations, reporter: oopReporter }); + if (mode === 'test') { + testServer.test({ configFile: config.configFile, locations, reporter: oopReporter, ...options }); token.onCancellationRequested(() => { testServer.stop({ configFile: config.configFile }); }); testServer.on('stdio', params => { if (params.type === 'stdout') - listener.onStdOut?.(unwrapString(params)); + reporter.onStdOut?.(unwrapString(params)); if (params.type === 'stderr') - listener.onStdErr?.(unwrapString(params)); + reporter.onStdErr?.(unwrapString(params)); }); } - await testServer.wireTestListener(listener, token); + await testServer.wireTestListener(mode, reporter, token); } async findRelatedTestFiles(config: TestConfig, files: string[]): Promise { @@ -324,7 +312,7 @@ export class PlaywrightTest { return await testServer.findRelatedTestFiles({ configFile: config.configFile, files }); } - async debugTests(vscode: vscodeTypes.VSCode, config: TestConfig, projectNames: string[], testDirs: string[], settingsEnv: NodeJS.ProcessEnv, locations: string[] | null, listener: TestListener, parametrizedTestTitle: string | undefined, token: vscodeTypes.CancellationToken) { + async debugTests(vscode: vscodeTypes.VSCode, config: TestConfig, projectNames: string[], testDirs: string[], settingsEnv: NodeJS.ProcessEnv, locations: string[] | null, reporter: reporterTypes.ReporterV2, parametrizedTestTitle: string | undefined, token: vscodeTypes.CancellationToken) { const configFolder = path.dirname(config.configFile); const configFile = path.basename(config.configFile); locations = locations || []; @@ -373,7 +361,7 @@ export class PlaywrightTest { program: config.cli, args, }); - await reporterServer.wireTestListener(listener, token); + await reporterServer.wireTestListener('test', reporter, token); } finally { await this._runHooks.onDidRunTests(true); } diff --git a/src/reporter.d.ts b/src/reporter.d.ts index c84c0a8f3..414b8acfd 100644 --- a/src/reporter.d.ts +++ b/src/reporter.d.ts @@ -16,25 +16,80 @@ /* eslint-disable quotes */ -export type Metadata = { [key: string]: any }; -export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped'; -export type FullConfig = {}; +export type Annotation = any; +export type Metadata = unknown; +export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'; +export type FullConfig = { + configFile: string; + version: string; + rootDir: string; + forbidOnly: unknown; + fullyParallel: unknown; + globalSetup?: unknown; + globalTeardown?: unknown; + globalTimeout?: unknown; + grep: string | RegExp | (string | RegExp)[]; + grepInvert: string | RegExp | (string | RegExp)[] | null; + maxFailures?: unknown; + metadata: Metadata; + preserveOutput?: unknown; + projects: FullProject[]; + quiet?: unknown; + reporter: unknown[]; + reportSlowTests?: unknown; + shard?: unknown; + updateSnapshots?: unknown; + webServer?: unknown; + workers?: number; +}; export type FullProject = { name: string; + testDir: string; + metadata: Metadata; outputDir: string; + teardown?: string; + dependencies: string[]; + testIgnore: string | RegExp | (string | RegExp)[]; + testMatch: string | RegExp | (string | RegExp)[]; + timeout: number; + grep: string | RegExp | (string | RegExp)[]; + grepInvert: string | RegExp | (string | RegExp)[] | null; + snapshotDir: string; + retries: number; + repeatEach: number; + use: unknown; }; +interface FullReporterV2 { + onConfigure(config: FullConfig): void; + onBegin(suite: Suite): void; + onTestBegin(test: TestCase, result: TestResult): void; + onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; + onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; + onTestEnd(test: TestCase, result: TestResult): void; + onEnd(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void; + onExit(): void | Promise; + onError(error: TestError): void; + onStepBegin(test: TestCase, result: TestResult, step: TestStep): void; + onStepEnd(test: TestCase, result: TestResult, step: TestStep): void; + printsToStdio(): boolean; + version(): 'v2'; +} + +export type ReporterV2 = Partial; + /** * `Suite` is a group of tests. All tests in Playwright Test form the following hierarchy: - * - Root suite has a child suite for each [TestProject]. + * - Root suite has a child suite for each {@link TestProject}. * - Project suite #1. Has a child suite for each test file in the project. * - File suite #1 - * - [TestCase] #1 - * - [TestCase] #2 + * - {@link TestCase} #1 + * - {@link TestCase} #2 * - Suite corresponding to a - * [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-1) group - * - [TestCase] #1 in a group - * - [TestCase] #2 in a group + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) + * group + * - {@link TestCase} #1 in a group + * - {@link TestCase} #2 in a group * - < more test cases ... > * - File suite #2 * - < more file suites ... > @@ -71,14 +126,15 @@ export interface Suite { parent?: Suite; /** - * Child suites. See [Suite] for the hierarchy of suites. + * Child suites. See {@link Suite} for the hierarchy of suites. */ suites: Array; /** * Test cases in the suite. Note that only test cases defined directly in this suite are in the list. Any test cases - * defined in nested [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-1) - * groups are listed in the child [suite.suites](https://playwright.dev/docs/api/class-suite#suite-suites). + * defined in nested + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) groups are + * listed in the child [suite.suites](https://playwright.dev/docs/api/class-suite#suite-suites). */ tests: Array; @@ -87,27 +143,31 @@ export interface Suite { * - Empty for root suite. * - Project name for project suite. * - File path for file suite. - * - Title passed to [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-1) - * for a group suite. + * - Title passed to + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) for a + * group suite. */ title: string; } /** * `TestCase` corresponds to every - * [test.(call)(title, testFunction)](https://playwright.dev/docs/api/class-test#test-call) call in a test file. When - * a single [test.(call)(title, testFunction)](https://playwright.dev/docs/api/class-test#test-call) is running in - * multiple projects or repeated multiple times, it will have multiple `TestCase` objects in corresponding projects' - * suites. + * [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) call in a test file. + * When a single [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) is + * running in multiple projects or repeated multiple times, it will have multiple `TestCase` objects in corresponding + * projects' suites. */ export interface TestCase { /** * Expected test status. - * - Tests marked as [test.skip(title, testFunction)](https://playwright.dev/docs/api/class-test#test-skip-1) or - * [test.fixme(title, testFunction)](https://playwright.dev/docs/api/class-test#test-fixme-1) are expected to be - * `'skipped'`. - * - Tests marked as [test.fail()](https://playwright.dev/docs/api/class-test#test-fail-1) are expected to be - * `'failed'`. + * - Tests marked as + * [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip) + * or + * [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme) + * are expected to be `'skipped'`. + * - Tests marked as + * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail) + * are expected to be `'failed'`. * - Other tests are expected to be `'passed'`. * * See also [testResult.status](https://playwright.dev/docs/api/class-testresult#test-result-status) for the actual @@ -133,9 +193,18 @@ export interface TestCase { titlePath(): Array; /** - * The list of annotations applicable to the current test. Includes annotations from the test, annotations from all - * [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-1) groups the test - * belongs to and file-level annotations for the test file. + * The list of annotations applicable to the current test. Includes: + * - annotations defined on the test or suite via + * [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) and + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe); + * - annotations implicitly added by methods + * [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip), + * [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme) + * and + * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail); + * - annotations appended to + * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations) during the test + * execution. * * Annotations are available during test execution through * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations). @@ -188,25 +257,35 @@ export interface TestCase { */ retries: number; + /** + * The list of tags defined on the test or suite via + * [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) or + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe), as well as + * `@`-tokens extracted from test and suite titles. + * + * Learn more about [test tags](https://playwright.dev/docs/test-annotations#tag-tests). + */ + tags: Array; + /** * The timeout given to the test. Affected by * [testConfig.timeout](https://playwright.dev/docs/api/class-testconfig#test-config-timeout), * [testProject.timeout](https://playwright.dev/docs/api/class-testproject#test-project-timeout), * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout), - * [test.slow()](https://playwright.dev/docs/api/class-test#test-slow-1) and + * [test.slow([condition, callback, description])](https://playwright.dev/docs/api/class-test#test-slow) and * [testInfo.setTimeout(timeout)](https://playwright.dev/docs/api/class-testinfo#test-info-set-timeout). */ timeout: number; /** * Test title as passed to the - * [test.(call)(title, testFunction)](https://playwright.dev/docs/api/class-test#test-call) call. + * [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) call. */ title: string; } /** - * A result of a single [TestCase] run. + * A result of a single {@link TestCase} run. */ export interface TestResult { /** @@ -311,6 +390,16 @@ export interface FullResult { * - 'interrupted' - interrupted by the user. */ status: 'passed' | 'failed' | 'timedout' | 'interrupted'; + + /** + * Test start wall time. + */ + startTime: Date; + + /** + * Test duration in milliseconds. + */ + duration: number; } /** @@ -322,7 +411,9 @@ export interface FullResult { * * ```js * // my-awesome-reporter.ts - * import { Reporter, FullConfig, Suite, TestCase, TestResult, FullResult } from '@playwright/test/reporter'; + * import type { + * Reporter, FullConfig, Suite, TestCase, TestResult, FullResult + * } from '@playwright/test/reporter'; * * class MyReporter implements Reporter { * constructor(options: { customOption?: string } = {}) { @@ -363,17 +454,18 @@ export interface FullResult { * * Here is a typical order of reporter calls: * - [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin) is called - * once with a root suite that contains all other suites and tests. Learn more about [suites hierarchy][Suite]. + * once with a root suite that contains all other suites and tests. Learn more about [suites hierarchy]{@link + * Suite}. * - [reporter.onTestBegin(test, result)](https://playwright.dev/docs/api/class-reporter#reporter-on-test-begin) is - * called for each test run. It is given a [TestCase] that is executed, and a [TestResult] that is almost empty. - * Test result will be populated while the test runs (for example, with steps and stdio) and will get final - * `status` once the test finishes. + * called for each test run. It is given a {@link TestCase} that is executed, and a {@link TestResult} that is + * almost empty. Test result will be populated while the test runs (for example, with steps and stdio) and will + * get final `status` once the test finishes. * - [reporter.onStepBegin(test, result, step)](https://playwright.dev/docs/api/class-reporter#reporter-on-step-begin) * and * [reporter.onStepEnd(test, result, step)](https://playwright.dev/docs/api/class-reporter#reporter-on-step-end) * are called for each executed step inside the test. When steps are executed, test run has not finished yet. * - [reporter.onTestEnd(test, result)](https://playwright.dev/docs/api/class-reporter#reporter-on-test-end) is - * called when test run has finished. By this time, [TestResult] is complete and you can use + * called when test run has finished. By this time, {@link TestResult} is complete and you can use * [testResult.status](https://playwright.dev/docs/api/class-testresult#test-result-status), * [testResult.error](https://playwright.dev/docs/api/class-testresult#test-result-error) and more. * - [reporter.onEnd(result)](https://playwright.dev/docs/api/class-reporter#reporter-on-end) is called once after @@ -395,15 +487,17 @@ export interface FullResult { */ export interface Reporter { /** - * Called once before running tests. All tests have been already discovered and put into a hierarchy of [Suite]s. + * Called once before running tests. All tests have been already discovered and put into a hierarchy of {@link + * Suite}s. * @param config Resolved configuration. * @param suite The root suite that contains all projects, files and test cases. */ onBegin?(config: FullConfig, suite: Suite): void; /** - * Called after all tests has been run, or testing has been interrupted. Note that this method may return a [Promise] - * and Playwright Test will await it. - * @param result Result of the full test run. + * Called after all tests have been run, or testing has been interrupted. Note that this method may return a [Promise] + * and Playwright Test will await it. Reporter is allowed to override the status and hence affect the exit code of the + * test runner. + * @param result Result of the full test run, `status` can be one of: * - `'passed'` - Everything went as expected. * - `'failed'` - Any test has failed. * - `'timedout'` - The @@ -411,7 +505,7 @@ export interface Reporter { * been reached. * - `'interrupted'` - Interrupted by the user. */ - onEnd?(result: FullResult): void | Promise; + onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void; /** * Called on some global error, for example unhandled exception in the worker process. * @param error The error. @@ -419,7 +513,7 @@ export interface Reporter { onError?(error: TestError): void; /** - * Called immediately before test runner exists. At this point all the reporters have recived the + * Called immediately before test runner exists. At this point all the reporters have received the * [reporter.onEnd(result)](https://playwright.dev/docs/api/class-reporter#reporter-on-end) signal, so all the reports * should be build. You can run the code that uploads the reports in this hook. */ @@ -495,6 +589,14 @@ export interface JSONReport { }; suites: JSONReportSuite[]; errors: TestError[]; + stats: { + startTime: string; // Date in ISO 8601 format. + duration: number; // In milliseconds; + expected: number; + unexpected: number; + flaky: number; + skipped: number; + } } export interface JSONReportSuite { @@ -542,7 +644,7 @@ export interface JSONReportTestResult { stderr: JSONReportSTDIOEntry[]; retry: number; steps?: JSONReportTestStep[]; - startTime: Date; + startTime: string; // Date in ISO 8601 format. attachments: { name: string; path?: string; @@ -566,7 +668,7 @@ export {}; /** - * Represents a location in the source code where [TestCase] or [Suite] is defined. + * Represents a location in the source code where {@link TestCase} or {@link Suite} is defined. */ export interface Location { /** diff --git a/src/reporterServer.ts b/src/reporterServer.ts index 5b23a413e..b8f84e177 100644 --- a/src/reporterServer.ts +++ b/src/reporterServer.ts @@ -14,13 +14,14 @@ * limitations under the License. */ +import path from 'path'; import * as http from 'http'; import WebSocket, { WebSocketServer } from 'ws'; -import { TestListener } from './playwrightTest'; import { ConnectionTransport } from './transport'; import { createGuid } from './utils'; import * as vscodeTypes from './vscodeTypes'; -import type { Location, TestError, Entry, StepBeginParams, StepEndParams, TestBeginParams, TestEndParams } from './oopReporter'; +import * as reporterTypes from './reporter'; +import { TeleReporterReceiver } from './upstream/teleReceiver'; export class ReporterServer { private _clientSocketPromise: Promise; @@ -33,14 +34,10 @@ export class ReporterServer { this._clientSocketPromise = new Promise(f => this._clientSocketCallback = f); } - reporterFile() { - return require.resolve('./oopReporter'); - } - async env() { const wsEndpoint = await this._listen(); return { - PW_TEST_REPORTER: this.reporterFile(), + PW_TEST_REPORTER: require.resolve('./oopReporter'), PW_TEST_REPORTER_WS_ENDPOINT: wsEndpoint, }; } @@ -69,7 +66,7 @@ export class ReporterServer { return wsEndpoint; } - async wireTestListener(listener: TestListener, token: vscodeTypes.CancellationToken) { + async wireTestListener(mode: 'test' | 'list', listener: reporterTypes.ReporterV2, token: vscodeTypes.CancellationToken) { let timeout: NodeJS.Timeout | undefined; const transport = await this._waitForTransport(); @@ -89,7 +86,20 @@ export class ReporterServer { if (token.isCancellationRequested) killTestProcess(); - transport.onmessage = message => translateMessage(this._vscode, message, listener, () => transport.close(), token); + const teleReceiver = new TeleReporterReceiver(listener, { + mergeProjects: true, + mergeTestCases: true, + resolvePath: (rootDir: string, relativePath: string) => this._vscode.Uri.file(path.join(rootDir, relativePath)).fsPath, + }); + + transport.onmessage = message => { + if (token.isCancellationRequested && message.method !== 'onEnd') + return; + if (message.method === 'onEnd') + transport.close(); + teleReceiver.dispatch(mode, message as any); + }; + await new Promise(f => transport.onclose = f); if (timeout) clearTimeout(timeout); @@ -128,41 +138,3 @@ export class ReporterServer { return transport; } } - -export function translateMessage(vscode: vscodeTypes.VSCode, message: any, listener: TestListener, onEnd: () => void, token: vscodeTypes.CancellationToken) { - if (token.isCancellationRequested && message.method !== 'onEnd') - return; - switch (message.method) { - case 'onBegin': { - (message.params as { projects: Entry[] }).projects.forEach((e: Entry) => patchLocation(vscode, e)); - listener.onBegin?.(message.params); - break; - } - case 'onTestBegin': listener.onTestBegin?.(patchLocation(vscode, message.params as TestBeginParams)); break; - case 'onTestEnd': listener.onTestEnd?.(patchLocation(vscode, message.params as TestEndParams)); break; - case 'onStepBegin': listener.onStepBegin?.(patchLocation(vscode, message.params as StepBeginParams)); break; - case 'onStepEnd': listener.onStepEnd?.(patchLocation(vscode, message.params as StepEndParams)); break; - case 'onError': listener.onError?.(patchLocation(vscode, message.params as { error: TestError })); break; - case 'onEnd': { - listener.onEnd?.(); - onEnd(); - break; - } - } -} - -function patchLocation(vscode: vscodeTypes.VSCode, object: T): T { - // Normalize all the location.file values using the Uri.file().fsPath normalization. - // vscode will normalize Windows drive letter, etc. - if (object.location) - object.location.file = vscode.Uri.file(object.location.file).fsPath; - if (object.error?.location) - object.error.location.file = vscode.Uri.file(object.error.location.file).fsPath; - if (object.errors) { - object.errors.forEach(e => { - if (e.location) - e.location.file = vscode.Uri.file(e.location.file).fsPath; - }); - } - return object; -} diff --git a/src/testModel.ts b/src/testModel.ts index 7d6b1c53d..adf822c42 100644 --- a/src/testModel.ts +++ b/src/testModel.ts @@ -14,12 +14,14 @@ * limitations under the License. */ -import { Entry, TestError } from './oopReporter'; -import { PlaywrightTest, TestConfig, TestListener } from './playwrightTest'; +import { PlaywrightTest, TestConfig } from './playwrightTest'; import { WorkspaceChange } from './workspaceObserver'; import * as vscodeTypes from './vscodeTypes'; import { resolveSourceMap } from './utils'; import { ProjectConfigWithFiles } from './listTests'; +import * as reporterTypes from './reporter'; + +export type TestEntry = reporterTypes.TestCase | reporterTypes.Suite; /** * This class builds the Playwright Test model in Playwright terms. @@ -34,7 +36,7 @@ import { ProjectConfigWithFiles } from './listTests'; export class TestFile { readonly project: TestProject; readonly file: string; - private _entries: Entry[] | undefined; + private _entries: TestEntry[] | undefined; private _revision = 0; constructor(project: TestProject, file: string) { @@ -42,11 +44,11 @@ export class TestFile { this.file = file; } - entries(): Entry[] | undefined { + entries(): TestEntry[] | undefined { return this._entries; } - setEntries(entries: Entry[]) { + setEntries(entries: TestEntry[]) { ++this._revision; this._entries = entries; } @@ -106,7 +108,7 @@ export class TestModel { return result; } - async listFiles(): Promise { + async listFiles(): Promise { const report = await this._playwrightTest.listFiles(this.config); if (report.error) return report.error; @@ -216,22 +218,22 @@ export class TestModel { this._didUpdate.fire(); } - async listTests(files: string[]): Promise { - const { entries, errors } = await this._playwrightTest.listTests(this.config, files); - this._updateProjects(entries, files); + async listTests(files: string[]): Promise { + const { rootSuite, errors } = await this._playwrightTest.listTests(this.config, files); + this._updateProjects(rootSuite.suites, files); return errors; } - private _updateProjects(projectEntries: Entry[], requestedFiles: string[]) { + private _updateProjects(projectSuites: reporterTypes.Suite[], requestedFiles: string[]) { for (const [projectName, project] of this._projects) { - const projectEntry = projectEntries.find(e => e.title === projectName); + const projectSuite = projectSuites.find(e => e.project()!.name === projectName); const filesToDelete = new Set(requestedFiles); - for (const fileEntry of projectEntry?.children || []) { - filesToDelete.delete(fileEntry.location.file); - const file = project.files.get(fileEntry.location.file); + for (const fileSuite of projectSuite?.suites || []) { + filesToDelete.delete(fileSuite.location!.file); + const file = project.files.get(fileSuite.location!.file); if (!file) continue; - file.setEntries(fileEntry.children || []); + file.setEntries([...fileSuite.suites, ...fileSuite.tests]); } // We requested update for those, but got no entries. for (const file of filesToDelete) { @@ -243,36 +245,36 @@ export class TestModel { this._didUpdate.fire(); } - updateFromRunningProjects(projectEntries: Entry[]) { - for (const projectEntry of projectEntries) { - const project = this._projects.get(projectEntry.title); + updateFromRunningProjects(projectSuites: reporterTypes.Suite[]) { + for (const projectSuite of projectSuites) { + const project = this._projects.get(projectSuite.project()!.name); if (project) - this._updateFromRunningProject(project, projectEntry); + this._updateFromRunningProject(project, projectSuite); } } - private _updateFromRunningProject(project: TestProject, projectEntry: Entry) { + private _updateFromRunningProject(project: TestProject, projectSuite: reporterTypes.Suite) { // When running tests, don't remove existing entries. - for (const fileEntry of projectEntry.children || []) { - if (!fileEntry.children) + for (const fileSuite of projectSuite.suites) { + if (!fileSuite.allTests().length) continue; - let file = project.files.get(fileEntry.location.file); + let file = project.files.get(fileSuite.location!.file); if (!file) - file = this._createFile(project, fileEntry.location.file); + file = this._createFile(project, fileSuite.location!.file); if (!file.entries()) - file.setEntries(fileEntry.children); + file.setEntries([...fileSuite.suites, ...fileSuite.tests]); } this._didUpdate.fire(); } - async runTests(projects: TestProject[], locations: string[] | null, testListener: TestListener, parametrizedTestTitle: string | undefined, token: vscodeTypes.CancellationToken) { + async runTests(projects: TestProject[], locations: string[] | null, reporter: reporterTypes.ReporterV2, parametrizedTestTitle: string | undefined, token: vscodeTypes.CancellationToken) { locations = locations || []; - await this._playwrightTest.runTests(this.config, projects.map(p => p.name), locations, testListener, parametrizedTestTitle, token); + await this._playwrightTest.runTests(this.config, projects.map(p => p.name), locations, reporter, parametrizedTestTitle, token); } - async debugTests(projects: TestProject[], locations: string[] | null, testListener: TestListener, parametrizedTestTitle: string | undefined, token: vscodeTypes.CancellationToken) { + async debugTests(projects: TestProject[], locations: string[] | null, reporter: reporterTypes.ReporterV2, parametrizedTestTitle: string | undefined, token: vscodeTypes.CancellationToken) { locations = locations || []; - await this._playwrightTest.debugTests(this._vscode, this.config, projects.map(p => p.name), projects.map(p => p.testDir), this._envProvider(), locations, testListener, parametrizedTestTitle, token); + await this._playwrightTest.debugTests(this._vscode, this.config, projects.map(p => p.name), projects.map(p => p.testDir), this._envProvider(), locations, reporter, parametrizedTestTitle, token); } private _mapFilesToSources(files: Set): Set { diff --git a/src/testServerController.ts b/src/testServerController.ts index d702af4bf..4fc42a8dc 100644 --- a/src/testServerController.ts +++ b/src/testServerController.ts @@ -13,13 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +import path from 'path'; import { BackendClient, BackendServer } from './backend'; import type { ConfigFindRelatedTestFilesReport } from './listTests'; -import type { TestConfig, TestListener } from './playwrightTest'; -import { translateMessage } from './reporterServer'; +import type { TestConfig } from './playwrightTest'; import type { TestServerEvents, TestServerInterface } from './testServerInterface'; import type * as vscodeTypes from './vscodeTypes'; +import type * as reporterTypes from './reporter'; +import { TeleReporterReceiver } from './upstream/teleReceiver'; export class TestServerController implements vscodeTypes.Disposable { private _vscode: vscodeTypes.VSCode; @@ -96,13 +97,23 @@ class TestServer extends BackendClient implements TestServerInterface, TestServe this.close(); } - async wireTestListener(listener: TestListener, token: vscodeTypes.CancellationToken) { + async wireTestListener(mode: 'test' | 'list', reporter: reporterTypes.ReporterV2, token: vscodeTypes.CancellationToken) { + const teleReceiver = new TeleReporterReceiver(reporter, { + mergeProjects: true, + mergeTestCases: true, + resolvePath: (rootDir: string, relativePath: string) => this.vscode.Uri.file(path.join(rootDir, relativePath)).fsPath, + }); return new Promise(f => { - const reportHandler = (message: any) => translateMessage(this.vscode, message, listener, () => { - this.off('report', reportHandler); - f(); - }, token); - this.on('report', reportHandler); + const handler = (message: any) => { + if (token.isCancellationRequested && message.method !== 'onEnd') + return; + teleReceiver.dispatch(mode, message); + if (message.method === 'onEnd') { + this.off('report', handler); + f(); + } + }; + this.on('report', handler); }); } } diff --git a/src/testServerInterface.ts b/src/testServerInterface.ts index 903145a05..31b8b4f40 100644 --- a/src/testServerInterface.ts +++ b/src/testServerInterface.ts @@ -62,5 +62,6 @@ export interface TestServerInterface { } export interface TestServerEvents { + on(event: 'report', listener: (params: any) => void): void; on(event: 'stdio', listener: (params: { type: 'stdout' | 'stderr', text?: string, buffer?: string }) => void): void; } diff --git a/src/testTree.ts b/src/testTree.ts index 8baba1f6b..55beaf3ce 100644 --- a/src/testTree.ts +++ b/src/testTree.ts @@ -16,12 +16,13 @@ import path from 'path'; import { MultiMap } from './multimap'; -import type { Location, Entry, EntryType } from './oopReporter'; -import { TestModel, TestProject } from './testModel'; +import { TestModel, TestProject, TestEntry } from './testModel'; import { createGuid } from './utils'; import * as vscodeTypes from './vscodeTypes'; +import * as reporterTypes from './reporter'; -type EntriesByTitle = MultiMap; +type EntriesByTitle = MultiMap; +type EntryType = 'test' | 'suite'; /** * This class maps a collection of TestModels into the UI terms, it merges @@ -171,14 +172,15 @@ export class TestTree { // Process each testItem exactly once. let testItem = existingItems.get(title); const firstEntry = entriesWithTag[0].entry; + const entryLocation = firstEntry.location!; if (!testItem) { // We sort by id in tests, so start with location. - testItem = this._testController.createTestItem(this._id(firstEntry.location.file + ':' + firstEntry.location.line + '|' + firstEntry.titlePath.join('|')), firstEntry.title, this._vscode.Uri.file(firstEntry.location.file)); - (testItem as any)[itemTypeSymbol] = firstEntry.type; + testItem = this._testController.createTestItem(this._id(entryLocation.file + ':' + entryLocation.line + '|' + firstEntry.titlePath().slice(3).join('|')), firstEntry.title, this._vscode.Uri.file(entryLocation.file)); + (testItem as any)[itemTypeSymbol] = 'id' in firstEntry ? 'test' : 'suite'; collection.add(testItem); } - if (!testItem.range || testItem.range.start.line + 1 !== firstEntry.location.line) { - const line = firstEntry.location.line; + if (!testItem.range || testItem.range.start.line + 1 !== entryLocation.line) { + const line = entryLocation.line; testItem.range = new this._vscode.Range(line - 1, 0, line, 0); } @@ -186,10 +188,12 @@ export class TestTree { for (const { projectTag, entry } of entriesWithTag) { if (!testItem.tags.includes(projectTag)) testItem.tags = [...testItem.tags, projectTag]; - if (entry.testId) - addTestIdToTreeItem(testItem, entry.testId); - for (const child of entry.children || []) - childEntries.set(child.title, { entry: child, projectTag }); + if ('id' in entry) { + addTestIdToTreeItem(testItem, entry.id); + } else { + for (const child of [...entry.suites, ...entry.tests]) + childEntries.set(child.title, { entry: child, projectTag }); + } } itemsToDelete.delete(testItem); this._updateTestItems(testItem.children, childEntries); @@ -229,7 +233,7 @@ export class TestTree { return folderItem; } - testItemForLocation(location: Location, titlePath: string[]): vscodeTypes.TestItem | undefined { + testItemForLocation(location: reporterTypes.Location, titlePath: string[]): vscodeTypes.TestItem | undefined { const fileItem = this._fileItems.get(location.file); if (!fileItem) return; diff --git a/src/upstream/teleEmitter.ts b/src/upstream/teleEmitter.ts new file mode 100644 index 000000000..357477285 --- /dev/null +++ b/src/upstream/teleEmitter.ts @@ -0,0 +1,278 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; +import { createGuid } from '../utils'; +import type * as reporterTypes from '../reporter'; +import type * as teleReceiver from './teleReceiver'; +import { serializeRegexPatterns } from './teleReceiver'; +import type { ReporterV2 } from '../reporter'; + +export type TeleReporterEmitterOptions = { + omitOutput?: boolean; + omitBuffers?: boolean; +}; + +export class TeleReporterEmitter implements ReporterV2 { + private _messageSink: (message: teleReceiver.JsonEvent) => void; + private _rootDir!: string; + private _emitterOptions: TeleReporterEmitterOptions; + + constructor(messageSink: (message: teleReceiver.JsonEvent) => void, options: TeleReporterEmitterOptions = {}) { + this._messageSink = messageSink; + this._emitterOptions = options; + } + + version(): 'v2' { + return 'v2'; + } + + onConfigure(config: reporterTypes.FullConfig) { + this._rootDir = config.rootDir; + this._messageSink({ method: 'onConfigure', params: { config: this._serializeConfig(config) } }); + } + + onBegin(suite: reporterTypes.Suite) { + const projects = suite.suites.map(projectSuite => this._serializeProject(projectSuite)); + for (const project of projects) + this._messageSink({ method: 'onProject', params: { project } }); + this._messageSink({ method: 'onBegin', params: undefined }); + } + + onTestBegin(test: reporterTypes.TestCase, result: reporterTypes.TestResult): void { + (result as any)[idSymbol] = createGuid(); + this._messageSink({ + method: 'onTestBegin', + params: { + testId: test.id, + result: this._serializeResultStart(result) + } + }); + } + + onTestEnd(test: reporterTypes.TestCase, result: reporterTypes.TestResult): void { + const testEnd: teleReceiver.JsonTestEnd = { + testId: test.id, + expectedStatus: test.expectedStatus, + annotations: test.annotations, + timeout: test.timeout, + }; + this._messageSink({ + method: 'onTestEnd', + params: { + test: testEnd, + result: this._serializeResultEnd(result), + } + }); + } + + onStepBegin(test: reporterTypes.TestCase, result: reporterTypes.TestResult, step: reporterTypes.TestStep): void { + (step as any)[idSymbol] = createGuid(); + this._messageSink({ + method: 'onStepBegin', + params: { + testId: test.id, + resultId: (result as any)[idSymbol], + step: this._serializeStepStart(step) + } + }); + } + + onStepEnd(test: reporterTypes.TestCase, result: reporterTypes.TestResult, step: reporterTypes.TestStep): void { + this._messageSink({ + method: 'onStepEnd', + params: { + testId: test.id, + resultId: (result as any)[idSymbol], + step: this._serializeStepEnd(step) + } + }); + } + + onError(error: reporterTypes.TestError): void { + this._messageSink({ + method: 'onError', + params: { error } + }); + } + + onStdOut(chunk: string | Buffer, test?: reporterTypes.TestCase, result?: reporterTypes.TestResult): void { + this._onStdIO('stdout', chunk, test, result); + } + + onStdErr(chunk: string | Buffer, test?: reporterTypes.TestCase, result?: reporterTypes.TestResult): void { + this._onStdIO('stderr', chunk, test, result); + } + + private _onStdIO(type: teleReceiver.JsonStdIOType, chunk: string | Buffer, test: void | reporterTypes.TestCase, result: void | reporterTypes.TestResult): void { + if (this._emitterOptions.omitOutput) + return; + const isBase64 = typeof chunk !== 'string'; + const data = isBase64 ? chunk.toString('base64') : chunk; + this._messageSink({ + method: 'onStdIO', + params: { testId: test?.id, resultId: result ? (result as any)[idSymbol] : undefined, type, data, isBase64 } + }); + } + + async onEnd(result: reporterTypes.FullResult) { + const resultPayload: teleReceiver.JsonFullResult = { + status: result.status, + startTime: result.startTime.getTime(), + duration: result.duration, + }; + this._messageSink({ + method: 'onEnd', + params: { + result: resultPayload + } + }); + } + + async onExit() { + } + + printsToStdio() { + return false; + } + + private _serializeConfig(config: reporterTypes.FullConfig): teleReceiver.JsonConfig { + return { + configFile: this._relativePath(config.configFile), + globalTimeout: config.globalTimeout, + maxFailures: config.maxFailures, + metadata: config.metadata, + rootDir: config.rootDir, + version: config.version, + workers: config.workers, + }; + } + + private _serializeProject(suite: reporterTypes.Suite): teleReceiver.JsonProject { + const project = suite.project()!; + const report: teleReceiver.JsonProject = { + metadata: project.metadata, + name: project.name, + outputDir: this._relativePath(project.outputDir), + repeatEach: project.repeatEach, + retries: project.retries, + testDir: this._relativePath(project.testDir), + testIgnore: serializeRegexPatterns(project.testIgnore), + testMatch: serializeRegexPatterns(project.testMatch), + timeout: project.timeout, + suites: suite.suites.map(fileSuite => { + return this._serializeSuite(fileSuite); + }), + grep: serializeRegexPatterns(project.grep), + grepInvert: serializeRegexPatterns(project.grepInvert || []), + dependencies: project.dependencies, + snapshotDir: this._relativePath(project.snapshotDir), + teardown: project.teardown, + }; + return report; + } + + private _serializeSuite(suite: reporterTypes.Suite): teleReceiver.JsonSuite { + const result = { + title: suite.title, + location: this._relativeLocation(suite.location), + suites: suite.suites.map(s => this._serializeSuite(s)), + tests: suite.tests.map(t => this._serializeTest(t)), + }; + return result; + } + + private _serializeTest(test: reporterTypes.TestCase): teleReceiver.JsonTestCase { + return { + testId: test.id, + title: test.title, + location: this._relativeLocation(test.location), + retries: test.retries, + tags: test.tags, + repeatEachIndex: test.repeatEachIndex, + }; + } + + private _serializeResultStart(result: reporterTypes.TestResult): teleReceiver.JsonTestResultStart { + return { + id: (result as any)[idSymbol], + retry: result.retry, + workerIndex: result.workerIndex, + parallelIndex: result.parallelIndex, + startTime: +result.startTime, + }; + } + + private _serializeResultEnd(result: reporterTypes.TestResult): teleReceiver.JsonTestResultEnd { + return { + id: (result as any)[idSymbol], + duration: result.duration, + status: result.status, + errors: result.errors, + attachments: this._serializeAttachments(result.attachments), + }; + } + + _serializeAttachments(attachments: reporterTypes.TestResult['attachments']): teleReceiver.JsonAttachment[] { + return attachments.map(a => { + return { + ...a, + // There is no Buffer in the browser, so there is no point in sending the data there. + base64: (a.body && !this._emitterOptions.omitBuffers) ? a.body.toString('base64') : undefined, + }; + }); + } + + private _serializeStepStart(step: reporterTypes.TestStep): teleReceiver.JsonTestStepStart { + return { + id: (step as any)[idSymbol], + parentStepId: (step.parent as any)?.[idSymbol], + title: step.title, + category: step.category, + startTime: +step.startTime, + location: this._relativeLocation(step.location), + }; + } + + private _serializeStepEnd(step: reporterTypes.TestStep): teleReceiver.JsonTestStepEnd { + return { + id: (step as any)[idSymbol], + duration: step.duration, + error: step.error, + }; + } + + private _relativeLocation(location: reporterTypes.Location): reporterTypes.Location; + private _relativeLocation(location?: reporterTypes.Location): reporterTypes.Location | undefined; + private _relativeLocation(location: reporterTypes.Location | undefined): reporterTypes.Location | undefined { + if (!location) + return location; + return { + ...location, + file: this._relativePath(location.file), + }; + } + + private _relativePath(absolutePath: string): string; + private _relativePath(absolutePath?: string): string | undefined; + private _relativePath(absolutePath?: string): string | undefined { + if (!absolutePath) + return absolutePath; + return path.relative(this._rootDir, absolutePath); + } +} + +const idSymbol = Symbol('id'); diff --git a/src/upstream/teleReceiver.ts b/src/upstream/teleReceiver.ts new file mode 100644 index 000000000..625a9a327 --- /dev/null +++ b/src/upstream/teleReceiver.ts @@ -0,0 +1,630 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Annotation } from '../reporter'; +import type { FullProject, Metadata } from '../reporter'; +import type * as reporterTypes from '../reporter'; +import type { ReporterV2 } from '../reporter'; + +export type StringIntern = (s: string) => string; +export type JsonLocation = reporterTypes.Location; +export type JsonError = string; +export type JsonStackFrame = { file: string, line: number, column: number }; + +export type JsonStdIOType = 'stdout' | 'stderr'; + +export type JsonConfig = Pick; + +export type JsonPattern = { + s?: string; + r?: { source: string, flags: string }; +}; + +export type JsonProject = { + grep: JsonPattern[]; + grepInvert: JsonPattern[]; + metadata: Metadata; + name: string; + dependencies: string[]; + // This is relative to root dir. + snapshotDir: string; + // This is relative to root dir. + outputDir: string; + repeatEach: number; + retries: number; + suites: JsonSuite[]; + teardown?: string; + // This is relative to root dir. + testDir: string; + testIgnore: JsonPattern[]; + testMatch: JsonPattern[]; + timeout: number; +}; + +export type JsonSuite = { + title: string; + location?: JsonLocation; + suites: JsonSuite[]; + tests: JsonTestCase[]; +}; + +export type JsonTestCase = { + testId: string; + title: string; + location: JsonLocation; + retries: number; + tags?: string[]; + repeatEachIndex: number; +}; + +export type JsonTestEnd = { + testId: string; + expectedStatus: reporterTypes.TestStatus; + timeout: number; + annotations: { type: string, description?: string }[]; +}; + +export type JsonTestResultStart = { + id: string; + retry: number; + workerIndex: number; + parallelIndex: number; + startTime: number; +}; + +export type JsonAttachment = Omit & { base64?: string }; + +export type JsonTestResultEnd = { + id: string; + duration: number; + status: reporterTypes.TestStatus; + errors: reporterTypes.TestError[]; + attachments: JsonAttachment[]; +}; + +export type JsonTestStepStart = { + id: string; + parentStepId?: string; + title: string; + category: string, + startTime: number; + location?: reporterTypes.Location; +}; + +export type JsonTestStepEnd = { + id: string; + duration: number; + error?: reporterTypes.TestError; +}; + +export type JsonFullResult = { + status: reporterTypes.FullResult['status']; + startTime: number; + duration: number; +}; + +export type JsonEvent = { + method: string; + params: any +}; + +type TeleReporterReceiverOptions = { + mergeProjects: boolean; + mergeTestCases: boolean; + resolvePath: (rootDir: string, relativePath: string) => string; + configOverrides?: Pick; +}; + +export class TeleReporterReceiver { + private _rootSuite: TeleSuite; + private _options: TeleReporterReceiverOptions; + private _reporter: Partial; + private _tests = new Map(); + private _rootDir!: string; + private _listOnly = false; + private _clearPreviousResultsWhenTestBegins: boolean = false; + private _config!: reporterTypes.FullConfig; + + constructor(reporter: Partial, options: TeleReporterReceiverOptions) { + this._rootSuite = new TeleSuite('', 'root'); + this._options = options; + this._reporter = reporter; + } + + dispatch(mode: 'list' | 'test', message: JsonEvent): Promise | void { + const { method, params } = message; + if (method === 'onConfigure') { + this._onConfigure(params.config); + this._listOnly = mode === 'list'; + return; + } + if (method === 'onProject') { + this._onProject(params.project); + return; + } + if (method === 'onBegin') { + this._onBegin(); + return; + } + if (method === 'onTestBegin') { + this._onTestBegin(params.testId, params.result); + return; + } + if (method === 'onTestEnd') { + this._onTestEnd(params.test, params.result); + return; + } + if (method === 'onStepBegin') { + this._onStepBegin(params.testId, params.resultId, params.step); + return; + } + if (method === 'onStepEnd') { + this._onStepEnd(params.testId, params.resultId, params.step); + return; + } + if (method === 'onError') { + this._onError(params.error); + return; + } + if (method === 'onStdIO') { + this._onStdIO(params.type, params.testId, params.resultId, params.data, params.isBase64); + return; + } + if (method === 'onEnd') + return this._onEnd(params.result); + if (method === 'onExit') + return this._onExit(); + } + + _setClearPreviousResultsWhenTestBegins() { + this._clearPreviousResultsWhenTestBegins = true; + } + + private _onConfigure(config: JsonConfig) { + this._rootDir = config.rootDir; + this._config = this._parseConfig(config); + this._reporter.onConfigure?.(this._config); + } + + private _onProject(project: JsonProject) { + let projectSuite = this._options.mergeProjects ? this._rootSuite.suites.find(suite => suite.project()!.name === project.name) : undefined; + if (!projectSuite) { + projectSuite = new TeleSuite(project.name, 'project'); + this._rootSuite.suites.push(projectSuite); + projectSuite.parent = this._rootSuite; + } + // Always update project in watch mode. + projectSuite._project = this._parseProject(project); + this._mergeSuitesInto(project.suites, projectSuite); + + // Remove deleted tests when listing. Empty suites will be auto-filtered + // in the UI layer. + if (this._listOnly) { + const testIds = new Set(); + const collectIds = (suite: JsonSuite) => { + suite.tests.map(t => t.testId).forEach(testId => testIds.add(testId)); + suite.suites.forEach(collectIds); + }; + project.suites.forEach(collectIds); + + const filterTests = (suite: TeleSuite) => { + suite.tests = suite.tests.filter(t => testIds.has(t.id)); + suite.suites.forEach(filterTests); + }; + filterTests(projectSuite); + } + } + + private _onBegin() { + this._reporter.onBegin?.(this._rootSuite); + } + + private _onTestBegin(testId: string, payload: JsonTestResultStart) { + const test = this._tests.get(testId)!; + if (this._clearPreviousResultsWhenTestBegins) + test._clearResults(); + const testResult = test._createTestResult(payload.id); + testResult.retry = payload.retry; + testResult.workerIndex = payload.workerIndex; + testResult.parallelIndex = payload.parallelIndex; + testResult.setStartTimeNumber(payload.startTime); + this._reporter.onTestBegin?.(test, testResult); + } + + private _onTestEnd(testEndPayload: JsonTestEnd, payload: JsonTestResultEnd) { + const test = this._tests.get(testEndPayload.testId)!; + test.timeout = testEndPayload.timeout; + test.expectedStatus = testEndPayload.expectedStatus; + test.annotations = testEndPayload.annotations; + const result = test._resultsMap.get(payload.id)!; + result.duration = payload.duration; + result.status = payload.status; + result.errors = payload.errors; + result.error = result.errors?.[0]; + result.attachments = this._parseAttachments(payload.attachments); + this._reporter.onTestEnd?.(test, result); + // Free up the memory as won't see these step ids. + result._stepMap = new Map(); + } + + private _onStepBegin(testId: string, resultId: string, payload: JsonTestStepStart) { + const test = this._tests.get(testId)!; + const result = test._resultsMap.get(resultId)!; + const parentStep = payload.parentStepId ? result._stepMap.get(payload.parentStepId) : undefined; + + const location = this._absoluteLocation(payload.location); + const step = new TeleTestStep(payload, parentStep, location); + if (parentStep) + parentStep.steps.push(step); + else + result.steps.push(step); + result._stepMap.set(payload.id, step); + this._reporter.onStepBegin?.(test, result, step); + } + + private _onStepEnd(testId: string, resultId: string, payload: JsonTestStepEnd) { + const test = this._tests.get(testId)!; + const result = test._resultsMap.get(resultId)!; + const step = result._stepMap.get(payload.id)!; + step.duration = payload.duration; + step.error = payload.error; + this._reporter.onStepEnd?.(test, result, step); + } + + private _onError(error: reporterTypes.TestError) { + this._reporter.onError?.(error); + } + + private _onStdIO(type: JsonStdIOType, testId: string | undefined, resultId: string | undefined, data: string, isBase64: boolean) { + const chunk = isBase64 ? ((globalThis as any).Buffer ? Buffer.from(data, 'base64') : atob(data)) : data; + const test = testId ? this._tests.get(testId) : undefined; + const result = test && resultId ? test._resultsMap.get(resultId) : undefined; + if (type === 'stdout') { + result?.stdout.push(chunk); + this._reporter.onStdOut?.(chunk, test, result); + } else { + result?.stderr.push(chunk); + this._reporter.onStdErr?.(chunk, test, result); + } + } + + private async _onEnd(result: JsonFullResult): Promise { + await this._reporter.onEnd?.({ + status: result.status, + startTime: new Date(result.startTime), + duration: result.duration, + }); + } + + private _onExit(): Promise | void { + return this._reporter.onExit?.(); + } + + private _parseConfig(config: JsonConfig): reporterTypes.FullConfig { + const result = { ...baseFullConfig, ...config }; + if (this._options.configOverrides) { + result.configFile = this._options.configOverrides.configFile; + result.reportSlowTests = this._options.configOverrides.reportSlowTests; + result.quiet = this._options.configOverrides.quiet; + result.reporter = [...this._options.configOverrides.reporter]; + } + return result; + } + + private _parseProject(project: JsonProject): TeleFullProject { + return { + metadata: project.metadata, + name: project.name, + outputDir: this._absolutePath(project.outputDir), + repeatEach: project.repeatEach, + retries: project.retries, + testDir: this._absolutePath(project.testDir), + testIgnore: parseRegexPatterns(project.testIgnore), + testMatch: parseRegexPatterns(project.testMatch), + timeout: project.timeout, + grep: parseRegexPatterns(project.grep) as RegExp[], + grepInvert: parseRegexPatterns(project.grepInvert) as RegExp[], + dependencies: project.dependencies, + teardown: project.teardown, + snapshotDir: this._absolutePath(project.snapshotDir), + use: {}, + }; + } + + private _parseAttachments(attachments: JsonAttachment[]): reporterTypes.TestResult['attachments'] { + return attachments.map(a => { + return { + ...a, + body: a.base64 && (globalThis as any).Buffer ? Buffer.from(a.base64, 'base64') : undefined, + }; + }); + } + + private _mergeSuitesInto(jsonSuites: JsonSuite[], parent: TeleSuite) { + for (const jsonSuite of jsonSuites) { + let targetSuite = parent.suites.find(s => s.title === jsonSuite.title); + if (!targetSuite) { + targetSuite = new TeleSuite(jsonSuite.title, parent._type === 'project' ? 'file' : 'describe'); + targetSuite.parent = parent; + parent.suites.push(targetSuite); + } + targetSuite.location = this._absoluteLocation(jsonSuite.location); + this._mergeSuitesInto(jsonSuite.suites, targetSuite); + this._mergeTestsInto(jsonSuite.tests, targetSuite); + } + } + + private _mergeTestsInto(jsonTests: JsonTestCase[], parent: TeleSuite) { + for (const jsonTest of jsonTests) { + let targetTest = this._options.mergeTestCases ? parent.tests.find(s => s.title === jsonTest.title && s.repeatEachIndex === jsonTest.repeatEachIndex) : undefined; + if (!targetTest) { + targetTest = new TeleTestCase(jsonTest.testId, jsonTest.title, this._absoluteLocation(jsonTest.location), jsonTest.repeatEachIndex); + targetTest.parent = parent; + parent.tests.push(targetTest); + this._tests.set(targetTest.id, targetTest); + } + this._updateTest(jsonTest, targetTest); + } + } + + private _updateTest(payload: JsonTestCase, test: TeleTestCase): TeleTestCase { + test.id = payload.testId; + test.location = this._absoluteLocation(payload.location); + test.retries = payload.retries; + test.tags = payload.tags ?? []; + return test; + } + + private _absoluteLocation(location: reporterTypes.Location): reporterTypes.Location; + private _absoluteLocation(location?: reporterTypes.Location): reporterTypes.Location | undefined; + private _absoluteLocation(location: reporterTypes.Location | undefined): reporterTypes.Location | undefined { + if (!location) + return location; + return { + ...location, + file: this._absolutePath(location.file), + }; + } + + private _absolutePath(relativePath: string): string; + private _absolutePath(relativePath?: string): string | undefined; + private _absolutePath(relativePath?: string): string | undefined { + if (relativePath === undefined) + return; + return this._options.resolvePath(this._rootDir, relativePath); + } +} + +export class TeleSuite implements reporterTypes.Suite { + title: string; + location?: reporterTypes.Location; + parent?: TeleSuite; + _requireFile: string = ''; + suites: TeleSuite[] = []; + tests: TeleTestCase[] = []; + _timeout: number | undefined; + _retries: number | undefined; + _project: TeleFullProject | undefined; + _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none'; + readonly _type: 'root' | 'project' | 'file' | 'describe'; + + constructor(title: string, type: 'root' | 'project' | 'file' | 'describe') { + this.title = title; + this._type = type; + } + + allTests(): TeleTestCase[] { + const result: TeleTestCase[] = []; + const visit = (suite: TeleSuite) => { + for (const entry of [...suite.suites, ...suite.tests]) { + if (entry instanceof TeleSuite) + visit(entry); + else + result.push(entry); + } + }; + visit(this); + return result; + } + + titlePath(): string[] { + const titlePath = this.parent ? this.parent.titlePath() : []; + // Ignore anonymous describe blocks. + if (this.title || this._type !== 'describe') + titlePath.push(this.title); + return titlePath; + } + + project(): TeleFullProject | undefined { + return this._project ?? this.parent?.project(); + } +} + +export class TeleTestCase implements reporterTypes.TestCase { + title: string; + fn = () => {}; + results: TeleTestResult[] = []; + location: reporterTypes.Location; + parent!: TeleSuite; + + expectedStatus: reporterTypes.TestStatus = 'passed'; + timeout = 0; + annotations: Annotation[] = []; + retries = 0; + tags: string[] = []; + repeatEachIndex = 0; + id: string; + + _resultsMap = new Map(); + + constructor(id: string, title: string, location: reporterTypes.Location, repeatEachIndex: number) { + this.id = id; + this.title = title; + this.location = location; + this.repeatEachIndex = repeatEachIndex; + } + + titlePath(): string[] { + const titlePath = this.parent ? this.parent.titlePath() : []; + titlePath.push(this.title); + return titlePath; + } + + outcome(): 'skipped' | 'expected' | 'unexpected' | 'flaky' { + // Ignore initial skips that may be a result of "skipped because previous test in serial mode failed". + const results = [...this.results]; + while (results[0]?.status === 'skipped' || results[0]?.status === 'interrupted') + results.shift(); + + // All runs were skipped. + if (!results.length) + return 'skipped'; + + const failures = results.filter(result => result.status !== 'skipped' && result.status !== 'interrupted' && result.status !== this.expectedStatus); + if (!failures.length) // all passed + return 'expected'; + if (failures.length === results.length) // all failed + return 'unexpected'; + return 'flaky'; // mixed bag + } + + ok(): boolean { + const status = this.outcome(); + return status === 'expected' || status === 'flaky' || status === 'skipped'; + } + + _clearResults() { + this.results = []; + this._resultsMap.clear(); + } + + _createTestResult(id: string): TeleTestResult { + const result = new TeleTestResult(this.results.length); + this.results.push(result); + this._resultsMap.set(id, result); + return result; + } +} + +class TeleTestStep implements reporterTypes.TestStep { + title: string; + category: string; + location: reporterTypes.Location | undefined; + parent: reporterTypes.TestStep | undefined; + duration: number = -1; + steps: reporterTypes.TestStep[] = []; + + private _startTime: number = 0; + + constructor(payload: JsonTestStepStart, parentStep: reporterTypes.TestStep | undefined, location: reporterTypes.Location | undefined) { + this.title = payload.title; + this.category = payload.category; + this.location = location; + this.parent = parentStep; + this._startTime = payload.startTime; + } + + titlePath() { + const parentPath = this.parent?.titlePath() || []; + return [...parentPath, this.title]; + } + + get startTime(): Date { + return new Date(this._startTime); + } + + set startTime(value: Date) { + this._startTime = +value; + } +} + +class TeleTestResult implements reporterTypes.TestResult { + retry: reporterTypes.TestResult['retry']; + parallelIndex: reporterTypes.TestResult['parallelIndex'] = -1; + workerIndex: reporterTypes.TestResult['workerIndex'] = -1; + duration: reporterTypes.TestResult['duration'] = -1; + stdout: reporterTypes.TestResult['stdout'] = []; + stderr: reporterTypes.TestResult['stderr'] = []; + attachments: reporterTypes.TestResult['attachments'] = []; + status: reporterTypes.TestStatus = 'skipped'; + steps: TeleTestStep[] = []; + errors: reporterTypes.TestResult['errors'] = []; + error: reporterTypes.TestResult['error']; + + _stepMap: Map = new Map(); + + private _startTime: number = 0; + + constructor(retry: number) { + this.retry = retry; + } + + setStartTimeNumber(startTime: number) { + this._startTime = startTime; + } + + get startTime(): Date { + return new Date(this._startTime); + } + + set startTime(value: Date) { + this._startTime = +value; + } +} + +export type TeleFullProject = FullProject; + +export const baseFullConfig: reporterTypes.FullConfig = { + forbidOnly: false, + fullyParallel: false, + globalSetup: null, + globalTeardown: null, + globalTimeout: 0, + grep: /.*/, + grepInvert: null, + maxFailures: 0, + metadata: {}, + preserveOutput: 'always', + projects: [], + reporter: [[process.env.CI ? 'dot' : 'list']], + reportSlowTests: { max: 5, threshold: 15000 }, + configFile: '', + rootDir: '', + quiet: false, + shard: null, + updateSnapshots: 'missing', + version: '', + workers: 0, + webServer: null, +}; + +export function serializeRegexPatterns(patterns: string | RegExp | (string | RegExp)[]): JsonPattern[] { + if (!Array.isArray(patterns)) + patterns = [patterns]; + return patterns.map(s => { + if (typeof s === 'string') + return { s }; + return { r: { source: s.source, flags: s.flags } }; + }); +} + +export function parseRegexPatterns(patterns: JsonPattern[]): (string | RegExp)[] { + return patterns.map(p => { + if (p.s) + return p.s; + return new RegExp(p.r!.source, p.r!.flags); + }); +}