chore: optionally use test server (#421)

microsoft · Feb 21, 2024 · 7eb9392 · 7eb9392
1 parent d6a0523
commit 7eb9392
Show file tree

Hide file tree

Showing 21 changed files with 398 additions and 178 deletions.
diff --git a/playwright.config.ts b/playwright.config.ts
@@ -18,6 +18,7 @@ import { WorkerOptions } from './tests/utils';
 
 const config: PlaywrightTestConfig<WorkerOptions> = {
   testDir: './tests',
+  outputDir: './test-results/inner',
   fullyParallel: true,
   forbidOnly: !!process.env.CI,
   workers: process.env.CI ? 1 : undefined,

diff --git a/src/backend.ts b/src/backend.ts
@@ -20,41 +20,47 @@ import * as vscodeTypes from './vscodeTypes';
 import EventEmitter from 'events';
 import { WebSocketTransport } from './transport';
 
+export type BackendServerOptions<T extends BackendClient> = {
+  args: string[],
+  cwd: string,
+  envProvider: () => NodeJS.ProcessEnv,
+  clientFactory: () => T,
+  dumpIO?: boolean,
+};
+
 export class BackendServer<T extends BackendClient> {
   private _vscode: vscodeTypes.VSCode;
-  private _args: string[];
-  private _cwd: string;
-  private _envProvider: () => NodeJS.ProcessEnv;
-  private _clientFactory: () => T;
+  private _options: BackendServerOptions<T>;
 
-  constructor(vscode: vscodeTypes.VSCode, args: string[], cwd: string, envProvider: () => NodeJS.ProcessEnv, clientFactory: () => T) {
+  constructor(vscode: vscodeTypes.VSCode, options: BackendServerOptions<T>) {
     this._vscode = vscode;
-    this._args = args;
-    this._cwd = cwd;
-    this._envProvider = envProvider;
-    this._clientFactory = clientFactory;
+    this._options = options;
   }
 
   async start(): Promise<T | null> {
-    const node = await findNode(this._vscode, this._cwd);
-    const serverProcess = spawn(node, this._args, {
-      cwd: this._cwd,
+    const node = await findNode(this._vscode, this._options.cwd);
+    const serverProcess = spawn(node, this._options.args, {
+      cwd: this._options.cwd,
       stdio: 'pipe',
       env: {
         ...process.env,
-        ...this._envProvider(),
+        ...this._options.envProvider(),
       },
     });
 
-    const client = this._clientFactory();
+    const client = this._options.clientFactory();
 
-    serverProcess.stderr?.on('data', () => {});
+    serverProcess.stderr?.on('data', data => {
+      if (this._options.dumpIO)
+        console.log('[server err]', data.toString());
+    });
     serverProcess.on('error', error => {
       client._onErrorEvent.fire(error);
     });
-
     return new Promise(fulfill => {
       serverProcess.stdout?.on('data', async data => {
+        if (this._options.dumpIO)
+          console.log('[server out]', data.toString());
         const match = data.toString().match(/Listening on (.*)/);
         if (!match)
           return;

diff --git a/src/extension.ts b/src/extension.ts
@@ -30,6 +30,7 @@ import { NodeJSNotFoundError, ansiToHtml } from './utils';
 import * as vscodeTypes from './vscodeTypes';
 import { WorkspaceChange, WorkspaceObserver } from './workspaceObserver';
 import { TraceViewer } from './traceViewer';
+import { TestServerController } from './testServerController';
 
 const stackUtils = new StackUtils({
   cwd: '/ensure_absolute_paths'
@@ -87,6 +88,7 @@ export class Extension implements RunHooks {
   } | undefined;
   private _diagnostics: Record<'configErrors' | 'testErrors', vscodeTypes.DiagnosticCollection>;
   private _treeItemObserver: TreeItemObserver;
+  private _testServerController: TestServerController;
 
   constructor(vscode: vscodeTypes.VSCode) {
     this._vscode = vscode;
@@ -111,7 +113,8 @@ export class Extension implements RunHooks {
     this._settingsModel = new SettingsModel(vscode);
     this._reusedBrowser = new ReusedBrowser(this._vscode, this._settingsModel, this._envProvider.bind(this));
     this._traceViewer = new TraceViewer(this._vscode, this._settingsModel, this._envProvider.bind(this));
-    this._playwrightTest = new PlaywrightTest(this._vscode, this._settingsModel, this, this._isUnderTest, this._envProvider.bind(this));
+    this._testServerController = new TestServerController(this._vscode, this._envProvider.bind(this));
+    this._playwrightTest = new PlaywrightTest(this._vscode, this._settingsModel, this, this._isUnderTest, this._testServerController, this._envProvider.bind(this));
     this._testController = vscode.tests.createTestController('pw.extension.testController', 'Playwright');
     this._testController.resolveHandler = item => this._resolveChildren(item);
     this._testController.refreshHandler = () => this._rebuildModel(true).then(() => {});
@@ -214,6 +217,7 @@ export class Extension implements RunHooks {
       this._reusedBrowser,
       ...Object.values(this._diagnostics),
       this._treeItemObserver,
+      this._testServerController,
     ];
     await this._rebuildModel(false);
 

diff --git a/src/oopReporter.ts b/src/oopReporter.ts
@@ -66,13 +66,15 @@ class OopReporter implements Reporter {
 
   constructor() {
     this._transport = WebSocketTransport.connect(process.env.PW_TEST_REPORTER_WS_ENDPOINT!);
-    this._transport.then(t => {
-      t.onmessage = message => {
-        if (message.method === 'stop')
-          process.emit('SIGINT' as any);
-      };
-      t.onclose = () => process.exit(0);
-    });
+    if (process.env.PW_TEST_REPORTER_SELF_DESTRUCT) {
+      this._transport.then(t => {
+        t.onmessage = message => {
+          if (message.method === 'stop')
+            process.emit('SIGINT' as any);
+        };
+        t.onclose = () => process.exit(0);
+      });
+    }
   }
 
   printsToStdio() {
@@ -177,7 +179,8 @@ class OopReporter implements Reporter {
   async onEnd(result: FullResult) {
     this._emit('onEnd', {});
     // Embedder is responsible for terminating the connection.
-    await new Promise(() => {});
+    if (process.env.PW_TEST_REPORTER_SELF_DESTRUCT)
+      await new Promise(() => {});
   }
 
   private _emit(method: string, params: Object) {

diff --git a/src/playwrightTest.ts b/src/playwrightTest.ts
@@ -23,6 +23,7 @@ import { ReporterServer } from './reporterServer';
 import { findNode, spawnAsync } from './utils';
 import * as vscodeTypes from './vscodeTypes';
 import { SettingsModel } from './settingsModel';
+import { TestServerController } from './testServerController';
 
 export type TestConfig = {
   workspaceFolder: string;
@@ -47,8 +48,12 @@ export interface TestListener {
 const pathSeparator = process.platform === 'win32' ? ';' : ':';
 
 export type PlaywrightTestOptions = {
+  headed?: boolean,
+  oneWorker?: boolean,
+  trace?: 'on' | 'off',
   projects?: string[];
   grep?: string;
+  reuseContext?: boolean,
   connectWsEndpoint?: string;
 };
 
@@ -64,12 +69,14 @@ export class PlaywrightTest {
   private _envProvider: () => NodeJS.ProcessEnv;
   private _vscode: vscodeTypes.VSCode;
   private _settingsModel: SettingsModel;
+  private _testServerController: TestServerController;
 
-  constructor(vscode: vscodeTypes.VSCode, settingsModel: SettingsModel, runHooks: RunHooks, isUnderTest: boolean, envProvider: () => NodeJS.ProcessEnv) {
+  constructor(vscode: vscodeTypes.VSCode, settingsModel: SettingsModel, runHooks: RunHooks, isUnderTest: boolean, testServerController: TestServerController, envProvider: () => NodeJS.ProcessEnv) {
     this._vscode = vscode;
     this._settingsModel = settingsModel;
     this._runHooks = runHooks;
     this._isUnderTest = isUnderTest;
+    this._testServerController = testServerController;
     this._envProvider = envProvider;
   }
 
@@ -117,15 +124,33 @@ export class PlaywrightTest {
     const locationArg = locations ? locations : [];
     if (token?.isCancellationRequested)
       return;
-    const testOptions = await this._runHooks.onWillRunTests(config, false);
+    const externalOptions = await this._runHooks.onWillRunTests(config, false);
+    const showBrowser = this._settingsModel.showBrowser.get() && !!externalOptions.connectWsEndpoint;
+
+    let trace: 'on' | 'off' | undefined;
+    if (this._settingsModel.showTrace.get())
+      trace = 'on';
+    // "Show browser" mode forces context reuse that survives over multiple test runs.
+    // Playwright Test sets up `tracesDir` inside the `test-results` folder, so it will be removed between runs.
+    // When context is reused, its ongoing tracing will fail with ENOENT because trace files
+    // were suddenly removed. So we disable tracing in this case.
+    if (this._settingsModel.showBrowser.get())
+      trace = 'off';
+
+    const options: PlaywrightTestOptions = {
+      grep: parametrizedTestTitle,
+      projects: projectNames.length ? projectNames.filter(Boolean) : undefined,
+      headed: showBrowser && !this._isUnderTest,
+      oneWorker: showBrowser,
+      trace,
+      reuseContext: showBrowser,
+      connectWsEndpoint: showBrowser ? externalOptions.connectWsEndpoint : undefined,
+    };
+
     try {
       if (token?.isCancellationRequested)
         return;
-      await this._test(config, locationArg, 'run', {
-        grep: parametrizedTestTitle,
-        projects: projectNames.filter(Boolean),
-        ...testOptions
-      }, listener, token);
+      await this._test(config, locationArg, 'run', options, listener, token);
     } finally {
       await this._runHooks.onDidRunTests(false);
     }
@@ -146,6 +171,13 @@ export class PlaywrightTest {
   }
 
   private async _test(config: TestConfig, locations: string[], mode: 'list' | 'run', options: PlaywrightTestOptions, listener: TestListener, token: vscodeTypes.CancellationToken): Promise<void> {
+    if (process.env.PWTEST_VSCODE_SERVER)
+      await this._testWithServer(config, locations, mode, options, listener, token);
+    else
+      await this._testWithCLI(config, locations, mode, options, listener, token);
+  }
+
+  private async _testWithCLI(config: TestConfig, locations: string[], mode: 'list' | 'run', options: PlaywrightTestOptions, listener: TestListener, token: vscodeTypes.CancellationToken): Promise<void> {
     // Playwright will restart itself as child process in the ESM mode and won't inherit the 3/4 pipes.
     // Always use ws transport to mitigate it.
     const reporterServer = new ReporterServer(this._vscode);
@@ -176,21 +208,13 @@ export class PlaywrightTest {
       '--repeat-each', '1',
       '--retries', '0',
     ];
-    const showBrowser = this._settingsModel.showBrowser.get() && !!options.connectWsEndpoint;
-    if (mode === 'run') {
-      if (showBrowser && !this._isUnderTest)
-        allArgs.push('--headed');
-      if (showBrowser)
-        allArgs.push('--workers', '1');
-      if (this._settingsModel.showTrace.get())
-        allArgs.push('--trace', 'on');
-      // "Show browser" mode forces context reuse that survives over multiple test runs.
-      // Playwright Test sets up `tracesDir` inside the `test-results` folder, so it will be removed between runs.
-      // When context is reused, its ongoing tracing will fail with ENOENT because trace files
-      // were suddenly removed. So we disable tracing in this case.
-      if (this._settingsModel.showBrowser.get())
-        allArgs.push('--trace', 'off');
-    }
+
+    if (options.headed)
+      allArgs.push('--headed');
+    if (options.oneWorker)
+      allArgs.push('--workers', '1');
+    if (options.trace)
+      allArgs.push('--trace', options.trace);
 
     const childProcess = spawn(node, allArgs, {
       cwd: configFolder,
@@ -201,9 +225,9 @@ export class PlaywrightTest {
         // Don't debug tests when running them.
         NODE_OPTIONS: undefined,
         ...this._envProvider(),
-        PW_TEST_REUSE_CONTEXT: showBrowser ? '1' : undefined,
-        PW_TEST_CONNECT_WS_ENDPOINT: showBrowser ? options.connectWsEndpoint : undefined,
-        ...(await reporterServer.env()),
+        PW_TEST_REUSE_CONTEXT: options.reuseContext ? '1' : undefined,
+        PW_TEST_CONNECT_WS_ENDPOINT: options.connectWsEndpoint,
+        ...(await reporterServer.env({ selfDestruct: true })),
         // Reset VSCode's options that affect nested Electron.
         ELECTRON_RUN_AS_NODE: undefined,
         FORCE_COLOR: '1',
@@ -218,6 +242,33 @@ export class PlaywrightTest {
     await reporterServer.wireTestListener(listener, token);
   }
 
+  private async _testWithServer(config: TestConfig, locations: string[], mode: 'list' | 'run', options: PlaywrightTestOptions, listener: TestListener, token: vscodeTypes.CancellationToken): Promise<void> {
+    const reporterServer = new ReporterServer(this._vscode);
+    const testServer = await this._testServerController.testServerFor(config);
+    if (!testServer)
+      return;
+    if (token?.isCancellationRequested)
+      return;
+    const env = await reporterServer.env({ selfDestruct: false });
+    const reporter = reporterServer.reporterFile();
+    if (mode === 'list')
+      testServer.list({ locations, reporter, env });
+    if (mode === 'run') {
+      testServer.test({ locations, reporter, env, options });
+      token.onCancellationRequested(() => {
+        testServer.stop();
+      });
+      testServer.on('stdio', params => {
+        if (params.type === 'stdout')
+          listener.onStdOut?.(unwrapString(params));
+        if (params.type === 'stderr')
+          listener.onStdErr?.(unwrapString(params));
+      });
+    }
+
+    await reporterServer.wireTestListener(listener, token);
+  }
+
   async debugTests(vscode: vscodeTypes.VSCode, config: TestConfig, projectNames: string[], testDirs: string[], settingsEnv: NodeJS.ProcessEnv, locations: string[] | null, listener: TestListener, parametrizedTestTitle: string | undefined, token: vscodeTypes.CancellationToken) {
     const configFolder = path.dirname(config.configFile);
     const configFile = path.basename(config.configFile);
@@ -255,7 +306,7 @@ export class PlaywrightTest {
           CI: this._isUnderTest ? undefined : process.env.CI,
           ...settingsEnv,
           PW_TEST_CONNECT_WS_ENDPOINT: testOptions.connectWsEndpoint,
-          ...(await reporterServer.env()),
+          ...(await reporterServer.env({ selfDestruct: true })),
           // Reset VSCode's options that affect nested Electron.
           ELECTRON_RUN_AS_NODE: undefined,
           FORCE_COLOR: '1',
@@ -289,3 +340,7 @@ export class PlaywrightTest {
 function escapeRegex(text: string) {
   return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
 }
+
+function unwrapString(params: { text?: string, buffer?: string }): string | Buffer {
+  return params.buffer ? Buffer.from(params.buffer, 'base64') : params.text || '';
+}
diff --git a/src/reporterServer.ts b/src/reporterServer.ts
@@ -33,11 +33,16 @@ export class ReporterServer {
     this._clientSocketPromise = new Promise(f => this._clientSocketCallback = f);
   }
 
-  async env() {
+  reporterFile() {
+    return require.resolve('./oopReporter');
+  }
+
+  async env(options: { selfDestruct: boolean }) {
     const wsEndpoint = await this._listen();
     return {
-      PW_TEST_REPORTER: require.resolve('./oopReporter'),
+      PW_TEST_REPORTER: this.reporterFile(),
       PW_TEST_REPORTER_WS_ENDPOINT: wsEndpoint,
+      PW_TEST_REPORTER_SELF_DESTRUCT: options.selfDestruct ? '1' : '',
     };
   }
 

diff --git a/src/reusedBrowser.ts b/src/reusedBrowser.ts
@@ -88,7 +88,12 @@ export class ReusedBrowser implements vscodeTypes.Disposable {
       PW_EXTENSION_MODE: '1',
     });
 
-    const backendServer = new BackendServer<Backend>(this._vscode, args, cwd, envProvider, () => new Backend(this._vscode));
+    const backendServer = new BackendServer<Backend>(this._vscode, {
+      args,
+      cwd,
+      envProvider,
+      clientFactory: () => new Backend(this._vscode)
+    });
     const backend = await backendServer.start();
     if (!backend)
       return;