chore: allow using test server via option

.github/workflows/ci.yml

@@ -31,7 +31,7 @@ jobs:
     - run: npx playwright install chromium
     - run: npm run lint
     - run: npm run build
-    - run: npm run test -- --workers=1
+    - run: npm run test -- --project=cli --project=cli-reuse --workers=1
     - run: npx vsce package
       if: matrix.os == 'ubuntu-latest' && matrix.node-version == '20'
     - uses: actions/upload-artifact@v3

package.json

@@ -83,6 +83,10 @@
           "type": "boolean",
           "default": false,
           "description": "%configuration.playwright.showTrace%"
+        },
+        "playwright.useTestServer": {
+          "type": "boolean",
+          "default": false
         }
       }
     },

playwright.config.ts

@@ -30,15 +30,33 @@ const config: PlaywrightTestConfig<WorkerOptions> = {
   ],
   projects: [
     {
-      name: 'default',
+      name: 'cli',
+      use: {
+        useTestServer: false,
+        showBrowser: false,
+      }
+    },
+    {
+      name: 'cli-reuse',
+      use: {
+        useTestServer: false,
+        showBrowser: true,
+      }
     },
     {
-      name: 'reuse',
-      testIgnore: '**/settings.spec.ts',
+      name: 'server',
       use: {
-        mode: 'reuse',
+        useTestServer: true,
+        showBrowser: false,
       }
-    }
+    },
+    {
+      name: 'server-reuse',
+      use: {
+        useTestServer: true,
+        showBrowser: true,
+      }
+    },
   ]
 };
 export default config;

src/extension.ts

@@ -244,6 +244,7 @@ export class Extension implements RunHooks {
   private async _rebuildModel(showWarnings: boolean): Promise<vscodeTypes.Uri[]> {
     this._testTree.startedLoading();
     this._workspaceObserver.reset();
+    this._testServerController.reset();
     this._models = [];
 
     const configFiles = await this._vscode.workspace.findFiles('**/*playwright*.config.{ts,js,mjs}', '**/node_modules/**');

src/playwrightTest.ts

@@ -171,7 +171,7 @@ export class PlaywrightTest {
   }
 
   private async _test(config: TestConfig, locations: string[], mode: 'list' | 'run', options: PlaywrightTestOptions, listener: TestListener, token: vscodeTypes.CancellationToken): Promise<void> {
-    if (process.env.PWTEST_VSCODE_SERVER)
+    if (config.version >= 1.43 || this._settingsModel.useTestServer.get())
       await this._testWithServer(config, locations, mode, options, listener, token);
     else
       await this._testWithCLI(config, locations, mode, options, listener, token);

src/settingsModel.ts

@@ -24,6 +24,7 @@ export class SettingsModel implements vscodeTypes.Disposable {
   private _disposables: vscodeTypes.Disposable[] = [];
   showBrowser: Setting<boolean>;
   showTrace: Setting<boolean>;
+  useTestServer: Setting<boolean>;
 
   constructor(vscode: vscodeTypes.VSCode) {
     this._vscode = vscode;
@@ -32,6 +33,7 @@ export class SettingsModel implements vscodeTypes.Disposable {
 
     this.showBrowser = this._createSetting('reuseBrowser');
     this.showTrace = this._createSetting('showTrace');
+    this.useTestServer = this._createSetting('useTestServer');
 
     this.showBrowser.onChange(enabled => {
       if (enabled && this.showTrace.get())

src/testServerController.ts

@@ -36,7 +36,12 @@ export class TestServerController implements vscodeTypes.Disposable {
     const testServerBackend = new BackendServer<TestServer>(this._vscode, {
       args,
       cwd: config.workspaceFolder,
-      envProvider: this._envProvider,
+      envProvider: () => {
+        return {
+          ...this._envProvider(),
+          FORCE_COLOR: '1',
+        };
+      },
       clientFactory: () => new TestServer(this._vscode),
       dumpIO: false,
     });
@@ -48,6 +53,10 @@ export class TestServerController implements vscodeTypes.Disposable {
   }
 
   dispose() {
+    this.reset();
+  }
+
+  reset() {
     for (const backend of this._testServers.values())
       backend.close();
     this._testServers.clear();

tests/auto-close.spec.ts

@@ -16,8 +16,8 @@
 
 import { connectToSharedBrowser, expect, test, waitForPage } from './utils';
 
-test.beforeEach(async ({ mode }) => {
-  test.skip(mode !== 'reuse');
+test.beforeEach(async ({ showBrowser }) => {
+  test.skip(!showBrowser);
   process.env.PW_DEBUG_CONTROLLER_HEADLESS = '1';
 });
 

tests/highlight-locators.spec.ts

@@ -17,9 +17,9 @@
 import { chromium } from '@playwright/test';
 import { expect, test } from './utils';
 
-test.beforeEach(({ mode }) => {
+test.beforeEach(({ showBrowser }) => {
   // Locator highlighting is only relevant when the browser stays open.
-  test.skip(mode !== 'reuse');
+  test.skip(!showBrowser);
   // the x-pw-highlight element has otherwise a closed shadow root.
   process.env.PWTEST_UNDER_TEST = '1';
   process.env.PW_DEBUG_CONTROLLER_HEADLESS = '1';

tests/run-tests.spec.ts

@@ -436,8 +436,8 @@ test('should group projects by config', async ({ activate }) => {
   `);
 });
 
-test('should stop', async ({ activate, mode }) => {
-  test.fixme(mode === 'reuse', 'Times out');
+test('should stop', async ({ activate, showBrowser }) => {
+  test.fixme(showBrowser, 'Times out');
   const { vscode, testController } = await activate({
     'playwright.config.js': `module.exports = { testDir: 'tests' }`,
     'tests/test.spec.ts': `
@@ -1004,8 +1004,8 @@ test('should produce output twice', async ({ activate }) => {
   `);
 });
 
-test('should disable tracing when reusing context', async ({ activate, mode }) => {
-  test.skip(mode !== 'reuse');
+test('should disable tracing when reusing context', async ({ activate, showBrowser }) => {
+  test.skip(!showBrowser);
 
   const { testController } = await activate({
     'playwright.config.js': `module.exports = { testDir: 'tests', use: { trace: 'on' } }`,
@@ -1022,8 +1022,8 @@ test('should disable tracing when reusing context', async ({ activate, mode }) =
   expect(fs.existsSync(test.info().outputPath('test-results', 'test-one', 'trace.zip'))).toBe(false);
 });
 
-test('should force workers=1 when reusing the browser', async ({ activate, mode }) => {
-  test.skip(mode !== 'reuse');
+test('should force workers=1 when reusing the browser', async ({ activate, showBrowser }) => {
+  test.skip(!showBrowser);
 
   const { testController } = await activate({
     'playwright.config.js': `module.exports = { testDir: 'tests', workers: 2 }`,

tests/settings.spec.ts

@@ -16,6 +16,10 @@
 
 import { expect, test } from './utils';
 
+test.beforeEach(async ({ showBrowser }) => {
+  test.skip(showBrowser);
+});
+
 test('should toggle settings', async ({ activate }) => {
   const { vscode } = await activate({
     'playwright.config.js': `module.exports = {}`,

tests/utils.ts

@@ -30,7 +30,8 @@ type TestFixtures = {
 };
 
 export type WorkerOptions = {
-  mode: 'default' | 'reuse';
+  useTestServer: boolean;
+  showBrowser: boolean;
 };
 
 // Make sure connect tests work with the locally-rolled version.
@@ -50,7 +51,7 @@ export const expect = baseExpect.extend({
   },
 
   toHaveExecLog(vscode: VSCode, expectedLog: string) {
-    if (process.env.PWTEST_VSCODE_SERVER)
+    if (vscode.extensions[0]._settingsModel.useTestServer.get())
       return { pass: true, message: () => '' };
     try {
       expect(vscode.renderExecLog('  ')).toBe(expectedLog);
@@ -65,9 +66,10 @@ export const expect = baseExpect.extend({
 });
 
 export const test = baseTest.extend<TestFixtures, WorkerOptions>({
-  mode: ['default', { option: true, scope: 'worker' }],
+  useTestServer: [false, { option: true, scope: 'worker' }],
+  showBrowser: [false, { option: true, scope: 'worker' }],
 
-  activate: async ({ browser, mode }, use, testInfo) => {
+  activate: async ({ browser, showBrowser, useTestServer }, use, testInfo) => {
     const instances: VSCode[] = [];
     await use(async (files: { [key: string]: string }, options?: { rootDir?: string, workspaceFolders?: [string, any][] }) => {
       const vscode = new VSCode(path.resolve(__dirname, '..'), browser);
@@ -81,10 +83,14 @@ export const test = baseTest.extend<TestFixtures, WorkerOptions>({
       vscode.extensions.push(extension);
       await vscode.activate();
 
-      if (mode === 'reuse') {
+      if (showBrowser) {
         const configuration = vscode.workspace.getConfiguration('playwright');
         configuration.update('reuseBrowser', true);
       }
+      if (useTestServer) {
+        const configuration = vscode.workspace.getConfiguration('playwright');
+        configuration.update('useTestServer', true);
+      }
 
       instances.push(vscode);
       return {