smoke tests: add only script (#4900)

Per conversation with @testlabauto , it might be beneficial to have
ability to run a subset of tests in parallel locally. Now we can just by
tagging tests with `#only` in test description.

### QA notes
* updated README to reflect these changes
* confirmed with `mocah.dryRun()` that web, pr, all still run the
expected the of filtered tests.

package.json

@@ -50,6 +50,7 @@
     "smoketest-pr": "export BUILD_ARTIFACTSTAGINGDIRECTORY=../../.build/logs/smoke-tests-electron && yarn smoketest --pr",
     "smoketest-web": "export BUILD_ARTIFACTSTAGINGDIRECTORY=../../.build/logs/smoke-tests-browser && yarn smoketest --web",
     "smoketest-win": "export BUILD_ARTIFACTSTAGINGDIRECTORY=../../.build/logs/smoke-tests-win && yarn smoketest --win",
+    "smoketest-only": "export BUILD_ARTIFACTSTAGINGDIRECTORY=../../.build/logs/smoke-tests-electron && cd test/smoke && yarn compile && node run-tests.js --only",
     "xunit-to-junit": "sh ./scripts/convert-xunit-to-junit.sh",
     "download-builtin-extensions": "node build/lib/builtInExtensions.js",
     "download-builtin-extensions-cg": "node build/lib/builtInExtensionsCG.js",

test/smoke/README.md

@@ -202,18 +202,26 @@ _Note: Don't forget to remove the `.only()`s when you're done!_
 
 #### Command Line
 
-The command line is a faster way to run tests since it **allows for parallel execution**. However, note that `.only()` does **not** work when running tests in parallel mode. For example, to run the entire test suite against your branch with 4 parallel jobs, use:
+The command line is a faster way to run tests since it **allows for parallel execution**. However, note that `.only()` does **not** work when running tests in parallel mode. To overcome this limitation and run a subset of tests in parallel locally, we introduced a workaround:
+
+1. Add `#only` to the test descriptions you want to run.
+2. Execute the following command to trigger the subset of tests:
 
 ```bash
-yarn smoketest-all --parallel --jobs 4
+yarn somketest-only
 ```
 
-The following smoketest scripts are currently available:
+Remember to remove any `#only` from test titles before committing!
+
+#### Smoke Test Scripts
+
+The following smoke test scripts are available:
 
 - `smoketest-all`: Runs all smoke tests
 - `smoketest-web`: Runs tests tagged with `#web`
 - `smoketest-win`: Runs tests tagged with `#win` (Windows)
 - `smoketest-pr`: Runs tests tagged with `#pr`
+- `smoketest-only`: Runs tests tagged with `#only`
 
 #### Target a Positron Build
 

test/smoke/src/test-runner/config.ts

@@ -20,6 +20,7 @@ Object.assign(process.env, {
 	VERBOSE: OPTS['verbose'] || '',
 	WEB: OPTS['web'] || '',
 	WIN: OPTS['win'] || '',
+	ONLY: OPTS['only'] || '',
 	PR: OPTS['pr'] || '',
 	SKIP_CLEANUP: OPTS['skip-cleanup'] || '',
 	TEST_DATA_PATH: TEST_DATA_PATH,

test/smoke/src/test-runner/mocha-runner.ts

@@ -65,15 +65,20 @@ export function runMochaTests(OPTS) {
  */
 function applyTestFilters(mocha: Mocha): void {
 	// TODO: see if it's possible to use multiple filters
-	if (process.env.WEB) {
-		mocha.grep(/#web/);
-	}
-	else if (process.env.WIN) {
-		mocha.grep(/#win/);
-	}
-	else if (process.env.PR) {
-		mocha.grep(/#pr/);
-	} else if (process.env.INVERSE_FILTER) {
+	const filters = {
+		WEB: /#web/,
+		WIN: /#win/,
+		PR: /#pr/,
+		ONLY: /#only/
+	};
+
+	Object.keys(filters).forEach((key) => {
+		if (process.env[key]) {
+			mocha.grep(filters[key]);
+		}
+	});
+
+	if (process.env.INVERSE_FILTER) {
 		mocha.grep(process.env.INVERSE_FILTER).invert();
 	}
 }