Callback POST to URL (#20)

* first draft/testing callback post for each test run * fix header split * move callback to host side of arrangement * further cleanup from moving to host side * hide header param when set from environment * testing test * add new params to README * Change env parsing, update mock, PR feedback * update snapshots
w3c · Oct 12, 2023 · 3086279 · 3086279
1 parent 9b67b40
commit 3086279
Show file tree

Hide file tree

Showing 15 changed files with 584 additions and 49 deletions.
diff --git a/README.md b/README.md
@@ -35,7 +35,7 @@ Options:
 ## `aria-at-harness-host`
 
 ```
-$ bin/host.js run-plan --help
+$ bin/host.js run-plan --help --show-hidden
 host.js run-plan [plan-files..]
 
 Run test plans
@@ -54,9 +54,18 @@ Options:
   --reference-hostname                                    [default: "localhost"]
   --plan-workingdir           Directory "plan-files" are relative to
                                                                   [default: "."]
+  --plan-protocol               [choices: "fork", "developer"] [default: "fork"]
   --agent-web-driver-url                      [default: "http://localhost:4444"]
   --agent-web-driver-browser [choices: "chrome", "firefox"] [default: "firefox"]
   --agent-at-driver-url                       [default: "http://localhost:4382"]
+  --agent-protocol              [choices: "fork", "developer"] [default: "fork"]
+  --agent-quiet               Disable all logging
+  --agent-debug               Enable all logging
+  --agent-verbose             Enable a subset of logging messages
+  --agent-mock                                                         [boolean]
+  --agent-mock-open-page                            [choices: "request", "skip"]
+  --callback-url              URL to POST test results to as they complete
+  --callback-header           Header to send with callback request
   --show-hidden               Show hidden options                      [boolean]
 ```
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -16,6 +16,7 @@
   },
   "dependencies": {
     "express": "^4.17.1",
+    "node-fetch": "^3.3.2",
     "selenium-webdriver": "^4.1.0",
     "ws": "^8.3.0",
     "yargs": "^17.2.1"

diff --git a/src/agent/at-driver.js b/src/agent/at-driver.js
@@ -32,13 +32,21 @@ export class ATDriver {
     this.socket = socket;
     this.log = log;
     this.ready = new Promise(resolve => socket.once('open', () => resolve())).then(() =>
-      this._send({ method: 'session.new', params: { capabilities: {} } })
+      this._send({ method: 'session.new', params: { capabilities: {} } }).then(
+        ({ result: { capabilities } }) => {
+          this._capabilities = capabilities;
+        }
+      )
     );
     this.closed = new Promise(resolve => socket.once('close', () => resolve()));
-
     this._nextId = 0;
   }
 
+  async getCapabilities() {
+    await this.ready;
+    return this._capabilities;
+  }
+
   async quit() {
     this.log(AgentMessage.AT_DRIVER_COMMS, { direction: 'close' });
     this.socket.close();
@@ -63,7 +71,7 @@ export class ATDriver {
         if (message.error) {
           throw new Error(message.error);
         }
-        return;
+        return message;
       }
     }
   }

diff --git a/src/agent/driver-test-runner.js b/src/agent/driver-test-runner.js
@@ -31,6 +31,15 @@ export class DriverTestRunner {
     this.log = log;
     this.webDriver = webDriver;
     this.atDriver = atDriver;
+    this.collectedCapabilities = this.getCapabilities();
+  }
+
+  async getCapabilities() {
+    const capabilities = await this.webDriver.getCapabilities();
+    const browserName = capabilities.get('browserName');
+    const browserVersion = capabilities.get('browserVersion');
+    const { atName, atVersion, platformName } = await this.atDriver.getCapabilities();
+    return { atName, atVersion, browserName, browserVersion, platformName };
   }
 
   /**
@@ -77,6 +86,9 @@ export class DriverTestRunner {
    * @param {AriaATFile.CollectedTest} test
    */
   async run(test) {
+    const capabilities = await this.collectedCapabilities;
+    await this.log(AgentMessage.CAPABILITIES, { capabilities });
+
     await this.log(AgentMessage.START_TEST, { id: test.info.testId, title: test.info.task });
 
     await this.log(AgentMessage.OPEN_PAGE, { url: 'about:blank' });
@@ -135,8 +147,11 @@ export class DriverTestRunner {
       }
     }
 
+    const testId = test.info.testId;
+
     return {
-      testId: test.info.testId,
+      testId,
+      capabilities,
       commands: commandsOutput,
       results,
     };

diff --git a/src/agent/messages.js b/src/agent/messages.js
@@ -29,6 +29,8 @@ export const AgentMessage = {
   NO_RUN_TEST_SETUP: 'noRunTestSetup',
   /** @type {'atDriverComms'} */
   AT_DRIVER_COMMS: 'atDriverComms',
+  /** @type {'capabilities'} */
+  CAPABILITIES: 'capabilities',
 };
 
 export const AGENT_TEMPLATES = {
@@ -44,6 +46,8 @@ export const AGENT_TEMPLATES = {
   [AgentMessage.NO_RUN_TEST_SETUP]: ({ referencePage }) =>
     `Test reference, ${referencePage}, does not have a Run Test Setup button.`,
   [AgentMessage.AT_DRIVER_COMMS]: ({ direction, message }) => `AT-Driver: ${direction}: ${message}`,
+  [AgentMessage.CAPABILITIES]: ({ capabilities }) =>
+    `Capabilities: ${JSON.stringify(capabilities)}`,
 };
 
 export function createAgentLogger(messages = AGENT_TEMPLATES) {

diff --git a/src/agent/mock-test-runner.js b/src/agent/mock-test-runner.js
@@ -59,6 +59,7 @@ export class MockTestRunner {
     return {
       command: command.id,
       expectation: assertion.expectation,
+      output: `mocked output for ${assertion.expectation}`,
       pass: await this.testAssertion(command, assertion),
     };
   }
@@ -83,7 +84,14 @@ export class MockTestRunner {
     );
     return {
       testId: task.info.testId,
-      results: await task.commands.reduce(
+      capabilities: {
+        browserName: 'mock',
+        browserVersion: '1.0',
+        atName: 'mock',
+        atVersion: '1.0',
+        platformName: 'mock',
+      },
+      commands: await task.commands.reduce(
         async (carry, command) => [
           ...(await carry),
           ...(await task.assertions.reduce(

diff --git a/src/host/README.md b/src/host/README.md
@@ -69,6 +69,8 @@ Options:
   --agent-verbose             Enable a subset of logging messages
   --agent-mock                                                         [boolean]
   --agent-mock-open-page                            [choices: "request", "skip"]
+  --callback-url              URL to POST test results to as they complete
+  --callback-header           Header to send with callback request
   --show-hidden               Show hidden options                      [boolean]
 ```
 

diff --git a/src/host/cli-run-plan.js b/src/host/cli-run-plan.js
@@ -4,6 +4,7 @@
 
 import path from 'path';
 import { Readable } from 'stream';
+import fetch from 'node-fetch';
 
 import yargs from 'yargs';
 import { pickAgentCliOptions } from '../agent/cli.js';
@@ -25,6 +26,7 @@ export const describe = 'Run test plans';
 export const builder = (args = yargs) =>
   args
     .positional('plan-files', { describe: 'Files in a test plan' })
+    .env('ARIA_AT')
     .options({
       quiet: {
         conflicts: ['debug', 'verbose'],
@@ -136,6 +138,24 @@ export const builder = (args = yargs) =>
         choices: ['request', 'skip'],
         hidden: true,
       },
+      'callback-url': {
+        describe: 'URL to POST test results to as they complete',
+      },
+      'callback-header': {
+        describe: 'Header to send with callback request',
+        coerce(arg) {
+          if (!arg) {
+            return {};
+          }
+          if (String(arg).indexOf(':') == -1) {
+            throw new Error('callback header must include a : to separate header name from value');
+          }
+          // capture all non ":" characters, ignore :\s*, capture rest of string
+          const [, name, value] = arg.match(/^([^:]+):\s*(.*)$/);
+
+          return { [name]: value };
+        },
+      },
     })
     .showHidden('show-hidden')
     .middleware(verboseMiddleware)
@@ -169,14 +189,28 @@ async function verboseMiddleware(argv) {
 
 function mainMiddleware(argv) {
   argv.planWorkingdir = path.resolve(argv.planWorkingdir);
-
+  mainFetchMiddleware(argv);
   mainLoggerMiddleware(argv);
   mainTestPlanMiddleware(argv);
   mainServerMiddleware(argv);
   mainAgentMiddleware(argv);
   mainResultMiddleware(argv);
 }
 
+function mainFetchMiddleware(argv) {
+  if (!argv.fetch) {
+    if (!argv.agentMock) {
+      argv.fetch = fetch;
+    } else {
+      argv.fetch = (...params) =>
+        new Promise(resolve => {
+          console.log('Callback Fetch Mocked: ', ...params);
+          resolve();
+        });
+    }
+  }
+}
+
 function mainLoggerMiddleware(argv) {
   const { stderr, verbosity } = argv;