Log errors encountered when reporting responses (#33)

* Log errors encountered when reporting responses * fixup! Log errors encountered when reporting responses
w3c · Dec 6, 2023 · 35dea7e · 35dea7e
1 parent 298d7d4
commit 35dea7e
Show file tree

Hide file tree

Showing 6 changed files with 138 additions and 5 deletions.
diff --git a/src/host/cli-run-plan.js b/src/host/cli-run-plan.js
@@ -4,7 +4,7 @@
 
 import path from 'path';
 import { Readable } from 'stream';
-import fetch from 'node-fetch';
+import fetch, { Response } from 'node-fetch';
 
 import yargs from 'yargs';
 import { pickAgentCliOptions } from '../agent/cli.js';
@@ -202,10 +202,20 @@ function mainFetchMiddleware(argv) {
     if (!argv.agentMock) {
       argv.fetch = fetch;
     } else {
-      argv.fetch = (...params) =>
+      argv.fetch = (url, ...params) =>
         new Promise(resolve => {
-          console.log('Callback Fetch Mocked: ', ...params);
-          resolve();
+          const { searchParams } = new URL(url);
+          const status = parseInt(searchParams.get('TEST-STATUS'), 10) || 200;
+          const response = new Response('a body', { status });
+
+          if (searchParams.has('TEST-BAD-BODY')) {
+            // Disturb the response body stream in order to trigger failure in
+            // any future attempt to read.
+            response.text();
+          }
+
+          console.log('Callback Fetch Mocked: ', url, ...params);
+          resolve(response);
         });
     }
   }

diff --git a/src/host/main.js b/src/host/main.js
@@ -14,6 +14,19 @@ import {
   addTestResultToTestPlan,
 } from './plan-object.js';
 
+/**
+ * @param {AriaATCIHost.Log} log
+ * @param {Response} response
+ */
+const logUnsuccessfulHTTP = async (log, response) => {
+  if (!response.ok) {
+    const { status } = response;
+    const body = await response.text().catch(() => 'Unknown error - unable to read response body.');
+
+    log(HostMessage.REPORTING_ERROR, { status, body });
+  }
+};
+
 /**
  * @param {object} options
  * @param {AriaATCIHost.Log} options.log
@@ -84,7 +97,7 @@ export async function hostMain({
             method: 'post',
             body,
             headers,
-          })
+          }).then(logUnsuccessfulHTTP.bind(null, log))
         );
       }
       plan = addTestResultToTestPlan(plan, test.filepath, result);

diff --git a/src/host/messages.js b/src/host/messages.js
@@ -40,6 +40,8 @@ export const HostMessage = {
   AGENT_CRASHED: 'agentCrashed',
   /** @type {'startTest'} */
   START_TEST: 'startTest',
+  /** @type {'reportingError'} */
+  REPORTING_ERROR: 'reportingError',
 };
 
 export const HOST_TEMPLATES = {
@@ -60,6 +62,8 @@ export const HOST_TEMPLATES = {
   [HostMessage.AGENT_LOG]: ({ text }) => `[Agent]: ${text}`,
   [HostMessage.AGENT_CRASHED]: () => `Agent crashed.`,
   [HostMessage.START_TEST]: () => `Starting test.`,
+  [HostMessage.REPORTING_ERROR]: ({ status, body }) =>
+    `HTTP ${status} response received when reporting result: '${body}'.`,
 };
 
 /**

diff --git a/src/host/tests/cli-run-plan.js b/src/host/tests/cli-run-plan.js
@@ -51,6 +51,32 @@ test('plan3 with callback', async t => {
   );
 });
 
+test('plan3 with callback request which fails', async t => {
+  t.snapshot(
+    await spawnRunPlan([
+      '--plan-workingdir=fixtures/host-bin/plan3',
+      '"**"',
+      '--agent-mock',
+      '--debug',
+      '--callback-url=http://example.com/?TEST-STATUS=418',
+      '--callback-header=x:y',
+    ])
+  );
+});
+
+test('plan3 with callback request which fails with a faulty response body', async t => {
+  t.snapshot(
+    await spawnRunPlan([
+      '--plan-workingdir=fixtures/host-bin/plan3',
+      '"**"',
+      '--agent-mock',
+      '--debug',
+      '--callback-url="http://example.com/?TEST-STATUS=418&TEST-BAD-BODY"',
+      '--callback-header=x:y',
+    ])
+  );
+});
+
 async function spawnRunPlan(args) {
   const dirname = path.dirname(fileURLToPath(import.meta.url));
   const hostBin = path.join(dirname, '../../../bin/host.js');

diff --git a/src/host/tests/snapshots/cli-run-plan.js.md b/src/host/tests/snapshots/cli-run-plan.js.md
@@ -125,3 +125,83 @@ Generated by [AVA](https://avajs.dev).
       {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","expectation":"role up","output":"mocked output for role up","pass":true}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","output":"mocked output for role down","pass":true}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊
       `,
     }
+
+## plan3 with callback request which fails
+
+> Snapshot 1
+
+    {
+      stderr: `Starting...␊
+      Reference server listening on 'http://localhost:8888'.␊
+      Plan 'unknown' with 2 tests and 4 files read from source 'fork'.␊
+      Reference available on 'http://localhost:8888/static␊
+      Starting test agent.␊
+      [Agent]: Starting...␊
+      Agent running with protocol 'fork'.␊
+      Starting test.␊
+      [Server]: Serving '/static/reference/index.html'.␊
+      [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊
+      HTTP 418 response received when reporting result: 'a body'.␊
+      Starting test.␊
+      [Server]: Serving '/static/reference/index.html'.␊
+      [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊
+      HTTP 418 response received when reporting result: 'a body'.␊
+      Removing reference from 'http://localhost:8888/static␊
+      Stopping test agent.␊
+      [Agent]: Stopping...␊
+      Stopping reference server.␊
+      Stopping...␊
+      `,
+      stdout: `Callback Fetch Mocked:  http://example.com/?TEST-STATUS=418 {␊
+        method: 'post',␊
+        body: '{"testCsvRow":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for role up"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊
+        headers: { 'Content-Type': 'application/json', x: 'y' }␊
+      }␊
+      Callback Fetch Mocked:  http://example.com/?TEST-STATUS=418 {␊
+        method: 'post',␊
+        body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for role down"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊
+        headers: { 'Content-Type': 'application/json', x: 'y' }␊
+      }␊
+      {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","expectation":"role up","output":"mocked output for role up","pass":true}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","output":"mocked output for role down","pass":true}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊
+      `,
+    }
+
+## plan3 with callback request which fails with a faulty response body
+
+> Snapshot 1
+
+    {
+      stderr: `Starting...␊
+      Reference server listening on 'http://localhost:8888'.␊
+      Plan 'unknown' with 2 tests and 4 files read from source 'fork'.␊
+      Reference available on 'http://localhost:8888/static␊
+      Starting test agent.␊
+      [Agent]: Starting...␊
+      Agent running with protocol 'fork'.␊
+      Starting test.␊
+      [Server]: Serving '/static/reference/index.html'.␊
+      [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊
+      HTTP 418 response received when reporting result: 'Unknown error - unable to read response body.'.␊
+      Starting test.␊
+      [Server]: Serving '/static/reference/index.html'.␊
+      [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊
+      HTTP 418 response received when reporting result: 'Unknown error - unable to read response body.'.␊
+      Removing reference from 'http://localhost:8888/static␊
+      Stopping test agent.␊
+      [Agent]: Stopping...␊
+      Stopping reference server.␊
+      Stopping...␊
+      `,
+      stdout: `Callback Fetch Mocked:  http://example.com/?TEST-STATUS=418&TEST-BAD-BODY {␊
+        method: 'post',␊
+        body: '{"testCsvRow":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for role up"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊
+        headers: { 'Content-Type': 'application/json', x: 'y' }␊
+      }␊
+      Callback Fetch Mocked:  http://example.com/?TEST-STATUS=418&TEST-BAD-BODY {␊
+        method: 'post',␊
+        body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for role down"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊
+        headers: { 'Content-Type': 'application/json', x: 'y' }␊
+      }␊
+      {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","expectation":"role up","output":"mocked output for role up","pass":true}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","output":"mocked output for role down","pass":true}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊
+      `,
+    }
diff --git a/src/host/tests/snapshots/cli-run-plan.js.snap b/src/host/tests/snapshots/cli-run-plan.js.snap