diff --git a/src/host/main.js b/src/host/main.js index d2fb166..caf4142 100644 --- a/src/host/main.js +++ b/src/host/main.js @@ -66,7 +66,27 @@ export async function hostMain({ log(HostMessage.START_AGENT); await agent.start({ referenceBaseUrl: serverDirectory.baseUrl }); - const callbackRequests = []; + let lastCallbackRequest = Promise.resolve(); + + const postCallbackWhenEnabled = body => { + // ignore if not in callback mode + if (!callbackUrl) return; + const headers = { + 'Content-Type': 'application/json', + ...(callbackHeader || {}), + }; + const perTestUrl = callbackUrl.replace( + ':testRowNumber', + body.testCsvRow ?? body.presentationNumber + ); + lastCallbackRequest = lastCallbackRequest.then(() => + fetch(perTestUrl, { + method: 'post', + body: JSON.stringify(body), + headers, + }).then(logUnsuccessfulHTTP.bind(null, log)) + ); + }; for (const test of plan.tests) { log(HostMessage.START_TEST); @@ -78,41 +98,43 @@ export async function hostMain({ }); const file = plan.files.find(({ name }) => name === test.filepath); - const result = await agent.run(JSON.parse(textDecoder.decode(file.bufferData))); - if (callbackUrl) { - const headers = { - 'Content-Type': 'application/json', - ...(callbackHeader || {}), - }; - const { testId, presentationNumber, capabilities, commands } = result; - const body = JSON.stringify({ - testCsvRow: testId, - presentationNumber, + const testSource = JSON.parse(textDecoder.decode(file.bufferData)); + + const callbackBody = { + testCsvRow: testSource.info.testId, + presentationNumber: testSource.info.presentationNumber, + }; + + try { + postCallbackWhenEnabled({ ...callbackBody, status: 'RUNNING' }); + + const result = await agent.run(testSource); + + const { capabilities, commands } = result; + + postCallbackWhenEnabled({ + ...callbackBody, capabilities, + status: 'COMPLETED', responses: commands.map(({ output }) => output), - // a v2 of this API should allow the aria at app to parse the capabilities we are sending instead. - atVersionName: capabilities.atVersion, - browserVersionName: capabilities.browserVersion, }); - callbackRequests.push( - fetch(callbackUrl, { - method: 'post', - body, - headers, - }).then(logUnsuccessfulHTTP.bind(null, log)) - ); + + plan = addTestResultToTestPlan(plan, test.filepath, result); + } catch (exception) { + const error = `${exception.message ?? exception}`; + log(HostMessage.TEST_ERROR, { error }); + postCallbackWhenEnabled({ ...callbackBody, error, status: 'ERROR' }); + } finally { + await testLogJob.cancel(); } - plan = addTestResultToTestPlan(plan, test.filepath, result); - await Promise.allSettled(callbackRequests); - await testLogJob.cancel(); } server.removeFiles(serverDirectory); log(HostMessage.REMOVE_SERVER_DIRECTORY, { url: serverDirectory.baseUrl }); log(HostMessage.STOP_AGENT); + await lastCallbackRequest; await agent.stop(); - await emitPlanResults(plan); } diff --git a/src/host/messages.js b/src/host/messages.js index 6dea921..cf81447 100644 --- a/src/host/messages.js +++ b/src/host/messages.js @@ -43,6 +43,8 @@ export const HostMessage = { START_TEST: 'startTest', /** @type {'reportingError'} */ REPORTING_ERROR: 'reportingError', + /** @type {'testError'} */ + TEST_ERROR: 'testError', }; export const HOST_TEMPLATES = { @@ -63,6 +65,7 @@ export const HOST_TEMPLATES = { [HostMessage.AGENT_LOG]: ({ text }) => `[Agent]: ${text}`, [HostMessage.AGENT_CRASHED]: () => `Agent crashed.`, [HostMessage.START_TEST]: () => `Starting test.`, + [HostMessage.TEST_ERROR]: ({ error }) => `Test Error ${error}`, [HostMessage.REPORTING_ERROR]: ({ status, body }) => `HTTP ${status} response received when reporting result: '${body}'.`, }; diff --git a/src/host/tests/cli-run-plan.js b/src/host/tests/cli-run-plan.js index af9ed44..4bd09a1 100644 --- a/src/host/tests/cli-run-plan.js +++ b/src/host/tests/cli-run-plan.js @@ -39,7 +39,7 @@ test('plan3', async t => { ); }); -test('plan3 with callback', async t => { +test('plan3 with callback no url param', async t => { t.snapshot( await spawnRunPlan([ '--plan-workingdir=fixtures/host-bin/plan3', @@ -52,6 +52,19 @@ test('plan3 with callback', async t => { ); }); +test('plan3 with callback', async t => { + t.snapshot( + await spawnRunPlan([ + '--plan-workingdir=fixtures/host-bin/plan3', + '"**"', + '--agent-mock', + '--debug', + '--callback-url=http://callback.url/:testRowNumber', + '--callback-header=test:header:multiple:colon', + ]) + ); +}); + test('plan3 with callback request which fails', async t => { t.snapshot( await spawnRunPlan([ diff --git a/src/host/tests/snapshots/agent.js.md b/src/host/tests/snapshots/agent.js.md index cfb461e..7dbaf93 100644 --- a/src/host/tests/snapshots/agent.js.md +++ b/src/host/tests/snapshots/agent.js.md @@ -92,7 +92,7 @@ Generated by [AVA](https://avajs.dev). }, ] -> {"tests":[],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log +> {"tests":[],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log [ { @@ -210,7 +210,7 @@ Generated by [AVA](https://avajs.dev). }, ] -> {"tests":[],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log +> {"tests":[],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log [ { @@ -278,7 +278,7 @@ Generated by [AVA](https://avajs.dev). }, ] -> {"tests":[],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log +> {"tests":[],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log [ { @@ -479,7 +479,7 @@ Generated by [AVA](https://avajs.dev). }, ] -> {"tests":[1],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1) +> {"tests":[1],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1) { capabilities: { @@ -506,7 +506,7 @@ Generated by [AVA](https://avajs.dev). testId: 1, } -> {"tests":[1],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log +> {"tests":[1],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log [ { @@ -738,7 +738,7 @@ Generated by [AVA](https://avajs.dev). }, ] -> {"tests":[1],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1) +> {"tests":[1],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1) { capabilities: { @@ -765,7 +765,7 @@ Generated by [AVA](https://avajs.dev). testId: 1, } -> {"tests":[1],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log +> {"tests":[1],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log [ { @@ -958,7 +958,7 @@ Generated by [AVA](https://avajs.dev). }, ] -> {"tests":[1],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1) +> {"tests":[1],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1) { capabilities: { @@ -985,7 +985,7 @@ Generated by [AVA](https://avajs.dev). testId: 1, } -> {"tests":[1],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log +> {"tests":[1],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log [ { @@ -1300,7 +1300,7 @@ Generated by [AVA](https://avajs.dev). }, ] -> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1) +> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1) { capabilities: { @@ -1327,7 +1327,7 @@ Generated by [AVA](https://avajs.dev). testId: 1, } -> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2) +> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2) { capabilities: { @@ -1354,7 +1354,7 @@ Generated by [AVA](https://avajs.dev). testId: 2, } -> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log +> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log [ { @@ -1700,7 +1700,7 @@ Generated by [AVA](https://avajs.dev). }, ] -> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1) +> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1) { capabilities: { @@ -1727,7 +1727,7 @@ Generated by [AVA](https://avajs.dev). testId: 1, } -> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2) +> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2) { capabilities: { @@ -1754,7 +1754,7 @@ Generated by [AVA](https://avajs.dev). testId: 2, } -> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log +> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log [ { @@ -2072,7 +2072,7 @@ Generated by [AVA](https://avajs.dev). }, ] -> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1) +> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1) { capabilities: { @@ -2099,7 +2099,7 @@ Generated by [AVA](https://avajs.dev). testId: 1, } -> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2) +> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2) { capabilities: { @@ -2126,7 +2126,7 @@ Generated by [AVA](https://avajs.dev). testId: 2, } -> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log +> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log [ { diff --git a/src/host/tests/snapshots/agent.js.snap b/src/host/tests/snapshots/agent.js.snap index 81806d0..a987776 100644 Binary files a/src/host/tests/snapshots/agent.js.snap and b/src/host/tests/snapshots/agent.js.snap differ diff --git a/src/host/tests/snapshots/cli-run-plan.js.md b/src/host/tests/snapshots/cli-run-plan.js.md index cd89c99..86dccb8 100644 --- a/src/host/tests/snapshots/cli-run-plan.js.md +++ b/src/host/tests/snapshots/cli-run-plan.js.md @@ -88,7 +88,7 @@ Generated by [AVA](https://avajs.dev). `, } -## plan3 with callback +## plan3 with callback no url param > Snapshot 1 @@ -114,19 +114,29 @@ Generated by [AVA](https://avajs.dev). `, stdout: `Callback Fetch Mocked: http://callback.url/ {␊ method: 'post',␊ - body: '{"testCsvRow":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for UP_ARROW"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ + body: '{"testCsvRow":1,"status":"RUNNING"}',␊ + headers: { 'Content-Type': 'application/json', test: 'header:multiple:colon' }␊ + }␊ + Callback Fetch Mocked: http://callback.url/ {␊ + method: 'post',␊ + body: '{"testCsvRow":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"status":"COMPLETED","responses":["mocked output for UP_ARROW"]}',␊ headers: { 'Content-Type': 'application/json', test: 'header:multiple:colon' }␊ }␊ Callback Fetch Mocked: http://callback.url/ {␊ method: 'post',␊ - body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for UP_ARROW,DOWN_ARROW"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ + body: '{"testCsvRow":2,"status":"RUNNING"}',␊ + headers: { 'Content-Type': 'application/json', test: 'header:multiple:colon' }␊ + }␊ + Callback Fetch Mocked: http://callback.url/ {␊ + method: 'post',␊ + body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"status":"COMPLETED","responses":["mocked output for UP_ARROW,DOWN_ARROW"]}',␊ headers: { 'Content-Type': 'application/json', test: 'header:multiple:colon' }␊ }␊ {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ `, } -## plan3 with callback request which fails +## plan3 with callback > Snapshot 1 @@ -141,25 +151,85 @@ Generated by [AVA](https://avajs.dev). Starting test.␊ [Server]: Serving '/static/reference/index.html'.␊ [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ - HTTP 418 response received when reporting result: 'a body'.␊ Starting test.␊ [Server]: Serving '/static/reference/index.html'.␊ [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ + Removing reference from 'http://localhost:8888/static␊ + Stopping test agent.␊ + [Agent]: Stopping...␊ + Stopping reference server.␊ + Stopping...␊ + `, + stdout: `Callback Fetch Mocked: http://callback.url/1 {␊ + method: 'post',␊ + body: '{"testCsvRow":1,"status":"RUNNING"}',␊ + headers: { 'Content-Type': 'application/json', test: 'header:multiple:colon' }␊ + }␊ + Callback Fetch Mocked: http://callback.url/1 {␊ + method: 'post',␊ + body: '{"testCsvRow":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"status":"COMPLETED","responses":["mocked output for UP_ARROW"]}',␊ + headers: { 'Content-Type': 'application/json', test: 'header:multiple:colon' }␊ + }␊ + Callback Fetch Mocked: http://callback.url/2 {␊ + method: 'post',␊ + body: '{"testCsvRow":2,"status":"RUNNING"}',␊ + headers: { 'Content-Type': 'application/json', test: 'header:multiple:colon' }␊ + }␊ + Callback Fetch Mocked: http://callback.url/2 {␊ + method: 'post',␊ + body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"status":"COMPLETED","responses":["mocked output for UP_ARROW,DOWN_ARROW"]}',␊ + headers: { 'Content-Type': 'application/json', test: 'header:multiple:colon' }␊ + }␊ + {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ + `, + } + +## plan3 with callback request which fails + +> Snapshot 1 + + { + stderr: `Starting...␊ + Reference server listening on 'http://localhost:8888'.␊ + Plan 'unknown' with 2 tests and 4 files read from source 'fork'.␊ + Reference available on 'http://localhost:8888/static␊ + Starting test agent.␊ + [Agent]: Starting...␊ + Agent running with protocol 'fork'.␊ + Starting test.␊ HTTP 418 response received when reporting result: 'a body'.␊ + [Server]: Serving '/static/reference/index.html'.␊ + [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ + Starting test.␊ + HTTP 418 response received when reporting result: 'a body'.␊ + HTTP 418 response received when reporting result: 'a body'.␊ + [Server]: Serving '/static/reference/index.html'.␊ + [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ Removing reference from 'http://localhost:8888/static␊ Stopping test agent.␊ + HTTP 418 response received when reporting result: 'a body'.␊ [Agent]: Stopping...␊ Stopping reference server.␊ Stopping...␊ `, stdout: `Callback Fetch Mocked: http://example.com/?TEST-STATUS=418 {␊ method: 'post',␊ - body: '{"testCsvRow":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for UP_ARROW"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ + body: '{"testCsvRow":1,"status":"RUNNING"}',␊ + headers: { 'Content-Type': 'application/json', x: 'y' }␊ + }␊ + Callback Fetch Mocked: http://example.com/?TEST-STATUS=418 {␊ + method: 'post',␊ + body: '{"testCsvRow":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"status":"COMPLETED","responses":["mocked output for UP_ARROW"]}',␊ + headers: { 'Content-Type': 'application/json', x: 'y' }␊ + }␊ + Callback Fetch Mocked: http://example.com/?TEST-STATUS=418 {␊ + method: 'post',␊ + body: '{"testCsvRow":2,"status":"RUNNING"}',␊ headers: { 'Content-Type': 'application/json', x: 'y' }␊ }␊ Callback Fetch Mocked: http://example.com/?TEST-STATUS=418 {␊ method: 'post',␊ - body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for UP_ARROW,DOWN_ARROW"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ + body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"status":"COMPLETED","responses":["mocked output for UP_ARROW,DOWN_ARROW"]}',␊ headers: { 'Content-Type': 'application/json', x: 'y' }␊ }␊ {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ @@ -179,27 +249,39 @@ Generated by [AVA](https://avajs.dev). [Agent]: Starting...␊ Agent running with protocol 'fork'.␊ Starting test.␊ + HTTP 418 response received when reporting result: 'Unknown error - unable to read response body.'.␊ [Server]: Serving '/static/reference/index.html'.␊ [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ - HTTP 418 response received when reporting result: 'Unknown error - unable to read response body.'.␊ Starting test.␊ + HTTP 418 response received when reporting result: 'Unknown error - unable to read response body.'.␊ + HTTP 418 response received when reporting result: 'Unknown error - unable to read response body.'.␊ [Server]: Serving '/static/reference/index.html'.␊ [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ - HTTP 418 response received when reporting result: 'Unknown error - unable to read response body.'.␊ Removing reference from 'http://localhost:8888/static␊ Stopping test agent.␊ + HTTP 418 response received when reporting result: 'Unknown error - unable to read response body.'.␊ [Agent]: Stopping...␊ Stopping reference server.␊ Stopping...␊ `, stdout: `Callback Fetch Mocked: http://example.com/?TEST-STATUS=418&TEST-BAD-BODY {␊ method: 'post',␊ - body: '{"testCsvRow":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for UP_ARROW"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ + body: '{"testCsvRow":1,"status":"RUNNING"}',␊ + headers: { 'Content-Type': 'application/json', x: 'y' }␊ + }␊ + Callback Fetch Mocked: http://example.com/?TEST-STATUS=418&TEST-BAD-BODY {␊ + method: 'post',␊ + body: '{"testCsvRow":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"status":"COMPLETED","responses":["mocked output for UP_ARROW"]}',␊ + headers: { 'Content-Type': 'application/json', x: 'y' }␊ + }␊ + Callback Fetch Mocked: http://example.com/?TEST-STATUS=418&TEST-BAD-BODY {␊ + method: 'post',␊ + body: '{"testCsvRow":2,"status":"RUNNING"}',␊ headers: { 'Content-Type': 'application/json', x: 'y' }␊ }␊ Callback Fetch Mocked: http://example.com/?TEST-STATUS=418&TEST-BAD-BODY {␊ method: 'post',␊ - body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for UP_ARROW,DOWN_ARROW"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ + body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"status":"COMPLETED","responses":["mocked output for UP_ARROW,DOWN_ARROW"]}',␊ headers: { 'Content-Type': 'application/json', x: 'y' }␊ }␊ {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ diff --git a/src/host/tests/snapshots/cli-run-plan.js.snap b/src/host/tests/snapshots/cli-run-plan.js.snap index 94fa60a..b1aa412 100644 Binary files a/src/host/tests/snapshots/cli-run-plan.js.snap and b/src/host/tests/snapshots/cli-run-plan.js.snap differ