Support v2 test commands (#32)

* v2 expectation as assertionStatement

* validate commands for mock runner

* Update mock snapshots to include validation errors for UP_ARROW and DOWN_ARROW

* Handle v2 combo key presses and v2 expectations in mocked error keys

* Normalize UP_ARROW and DOWN_ARRROW before validating

* Normalization in validation for multi-press arrow keys

* Split multipresses in validation

* chain replace calls in key validator

* Attempt at clarifying comment in AriaATCIData.CollectedTest JSDoc type declaration

* Remove MockTestRunner.runAssertion

* Handle merge conflicts in ava snapshots

src/agent/driver-test-runner.js

@@ -180,7 +180,7 @@ export class DriverTestRunner {
         for (const assertion of test.assertions) {
           results.push({
             command: command.id,
-            expectation: assertion.expectation,
+            expectation: assertion.expectation || assertion.assertionStatement,
             pass: true,
           });
         }
@@ -253,8 +253,13 @@ export class DriverTestRunner {
 export function validateKeysFromCommand(command) {
   const errors = [];
   for (let { id } of command.keypresses) {
-    // PAGE_DOWN and PAGE_UP are the only commands that have the extra _ inside a key
-    id = id.replace(/(PAGE)_(DOWN|UP)/, '$1$2');
+    id = id
+      // PAGE_DOWN and PAGE_UP are the only commands that have the extra _ inside a key
+      .replace(/(PAGE)_(DOWN|UP)/, '$1$2')
+      // + is used to connect keys that are pressed simultaneously in v2 tests
+      .replace('+', '_')
+      // `UP_ARROW`, `DOWN_ARROW`, etc are sent as `up`, `down`, etc
+      .replace(/_ARROW/g, '');
     if (/\//.test(id)) {
       errors.push(`'${id}' cannot contain '/'.`);
     }
@@ -267,7 +272,7 @@ export function validateKeysFromCommand(command) {
     if (/\bfollowed\b/.test(id)) {
       errors.push(`'${id}' cannot contain 'followed' or 'followed by'.`);
     }
-    for (const part of id.split('_')) {
+    for (const part of id.split(/[_+,]/)) {
       // Some old test plans have keys that contain indications of unspecified
       // instructions ('/') or additional instructions that are not standardized
       // in test plans. These keys should be updated to be separate commands or
@@ -296,6 +301,7 @@ export function atKeysFromCommand(command) {
       ATKey.chord(
         ...id
           .replace(/(PAGE)_(DOWN|UP)/, '$1$2')
+          .replace('+', '_') // + is used to connect keys that are pressed simultaneously in v2 tests
           .split('_')
           .map(key => key.trim().toLowerCase())
           // `up arrow`, `down arrow`, etc are sent as `up`, `down`, etc

src/agent/mock-test-runner.js

@@ -8,6 +8,7 @@
 
 import { request } from 'http';
 import { AgentMessage } from './messages.js';
+import { validateKeysFromCommand } from './driver-test-runner.js';
 
 /**
  * @implements {AriaATCIAgent.TestRunner}
@@ -51,19 +52,6 @@ export class MockTestRunner {
     this.log(AgentMessage.OPEN_PAGE, { url });
   }
 
-  /**
-   * @param {CollectedTestCommand} command
-   * @param {CollectedTestAssertion} assertion
-   */
-  async runAssertion(command, assertion) {
-    return {
-      command: command.id,
-      expectation: assertion.expectation,
-      output: `mocked output for ${assertion.expectation}`,
-      pass: await this.testAssertion(command, assertion),
-    };
-  }
-
   /**
    * @param {CollectedTestCommand} command
    * @param {CollectedTestAssertion} assertion
@@ -82,6 +70,50 @@ export class MockTestRunner {
         this.baseUrl
       )
     );
+
+    const commandsOutput = [];
+    const results = [];
+
+    for (const command of task.commands) {
+      const { value: validCommand, errors } = validateKeysFromCommand(command);
+      if (validCommand) {
+        const mockOutput = `mocked output for ${command.id}`;
+        commandsOutput.push({
+          command: validCommand.id,
+          output: mockOutput,
+        });
+
+        for (const assertion of task.assertions) {
+          const expectationText = assertion.expectation || assertion.assertionStatement;
+
+          results.push({
+            command: validCommand.id,
+            expectation: expectationText,
+            pass: await this.testAssertion(validCommand, assertion),
+            output: `mocked output for ${expectationText}`,
+          });
+        }
+      } else {
+        await this.log(AgentMessage.INVALID_KEYS, { command, errors });
+
+        commandsOutput.push({
+          command: command.id,
+          errors,
+        });
+
+        for (const assertion of task.assertions) {
+          const expectationText = assertion.expectation || assertion.assertionStatement;
+
+          results.push({
+            command: command.id,
+            expectation: expectationText,
+            output: `mocked output for ${expectationText}`,
+            pass: false,
+          });
+        }
+      }
+    }
+
     return {
       testId: task.info.testId,
       capabilities: {
@@ -91,19 +123,8 @@ export class MockTestRunner {
         atVersion: '1.0',
         platformName: 'mock',
       },
-      commands: await task.commands.reduce(
-        async (carry, command) => [
-          ...(await carry),
-          ...(await task.assertions.reduce(
-            async (carry, assertion) => [
-              ...(await carry),
-              await this.runAssertion(command, assertion),
-            ],
-            Promise.resolve([])
-          )),
-        ],
-        Promise.resolve([])
-      ),
+      commands: commandsOutput,
+      results,
     };
   }
 }

src/data/types.js

@@ -41,7 +41,8 @@
  * @property {string} [commands[].extraInstruction] human-readable additional instruction to follow
  * @property {object[]} assertions[]
  * @property {1 | 2} assertions[].priority
- * @property {string} assertions[].expectation
+ * @property {string} [assertions[].expectation] assertion statement string, this property only exists on v1 tests
+ * @property {string} [assertions[].assertionStatement] assertion statement string, this property only exists on v2 tests
  */
 
 /**