Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

configurable timeout and delay values #61

Merged
merged 8 commits into from
Jul 23, 2024
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions src/agent/browser-driver/create-safari-apple-script-driver.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
import { execFile } from 'child_process';

/** Number of milliseconds to wait for document to be ready before giving up. */
const DOCUMENT_READY_TIMEOUT = 2000;
import { timesOption } from '../../shared/times-option.js';

/**
* @param {string} source
Expand Down Expand Up @@ -60,7 +58,7 @@ export default async () => {
async documentReady() {
const start = Date.now();

while (Date.now() - start < DOCUMENT_READY_TIMEOUT) {
while (Date.now() - start < timesOption.docReady) {
const readyState = await evalJavaScript('document.readyState');
if (readyState === 'complete') {
return;
Expand Down
3 changes: 3 additions & 0 deletions src/agent/cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { iterateEmitter } from '../shared/iterate-emitter.js';
import { createRunner } from './create-test-runner.js';
import { agentMain } from './main.js';
import { AgentMessage, createAgentLogger } from './messages.js';
import { timesArgs, timesOptionsConfig } from '../shared/times-option.js';

/** @param {yargs} args */
export function buildAgentCliOptions(args = yargs) {
Expand Down Expand Up @@ -76,6 +77,7 @@ export function buildAgentCliOptions(args = yargs) {
choices: ['request', 'skip'],
hidden: true,
},
...timesOptionsConfig,
})
.showHidden('show-hidden');
}
Expand All @@ -86,6 +88,7 @@ export function buildAgentCliOptions(args = yargs) {
*/
export function agentCliArgsFromOptionsMap(options) {
const args = [];
args.push(...timesArgs());
for (const key of Object.keys(options)) {
const value = options[key];
switch (key) {
Expand Down
23 changes: 6 additions & 17 deletions src/agent/driver-test-runner.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
/// <reference path="types.js" />

import { startJob } from '../shared/job.js';
import { timesOption } from '../shared/times-option.js';

import { ATDriver, ATKey, webDriverCodePoints } from './at-driver.js';
import { AgentMessage } from './messages.js';
Expand All @@ -11,10 +12,6 @@ import { AgentMessage } from './messages.js';
* @module agent
*/

const AFTER_NAVIGATION_DELAY = 1000;
const AFTER_KEYS_DELAY = 5000;
const RUN_TEST_SETUP_BUTTON_TIMEOUT = 1000;

export class DriverTestRunner {
/**
* @param {object} options
Expand Down Expand Up @@ -49,10 +46,7 @@ export class DriverTestRunner {
await this.browserDriver.documentReady();

try {
await this.browserDriver.clickWhenPresent(
'.button-run-test-setup',
RUN_TEST_SETUP_BUTTON_TIMEOUT
);
await this.browserDriver.clickWhenPresent('.button-run-test-setup', timesOption.testSetup);
} catch ({}) {
await this.log(AgentMessage.NO_RUN_TEST_SETUP, { referencePage });
}
Expand All @@ -71,15 +65,10 @@ export class DriverTestRunner {
* @param {string} desiredResponse
*/
async pressKeysToToggleSetting(sequence, desiredResponse) {
// This timeout may be reached as many as two times for every test.
// Delays of over 500ms have been observed during local testing in a
// Windows virtual machine.
const MODE_SWITCH_SPEECH_TIMEOUT = 750;

let unknownCollected = '';
// there are 2 modes, so we will try pressing mode switch up to twice
for (let triesRemain = 2; triesRemain > 0; triesRemain--) {
const speechResponse = await this._collectSpeech(MODE_SWITCH_SPEECH_TIMEOUT, () =>
const speechResponse = await this._collectSpeech(timesOption.modeSwitch, () =>
this.sendKeys(sequence)
);
while (speechResponse.length) {
Expand Down Expand Up @@ -202,7 +191,7 @@ export class DriverTestRunner {
const { value: validCommand, errors } = validateKeysFromCommand(command);

if (validCommand) {
await this._collectSpeech(AFTER_NAVIGATION_DELAY, () =>
await this._collectSpeech(timesOption.afterNav, () =>
this.openPage({
url: this._appendBaseUrl(test.target.referencePage),
referencePage: test.target.referencePage,
Expand All @@ -217,11 +206,11 @@ export class DriverTestRunner {
await this.ensureMode(test.target.mode);
}

const spokenOutput = await this._collectSpeech(AFTER_KEYS_DELAY, () =>
const spokenOutput = await this._collectSpeech(timesOption.afterKeys, () =>
this.sendKeys(atKeysFromCommand(validCommand))
);

await this._collectSpeech(AFTER_NAVIGATION_DELAY, async () => {
await this._collectSpeech(timesOption.afterNav, async () => {
await this.log(AgentMessage.OPEN_PAGE, { url: 'about:blank' });
await this.browserDriver.navigate('about:blank');
});
Expand Down
2 changes: 2 additions & 0 deletions src/host/cli-run-plan.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import { hostMain } from './main.js';
import { HostMessage, createHostLogger } from './messages.js';
import { plansFrom } from './plan-from.js';
import { HostServer } from './server.js';
import { timesOptionsConfig } from '../shared/times-option.js';

export const command = 'run-plan [plan-files..]';

Expand Down Expand Up @@ -156,6 +157,7 @@ export const builder = (args = yargs) =>
return { [name]: value };
},
},
...timesOptionsConfig,
})
.showHidden('show-hidden')
.middleware(verboseMiddleware)
Expand Down
91 changes: 91 additions & 0 deletions src/shared/times-option.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/// <reference path="./types.js" />

/**
* @module shared
*/

/**
* @type AriaATCIShared.timesOption
*/
export const timesOption = {
afterNav: 1000,
afterKeys: 5000,
testSetup: 1000,
modeSwitch: 750,
docReady: 2000,
};
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Stateful modules are a source of tech debt in my experience. This clearly works for our current needs, but at some point in the future (e.g. when we go to write a new kind of test or re-use something), the timesOption singleton may make the change more involved than if we had explicitly passed the value around to begin with. Think we can treat these values a little more like the rest of the command-line arguments?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's possible, the reason I was sort of avoiding it is this would necessitate passing these options into modules everywhere, I feel like the usability/dev experience of adding a timeout to it in the future might be pretty large. I'll do this as a separate commit to kind of weigh it out (not having done it yet, it might not be "that bad")

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the end this didn't feel too terrible. ea8d351


/**
* @type AriaATCIShared.timesOption
*/
const timesDefaults = { ...timesOption };

/**
* Convert from 'afterNav' to 'time-after-nav'.
* @param {keyof AriaATCIShared.timesOption} optionName
* @returns string
*/
function makeSnakeCasedOption(optionName) {
const snakeCased = optionName.replace(/[A-Z]/g, cap => '-' + cap.toLowerCase());
const optionText = `time-${snakeCased}`;
return optionText;
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I kinda have a thing for grep-able source code. I'd much rather maintain a bit of redundancy (e.g. an argument list like "afterNav", "time-after-nav") than an abstraction like this.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pushed this change in


/**
* Create a yargs description for the specified timesOption.
* @param {keyof AriaATCIShared.timesOption} optionName Key from timesOption
* @param {string} describe Description to be used in --show-help
*/
function addOptionConfig(optionName, describe) {
timesOptionsConfig[makeSnakeCasedOption(optionName)] = {
hidden: true,
default: timesOption[optionName],
describe,
coerce(arg) {
const isNumber = typeof arg === 'number';
if (!isNumber && !arg.match(/^\d+$/)) {
throw new Error('option value not a number');
}
const time = isNumber ? arg : parseInt(arg, 10);
if (time <= 0) {
throw new Error('time must be positive and non-zero');
}
timesOption[optionName] = time;
},
};
}

/**
* the yargs configuration for the time options
*/
export const timesOptionsConfig = {};
addOptionConfig('afterNav', 'Timeout used after navigation to collect and discard speech.');
addOptionConfig('afterKeys', 'Timeout used to wait for speech to finish after pressing keys.');
addOptionConfig(
'testSetup',
'Timeout used after pressing test setup button to collect and discard speech.'
);
addOptionConfig(
'modeSwitch',
'Timeout used after switching modes to check resulting speech (NVDA).'
);
addOptionConfig('docReady', 'Timeout used waiting for document ready (Safari).');

/**
* Convert the times dictionary to an array of strings to pass back to args.
* @param {AriaATCIShared.timesOption} opts
* @returns [string]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[string] means "an array with exactly one string element"; what we want here is

Suggested change
* @returns [string]
* @returns {string[]}

(The compiler isn't flagging this right now, presumably due to the absence of curly braces. I guess it considers that "not JSDoc" rather than "invalid JSDoc".)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oops! thanks! (still not super used to the jsdoc format)

*/
export function timesArgs(opts = timesOption) {
const args = [];
for (const key of Object.keys(timesOption)) {
const value = timesOption[key];
// no need to pass on "default" value
if (value === timesDefaults[key]) continue;
// casting in jsdoc syntax is complicated - the extra () around key are
// required to make the type annotation work.
args.push(makeSnakeCasedOption(/** @type {keyof AriaATCIShared.timesOption} */ (key)));
args.push(String(value));
}
return args;
}
9 changes: 9 additions & 0 deletions src/shared/types.js
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,12 @@
* @param {AriaATCIShared.JobBinding<*>} binding
* @returns {Promise<T>}
*/

/**
* @typedef AriaATCIShared.timesOption
* @property {number} afterNav Timeout used after navigation to collect and discard speech.
* @property {number} afterKeys Timeout used to wait for speech to finish after pressing keys.
* @property {number} testSetup Timeout used after pressing test setup button to collect and discard speech.
* @property {number} modeSwitch Timeout used after switching modes to check resulting speech (NVDA).
* @property {number} docReady Timeout used waiting for document ready (Safari).
*/
Loading