From 7a8d13adb55afbaec57a64826584a1cab1d6cb2f Mon Sep 17 00:00:00 2001 From: Lei Zhu Date: Wed, 8 Mar 2017 16:53:17 -0800 Subject: [PATCH] Magellan Executor (#206) * print executor help with --help * unify help handler * allow magellan to solve executor params * turn profiles to capabilities * propagate profile and executor to plugin * local executor * local goes through * pass opts in profile * remove fork * config for sauce executor * clean up aged change * prototype of working sauce executor * run enableExecutor per plugin if exists * more tunnel params support * add locks * remove src/sauce * move expensive call out * remove unnecessary code * add logger * change log order * remove dup code * fix lint issue * remove executors * double check for executor config * fix bug * adapt executor method name * remove failed test * bump version * unit test for profile * remove lint in test, add cli_help unit test * runner unitest * add unit test for test_runner * version bump * update readme * update readme * fix lint * remove unncessary check * use marge to capture config in magellan.json * update readme * fix lint --- README.md | 195 ++---- bin/magellan | 15 - magellan.json | 17 +- package.json | 4 +- src/cli.js | 392 +++++++----- src/cli_help.js | 136 ++-- src/detect_browsers.js | 207 ------ src/help.js | 121 ++++ src/logger.js | 36 ++ src/profiles.js | 174 +++++ src/sauce/browsers.js | 67 -- src/sauce/settings.js | 104 --- src/sauce/tunnel.js | 145 ----- src/sauce/worker_allocator.js | 304 --------- src/settings.js | 3 +- src/test.js | 13 +- src/test_runner.js | 318 +++++----- src/util/check_ports.js | 6 +- src/util/load_relative_module.js | 9 +- src/util/process_cleanup.js | 14 +- src/worker_allocator.js | 13 +- test/cli.js | 852 +++++++++++-------------- test/cli_help.js | 39 ++ test/detect_browsers.js | 178 ------ test/profiles.js | 290 +++++++++ test/sauce/browsers.js | 46 -- test/sauce/settings.js | 134 ---- test/sauce/tunnel.js | 255 -------- test/sauce/worker_allocator.js | 507 --------------- test/test.js | 6 +- test/test_runner.js | 1024 +++++++++++++++--------------- test/utils/port_util.js | 4 +- 32 files changed, 2097 insertions(+), 3531 deletions(-) delete mode 100644 src/detect_browsers.js create mode 100644 src/help.js create mode 100644 src/logger.js create mode 100644 src/profiles.js delete mode 100644 src/sauce/browsers.js delete mode 100644 src/sauce/settings.js delete mode 100644 src/sauce/tunnel.js delete mode 100644 src/sauce/worker_allocator.js create mode 100644 test/cli_help.js delete mode 100644 test/detect_browsers.js create mode 100644 test/profiles.js delete mode 100644 test/sauce/browsers.js delete mode 100644 test/sauce/settings.js delete mode 100644 test/sauce/tunnel.js delete mode 100644 test/sauce/worker_allocator.js diff --git a/README.md b/README.md index 7130e5d..c9bb672 100644 --- a/README.md +++ b/README.md @@ -17,19 +17,38 @@ Features - Testing and debugging workflows. Run many tests at once, one test at a time, filter by tags, groups, etc. - Suite run control: Bail likely-failing suite runs early, or bail upon first failure. - Run many different parallel **local** browsers (eg: Chrome, Firefox, etc) all at the same time. - - Run many different parallel **remote** (SauceLabs) browsers. + - Run many different parallel **remote** (SauceLabs, Browserstack, etc.) browsers. - **Integration Support** - Status reporter API with events streamed from workers, with some included reporters. - Slack reporting support. - MongoDB event export. - [Admiral](https://github.com/TestArmada/admiral) reporting support. - Plays well with CI (Jenkins, etc). - - SauceLabs Remote Browser and Device Support: - - Optional Sauce Connect tunnel management (`--create_tunnels`). - - Create lists of browser tiers or browser testing groups with browser profiles (eg: tier1 browsers, tier2 browsers, mobile browsers, vintage IE versions, etc). - - Manage not just browsers, but also devices for native application testing (iOS and Android) + - Runs test over the cloud like saucelabs via magellan executor (configurable and in parallel) - Can talk to a [locks service](https://github.com/TestArmada/locks) to control saucelabs virtual machine usage (beta). +------------------**BREAKING CHANGE in v10.0.0**------------------ +### Magellan Executor + +Executor is a mid layer between magellan and test framework to drive test run (via framework) based on a specific need (differentiated by executing environments). Magellan doesn't provide a default executor, so you need to pick at least one executor from the existing executor list, or implement one yourself. + +#### What is an executor +1. middle layer between magellan and test framework +2. bridge to connect magellan and plugins + +#### What can an executor do +1. resolve profiles (env info, test info, capabilities for selenium test) +2. patch setup and teardown event on the magellan test runner +3. patch setup and teardown event on a magellan worker +4. do some extra work in test's lifecycle +5. communicate to a specific test env + +#### Existing executors + * [magellan-local-executor](https://github.com/TestArmada/magellan-local-executor) + * [magellan-saucelabs-executor](https://github.com/TestArmada/magellan-saucelabs-executor) + * [magellan-browserstack-executor](https://github.com/TestArmada/magellan-browserstack-executor)(early beta) + + Test Framework Compatibility and Installation ============================================= @@ -37,29 +56,11 @@ Magellan supports test frameworks like Mocha and Nightwatch via the usage of **p #### Mocha -Plugin: - - https://github.com/TestArmada/magellan-mocha-plugin - -Boilerplate / example projects: +------------------BREAKING CHANGE in v10.0.0------------------ - - `wd` ( [example Mocha/wd project](https://github.com/TestArmada/boilerplate-mocha-wd) ) - - `webdriver.io` ( [example Mocha/webdriver.io project](https://github.com/TestArmada/boilerplate-mocha-webdriverio) ) - - `node.js` test suites - see `magellan`'s own test suite - - `appium.js` - example project coming soon. +magellan@10.0.0 doesn't support the mocha plugin for now. If you're using magellan version 9 or lower to run mocha test please don't upgrade. Or if you're seeking for mocha support please use magellan version 9 or lower. -Installation: - -```shell -npm install --save-dev testarmada-magellan -npm install --save-dev testarmada-magellan-mocha-plugin -``` - -`magellan.json` -```json -{ - "framework": "testarmada-magellan-mocha-plugin" -} -``` +All magellan mocha supports can be found [here](https://github.com/TestArmada/magellan/blob/v8.8.5/README.md#mocha) #### Nightwatch @@ -72,17 +73,30 @@ Boilerplate / example project: Helper Library: (note: this is not required for nightwatch support) - https://github.com/TestArmada/magellan-nightwatch +Executor: + - must have + - https://github.com/TestArmada/magellan-local-executor + - optional + - https://github.com/TestArmada/magellan-saucelabs-executor + - https://github.com/TestArmada/magellan-browserstack-executor (early beta) + Installation: ```shell npm install --save-dev testarmada-magellan npm install --save-dev testarmada-magellan-nightwatch-plugin +npm install --save-dev testarmada-magellan-local-executor +npm install --save-dev testarmada-magellan-saucelabs-executor ``` `magellan.json` ```json { - "framework": "testarmada-magellan-nightwatch-plugin" + "framework": "testarmada-magellan-nightwatch-plugin", + "executors": [ + "testarmada-magellan-local-executor", + "testarmada-magellan-saucelabs-executor" + ] } ``` @@ -130,45 +144,28 @@ Quick Reference Guide for Command-Line Use #### Running Many Tests in Parallel (Default) -By default, `magellan` will try to run your test suite the fastest way possible, in parallel, in the `phantomjs` browser. +By default, `magellan` will try to run your test suite the fastest way possible, in parallel -To execute your tests, run: +You can also run parallel tests on a real local browser (with `magellan-local-executor`): ```console -$ magellan -``` - -You can also run parallel tests on a real local browser: -```console -# launch several instances of Chrome at once and run tests in parallel -$ magellan --browser=chrome +# launch several instances of phantomjs at once and run tests in parallel +$ magellan --local_browser=phantomjs # launch several instances of Firefox at once and run tests in parallel -$ magellan --browser=firefox +$ magellan --local_browser=firefox ``` #### Testing in Multiple Browsers `magellan` can run your test suite across multiple browsers with one command: ```console -# Run tests locally in both PhantomJS and Chrome -$ magellan --browser=chrome,phantomjs -# Run tests locally in Chrome and Firefox -$ magellan --browser=chrome,firefox +# Run tests locally in Chrome,phantomjs and Firefox +$ magellan --local_browsers=chrome,firefox,phantomjs ``` #### Controlling Which Tests Run -##### Tag Support in Mocha-based Tests - -In Mocha-based tests, Magellan will find tags in `it()` strings in the form of a tagname preceded by an `@`. For example, to define a test that would match with `--tags=commerce` or `--tags=smoke`, we would write: - -```javascript -it("should submit the purchase @commerce @smoke", function (done) { - ... -}); -``` - ##### Tag Support in Nightwatch.js-based Tests In Nightwatch-based tests, Magellan supports the standard Nightwatch convention of tags. To define a test that Magellan can match with `--tags=commerce` or `--tags=smoke`, we would write: @@ -440,100 +437,8 @@ module.exports = SetupTeardown; Note: Magellan should ensure that `flush()` always runs even if your test suite fails. -SauceLabs Support -================= - -Magellan supports running tests through SauceLabs remote browsers. To do this, the following environment variables should be set: - -```shell -# Set SauceLabs Credentials -export SAUCE_USERNAME='xxxxxxxxxx' -export SAUCE_ACCESS_KEY='xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx' - -# Set Secure Tunnel Settings - -# SauceConnect version for download -export SAUCE_CONNECT_VERSION=4.3.10 -``` - -A Sauce Connect tunnel prefix must be set for tunnel management to work when using `--create_tunnels` (see more on this below). - -``` -# Tunnel id prefix, Example: "my_tunnel", "qa_tunnel", etc -export SAUCE_TUNNEL_ID="xxxxxxxxx" -``` - -Listing and Using SauceLabs Browsers -==================================== - -Magellan can query the SauceLabs API for a list of available browsers (for web testing) and devices (for native app testing in iOS and Android), and present them as a list of friendly browser ids: - -```console -$ magellan --list_browsers -``` - -To use a given SauceLabs browser, specify it when using the `--sauce` option: - -```console -$ magellan --sauce --browser=chrome_42_Windows_2012_R2_Desktop -``` - -To use multiple SauceLabs browsers and environments at the same time, simply list multiple ids: -``` -$ magellan --sauce --browsers=chrome_42_Windows_2012_R2_Desktop,safari_7_OS_X_10_9_Desktop -``` - -Note: If you are building reporting or CI tools and want to use the same SauceLabs API and browser naming support toolset, check out [guacamole](https://github.com/TestArmada/guacamole). - -SauceLabs Tunnelling Support (Sauce Connect) -============================================ - -**NOTE**: By default, Magellan assumes that tests run on an open network visible to SauceLabs. - -If your tests are running in a closed CI environment not visible to the Internet, a tunnel is required from SauceLabs to your test machine (when using `--sauce` mode). To activate tunnel creation, use `--create_tunnels`. Magellan will create a tunnel for the test run. If you want to distribute your work load across more than one tunnel, specify the `--max_tunnels=N` option, like so: - -```console -$ magellan --sauce --browser=chrome_42_Windows_2012_R2_Desktop --create_tunnels --max_tunnels=4 --max_workers=16 -``` - -In the above example, 4 tunnels will be distributed amongst 16 workers. - -SauceLabs VM Traffic Control (`locks`) -====================================== - -To specify a `locks` server, set the environment variable `LOCKS_SERVER`: - -``` -export LOCKS_SERVER=http://locks.server.example:4765/ -``` - -or use the `--locks_server` option: -``` -$ magellan --locks_server=http://locks.server.example:4765 --sauce --browser=chrome_42_Windows_2012_R2_Desktop --create_tunnels --max_tunnels=4 --max_workers=16 -``` - -Display Resolution and Orientation Support (SauceLabs Browsers) -=============================================================== - -To ensure that the SauceLabs display being used has enough resolution to support a given browser window size, use the `--resolution` option: - -Single Sauce browser: -```console -$ magellan --sauce --browser=chrome_42_Windows_2012_R2_Desktop --resolution=1024x768 -``` - -Multiple Sauce browsers: -```console -$ magellan --sauce --browsers=chrome_42_Windows_2012_R2_Desktop,safari_7_OS_X_10_9_Desktop --resolution=1024x768 -``` - -In this case, `1024x768` is selected for `chrome_42_Windows_2012_R2_Desktop` and `safari_7_OS_X_10_9_Desktop`. If this resolution isn't available in all Sauce browser environments specified, Magellan will return an error. - -For Sauce devices that support it, orientation is also supported with the `--orientation` option: - -```console -$ magellan --sauce --browser=iphone_8_2_iOS_iPhone_Simulator --orientation=landscape -``` +Local Browser Profiles +======================= Sometimes it's useful to specify a list of environments with differing resolutions or orientations. For this case, Magellan supports profiles stored in `magellan.json`: diff --git a/bin/magellan b/bin/magellan index 670b2ed..14b924c 100755 --- a/bin/magellan +++ b/bin/magellan @@ -2,21 +2,6 @@ /* eslint no-process-exit: 0, no-console: 0 */ "use strict"; -const yargs = require("yargs"); -const margs = require("marge"); - -const defaultConfigFilePath = "./magellan.json"; -const configFilePath = yargs.argv.config; - -if (configFilePath) { - console.log(`Will try to load configuration from ${configFilePath}`); -} else { - console.log(`Will try to load configuration from default of ${defaultConfigFilePath}`); -} - -// NOTE: marge can throw an error here if --config points at a file that doesn't exist -// FIXME: handle this error nicely instead of printing an ugly stack trace -margs.init(defaultConfigFilePath, configFilePath); require("../src/cli")() .then(() => process.exit(0)) diff --git a/magellan.json b/magellan.json index ae8fd96..f1a031b 100644 --- a/magellan.json +++ b/magellan.json @@ -1,5 +1,16 @@ { - "mocha_tests": ["./integration"], + "mocha_tests": [ + "./integration" + ], "framework": "vanilla-mocha", - "max_workers": 7 -} + "max_workers": 7, + "executors": [], + "profiles": { + "sauce-chrome": [ + { + "browser": "chrome_latest_Windows_10_Desktop", + "resolution": "1280x1024" + } + ] + } +} \ No newline at end of file diff --git a/package.json b/package.json index b18dcad..1e8bdbb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "testarmada-magellan", - "version": "8.8.4", + "version": "10.0.0", "description": "Massively parallel automated testing", "main": "src/main", "directories": { @@ -29,7 +29,7 @@ "test": "eslint src/** bin/** && mocha --recursive && npm run coverage && npm run check-coverage", "dev-test": "mocha --recursive && eslint src/** bin/**", "integration": "eslint src/** bin/** && ./bin/magellan", - "lint": "eslint src/** bin/** test/**", + "lint": "eslint src/** bin/**", "coverage": "istanbul cover _mocha -- --recursive", "check-coverage": "istanbul check-coverage --statement 95 --function 95 --branch 90" }, diff --git a/src/cli.js b/src/cli.js index a314287..bf4b286 100644 --- a/src/cli.js +++ b/src/cli.js @@ -15,7 +15,6 @@ const path = require("path"); const _ = require("lodash"); const margs = require("marge"); const async = require("async"); -const clc = require("cli-color"); const Q = require("q"); const analytics = require("./global_analytics"); @@ -23,63 +22,57 @@ const TestRunner = require("./test_runner"); const getTests = require("./get_tests"); const testFilters = require("./test_filter"); const WorkerAllocator = require("./worker_allocator"); -const SauceWorkerAllocator = require("./sauce/worker_allocator"); -const browserOptions = require("./detect_browsers"); const settings = require("./settings"); -const sauceSettings = require("./sauce/settings")(); -const browsers = require("./sauce/browsers"); +const profiles = require("./profiles"); const loadRelativeModule = require("./util/load_relative_module"); const processCleanup = require("./util/process_cleanup"); +const magellanArgs = require("./help").help; +const logger = require("./logger"); module.exports = (opts) => { const defer = Q.defer(); const runOpts = _.assign({ require, - console, analytics, settings, - sauceSettings, - browsers, yargs, margs, - SauceWorkerAllocator, WorkerAllocator, TestRunner, process, getTests, testFilters, - browserOptions, processCleanup, + profiles, path, loadRelativeModule }, opts); const project = runOpts.require("../package.json"); - runOpts.console.log("Magellan " + project.version); + logger.log("Magellan " + project.version); const defaultConfigFilePath = "./magellan.json"; const configFilePath = runOpts.yargs.argv.config; if (configFilePath) { - runOpts.console.log("Will try to load configuration from " + configFilePath); + logger.log("Will try to load configuration from " + configFilePath); } else { - runOpts.console.log("Will try to load configuration from default of " + defaultConfigFilePath); + logger.log("Will try to load configuration from default of " + defaultConfigFilePath); } // NOTE: marge can throw an error here if --config points at a file that doesn't exist // FIXME: handle this error nicely instead of printing an ugly stack trace runOpts.margs.init(defaultConfigFilePath, configFilePath); - const isSauce = runOpts.margs.argv.sauce ? true : false; - const isNodeBased = runOpts.margs.argv.framework && - runOpts.margs.argv.framework.indexOf("mocha") > -1; + // const isNodeBased = runOpts.margs.argv.framework && + // runOpts.margs.argv.framework.indexOf("mocha") > -1; const debug = runOpts.margs.argv.debug || false; const useSerialMode = runOpts.margs.argv.serial; const MAX_TEST_ATTEMPTS = parseInt(runOpts.margs.argv.max_test_attempts) || 3; - let selectedBrowsers; + let targetProfiles; let workerAllocator; let MAX_WORKERS; @@ -93,7 +86,7 @@ module.exports = (opts) => { // // Initialize Framework Plugins // ============================ - // + // TODO: move to a function // We translate old names like "mocha" to the new module names for the // respective plugins that provide support for those frameworks. Officially, @@ -139,6 +132,88 @@ module.exports = (opts) => { frameworkInitializationException = e; } + if (!runOpts.settings.testFramework || + frameworkLoadException || + frameworkInitializationException) { + logger.err("Could not start Magellan."); + if (frameworkLoadException) { + logger.err("Could not load the testing framework plugin '" + + runOpts.settings.framework + "'."); + logger.err("Check and make sure your package.json includes a module named '" + + runOpts.settings.framework + "'."); + logger.err("If it does not, you can remedy this by typing:" + + "\nnpm install --save " + runOpts.settings.framework); + logger.err(frameworkLoadException); + } else /* istanbul ignore else */ if (frameworkInitializationException) { + logger.err("Could not initialize the testing framework plugin '" + + runOpts.settings.framework + "'."); + logger.err("This plugin was found and loaded, but an error occurred during initialization:"); + logger.err(frameworkInitializationException); + } + + defer.reject({ error: "Couldn't start Magellan" }); + } + + logger.log("Loaded test framework: "); + logger.log(" " + runOpts.settings.framework); + // + // Initialize Executor + // ============================ + // TODO: move to a function + // TODO: move to a function + // let formalExecutor = ["local"]; + let formalExecutors = ["testarmada-magellan-local-executor"]; + + // executors is as array from magellan.json by default + if (runOpts.margs.argv.executors) { + if (_.isArray(runOpts.margs.argv.executors)) { + formalExecutors = runOpts.margs.argv.executors; + } else if (_.isString(runOpts.margs.argv.executors)) { + formalExecutors = [runOpts.margs.argv.executors]; + } else { + logger.err("Executors only accepts string and array"); + logger.warn("Setting executor to \"local\" by default"); + } + } else { + logger.warn("No executor is passed in"); + logger.warn("Setting executor to \"local\" by default"); + } + + runOpts.settings.executors = formalExecutors; + + // load executor + const executorLoadExceptions = []; + runOpts.settings.testExecutors = {}; + + _.forEach(runOpts.settings.executors, (executor) => { + try { + const targetExecutor = runOpts.require(executor); + targetExecutor.validateConfig(runOpts); + runOpts.settings.testExecutors[targetExecutor.shortName] = targetExecutor; + } catch (e) { + executorLoadExceptions.push(e); + } + }); + + if (executorLoadExceptions.length > 0) { + // error happens while loading executor + logger.err("There are errors in loading executors"); + _.forEach(executorLoadExceptions, (exception) => { + logger.err(exception.toString()); + }); + + defer.reject({ error: "Couldn't start Magellan" }); + } + + logger.log("Loaded test executors: "); + _.forEach(runOpts.settings.testExecutors, (executor) => { + logger.log(" " + executor.name); + }); + + const testExecutors = runOpts.settings.testExecutors; + + // finish processing all params =========================== + // Show help and exit if it's asked for if (runOpts.margs.argv.help) { const help = runOpts.require("./cli_help"); @@ -147,43 +222,31 @@ module.exports = (opts) => { return defer.promise; } - if (runOpts.margs.argv.list_browsers) { - runOpts.browsers.initialize(true).then(() => { - if (runOpts.margs.argv.device_additions) { - runOpts.browsers.addDevicesFromFile(runOpts.margs.argv.device_additions); + // handle executor specific params + const executorParams = _.omit(runOpts.margs.argv, _.keys(magellanArgs)); + + // ATTENTION: there should only be one executor param matched for the function call + _.forEach(runOpts.settings.testExecutors, (v, k) => { + _.forEach(executorParams, (epValue, epKey) => { + if (v.help[epKey] && v.help[epKey].type === "function") { + // we found a match in current executor + // method name convention for an executor: PREFIX_string_string_string_... + let names = epKey.split("_"); + names = names.slice(1, names.length); + const executorMethodName = _.camelCase(names.join(" ")); + + if (_.has(v, executorMethodName)) { + // method found in current executor + v[executorMethodName](runOpts, () => { + defer.resolve(); + }); + } else { + logger.err("Error: executor" + k + " doesn't has method " + executorMethodName + "."); + defer.resolve(); + } } - runOpts.browsers.listBrowsers(); - defer.resolve(); - }).catch((err) => { - runOpts.console.log("Couldn't fetch runOpts.browsers. Error: ", err); - runOpts.console.log(err.stack); - defer.reject(err); }); - return defer.promise; - } - - if (!runOpts.settings.testFramework || - frameworkLoadException || - frameworkInitializationException) { - runOpts.console.error(clc.redBright("Error: Could not start Magellan.")); - if (frameworkLoadException) { - runOpts.console.error(clc.redBright("Error: Could not load the testing framework plugin '" - + runOpts.settings.framework + "'." - + "\nCheck and make sure your package.json includes a module named '" - + runOpts.settings.framework + "'." - + "\nIf it does not, you can remedy this by typing:" - + "\n\nnpm install --save " + runOpts.settings.framework)); - runOpts.console.log(frameworkLoadException); - } else /* istanbul ignore else */ if (frameworkInitializationException) { - runOpts.console.error( - clc.redBright("Error: Could not initialize the testing framework plugin '" - + runOpts.settings.framework + "'." - + "\nThis plugin was found and loaded, but an error occurred during initialization:")); - runOpts.console.log(frameworkInitializationException); - } - - defer.reject({error: "Couldn't start Magellan"}); - } + }); // // Initialize Listeners @@ -261,13 +324,15 @@ module.exports = (opts) => { const tests = runOpts.getTests(runOpts.testFilters.detectFromCLI(runOpts.margs.argv)); if (_.isEmpty(tests)) { - runOpts.console.log("Error: no tests found"); - defer.reject({error: "No tests found"}); + logger.log("Error: no tests found"); + defer.reject({ error: "No tests found" }); return defer.promise; } const initializeListeners = () => { const deferred = Q.defer(); + magellanGlobals.workerAmount = MAX_WORKERS; + async.each(listeners, (listener, done) => { listener.initialize(magellanGlobals) .then(() => done()) @@ -285,122 +350,127 @@ module.exports = (opts) => { const startSuite = () => { const deferred = Q.defer(); - workerAllocator.initialize((err) => { - if (err) { - runOpts.console.error( - clc.redBright("Could not start Magellan. Got error while initializing" - + " worker allocator")); - deferred.reject(err); - return defer.promise; - } - - const testRunner = new runOpts.TestRunner(tests, { - debug, - - maxWorkers: MAX_WORKERS, - - maxTestAttempts: MAX_TEST_ATTEMPTS, - - browsers: selectedBrowsers, - - listeners, - - bailFast: runOpts.margs.argv.bail_fast ? true : false, - bailOnThreshold: runOpts.margs.argv.bail_early ? true : false, - - serial: useSerialMode, + Promise + .all(_.map(testExecutors, (executor) => executor.setupRunner())) + .then(() => { + workerAllocator.initialize((workerInitErr) => { + if (workerInitErr) { + logger.err("Could not start Magellan. Got error while initializing" + + " worker allocator"); + deferred.reject(workerInitErr); + return defer.promise; + } + + const testRunner = new runOpts.TestRunner(tests, { + debug, + + maxWorkers: MAX_WORKERS, + + maxTestAttempts: MAX_TEST_ATTEMPTS, + + profiles: targetProfiles, + executors: testExecutors, + + listeners, + + bailFast: runOpts.margs.argv.bail_fast ? true : false, + bailOnThreshold: runOpts.margs.argv.bail_early ? true : false, + + serial: useSerialMode, + + allocator: workerAllocator, + + onSuccess: () => { + /*eslint-disable max-nested-callbacks*/ + workerAllocator.teardown(() => { + Promise + .all(_.map(testExecutors, (executor) => executor.teardownRunner())) + .then(() => { + runOpts.processCleanup(() => { + deferred.resolve(); + }); + }) + .catch((err) => { + // we eat error here + logger.warn("executor teardownRunner error: " + err); + runOpts.processCleanup(() => { + deferred.resolve(); + }); + }); + }); + }, + + onFailure: (/*failedTests*/) => { + /*eslint-disable max-nested-callbacks*/ + workerAllocator.teardown(() => { + Promise + .all(_.map(testExecutors, (executor) => executor.teardownRunner())) + .then(() => { + runOpts.processCleanup(() => { + // Failed tests are not a failure in Magellan itself, + // so we pass an empty error here so that we don't + // confuse the user. Magellan already outputs a failure + // report to the screen in the case of failed tests. + deferred.reject(null); + }); + }) + .catch((err) => { + logger.warn("executor teardownRunner error: " + err); + // we eat error here + runOpts.processCleanup(() => { + deferred.reject(null); + }); + }); + }); + } + }); - allocator: workerAllocator, + testRunner.start(); + }); + }) + .catch((err) => { + deferred.reject(err); + }); - sauceSettings: isSauce ? runOpts.sauceSettings : undefined, + return deferred.promise; + }; - onSuccess: () => { - workerAllocator.teardown(() => { - runOpts.processCleanup(() => { - deferred.resolve(); - }); - }); - }, - - onFailure: (/*failedTests*/) => { - workerAllocator.teardown(() => { - runOpts.processCleanup(() => { - // Failed tests are not a failure in Magellan itself, so we pass an empty error - // here so that we don't confuse the user. Magellan already outputs a failure - // report to the screen in the case of failed tests. - deferred.reject(null); - }); - }); - } - }); + const enableExecutors = (_targetProfiles) => { + // this is to allow magellan to double check with profile that + // is retrieved by --profile or --profiles + targetProfiles = _targetProfiles; - testRunner.start(); - }); + const deferred = Q.defer(); + try { + _.forEach( + _.uniq(_.map(_targetProfiles, (targetProfile) => targetProfile.executor)), + (shortname) => { + if (runOpts.settings.testExecutors[shortname]) { + runOpts.settings.testExecutors[shortname].validateConfig({ isEnabled: true }); + } + }); + + deferred.resolve(); + } catch (err) { + deferred.reject(err); + } return deferred.promise; }; - runOpts.browsers.initialize(isSauce) - .then(() => { - if (runOpts.margs.argv.device_additions) { - runOpts.browsers.addDevicesFromFile(runOpts.margs.argv.device_additions); - } - }) - .then(runOpts.browserOptions.detectFromCLI.bind({}, runOpts.margs.argv, isSauce, isNodeBased)) - .then((_selectedBrowsers) => { - selectedBrowsers = _selectedBrowsers; - if (!_selectedBrowsers) { - // If this list comes back completely undefined, it's because we didn't - // get anything back from either profile lookup or the saucelabs API, which - // likely means we requested a browser that doesn't exist or no longer exists. - runOpts.console.log(clc.redBright("\nError: No matching browsers have been found." - + "\nTo see a list of sauce browsers, use the --list_browsers option.\n")); - throw new Error("Invalid browser specified for Sauce support"); - } else if (_selectedBrowsers.length === 0) { - runOpts.console.log( - clc.redBright("\nError: To use --sauce mode, you need to specify a browser." - + "\nTo see a list of sauce browsers, use the --list_browsers option.\n")); - throw new Error("No browser specified for Sauce support"); - } else if (debug) { - runOpts.console.log("Selected browsers: "); - runOpts.console.log(_selectedBrowsers.map((b) => { - return [ - b.browserId, - b.resolution ? b.resolution : "(any resolution)", - b.orientation ? b.orientation : "(any orientation)" - ].join(" "); - }).join("\n")); - } - }) + runOpts.profiles + .detectFromCLI(runOpts) + .then(enableExecutors) .then(() => { // // Worker Count: // ============= // - // Non-sauce mode: - // Default to 8 workers if we're running phantomjs and *only* phantomjs, - // otherwise 3 if other browsers are involved - // Default to 1 worker in serial mode. - // - // Sauce mode: // Default to 3 workers in parallel mode (default). // Default to 1 worker in serial mode. // - /*eslint-disable no-extra-parens*/ - if (isSauce) { - MAX_WORKERS = useSerialMode ? 1 : (parseInt(runOpts.margs.argv.max_workers) || 3); - } else { - const DEFAULT_MAX_WORKERS = (selectedBrowsers.length === 1 - && selectedBrowsers[0] === "phantomjs") ? 8 : 3; - MAX_WORKERS = useSerialMode ? - 1 : (parseInt(runOpts.margs.argv.max_workers) || DEFAULT_MAX_WORKERS); - } - - if (isSauce) { - workerAllocator = new runOpts.SauceWorkerAllocator(MAX_WORKERS); - } else { - workerAllocator = new runOpts.WorkerAllocator(MAX_WORKERS); - } + MAX_WORKERS = useSerialMode ? 1 : parseInt(runOpts.margs.argv.max_workers) || 3; + workerAllocator = new runOpts.WorkerAllocator(MAX_WORKERS); }) .then(initializeListeners) // NOTE: if we don't end up in catch() below, magellan exits with status code 0 naturally @@ -410,17 +480,17 @@ module.exports = (opts) => { }) .catch((err) => { if (err) { - runOpts.console.error(clc.redBright("Error initializing Magellan")); - runOpts.console.log(clc.redBright("\nError description:")); - runOpts.console.error(err.toString()); - runOpts.console.log(clc.redBright("\nError stack trace:")); - runOpts.console.log(err.stack); + logger.err("Error initializing Magellan"); + logger.err("Error description:"); + logger.err(err.toString()); + logger.err("Error stack trace:"); + logger.err(err.stack); } else { // No err object means we didn't have an internal crash while setting up / tearing down } // Fail the test suite or fail because of an internal crash - defer.reject({error: "Internal crash"}); + defer.reject({ error: "Internal crash" }); }); return defer.promise; diff --git a/src/cli_help.js b/src/cli_help.js index a697080..96e805f 100644 --- a/src/cli_help.js +++ b/src/cli_help.js @@ -4,92 +4,88 @@ const _ = require("lodash"); const project = require("../package.json"); const settings = require("./settings"); +const magellanHelp = require("./help").help; +const logger = require("./logger"); + +const MAX_HELP_KEY_WIDTH = 40; /*eslint max-len: 0*/ /*eslint max-statements: 0*/ module.exports = { help: (opts) => { const runOpts = _.assign({ - console, settings }, opts); - runOpts.console.log("Usage: magellan [options]"); - runOpts.console.log(""); - runOpts.console.log("By default, magellan will run all available tests in parallel with phantomjs."); - runOpts.console.log(""); - runOpts.console.log(" Parallelism, Workflow and Filtering:"); - runOpts.console.log(" --serial Run tests one at a time with detailed output."); - runOpts.console.log(" --max_workers=N Set maximum number of parallel works to (see defaults below)."); - runOpts.console.log(" --max_test_attempts=N Retry tests N times (default: 3)."); - runOpts.console.log(" --bail_early Kill builds that have failed at least 10% of tests, after 10 or more test runs."); - runOpts.console.log(" --bail_fast Kill builds that fail any test."); - runOpts.console.log(" --bail_time Set test kill time in milliseconds. *CAN* be used without bail_early/bail_fast."); - runOpts.console.log(" --early_bail_threshold A decimal ratio (eg 0.25 for 25%) how many tests to fail before bail_early"); - runOpts.console.log(" --early_bail_min_attempts How many test runs to run before applying bail_early rule."); - runOpts.console.log(" --debug Enable debugging magellan messages (dev mode)."); - runOpts.console.log(""); - runOpts.console.log(" Configuration:"); - runOpts.console.log(" --config=config-path Specify Magellan configuration location."); - runOpts.console.log(" --temp_dir=path Specify temporary file directory for Magellan (default: temp/)."); - runOpts.console.log(""); - runOpts.console.log(" Reporting and CI Integration:"); - runOpts.console.log(" --aggregate_screenshots Activate harvesting of screenshots for uploading to a screenshot service."); - runOpts.console.log(" --screenshot_aggregator_url Specify the URL to the screenshot service endpoint."); - runOpts.console.log(" --external_build_id Use an external build id, i.e. from CI. Must be filename and URL-safe."); - runOpts.console.log(""); - runOpts.console.log(" Browser and SauceLabs Support:"); - runOpts.console.log(" --sauce Run tests on SauceLabs cloud."); - runOpts.console.log(" --list_browsers List the available browsers configured."); - runOpts.console.log(" --browser=browsername Run tests in chrome, firefox, etc (default: phantomjs)."); - runOpts.console.log(" --browsers=b1,b2,.. Run multiple browsers in parallel."); - runOpts.console.log(" --browsers=all Run all available browsers (sauce only)."); - runOpts.console.log(" --create_tunnels Create secure tunnels in sauce mode (for use with --sauce only)"); - runOpts.console.log(" --sauce_tunnel_id Use an existing secure tunnel (for use with --sauce only, exclusive with --create_tunnels)"); - runOpts.console.log(" --shared_sauce_parent_account Specify parent account name if existing shared secure tunnel is in use (for use with --sauce only, exclusive with --create_tunnels)"); - runOpts.console.log(" --profile=p1,p2,.. Specify lists of browsers to use defined in profiles in magellan.json config."); - runOpts.console.log(" --profile=http://abc/p#p1,p2 Use profiles p1 and p2 hosted at JSON file http://abc/p (see README for details)."); - - let help; + logger.loghelp(""); + logger.loghelp("Usage: magellan [options]"); + logger.loghelp(""); + logger.loghelp("By default, magellan will run all available tests in parallel with phantomjs."); + logger.loghelp(""); + logger.loghelp("Available options:"); + logger.loghelp(""); + + const help = {}; + // load magellan help by default + _.forEach(magellanHelp, (v, k) => { + if (v.visible === undefined || v.visible) { + if (!help[v.category]) { + help[v.category] = {}; + } + help[v.category][k] = v; + } + }); + + // load desire framework help if (runOpts.settings.testFramework && runOpts.settings.testFramework.help) { - help = runOpts.settings.testFramework.help; - } + help[" Framework-specific (" + runOpts.settings.framework + ")"] = {}; - if (help) { - runOpts.console.log(""); - runOpts.console.log(" Framework-specific (" + runOpts.settings.framework + "):"); - const maxWidth = 31; - - Object.keys(help).forEach((key) => { - let str = " --" + key; - if (help[key].example) { - str += "=" + help[key].example; + _.forEach(runOpts.settings.testFramework.help, (v, k) => { + if (v.visible === undefined || v.visible) { + help[" Framework-specific (" + runOpts.settings.framework + ")"][k] = v; } - // pad - while (str.length < maxWidth) { - str += " "; + }); + } + + // load desire executor(s) help + if (runOpts.settings.testExecutors) { + _.forEach(runOpts.settings.testExecutors, (v) => { + if (v.help) { + help[" Executor-specific (" + v.name + ")"] = {}; + + _.forEach(v.help, (itemValue, itemKey) => { + if (itemValue.visible === undefined || itemValue.visible) { + help[" Executor-specific (" + v.name + ")"][itemKey] = itemValue; + } + }); } - // truncate just in case the example was too long to begin with - str = str.substr(0, maxWidth); - str += help[key].description; - runOpts.console.log(str); }); } - runOpts.console.log(""); - runOpts.console.log(" +------------------------------------------+-----------+-------------------+"); - runOpts.console.log(" | Workflow / Output Examples | # workers | output style |"); - runOpts.console.log(" +------------------------------------------+-----------+-------------------+"); - runOpts.console.log(" |--browser=phantomjs | 8 | summary, failures |"); - runOpts.console.log(" |--browser= --sauce | 3 | summary, failures |"); - runOpts.console.log(" |--browser= | 3 | summary, failures |"); - runOpts.console.log(" |--sauce --browser= --serial | 1 | detail, all tests |"); - runOpts.console.log(" |--sauce --browser= --serial | 1 | detail, all tests |"); - runOpts.console.log(" |--browser= --serial | 1 | detail, all tests |"); - runOpts.console.log(" +------------------------------------------+-----------+-------------------+"); - - runOpts.console.log(""); - runOpts.console.log("magellan v" + project.version); + if (help) { + _.forEach(help, (helpValue, helpKey) => { + logger.loghelp(" " + helpKey); + + _.forEach(helpValue, (itemValue, itemKey) => { + let str = " --" + itemKey; + if (itemValue.example) { + str += "=" + itemValue.example; + } + + while (str.length < MAX_HELP_KEY_WIDTH) { + str += " "; + } + + // truncate just in case the example was too long to begin with + str = str.substr(0, MAX_HELP_KEY_WIDTH); + str += itemValue.description; + logger.loghelp(str); + }); + logger.loghelp(""); + }); + } + + logger.log("magellan v" + project.version); } }; diff --git a/src/detect_browsers.js b/src/detect_browsers.js deleted file mode 100644 index c345531..0000000 --- a/src/detect_browsers.js +++ /dev/null @@ -1,207 +0,0 @@ -"use strict"; - -const Q = require("q"); -const _ = require("lodash"); - -const sauceBrowsers = require("./sauce/browsers.js"); -const hostedProfiles = require("./hosted_profiles"); - -class Slug { - constructor(id, resolution, orientation) { - this.browserId = id; - this.resolution = resolution ? resolution.trim() : undefined; - this.orientation = orientation ? orientation.trim() : undefined; - } - - slug() { - return this.browserId - + (this.resolution ? "_" + this.resolution : "") - + (this.orientation ? "_" + this.orientation : ""); - } - - toString() { - return this.browserId - + (this.resolution ? " @" + this.resolution : "") - + (this.orientation ? " orientation: " + this.orientation : ""); - } -} - -const createBrowser = (id, resolution, orientation) => { - return new Slug(id, resolution, orientation); -}; - -module.exports = { - createBrowser, - - // FIXME: reduce complexity so we can pass lint without this disable - /*eslint-disable complexity*/ - /*eslint-disable no-magic-numbers*/ - // Return a promise that we'll resolve with a list of browsers selected - // by the user from command line arguments - detectFromCLI: (argv, sauceEnabled, isNodeBased, opts) => { - const runOpts = _.assign({ - console - }, opts); - - const deferred = Q.defer(); - let browsers; - - // If a profile key is specified, look to argv for it and use it. If - // a browser is set ia CLI, we assume details from the stored profile - // and override with anything else explicitly set. - if (argv.profile) { - - if (argv.profile.indexOf("http:") > -1 || argv.profile.indexOf("https:") > -1) { - // We fetch profiles from an URL if it starts with http: or https: - // We assume it will have a #fragment to identify a given desired profile. - // Note: The hosted profiles are merged on top of any local profiles. - const fetchedProfiles = hostedProfiles.getProfilesAtURL(argv.profile.split("#")[0], opts); - /* istanbul ignore else */ - if (fetchedProfiles && fetchedProfiles.profiles) { - argv.profiles = _.extend({}, argv.profiles, fetchedProfiles.profiles); - - runOpts.console.log("Loaded hosted profiles from " + argv.profile.split("#")[0]); - } - - argv.profile = hostedProfiles.getProfileNameFromURL(argv.profile); - } - - runOpts.console.log("Requested profile(s): ", argv.profile); - - // NOTE: We check "profiles" (plural) here because that's what has - // the actual profile definition. "profile" is the argument from the - // command line. "profiles" is the list structure in magellan.json. - if (argv.profiles && Object.keys(argv.profiles).length > 0) { - let requestedProfiles; - - // Generate a list of profiles, which may typically be just one profile. - if (argv.profile.indexOf(",") > -1) { - requestedProfiles = argv.profile.split(","); - } else if (argv.profiles.hasOwnProperty(argv.profile)) { - requestedProfiles = [argv.profile]; - } - - // Search for the requested profiles and copy their browsers to browsers[] - if (requestedProfiles) { - browsers = []; - const notFoundProfiles = []; - - requestedProfiles.forEach((requestedProfile) => { - if (argv.profiles.hasOwnProperty(requestedProfile)) { - argv.profiles[requestedProfile].forEach((b) => { - browsers.push(createBrowser(b.browser, b.resolution, b.orientation)); - }); - } else { - notFoundProfiles.push(requestedProfile); - } - }); - - if (notFoundProfiles.length > 0) { - deferred.reject("Profile(s) " + notFoundProfiles.join(",") + " not found!"); - return; - } - } else { - deferred.reject("Profile " + argv.profile + " not found!"); - return; - } - } else { - deferred.reject("Profile " + argv.profile + " not found!"); - } - } - - if (argv.browser && argv.browser.indexOf(",") > -1) { - argv.browsers = argv.browser; - } - - // Note: "browsers" always trumps a single "browser" and will overwrite - // anything from a profile completely. - if (argv.browsers) { - browsers = argv.browsers.split(",").map((browser) => { - // NOTE: This applies the same orientation value to all browsers, regardless of whether it's - // appropriate or not. For better per-browser control, it's better to use browser profiles. - return createBrowser(browser.trim(), argv.resolution, argv.orientation); - }); - } else if (argv.browser) { - const singleBrowser = createBrowser(argv.browser, argv.resolution, argv.orientation); - - if (argv.profile) { - // If we've loaded a profile from magellan.json, the --browser option - // merely narrows down which of the profiles we're using. - // Select argv.browser *from* the existing list of browsers, which - // have already been loaded via argv.profile - browsers = browsers.filter((b) => { - return b.browserId === argv.browser.trim(); - }); - } else { - browsers = [singleBrowser]; - } - } else { - /*eslint-disable no-lonely-if*/ - // If we don't have a browser list yet from profiles or wherever else, then - // we fall back on default behavior. - if (browsers) { - if (!sauceEnabled) { - // - // TODO: Check browsers from profile are correct for not-sauce - // - } else { - // - // TODO: check sauce mode browsers here? or have they already been checked? - // - } - } else { - if (sauceEnabled) { - browsers = []; - } else { - let fallbackBrowser; - if (isNodeBased) { - fallbackBrowser = createBrowser("nodejs", argv.resolution, argv.orientation); - } else { - fallbackBrowser = createBrowser("phantomjs", argv.resolution, argv.orientation); - } - browsers = [fallbackBrowser]; - } - } - } - - // validate browser list if Sauce is enabled - if (sauceEnabled) { - sauceBrowsers.initialize(sauceEnabled).then(() => { - const unrecognizedBrowsers = browsers.filter((browser) => { - const browserId = browser.browserId; - // If we've specified a resolution, validate that this browser supports that exact one. - if (browser.resolution) { - const b = sauceBrowsers.browser(browserId); - if (b && b.resolutions && b.resolutions.indexOf(browser.resolution) > -1) { - // recognized - return false; - } - // unrecognized - return true; - } else { - return !sauceBrowsers.browser(browserId); - } - }); - - if (unrecognizedBrowsers.length > 0) { - runOpts.console.log("Error! Unrecognized saucelabs browsers specified:"); - unrecognizedBrowsers.forEach((browser) => { - runOpts.console.log( - " " + browser.browserId + " " + (browser.resolution ? " at resolution: " - + browser.resolution : "")); - }); - - // invalidate our result - browsers = []; - } - - deferred.resolve(browsers); - }); - } else { - deferred.resolve(browsers); - } - - /*eslint-disable consistent-return*/ - return deferred.promise; - } -}; diff --git a/src/help.js b/src/help.js new file mode 100644 index 0000000..f48ce00 --- /dev/null +++ b/src/help.js @@ -0,0 +1,121 @@ +"use strict"; + +/*eslint-disable max-len*/ +/*eslint-disable no-dupe-keys*/ +module.exports = { + name: "testarmada-magellan", + shortName: "magellan", + + help: { + "help": { + "category": "Usability", + "visible": false, + "description": "List options of magellan and all configured frameworks, executors and plugins in magellan.json." + }, + "serial": { + "category": "Parallelism, Workflow and Filtering", + "visible": true, + "description": "Run tests one at a time with detailed output." + }, + "max_workers": { + "category": "Parallelism, Workflow and Filtering", + "example": "N", + "visible": true, + "description": "Set maximum number of parallel works to (see defaults below)." + }, + "max_test_attempts": { + "category": "Parallelism, Workflow and Filtering", + "example": "N", + "visible": true, + "description": "Retry tests N times (default: 3)." + }, + "bail_early": { + "category": "Parallelism, Workflow and Filtering", + "visible": true, + "description": "Kill builds that have failed at least 10% of tests, after 10 or more test runs." + }, + "bail_fast": { + "category": "Parallelism, Workflow and Filtering", + "visible": true, + "description": "Kill builds that fail any test." + }, + "bail_time": { + "category": "Parallelism, Workflow and Filtering", + "visible": true, + "description": "Set test kill time in milliseconds. *CAN* be used without bail_early/bail_fast." + }, + "early_bail_threshold": { + "category": "Parallelism, Workflow and Filtering", + "visible": true, + "description": "A decimal ratio (eg 0.25 for 25%) how many tests to fail before bail_early" + }, + "early_bail_min_attempts": { + "category": "Parallelism, Workflow and Filtering", + "visible": true, + "description": "How many test runs to run before applying bail_early rule." + }, + "debug": { + "category": "Parallelism, Workflow and Filtering", + "visible": true, + "description": "Enable debugging magellan messages (dev mode)." + }, + "config": { + "category": "Configuration", + "visible": true, + "example": "config-path", + "description": "Specify Magellan configuration location." + }, + "temp_dir": { + "category": "Configuration", + "visible": true, + "example": "path", + "description": "Specify temporary file directory for Magellan (default: temp/)." + }, + "aggregate_screenshots": { + "category": "Reporting and CI Integration", + "visible": true, + "description": "Activate harvesting of screenshots for uploading to a screenshot service." + }, + "screenshot_aggregator_url": { + "category": "Reporting and CI Integration", + "visible": true, + "example": "http://some.image.url", + "description": "Specify the URL to the screenshot service endpoint." + }, + "external_build_id": { + "category": "Reporting and CI Integration", + "visible": true, + "example": "magellanBuildId312123", + "description": "Use an external build id, i.e. from CI. Must be filename and URL-safe." + }, + "profile": { + "category": "Environment support", + "visible": true, + "example": "p1,p2,..", + "description": "Specify lists of browsers to use defined in profiles in magellan.json config." + }, + "profile": { + "category": "Environment support", + "visible": true, + "example": "http://abc/p#p1,p2 ", + "description": "Use profiles p1 and p2 hosted at JSON file http://abc/p (see README for details)." + }, + "profiles": { + "category": "Environment support", + "visible": false, + "description": "A JSON object contains desiredCapabilities" + }, + "framework": { + "category": "Framework", + "visible": false, + "example": "nightwatch", + "description": "Framework which magellan will drive tests with." + }, + "executors": { + "category": "Executors", + "visible": false, + "example": "local", + "description": "Executors which magellan will use to execute test" + } + } +}; diff --git a/src/logger.js b/src/logger.js new file mode 100644 index 0000000..76abe1b --- /dev/null +++ b/src/logger.js @@ -0,0 +1,36 @@ +"use strict"; + +const argvs = require("marge"); +const util = require("util"); +const clc = require("cli-color"); + +const debug = argvs.argv.debug; + +const PREFIX = "Magellan"; + +module.exports = { + output: console, + + debug(msg) { + /* istanbul ignore if */ + if (debug) { + const deb = clc.blueBright("[DEBUG]"); + this.output.log(util.format("%s [%s] %s", deb, PREFIX, msg)); + } + }, + log(msg) { + const info = clc.greenBright("[INFO]"); + this.output.log(util.format("%s [%s] %s", info, PREFIX, msg)); + }, + warn(msg) { + const warn = clc.yellowBright("[WARN]"); + this.output.warn(util.format("%s [%s] %s", warn, PREFIX, msg)); + }, + err(msg) { + const err = clc.redBright("[ERROR]"); + this.output.error(util.format("%s [%s] %s", err, PREFIX, msg)); + }, + loghelp(msg) { + this.output.log(msg); + } +}; diff --git a/src/profiles.js b/src/profiles.js new file mode 100644 index 0000000..73ea29a --- /dev/null +++ b/src/profiles.js @@ -0,0 +1,174 @@ +"use strict"; + +const _ = require("lodash"); +const hostedProfiles = require("./hosted_profiles"); +const logger = require("./logger"); + +class Profile { + constructor(p) { + _.forEach(p, (v, k) => { + this[k] = v; + }); + } + + toString() { + const cap = this.desiredCapabilities || this; + + /* istanbul ignore next */ + return cap.browserName + + (cap.version ? "|version:" + cap.version : "") + + (cap.resolution ? "|resolution:" + cap.resolution : "") + + (cap.orientation ? "|orientation:" + cap.orientation : "") + + "|executor:" + this.executor; + } +} + +module.exports = { + detectFromCLI: (opts) => { + // runOpts.margs.argv, runOpts.settings.testExecutors + /** + * Handle following command argument + * --profile + * + */ + const runOpts = _.assign({}, opts); + + const argv = runOpts.margs.argv; + const testExecutors = runOpts.settings.testExecutors; + + return new Promise((resolve, reject) => { + let profiles = []; + + if (argv.profile) { + // If a profile key is specified, look to argv for it and use it. If + // a browser is set ia CLI, we assume details from the stored profile + // and override with anything else explicitly set. + if (argv.profile.indexOf("http:") > -1 || argv.profile.indexOf("https:") > -1) { + // We fetch profiles from an URL if it starts with http: or https: + // We assume it will have a #fragment to identify a given desired profile. + // Note: The hosted profiles are merged on top of any local profiles. + const remoteProfileURL = argv.profile.split("#")[0]; + const fetchedProfiles = hostedProfiles.getProfilesAtURL(remoteProfileURL, opts); + + if (fetchedProfiles && fetchedProfiles.profiles) { + argv.profiles = _.extend({}, argv.profiles, fetchedProfiles.profiles); + + logger.log("Loaded hosted profiles from " + remoteProfileURL); + } + + argv.profile = hostedProfiles.getProfileNameFromURL(argv.profile); + } + + logger.log("Requested profile(s): " + argv.profile); + + // NOTE: We check "profiles" (plural) here because that's what has + // the actual profile definition. "profile" is the argument from the + // command line. "profiles" is the list structure in magellan.json. + if (argv.profiles && Object.keys(argv.profiles).length > 0) { + let requestedProfiles; + + // Generate a list of profiles, which may typically be just one profile. + if (argv.profile.indexOf(",") > -1) { + requestedProfiles = argv.profile.split(","); + } else if (argv.profiles.hasOwnProperty(argv.profile)) { + requestedProfiles = [argv.profile]; + } + + // Search for the requested profiles and resolve their profiles + if (requestedProfiles) { + const notFoundProfiles = []; + + _.forEach(requestedProfiles, (requestedProfile) => { + if (argv.profiles[requestedProfile]) { + profiles = _.concat(profiles, argv.profiles[requestedProfile]); + } else { + notFoundProfiles.push(requestedProfile); + } + }); + + if (notFoundProfiles.length > 0) { + reject("Profile(s) " + notFoundProfiles.join(",") + " not found!"); + } + + logger.debug(" Selected profiles: "); + _.forEach(profiles, (p) => { + const str = []; + + _.map(p, (v, k) => { + str.push(k + ": " + v); + }); + logger.debug(" " + str.join(", ")); + }); + + // convert profile to an executor-understandable capabilities + const profileResolvePromises = []; + + _.forEach(profiles, (profile) => { + // we treat all missing profile.executor with sauce executor by default + if (!profile.executor) { + profile.executor = "sauce"; + } + + if (testExecutors[profile.executor]) { + profileResolvePromises.push( + testExecutors[profile.executor].getCapabilities(profile, opts)); + } else { + reject("Executor " + profile.executor + " not found! You'll need to configure" + + " it in magellan.json"); + } + }); + + Promise + .all(profileResolvePromises) + .then((targetProfiles) => { + const resolvedprofiles = []; + + if (targetProfiles && targetProfiles.length > 0) { + _.forEach(targetProfiles, (tp) => { + resolvedprofiles.push(new Profile(tp)); + }); + } + resolve(resolvedprofiles); + }) + .catch((err) => { + reject(err); + }); + + } else { + reject("Profile " + argv.profile + " not found!"); + } + } else { + reject("Profile " + argv.profile + " not found!"); + } + } else { + // user passes profile information from command line directly, + // like --local_browser or some params tighted to an executor + const profileResolvePromises = []; + + _.forEach(testExecutors, (executor) => { + profileResolvePromises.push(executor.getProfiles(opts)); + }); + + Promise + .all(profileResolvePromises) + .then((targetProfiles) => { + const resolvedprofiles = []; + + if (targetProfiles && targetProfiles.length > 0) { + const flattenTargetProfiles = _.flatten(targetProfiles); + + _.forEach(flattenTargetProfiles, (tp) => { + if (tp) { + resolvedprofiles.push(new Profile(tp)); + } + }); + } + resolve(resolvedprofiles); + }) + .catch((err) => { + reject(err); + }); + } + }); + } +}; diff --git a/src/sauce/browsers.js b/src/sauce/browsers.js deleted file mode 100644 index 2549686..0000000 --- a/src/sauce/browsers.js +++ /dev/null @@ -1,67 +0,0 @@ -"use strict"; - -const Q = require("q"); -const _ = require("lodash"); -const SauceBrowsers = require("guacamole"); -const listSauceCliBrowsers = require("guacamole/src/cli_list"); - -module.exports = { - - // - // TODO: the actual listing of browsers should be provided by guacamole - // - listBrowsers: (opts) => { - const runOpts = _.assign({}, { - console, - listSauceCliBrowsers - }, opts); - - runOpts.listSauceCliBrowsers((browserTable) => { - // convert table heading - browserTable.options.head[1] = "Copy-Paste Command-Line Option"; - runOpts.console.log(browserTable.toString()); - runOpts.console.log(""); - runOpts.console.log("Non-Sauce Browser List:"); - runOpts.console.log(" --browser=chrome\t\tLocal Chrome browser"); - runOpts.console.log(" --browser=firefox\t\tLocal Firefox browser"); - runOpts.console.log(" --browser=safari\t\tLocal Safari browser"); - runOpts.console.log( - " --browser=phantomjs\t\tLocal Phantomjs browser [default in non-sauce mode]"); - }); - }, - - // Return a browser by id if it exists in our browser list. Optionally return that browser - // only if a resolution is supported by that browser environment - browser: (id, resolution, orientation) => { - const results = SauceBrowsers.get({ - id, - screenResolution: resolution, - deviceOrientation: orientation - }, true); - - let result; - if (results.length > 0) { - const browser = results[0]; - result = _.extend({ - id, - resolutions: browser.resolutions - }, browser.desiredCapabilities); - } - - return result; - }, - - addDevicesFromFile: (filePath) => { - SauceBrowsers.addNormalizedBrowsersFromFile(filePath); - }, - - initialize: (fetchSauceBrowsers) => { - if (fetchSauceBrowsers) { - return SauceBrowsers.initialize(); - } else { - const d = Q.defer(); - d.resolve(); - return d.promise; - } - } -}; diff --git a/src/sauce/settings.js b/src/sauce/settings.js deleted file mode 100644 index 2779ebf..0000000 --- a/src/sauce/settings.js +++ /dev/null @@ -1,104 +0,0 @@ -/* eslint complexity: 0, no-console: 0 */ -"use strict"; -const _ = require("lodash"); - -// Sauce Settings -// -// Cobble together settings for sauce either from process.env or from a sauce configuration file - -const argv = require("marge").argv; -const clc = require("cli-color"); - -module.exports = (opts) => { - const runOpts = _.assign({}, { - argv, - console, - env: process.env - }, opts); - - /*eslint-disable no-magic-numbers*/ - const config = { - // required: - username: runOpts.env.SAUCE_USERNAME, - accessKey: runOpts.env.SAUCE_ACCESS_KEY, - sauceConnectVersion: runOpts.env.SAUCE_CONNECT_VERSION, - - // optional: - sauceTunnelId: runOpts.argv.sauce_tunnel_id, - sharedSauceParentAccount: runOpts.argv.shared_sauce_parent_account, - tunnelTimeout: runOpts.env.SAUCE_TUNNEL_CLOSE_TIMEOUT, - useTunnels: !!runOpts.argv.create_tunnels, - maxTunnels: runOpts.argv.num_tunnels || 1, - fastFailRegexps: runOpts.env.SAUCE_TUNNEL_FAST_FAIL_REGEXPS, - - locksServerLocation: runOpts.argv.locks_server || runOpts.env.LOCKS_SERVER, - locksOutageTimeout: 1000 * 60 * 5, - locksPollingInterval: 2500, - locksRequestTimeout: 2500 - }; - - // Remove trailing / in locks server location if it's present. - if (typeof config.locksServerLocation === "string" && config.locksServerLocation.length > 0) { - if (config.locksServerLocation.charAt(config.locksServerLocation.length - 1) === "/") { - config.locksServerLocation = config.locksServerLocation.substr(0, - config.locksServerLocation.length - 1); - } - } - - const parameterWarnings = { - username: { - required: true, - envKey: "SAUCE_USERNAME" - }, - accessKey: { - required: true, - envKey: "SAUCE_ACCESS_KEY" - }, - sauceConnectVersion: { - required: false, - envKey: "SAUCE_CONNECT_VERSION" - } - }; - - // Validate configuration if we have --sauce - if (runOpts.argv.sauce) { - let valid = true; - Object.keys(parameterWarnings).forEach((key) => { - const param = parameterWarnings[key]; - - if (!config[key]) { - if (param.required) { - runOpts.console.log( - clc.redBright("Error! Sauce requires " + key + " to be set. Check if the" - + " environment variable $" + param.envKey + " is defined.")); - valid = false; - } else { - runOpts.console.log(clc.yellowBright("Warning! No " + key + " is set. This is set via the" - + " environment variable $" + param.envKey + " . This isn't required, but can cause " - + "problems with Sauce if not set")); - } - } - }); - - if (!valid) { - throw new Error("Missing configuration for Saucelabs connection."); - } - - if (runOpts.argv.sauce_tunnel_id && runOpts.argv.create_tunnels) { - throw new Error("Only one Saucelabs tunnel arg is allowed, --sauce_tunnel_id " + - "or --create_tunnels."); - } - - if (runOpts.argv.shared_sauce_parent_account && runOpts.argv.create_tunnels) { - throw new Error("--shared_sauce_parent_account only works with --sauce_tunnel_id."); - } - } - - if (runOpts.argv.debug) { - runOpts.console.log("Sauce configuration: ", config); - } - - runOpts.console.log("Sauce configuration OK"); - - return config; -}; diff --git a/src/sauce/tunnel.js b/src/sauce/tunnel.js deleted file mode 100644 index e9d9c10..0000000 --- a/src/sauce/tunnel.js +++ /dev/null @@ -1,145 +0,0 @@ -/* eslint no-console: 0 */ -"use strict"; - -const clc = require("cli-color"); -const _ = require("lodash"); -const path = require("path"); -const sauceConnectLauncher = require("sauce-connect-launcher"); - -const settings = require("../settings"); -const sauceSettingsFunc = require("./settings"); -const analytics = require("../global_analytics"); - -let username; -let accessKey; - -let connectFailures = 1; -/*eslint-disable no-magic-numbers*/ -const MAX_CONNECT_RETRIES = process.env.SAUCE_CONNECT_NUM_RETRIES || 10; -let BAILED = false; - -module.exports = { - initialize: (callback, opts) => { - const sauceSettings = sauceSettingsFunc(opts); - username = sauceSettings.username; - accessKey = sauceSettings.accessKey; - - const runOpts = _.assign({ - console, - analytics, - sauceConnectLauncher - }, opts); - - if (!username) { - return callback("Sauce tunnel support is missing configuration: Sauce username."); - } - - if (!accessKey) { - return callback("Sauce tunnel support is missing configuration: Sauce access key."); - } - - runOpts.analytics.push("sauce-connect-launcher-download"); - runOpts.sauceConnectLauncher.download({ - logger: console.log.bind(console) - }, (err) => { - if (err) { - runOpts.analytics.mark("sauce-connect-launcher-download", "failed"); - runOpts.console.log(clc.redBright("Failed to download sauce connect binary:")); - runOpts.console.log(clc.redBright(err)); - runOpts.console.log(clc.redBright("sauce-connect-launcher will attempt to re-download " + - "next time it is run.")); - } else { - runOpts.analytics.mark("sauce-connect-launcher-download"); - } - callback(err); - }); - }, - - open: (options, opts) => { - const runOpts = _.assign({ - console, - settings, - sauceConnectLauncher - }, opts); - - const tunnelInfo = {}; - const tunnelId = options.tunnelId; - const callback = options.callback; - - runOpts.console.info("Opening Sauce Tunnel ID: " + tunnelId + " for user " + username); - - const connect = (/*runDiagnostics*/) => { - const logFilePath = path.resolve(settings.tempDir) + "/build-" - + settings.buildId + "_sauceconnect_" + tunnelId + ".log"; - const sauceOptions = { - username, - accessKey, - tunnelIdentifier: tunnelId, - readyFileId: tunnelId, - verbose: settings.debug, - verboseDebugging: settings.debug, - logfile: logFilePath - }; - - if (runOpts.settings.fastFailRegexps) { - sauceOptions.fastFailRegexps = runOpts.settings.fastFailRegexps; - } - - const seleniumPort = options.seleniumPort; - if (seleniumPort) { - sauceOptions.port = seleniumPort; - } - - if (runOpts.settings.debug) { - runOpts.console.log("calling sauceConnectLauncher() w/ ", sauceOptions); - } - runOpts.sauceConnectLauncher(sauceOptions, (err, sauceConnectProcess) => { - if (err) { - if (runOpts.settings.debug) { - runOpts.console.log("Error from sauceConnectLauncher():"); - } - runOpts.console.error(err.message); - if (err.message && err.message.indexOf("Could not start Sauce Connect") > -1) { - return callback(err.message); - } else if (BAILED) { - connectFailures++; - // If some other parallel tunnel construction attempt has tripped the BAILED flag - // Stop retrying and report back a failure. - return callback(new Error("Bailed due to maximum number of tunnel retries.")); - } else { - connectFailures++; - - if (connectFailures >= MAX_CONNECT_RETRIES) { - // We've met or exceeded the number of max retries, stop trying to connect. - // Make sure other attempts don't try to re-state this error. - BAILED = true; - return callback(new Error("Failed to create a secure sauce tunnel after " - + connectFailures + " attempts.")); - } else { - // Otherwise, keep retrying, and hope this is merely a blip and not an outage. - runOpts.console.log(">>> Sauce Tunnel Connection Failed! Retrying " - + connectFailures + " of " + MAX_CONNECT_RETRIES + " attempts..."); - connect(); - } - } - } else { - tunnelInfo.process = sauceConnectProcess; - return callback(null, tunnelInfo); - } - }); - }; - - connect(); - }, - - close: (tunnelInfo, callback, opts) => { - const runOpts = _.assign({ - console - }, opts); - - tunnelInfo.process.close(() => { - runOpts.console.log("Closed Sauce Connect process"); - callback(); - }); - } -}; diff --git a/src/sauce/worker_allocator.js b/src/sauce/worker_allocator.js deleted file mode 100644 index 4aedf4e..0000000 --- a/src/sauce/worker_allocator.js +++ /dev/null @@ -1,304 +0,0 @@ -/* eslint no-invalid-this: 0 */ -"use strict"; - -const BaseWorkerAllocator = require("../worker_allocator"); -const _ = require("lodash"); -const request = require("request"); - -const sauceSettings = require("./settings")(); -const settings = require("../settings"); -const analytics = require("../global_analytics"); -const guid = require("../util/guid"); -const tunnel = require("./tunnel"); - -const BASE_SELENIUM_PORT_OFFSET = 56000; -const SECOND_MS = 1000; -const SECONDS_MINUTE = 60; - -class SauceWorkerAllocator extends BaseWorkerAllocator { - constructor(_MAX_WORKERS, opts) { - super(_MAX_WORKERS, opts); - - _.assign(this, { - console, - sauceSettings, - request, - clearTimeout, - setTimeout, - tunnel, - analytics, - settings, - delay: _.delay - }, opts); - - this.tunnels = []; - this.tunnelErrors = []; - this.MAX_WORKERS = _MAX_WORKERS; - this.maxTunnels = this.sauceSettings.maxTunnels; - this.tunnelPrefix = guid(); - - if (this.sauceSettings.locksServerLocation) { - this.console.log("Using locks server at " + this.sauceSettings.locksServerLocation - + " for VM traffic control."); - } - } - - initialize(callback) { - this.initializeWorkers(this.MAX_WORKERS); - - if (!this.sauceSettings.useTunnels && !this.sauceSettings.sauceTunnelId) { - return callback(); - } else if (this.sauceSettings.sauceTunnelId) { - // Aoint test to a tunnel pool, no need to initialize tunnel - // TODO: verify if sauce connect pool is avaiable and if at least one - // tunnel in the pool is ready - this.tunnels.push({ name: "fake sc process" }); - this.console.log("Connected to sauce tunnel pool with Tunnel ID", - this.sauceSettings.sauceTunnelId); - this.assignTunnelsToWorkers(this.tunnels.length); - return callback(); - } else { - this.tunnel.initialize((initErr) => { - if (initErr) { - return callback(initErr); - } else { - this.analytics.push("sauce-open-tunnels"); - this.openTunnels((openErr) => { - if (openErr) { - this.analytics.mark("sauce-open-tunnels", "failed"); - return callback(new Error("Cannot initialize worker allocator: " + - openErr.toString())); - } else { - // NOTE: We wait until we know how many tunnels we actually got before - // we assign tunnel ids to workers. - this.analytics.mark("sauce-open-tunnels"); - this.assignTunnelsToWorkers(this.tunnels.length); - return callback(); - } - }); - } - }); - } - } - - release(worker) { - if (this.sauceSettings.locksServerLocation) { - this.request({ - method: "POST", - json: true, - timeout: this.sauceSettings.locksRequestTimeout, - body: { - token: worker.token - }, - url: this.sauceSettings.locksServerLocation + "/release" - }, () => { - // TODO: decide whether we care about an error at this stage. We're releasing - // this worker whether the remote release is successful or not, since it will - // eventually be timed out by the locks server. - super.release.call(this, worker); - }); - } else { - super.release.call(this, worker); - } - } - - get(callback) { - // - // http://0.0.0.0:3000/claim - // - // {"accepted":false,"message":"Claim rejected. No VMs available."} - // {"accepted":true,"token":null,"message":"Claim accepted"} - // - if (this.sauceSettings.locksServerLocation) { - const pollingStartTime = Date.now(); - - // Poll the worker allocator until we have a known-good port, then run this test - const poll = () => { - if (this.settings.debug) { - this.console.log("asking for VM.."); - } - this.request.post({ - url: this.sauceSettings.locksServerLocation + "/claim", - timeout: this.sauceSettings.locksRequestTimeout, - form: {} - }, (error, response, body) => { - try { - if (error) { - throw new Error(error); - } - - const result = JSON.parse(body); - if (result) { - if (result.accepted) { - if (this.settings.debug) { - this.console.log("VM claim accepted, token: " + result.token); - } - super.get.call(this, (getWorkerError, worker) => { - if (worker) { - worker.token = result.token; - } - callback(getWorkerError, worker); - }); - } else { - if (this.settings.debug) { - this.console.log("VM claim not accepted, waiting to try again .."); - } - // If we didn't get a worker, try again - throw new Error("Request not accepted"); - } - } else { - throw new Error("Result from locks server is invalid or empty: '" + result + "'"); - } - } catch (e) { - // NOTE: There are several errors that can happen in the above code: - // - // 1. Parsing - we got a response from locks, but it's malformed - // 2. Interpretation - we could parse a result, but it's empty or weird - // 3. Connection - we attempted to connect, but timed out, 404'd, etc. - // - // All of the above errors end up here so that we can indiscriminately - // choose to tolerate all types of errors until we've waited too long. - // This allows for the locks server to be in a bad state (whether due - // to restart, failure, network outage, or whatever) for some amount of - // time before we panic and start failing tests due to an outage. - if (Date.now() - pollingStartTime > this.sauceSettings.locksOutageTimeout) { - // we've been polling for too long. Bail! - return callback(new Error("Gave up trying to get " - + "a saucelabs VM from locks server. " + e)); - } else { - if (this.settings.debug) { - this.console.log("Error from locks server, tolerating error and" + - " waiting " + this.sauceSettings.locksPollingInterval + - "ms before trying again"); - } - this.setTimeout(poll, this.sauceSettings.locksPollingInterval); - } - } - }); - }; - - poll(); - } else { - super.get.call(this, callback); - } - } - - assignTunnelsToWorkers(numOpenedTunnels) { - // Assign a tunnel id for each worker. - this.workers.forEach((worker, i) => { - worker.tunnelId = this.getTunnelId(i % numOpenedTunnels); - this.console.log("Assigning worker " + worker.index + " to tunnel " + worker.tunnelId); - }); - } - - getTunnelId(tunnelIndex) { - if (this.sauceSettings.sauceTunnelId) { - // if sauce tunnel id exists - return this.sauceSettings.sauceTunnelId; - } else { - return this.tunnelPrefix + "_" + tunnelIndex; - } - } - - teardown(callback) { - if (this.sauceSettings.useTunnels) { - this.teardownTunnels(callback); - } else { - return callback(); - } - } - - openTunnels(callback) { - const tunnelOpened = (err, tunnelInfo) => { - - if (err) { - this.tunnelErrors.push(err); - } else { - this.tunnels.push(tunnelInfo); - } - - if (this.tunnels.length === this.maxTunnels) { - this.console.log("All tunnels open! Continuing..."); - return callback(); - } else if (this.tunnels.length > 0 - && this.tunnels.length + this.tunnelErrors.length === this.maxTunnels) { - // We've accumulated some tunnels and some errors. Continue - // with a limited number of workers? - this.console.log("Opened only " + this.tunnels.length + " tunnels out of " - + this.maxTunnels + " requested (due to errors)."); - this.console.log("Continuing with a reduced number of workers (" - + this.tunnels.length + ")."); - return callback(); - } else if (this.tunnelErrors.length === this.maxTunnels) { - // We've tried to open N tunnels but instead got N errors. - return callback(new Error("\nCould not open any sauce tunnels (attempted to open " - + this.maxTunnels + " total tunnels): \n" + - this.tunnelErrors.map((tunnelErr) => { - return tunnelErr.toString(); - }).join("\n") + "\nPlease check that there are no " - + "sauce-connect-launcher (sc) processes running." - )); - } else { - if (err) { - this.console.log("Failed to open a tunnel, number of failed tunnels: " - + this.tunnelErrors.length); - } - this.console.log( - this.tunnels.length + " of " + this.maxTunnels + " tunnels open. Waiting..." - ); - } - }; - - const openTunnel = (tunnelIndex) => { - - const tunnelId = this.getTunnelId(tunnelIndex); - this.console.log("Opening tunnel " + tunnelIndex + " of " - + this.maxTunnels + " [id = " + tunnelId + "]"); - - const options = { - tunnelId, - seleniumPort: BASE_SELENIUM_PORT_OFFSET + (tunnelIndex + 1), - callback: tunnelOpened - }; - - this.tunnel.open(options); - }; - - _.times(this.maxTunnels, (n) => { - // worker numbers are 1-indexed - this.console.log("Waiting " + n + " sec to open tunnel #" + n); - this.delay(() => openTunnel(n), n * SECOND_MS); - }); - } - - teardownTunnels(callback) { - const tunnelsOriginallyOpen = this.tunnels.length; - let tunnelsOpen = this.tunnels.length; - const tunnelCloseTimeout = (this.sauceSettings.tunnelTimeout || SECONDS_MINUTE) * SECOND_MS; - - const closeTimer = this.setTimeout(() => { - // NOTE: We *used to* forcefully clean up stuck tunnels in here, but instead, - // we now leave the tunnel processes for process_cleanup to clean up. - this.console.log("Timeout reached waiting for tunnels to close... Continuing..."); - return callback(); - }, tunnelCloseTimeout); - - const tunnelClosed = () => { - - if (--tunnelsOpen === 0) { - this.console.log("All tunnels closed! Continuing..."); - this.clearTimeout(closeTimer); - return callback(); - } else { - this.console.log(tunnelsOpen + " of " + tunnelsOriginallyOpen - + " tunnels still open... waiting..."); - } - }; - - _.each(this.tunnels, (tunnelInfo) => { - this.tunnel.close(tunnelInfo, tunnelClosed); - }); - } -} - -module.exports = SauceWorkerAllocator; diff --git a/src/settings.js b/src/settings.js index 1802d86..dcf9e76 100644 --- a/src/settings.js +++ b/src/settings.js @@ -7,6 +7,7 @@ const argv = require("marge").argv; const env = process.env; const fs = require("fs"); const path = require("path"); +const logger = require("./logger"); // Allow an external build id (eg: from CI system, for example) to be used. If we're not given one, // we generate a random build id instead. NOTE: This build id must work as a part of a filename. @@ -40,7 +41,7 @@ try { if (fs.accessSync) { fs.accessSync(TEMP_DIR, fs.R_OK | fs.W_OK); } - console.log("Magellan is creating temporary files at: " + TEMP_DIR); + logger.log("Magellan is creating temporary files at: " + TEMP_DIR); } catch (e) { /* istanbul ignore next */ throw new Error("Magellan cannot write to or create the temporary directory: " + TEMP_DIR); diff --git a/src/test.js b/src/test.js index 0ec33f1..896608a 100644 --- a/src/test.js +++ b/src/test.js @@ -5,7 +5,7 @@ const TEST_STATUS_FAILED = 2; const TEST_STATUS_SUCCESSFUL = 3; class Test { - constructor(locator, browser, sauceBrowserSettings, maxAttempts) { + constructor(locator, profile, executor, maxAttempts) { // // note: this locator object is an instance of an object which is defined by whichever test // framework plugin is currently loaded. The implementation of locator could be almost any @@ -18,14 +18,13 @@ class Test { this.attempts = 0; this.status = TEST_STATUS_NEW; - this.browser = browser; + this.profile = profile; + this.executor = executor; this.workerIndex = -1; this.error = undefined; this.stdout = ""; this.stderr = ""; - - this.sauceBrowserSettings = sauceBrowserSettings; } // Return true if we've either: @@ -56,11 +55,9 @@ class Test { this.runningTime = (new Date()).getTime() - this.startTime; } - // return an unambiguous representation of this test: path, browserId, resolution, orientation + // return an unambiguous representation of this test: path, profile information toString() { - return this.locator.toString() + " @" + this.browser.browserId - + " " + (this.browser.resolution ? "res:" + this.browser.resolution : "") - + (this.browser.orientation ? "orientation:" + this.browser.orientation : ""); + return this.locator.toString() + " @" + this.profile.toString(); } getRuntime() { diff --git a/src/test_runner.js b/src/test_runner.js index b40b7ab..1aea6b3 100644 --- a/src/test_runner.js +++ b/src/test_runner.js @@ -4,7 +4,6 @@ // TODO: Extract trending into another class // TODO: Move bailFast to a strategy pattern implementation -const fork = require("child_process").fork; const async = require("async"); const _ = require("lodash"); const clc = require("cli-color"); @@ -18,11 +17,11 @@ const mkdirSync = require("./mkdir_sync"); const guid = require("./util/guid"); const logStamp = require("./util/logstamp"); const sanitizeFilename = require("sanitize-filename"); -const sauceBrowsers = require("./sauce/browsers"); const analytics = require("./global_analytics"); const settings = require("./settings"); const Test = require("./test"); +const logger = require("./logger"); const WORKER_START_DELAY = 1000; const WORKER_STOP_DELAY = 1500; @@ -60,16 +59,14 @@ const strictness = { class TestRunner { constructor(tests, options, opts) { _.assign(this, { - console, fs, mkdirSync, - fork, - sauceBrowsers, settings, setTimeout, clearInterval, setInterval, prettyMs, + path, analytics }, opts); @@ -104,13 +101,12 @@ class TestRunner { this.hasBailed = false; - this.browsers = options.browsers; + this.profiles = options.profiles; + this.executors = options.executors; this.debug = options.debug; this.serial = options.serial || false; - this.sauceSettings = options.sauceSettings; - this.listeners = options.listeners || []; this.onFailure = options.onFailure; @@ -120,12 +116,9 @@ class TestRunner { // For each actual test path, split out this.tests = _.flatten(tests.map((testLocator) => { - return options.browsers.map((requestedBrowser) => { - // Note: For non-sauce browsers, this can come back empty, which is just fine. - const sauceBrowserSettings = this.sauceBrowsers.browser(requestedBrowser.browserId, - requestedBrowser.resolution, requestedBrowser.orientation); - return new Test(testLocator, requestedBrowser, sauceBrowserSettings, - this.MAX_TEST_ATTEMPTS); + return options.profiles.map((profile) => { + return new Test(testLocator, profile, + this.executors[profile.executor], this.MAX_TEST_ATTEMPTS); }); })); @@ -133,7 +126,7 @@ class TestRunner { this.trends = { failures: {} }; - this.console.log("Gathering trends to ./trends.json"); + logger.log("Gathering trends to ./trends.json"); } this.numTests = this.tests.length; @@ -150,16 +143,14 @@ class TestRunner { start() { this.startTime = (new Date()).getTime(); - let browserStatement = " with "; - browserStatement += this.browsers.map((b) => b.toString()).join(", "); + let profileStatement = this.profiles.map((b) => b.toString()).join(", "); if (this.serial) { - this.console.log( - "\nRunning " + this.numTests + " tests in serial mode" + browserStatement + "\n" - ); + logger.log("Running " + this.numTests + " tests in serial mode with [" + + profileStatement + "]"); } else { - this.console.log("\nRunning " + this.numTests + " tests with " + this.MAX_WORKERS - + " workers" + browserStatement + "\n"); + logger.log("Running " + this.numTests + " tests with " + this.MAX_WORKERS + + " workers with [" + profileStatement + "]"); } if (this.tests.length === 0) { @@ -197,66 +188,84 @@ class TestRunner { this.analytics.push("acquire-worker-" + analyticsGuid); - this.allocator.get((error, worker) => { - if (!error) { - this.analytics.mark("acquire-worker-" + analyticsGuid); + const failTest = (error) => { + this.analytics.mark("acquire-worker-" + analyticsGuid, "failed"); + // If the allocator could not give us a worker, pass + // back a failed test result with the allocator's error. + logger.err("Worker allocator error: " + error); + logger.err(error.stack); - this.runTest(test, worker) - .then((runResults) => { - // Give this worker back to the allocator - this.allocator.release(worker); + /*eslint-disable no-magic-numbers*/ + test.workerIndex = -1; + test.error = undefined; + test.stdout = ""; + test.stderr = error; - test.workerIndex = worker.index; - test.error = runResults.error; - test.stdout = runResults.stdout; - test.stderr = runResults.stderr; + test.fail(); - // Pass or fail the test - if (runResults.error) { - test.fail(); - } else { - test.pass(); - } + onTestComplete(null, test); + }; - onTestComplete(null, test); - }) - .catch((runTestError) => { - // Catch a testing infrastructure error unrelated to the test itself failing. - // This indicates something went wrong with magellan itself. We still need - // to drain the queue, so we fail the test, even though the test itself may - // have not actually failed. - this.console.log(clc.redBright( - "Fatal internal error while running a test:", runTestError - )); - this.console.log(clc.redBright(runTestError.stack)); - - // Give this worker back to the allocator - this.allocator.release(worker); - - test.workerIndex = worker.index; - test.error = runTestError; - test.stdout = ""; - test.stderr = runTestError; - - test.fail(); - onTestComplete(runTestError, test); - }); + test.executor.setupTest((stageExecutorError, token) => { + if (!stageExecutorError) { + + this.allocator.get((getWorkerError, worker) => { + if (!getWorkerError) { + + this.analytics.mark("acquire-worker-" + analyticsGuid); + + this.runTest(test, worker) + .then((runResults) => { + // Give this worker back to the allocator + /*eslint-disable max-nested-callbacks*/ + test.executor.teardownTest(token, () => { + this.allocator.release(worker); + }); + + test.workerIndex = worker.index; + test.error = runResults.error; + test.stdout = runResults.stdout; + test.stderr = runResults.stderr; + + // Pass or fail the test + if (runResults.error) { + test.fail(); + } else { + test.pass(); + } + + onTestComplete(null, test); + }) + .catch((runTestError) => { + // Catch a testing infrastructure error unrelated to the test itself failing. + // This indicates something went wrong with magellan itself. We still need + // to drain the queue, so we fail the test, even though the test itself may + // have not actually failed. + logger.err("Fatal internal error while running a test:" + runTestError); + logger.err(runTestError.stack); + + // Give this worker back to the allocator + /*eslint-disable max-nested-callbacks*/ + test.executor.wrapup(() => { + this.allocator.release(worker); + }); + + test.workerIndex = worker.index; + test.error = runTestError; + test.stdout = ""; + test.stderr = runTestError; + + test.fail(); + onTestComplete(runTestError, test); + }); + } else { + // fail test due to failure of allocator.get() + failTest(getWorkerError); + } + }); } else { - this.analytics.mark("acquire-worker-" + analyticsGuid, "failed"); - // If the allocator could not give us a worker, pass - // back a failed test result with the allocator's error. - this.console.error("Worker allocator error: " + error); - this.console.error(error.stack); - - /*eslint-disable no-magic-numbers*/ - test.workerIndex = -1; - test.error = undefined; - test.stdout = ""; - test.stderr = error; - - test.fail(); - - onTestComplete(null, test); + // fail test due to failure of test.executor.stage() + failTest(stageExecutorError); } }); } @@ -266,9 +275,15 @@ class TestRunner { // Rejections only happen if we encounter a problem with magellan itself, not // Rejections only happen if we encounter a problem with magellan itself, not // the test. The test will resolve with a test result whether it fails or passes. - spawnTestProcess(testRun, test) { + execute(testRun, test) { const deferred = Q.defer(); + if (testRun.enableExecutor + && typeof testRun.enableExecutor === "function") { + // if we have addExecutor defined in test run (new in magellan 10.0.0) + testRun.enableExecutor(test.executor); + } + let env; try { env = testRun.getEnvironment(this.settings.environment); @@ -284,9 +299,10 @@ class TestRunner { stdio: ["pipe", "pipe", "pipe", "ipc"] }; - let childProcess; + let handler; try { - childProcess = this.fork(testRun.getCommand(), testRun.getArguments(), options); + ////////////////////////////////////////////////// + handler = this.executors[test.profile.executor].execute(testRun, options); this.notIdle(); } catch (e) { deferred.reject(e); @@ -296,15 +312,15 @@ class TestRunner { // Simulate some of the aspects of a node process by adding stdout and stderr streams // that can be used by listeners and reporters. const statusEmitter = new EventEmitter(); - statusEmitter.stdout = childProcess.stdout; - statusEmitter.stderr = childProcess.stderr; + statusEmitter.stdout = handler.stdout; + statusEmitter.stderr = handler.stderr; const statusEmitterEmit = (type, message) => { statusEmitter.emit(type, message); }; let sentry; - let seleniumSessionId; + let testMetadata; let stdout = clc.greenBright(logStamp()) + " Magellan child process start\n\n"; let stderr = ""; @@ -340,7 +356,7 @@ class TestRunner { metadata: { test: test.locator.toString(), - browser: test.browser.browserId, + profile: test.profile.id, // NOTE: attempt numbers are 1-indexed attemptNumber: (test.attempts + 1) } @@ -375,40 +391,39 @@ class TestRunner { status: "finished", name: test.locator.toString(), passed: code === 0, - metadata: { - // - // TODO: move the generation of this resultURL to sauce support modules - // TODO: leave it open to have result URLs for anything including non-sauce tests - // right now this is directly tied to sauce since sauce is the only thing that - // generates a resultURL, but in the future, we may have resultURLs that - // originate from somewhere else. - // - resultURL: "https://saucelabs.com/tests/" + seleniumSessionId - } + metadata: testMetadata }); // Detach ALL listeners that may have been attached - childProcess.stdout.removeAllListeners(); - childProcess.stderr.removeAllListeners(); - childProcess.stdout.unpipe(); - childProcess.stderr.unpipe(); - childProcess.removeAllListeners(); + handler.stdout.removeAllListeners(); + handler.stderr.removeAllListeners(); + handler.stdout.unpipe(); + handler.stderr.unpipe(); + handler.removeAllListeners(); statusEmitter.stdout = null; statusEmitter.stderr = null; - // Resolve the promise - deferred.resolve({ - error: (code === 0) ? null : "Child test run process exited with code " + code, - stderr, - stdout - }); + test.executor.summerizeTest( + this.buildId, + { + result: code === 0, + metadata: testMetadata + }, + () => { + // Resolve the promise + deferred.resolve({ + error: (code === 0) ? null : "Child test run process exited with code " + code, + stderr, + stdout + }); + }); }); if (this.debug) { // For debugging purposes. - childProcess.on("message", (msg) => { - this.console.log("Message from worker:", msg); + handler.on("message", (msg) => { + logger.debug("Message from worker:" + JSON.stringify(msg)); }); } @@ -419,18 +434,19 @@ class TestRunner { // // FIXME: make it possible to receive this information from test frameworks not based on nodejs // - childProcess.on("message", (message) => { - if (message.type === "selenium-session-info") { - seleniumSessionId = message.sessionId; + handler.on("message", (message) => { + if (message.type === "test-meta-data") { + testMetadata = message.metadata; } }); - childProcess.stdout.on("data", (data) => { + handler.stdout.on("data", (data) => { let text = ("" + data); if (text.trim() !== "") { text = text .split("\n") .filter((line) => { + /* istanbul ignore next */ return line.trim() !== "" || line.indexOf("\n") > -1; }) .map((line) => { @@ -447,12 +463,13 @@ class TestRunner { } }); - childProcess.stderr.on("data", (data) => { + handler.stderr.on("data", (data) => { let text = ("" + data); if (text.trim() !== "") { text = text .split("\n") .filter((line) => { + /* istanbul ignore next */ return line.trim() !== "" || line.indexOf("\n") > -1; }) .map((line) => { @@ -469,7 +486,7 @@ class TestRunner { } }); - childProcess.on("close", workerClosed); + handler.on("close", workerClosed); // A sentry monitors how long a given worker has been working. In every // strictness level except BAIL_NEVER, we kill a worker process and its @@ -493,10 +510,10 @@ class TestRunner { this.clearInterval(sentry); // Tell the child to shut down the running test immediately - childProcess.send({ + handler.send({ signal: "bail", customMessage: "Killed by magellan after " + strictness.LONG_RUNNING_TEST - + "ms (long running test)" + + "ms (long running test)" }); this.setTimeout(() => { @@ -521,10 +538,6 @@ class TestRunner { msg.push("-->"); msg.push((this.serial ? "Serial mode" : "Worker " + worker.index) + ","); - if (this.sauceSettings && worker.tunnelId) { - msg.push("tunnel id: " + worker.tunnelId + ","); - } - msg.push("mock port:" + worker.portOffset + ","); if (worker.token) { @@ -533,7 +546,7 @@ class TestRunner { msg.push("running test: " + test.toString()); - this.console.log(msg.join(" ")); + logger.log(msg.join(" ")); } let testRun; @@ -543,7 +556,7 @@ class TestRunner { const childBuildId = guid(); // Note: we must sanitize the buildid because it might contain slashes or "..", etc - const tempAssetPath = path.resolve(this.settings.tempDir + "/build-" + const tempAssetPath = this.path.resolve(this.settings.tempDir + "/build-" + sanitizeFilename(this.buildId) + "_" + childBuildId + "__temp_assets"); this.mkdirSync(tempAssetPath); @@ -564,17 +577,15 @@ class TestRunner { // Magellan environment id (i.e. id of browser, id of device, version, etc.), // typically reflects one of the items from --browsers=item1,item2,item3 options - environmentId: test.browser.browserId, + // environmentId: test.browser.browserId, + profile: test.profile, + // executor: this.executors[test.profile.executor], // The locator object originally generated by the plugin itself locator: test.locator, seleniumPort: worker.portOffset + 1, - mockingPort: worker.portOffset, - - tunnelId: worker.tunnelId, - sauceSettings: this.sauceSettings, - sauceBrowserSettings: test.sauceBrowserSettings + mockingPort: worker.portOffset }); } catch (e) { deferred.reject(e); @@ -582,7 +593,7 @@ class TestRunner { if (testRun) { this.setTimeout(() => { - this.spawnTestProcess(testRun, test) + this.execute(testRun, test) .then(deferred.resolve) .catch(deferred.reject); }, WORKER_START_DELAY); @@ -593,14 +604,14 @@ class TestRunner { gatherTrends() { if (this.settings.gatherTrends) { - this.console.log("Updating trends ..."); + logger.log("Updating trends ..."); let existingTrends; try { existingTrends = JSON.parse(this.fs.readFileSync("./trends.json")); } catch (e) { - existingTrends = {failures: {}}; + existingTrends = { failures: {} }; } Object.keys(this.trends.failures).forEach((key) => { @@ -612,21 +623,20 @@ class TestRunner { this.fs.writeFileSync("./trends.json", JSON.stringify(existingTrends, null, 2)); - this.console.log("Updated trends at ./trends.json"); + logger.log("Updated trends at ./trends.json"); } } logFailedTests() { - this.console.log(clc.redBright("\n============= Failed Tests: =============\n")); + logger.log(clc.redBright("============= Failed Tests: =============")); this.failedTests.forEach((failedTest) => { - this.console.log("\n- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -" - + " - - - - - - - - - - - - - - - "); - this.console.log("Failed Test: " + failedTest.toString()); - this.console.log(" # attempts: " + failedTest.attempts); - this.console.log(" output: "); - this.console.log(failedTest.stdout); - this.console.log(failedTest.stderr); + logger.log("- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -"); + logger.log("Failed Test: " + failedTest.toString()); + logger.log(" # attempts: " + failedTest.attempts); + logger.log(" output: "); + logger.log(failedTest.stdout); + logger.log(failedTest.stderr); }); } @@ -667,23 +677,23 @@ class TestRunner { } }); - this.console.log(clc.greenBright("\n============= Suite Complete =============\n")); - this.console.log(" Status: " + status); - this.console.log(" Runtime: " + this.prettyMs((new Date()).getTime() - this.startTime)); - this.console.log("Total tests: " + this.numTests); - this.console.log(" Successful: " + this.passedTests.length + " / " + this.numTests); + logger.log(clc.greenBright("============= Suite Complete =============")); + logger.log(" Status: " + status); + logger.log(" Runtime: " + this.prettyMs((new Date()).getTime() - this.startTime)); + logger.log("Total tests: " + this.numTests); + logger.log(" Successful: " + this.passedTests.length + " / " + this.numTests); _.forOwn(retryMetrics, (testCount, numRetries) => { - this.console.log(testCount + " test(s) have retried: " + numRetries + " time(s)"); + logger.log(testCount + " test(s) have retried: " + numRetries + " time(s)"); }); if (this.failedTests.length > 0) { - this.console.log(" Failed: " + this.failedTests.length + " / " + this.numTests); + logger.log(" Failed: " + this.failedTests.length + " / " + this.numTests); } const skipped = this.numTests - (this.passedTests.length + this.failedTests.length); if (this.hasBailed && skipped > 0) { - this.console.log(" Skipped: " + skipped); + logger.log(" Skipped: " + skipped); } const flushNextListener = () => { @@ -702,7 +712,7 @@ class TestRunner { promise .then(flushNextListener) .catch((error) => { - this.console.log("Error when flushing listener output: ", error); + logger.log("Error when flushing listener output: ", error); flushNextListener(); }); } else { @@ -740,7 +750,7 @@ class TestRunner { if (this.hasBailed) { // Ignore results from this test if we've bailed. This is likely a test that // was killed when the build went into bail mode. - this.console.log("\u2716 " + clc.redBright("KILLED ") + " " + test.toString() + logger.log("\u2716 " + clc.redBright("KILLED ") + " " + test.toString() + (this.serial ? "\n" : "")); return; } @@ -780,9 +790,9 @@ class TestRunner { let suffix; if (this.serial) { - prefix = "\n(" + (this.passedTests.length + this.failedTests.length) + " / " + prefix = "(" + (this.passedTests.length + this.failedTests.length) + " / " + this.numTests + ")"; - suffix = "\n"; + suffix = ""; } else { prefix = "(" + (this.passedTests.length + this.failedTests.length) + " / " + this.numTests + ") <-- Worker " + test.workerIndex; @@ -790,8 +800,8 @@ class TestRunner { } const requeueNote = testRequeued ? clc.cyanBright("(will retry). Spent " - + test.getRuntime() + " msec") : ""; - this.console.log(prefix + " " + + test.getRuntime() + " msec") : ""; + logger.log(prefix + " " + (successful ? clc.greenBright("PASS ") : clc.redBright("FAIL ")) + requeueNote + " " + test.toString() + " " + suffix); @@ -842,7 +852,7 @@ class TestRunner { if (totalAttempts > strictness.THRESHOLD_MIN_ATTEMPTS) { if (ratio > strictness.THRESHOLD) { - this.console.log("Magellan has seen at least " + (strictness.THRESHOLD * 100) + "% of " + logger.log("Magellan has seen at least " + (strictness.THRESHOLD * 100) + "% of " + " tests fail after seeing at least " + strictness.THRESHOLD_MIN_ATTEMPTS + " tests run. Bailing early."); return true; diff --git a/src/util/check_ports.js b/src/util/check_ports.js index d7a856e..d192bd9 100644 --- a/src/util/check_ports.js +++ b/src/util/check_ports.js @@ -3,6 +3,7 @@ const _ = require("lodash"); const request = require("request"); const portscanner = require("portscanner"); +const logger = require("../logger"); const PORT_STATUS_IN_USE = 0; const PORT_STATUS_AVAILABLE = 1; @@ -10,8 +11,7 @@ const PORT_STATUS_AVAILABLE = 1; const checkPortStatus = (desiredPort, callback, opts) => { const runOpts = _.assign({ request, - portscanner, - console + portscanner }, opts); runOpts.request("http://127.0.0.1:" + desiredPort + @@ -25,7 +25,7 @@ const checkPortStatus = (desiredPort, callback, opts) => { } }); } else { - runOpts.console.log( + logger.log( "Found selenium HTTP server at port " + desiredPort + ", port is in use."); return callback(PORT_STATUS_IN_USE); } diff --git a/src/util/load_relative_module.js b/src/util/load_relative_module.js index e6d72af..357dba0 100644 --- a/src/util/load_relative_module.js +++ b/src/util/load_relative_module.js @@ -1,16 +1,15 @@ "use strict"; const path = require("path"); -const clc = require("cli-color"); const _ = require("lodash"); +const logger = require("../logger"); module.exports = (mPath, moduleIsOptional, opts) => { let resolvedRequire; mPath = mPath.trim(); const runOpts = _.assign({ - require, - console + require }, opts); if (mPath.charAt(0) === ".") { @@ -25,8 +24,8 @@ module.exports = (mPath, moduleIsOptional, opts) => { RequiredModule = runOpts.require(resolvedRequire); } catch (e) { if (e.code === "MODULE_NOT_FOUND" && moduleIsOptional !== true) { - runOpts.console.error(clc.redBright("Error loading a module from user configuration.")); - runOpts.console.error(clc.redBright("Cannot find module: " + resolvedRequire)); + logger.err("Error loading a module from user configuration."); + logger.err("Cannot find module: " + resolvedRequire); throw new Error(e); } else if (e.code === "MODULE_NOT_FOUND" && moduleIsOptional === true) { // Do nothing diff --git a/src/util/process_cleanup.js b/src/util/process_cleanup.js index c162a06..121eff5 100644 --- a/src/util/process_cleanup.js +++ b/src/util/process_cleanup.js @@ -5,6 +5,7 @@ const _ = require("lodash"); const pid = process.pid; const settings = require("../settings"); +const logger = require("../logger"); // Max time before we forcefully kill child processes left over after a suite run const ZOMBIE_POLLING_MAX_TIME = 15000; @@ -12,34 +13,31 @@ const ZOMBIE_POLLING_MAX_TIME = 15000; module.exports = (callback, opts) => { const runOpts = _.assign({ settings, - console, treeUtil }, opts); if (runOpts.settings.debug) { - runOpts.console.log("Checking for zombie processes..."); + logger.log("Checking for zombie processes..."); } runOpts.treeUtil.getZombieChildren(pid, ZOMBIE_POLLING_MAX_TIME, (zombieChildren) => { if (zombieChildren.length > 0) { - runOpts.console.log("Giving up waiting for zombie child processes to die. Cleaning up.."); + logger.log("Giving up waiting for zombie child processes to die. Cleaning up.."); const killNextZombie = () => { if (zombieChildren.length > 0) { const nextZombieTreePid = zombieChildren.shift(); - runOpts.console.log("Killing pid and its child pids: " + nextZombieTreePid); + logger.log("Killing pid and its child pids: " + nextZombieTreePid); runOpts.treeUtil.kill(nextZombieTreePid, "SIGKILL", killNextZombie); } else { - runOpts.console.log("Done killing zombies."); + logger.log("Done killing zombies."); return callback(); } }; return killNextZombie(); } else { - if (runOpts.settings.debug) { - runOpts.console.log("No zombies found."); - } + logger.debug("No zombies found."); return callback(); } }); diff --git a/src/worker_allocator.js b/src/worker_allocator.js index 7eb3dc0..ab71d6c 100644 --- a/src/worker_allocator.js +++ b/src/worker_allocator.js @@ -1,9 +1,9 @@ "use strict"; const _ = require("lodash"); -const clc = require("cli-color"); const settings = require("./settings"); const portUtil = require("./util/port_util"); +const logger = require("./logger"); const MAX_ALLOCATION_ATTEMPTS = 120; const WORKER_START_DELAY = 1000; @@ -14,7 +14,6 @@ const WORKER_START_DELAY = 1000; class Allocator { constructor(MAX_WORKERS, opts) { _.assign(this, { - console, setTimeout, checkPorts: portUtil.checkPorts, getNextPort: portUtil.getNextPort, @@ -22,8 +21,8 @@ class Allocator { }, opts); if (this.debug) { - this.console.log("Worker Allocator starting."); - this.console.log("Port allocation range from: " + settings.BASE_PORT_START + " to " + logger.log("Worker Allocator starting."); + logger.log("Port allocation range from: " + settings.BASE_PORT_START + " to " + (settings.BASE_PORT_START + settings.BASE_PORT_RANGE - 1) + " with " + settings.BASE_PORT_SPACING + " ports available to each worker."); } @@ -105,11 +104,11 @@ class Allocator { // Print a message that ports are not available, show which ones in the range availableWorker.occupied = false; - this.console.log(clc.yellowBright("Detected port contention while spinning up worker: ")); + logger.warn("Detected port contention while spinning up worker: "); statuses.forEach((status, portIndex) => { if (!status.available) { - this.console.log(clc.yellowBright(" in use: #: " + status.port + " purpose: " - + (desiredPortLabels[portIndex] ? desiredPortLabels[portIndex] : "generic"))); + logger.warn(" in use: #: " + status.port + " purpose: " + + (desiredPortLabels[portIndex] ? desiredPortLabels[portIndex] : "generic")); } }); diff --git a/test/cli.js b/test/cli.js index 0a2bd9a..298cf24 100644 --- a/test/cli.js +++ b/test/cli.js @@ -1,70 +1,16 @@ -/* eslint no-undef: 0, no-unused-expressions: 0, no-invalid-this: 0, - no-throw-literal: 0, no-empty: 0, camelcase: 0, no-unused-vars: 0 */ "use strict"; -const expect = require("chai").expect; -const _ = require("lodash"); -const Q = require("q"); -const sinon = require("sinon"); -const cli = require("../src/cli"); - -class FakeAllocator { - constructor() { - } - initialize(cb) { - cb(null); - } - teardown(cb) { - cb(); - } -} -class FakeTestRunner { - constructor(tests, opts) { - this.tests = tests; - this.opts = opts; - } - start() { - this.opts.onSuccess(); - } -} +const chai = require("chai"); +const chaiAsPromise = require("chai-as-promised"); +const _ = require("lodash"); -class FailingTestRunner { - constructor(tests, opts) { - this.tests = tests; - this.opts = opts; - } - start() { - this.opts.onFailure(); - } -} +const cli = require("../src/cli.js"); +const logger = require("../src/logger"); -class FakeReporter { - initialize() { - const defer = Q.defer(); - defer.resolve(); - return defer.promise; - } - listenTo() { - } - flush() { - const defer = Q.defer(); - defer.resolve(); - return defer.promise; - } -} +chai.use(chaiAsPromise); -class BadReporter { - initialize() { - throw new Error("Bad!"); - } - listenTo() { - } - flush() { - const defer = Q.defer(); - defer.resolve(); - return defer.promise; - } -} +const expect = chai.expect; +const assert = chai.assert; const _fakeRequire = (overrides) => { return (name) => { @@ -85,33 +31,111 @@ const _fakeRequire = (overrides) => { } if (name === "./cli_help") { return { - help: () => {} + help: () => { } }; } if (name === "./reporters/slack/settings") { return {}; } if (name === "./reporters/slack/slack" || - name === "./reporters/screenshot_aggregator/reporter" || - name === "./reporters/stdout/reporter") { + name === "./reporters/screenshot_aggregator/reporter" || + name === "./reporters/stdout/reporter") { return new FakeReporter(); } + if (name === "testarmada-magellan-local-executor") { + return fakeExecutor; + } + if (name.indexOf("error") > -1) { + throw new Error("FAKE FRAMEWORK EXCEPTION"); + } + if (name.match(/\/index/)) { return { - initialize: () => {}, - getPluginOptions: () => {} + initialize: () => { }, + getPluginOptions: () => { } }; } return { }; - }; + } +}; + +const fakeExecutor = { + name: "testarmada-magellan-local-executor", + shortName: "local", + help: { + "local_list_browsers": { + "visible": true, + "type": "function", + "description": "List the available browsers configured." + }, + "local_list_fakes": { + "visible": true, + "type": "function", + "description": "List the available browsers configured." + } + }, + validateConfig() { }, + setupRunner() { + return new Promise((resolve) => { + resolve(); + }); + }, + teardownRunner() { + return new Promise((resolve) => { + resolve(); + }); + }, + listBrowsers(param, callback) { + callback(); + } +}; + +const FakeProfiles = { + detectFromCLI() { + return new Promise((resolve) => { + resolve([{ executor: "local" }]); + }); + } }; +class FakeAllocator { + constructor() { + } + initialize(cb) { + cb(null); + } + teardown(cb) { + cb(); + } +} + +class FakeTestRunner { + constructor(tests, opts) { + this.tests = tests; + this.opts = opts; + } + start() { + this.opts.onSuccess(); + } +} + +class FakeReporter { + initialize() { + return new Promise((resolve) => { resolve() }); + } + listenTo() { + } + flush() { + return new Promise((resolve) => { resolve() }); + } +} + const _testConfig = (overrides) => { return _.merge({ console: { - log: () => {}, - error: () => {} + log: () => { }, + error: () => { } }, require: _fakeRequire(), process: { @@ -122,23 +146,27 @@ const _testConfig = (overrides) => { } }, analytics: { - mark: () => {}, - push: () => {} + mark: () => { }, + push: () => { } }, getTests: () => { return [ - {test: "a"}, - {test: "b"}, - {test: "c"} + { test: "a" }, + { test: "b" }, + { test: "c" } ]; }, margs: { - init: () => {}, + init: () => { }, argv: { + debug: true } }, settings: { - framework: "foo" + framework: "foo", + testExecutors: { + "local": fakeExecutor + } }, processCleanup: (cb) => { cb(); @@ -152,235 +180,225 @@ const _testConfig = (overrides) => { return str; } }, - SauceWorkerAllocator: FakeAllocator, WorkerAllocator: FakeAllocator, TestRunner: FakeTestRunner, - browsers: { - initialize: () => { - const defer = Q.defer(); - defer.resolve(); - return defer.promise; - } - }, + profiles: FakeProfiles, testFilters: { - detectFromCLI: () => {} + detectFromCLI: () => { } }, - browserOptions: { - detectFromCLI: () => { - return [ - {browserId: "chrome", resolution: 1024, orientation: "portrait"}, - {browserId: "foo"} - ]; - } - } + loadRelativeModule: () => { return new FakeReporter(); } }, overrides); }; -describe("CLI", () => { - it("startup", (done) => { - const spy = sinon.spy(); - cli(_testConfig({ - console: { - log: spy - } - })).then(() => { - expect(spy.called).to.be.true; - done(); - }); - }); - - it("allow for config path", (done) => { - let text = ""; - cli(_testConfig({ +describe("pure_cli", () => { + it("allow for config path", () => { + return cli(_testConfig({ yargs: { argv: { config: "FOOBAR_CONFIG" } - }, - console: { - log: (t) => { - text += t; - } } - })).then(() => { - expect(text.match(/FOOBAR_CONFIG/).length).to.eql(1); - done(); - }); + })) + .then() + .catch(err => assert(false, "shouldn't be here")); }); - it("check for mocha", (done) => { - cli(_testConfig({ - margs: { - argv: { - framework: "mocha" - } - }, - browserOptions: { - detectFromCLI: (a, b, c) => { - expect(c).to.be.true; - return [ - {browserId: "chrome", resolution: 1024, orientation: "portrait"} - ]; + describe("resolve framework", () => { + it("legacy framework name translation", () => { + return cli(_testConfig({ + settings: { + framework: "vanilla-mocha" } - } - })).then(() => { - done(); + })) + .then() + .catch(err => assert(false, "shouldn't be here")); }); - }); - it("check for rowdy-mocha", (done) => { - let sawRequire = false; - cli(_testConfig({ - settings: { - framework: "rowdy-mocha" - }, - require: _fakeRequire((name) => { - if (name === "./node_modules/testarmada-magellan-mocha-plugin/index") { - sawRequire = true; + it("handle framework load exception", () => { + return cli(_testConfig({ + settings: { + framework: "error" } - }) - })).then(() => { - expect(sawRequire).to.be.true; - done(); + })) + .then(() => assert(false, "shouldn't be here")) + .catch(err => { }); }); - try { - cli(_testConfig({ + it("handle framework init exception", () => { + return cli(_testConfig({ settings: { - framework: "rowdy-mocha" + framework: "local" }, require: _fakeRequire((name) => { - if (name === "./node_modules/testarmada-magellan-mocha-plugin/index") { - throw "Boom!"; + if (name.match(/\/index/)) { + return { + initialize: () => { }, + getPluginOptions: () => { throw new Error("FAKE INIT ERROR") } + }; } }) - })).then(() => { - expect(sawRequire).to.be.true; - done(); - }); - } catch (e) { - } + })) + .then(() => assert(false, "shouldn't be here")) + .catch(err => { }); + }); }); - it("allow for sauce", (done) => { - cli(_testConfig({ + it("get help", () => { + return cli(_testConfig({ margs: { argv: { - sauce: true - } - }, - browsers: { - initialize: (sauce) => { - expect(sauce).to.be.true; - const defer = Q.defer(); - defer.resolve(); - return defer.promise; + help: true } } - })).then(() => { - done(); - }); + })) + .then() + .catch(err => assert(false, "shouldn't be here")); }); - it("show help", (done) => { - const spy = sinon.spy(); - cli(_testConfig({ + it("setup_teardown", () => { + return cli(_testConfig({ margs: { argv: { - help: true + setup_teardown: "something" } - }, - require: _fakeRequire((name) => { - if (name === "./cli_help") { - return { - help: spy - }; + } + })) + .then() + .catch(err => assert(false, "shouldn't be here")); + }); + + + + describe("resolve executor", () => { + it("as string", () => { + return cli(_testConfig({ + margs: { + argv: { + executors: "testarmada-magellan-local-executor" + } } - return null; - }) - })).then(() => { - expect(spy.called).to.be.true; - done(); + })) + .then() + .catch(err => assert(false, "shouldn't be here")); }); - }); - it("allow for no plugin options", (done) => { - cli(_testConfig({ - require: _fakeRequire((name) => { - if (name.match(/\/index/)) { - return { - initialize: () => {} - }; + it("as array", () => { + return cli(_testConfig({ + margs: { + argv: { + executors: ["testarmada-magellan-local-executor"] + } } - }) - })).then(() => { - done(); + })) + .then() + .catch(err => assert(false, "shouldn't be here")); + }); + + it("malformed", () => { + return cli(_testConfig({ + margs: { + argv: { + executors: {} + } + } + })) + .then() + .catch(err => assert(false, "shouldn't be here")); + }); + + it("executor method", () => { + return cli(_testConfig({ + margs: { + argv: { + executors: ["testarmada-magellan-local-executor"], + local_list_browsers: true + } + } + })) + .then() + .catch(err => assert(false, "shouldn't be here")); + }); + + it("executor method no matches", () => { + return cli(_testConfig({ + margs: { + argv: { + executors: ["testarmada-magellan-local-executor"], + local_list_fakes: true + } + } + })) + .then() + .catch(err => assert(false, "shouldn't be here")); + }); + + it("executor load exception", () => { + return cli(_testConfig({ + margs: { + argv: { + executors: ["testarmada-magellan-local-executor"] + } + }, + require: _fakeRequire((name) => { + if (name === "testarmada-magellan-local-executor") { + throw new Error("FAKE EXECUTOR INIT ERROR"); + } + }) + })) + .then(() => assert(false, "shouldn't be here")) + .catch(err => { }); }); }); - it("throw an exception in initialization", (done) => { - cli(_testConfig({ + it("enable slack", () => { + return cli(_testConfig({ require: _fakeRequire((name) => { - if (name.match(/\/index/)) { - return BadReporter; + if (name === "./reporters/slack/settings") { + return { + enabled: true + }; + } + if (name === "./reporters/slack/slack") { + return FakeReporter; } }) - })).then(() => { - }).catch(() => { - done(); - }); + })) + .then() + .catch(err => assert(false, "shouldn't be here")); }); - it("allow for setup_teardown", (done) => { - cli(_testConfig({ + it("reporter as array", () => { + return cli(_testConfig({ margs: { argv: { - setup_teardown: "hola!" + reporters: ["a", "b", "c"] } }, loadRelativeModule: () => { return new FakeReporter(); } - })).then(() => { - done(); - }).catch((e) => { - }); + })) + .then() + .catch(err => assert(false, "shouldn't be here")); }); - it("allow for reporters", (done) => { - cli(_testConfig({ + it("allow optional reporter", () => { + return cli(_testConfig({ margs: { argv: { - reporters: ["a", "b", "c"] + optional_reporters: ["a", "b", "c"] } }, loadRelativeModule: () => { return new FakeReporter(); } - })).then(() => { - done(); - }).catch((e) => { - }); - }); - - it("allow for aggregateScreenshots", (done) => { - cli(_testConfig({ - settings: { - aggregateScreenshots: true - }, - require: _fakeRequire((name) => { - if (name === "./reporters/screenshot_aggregator/reporter") { - return FakeReporter; - } - }) - })).then(() => { - done(); - }).catch((e) => { - }); + })) + .then() + .catch(err => assert(false, "shouldn't be here")); }); - it("allow for serial", (done) => { - cli(_testConfig({ + it("enable serial", () => { + return cli(_testConfig({ margs: { argv: { serial: true @@ -391,277 +409,167 @@ describe("CLI", () => { return FakeReporter; } }) - })).then(() => { - done(); - }).catch((e) => { - }); + })) + .then() + .catch(err => assert(false, "shouldn't be here")); }); - it("allow for serial and sauce", (done) => { - cli(_testConfig({ - margs: { - argv: { - serial: true, - sauce: true - } + it("enable screenshot", () => { + return cli(_testConfig({ + settings: { + aggregateScreenshots: true }, require: _fakeRequire((name) => { - if (name === "./reporters/stdout/reporter") { + if (name === "./reporters/screenshot_aggregator/reporter") { return FakeReporter; } }) - })).then(() => { - done(); - }).catch((e) => { - }); + })) + .then() + .catch(err => assert(false, "shouldn't be here")); }); - it("allow for optional_reporters", (done) => { - cli(_testConfig({ - margs: { - argv: { - optional_reporters: ["a", "b", "c"] - } - }, - loadRelativeModule: () => { - return new FakeReporter(); + it("allow no test", () => { + return cli(_testConfig({ + getTests: () => { + return []; } - })).then(() => { - done(); - }).catch((e) => { - }); + })) + .then(() => assert(false, "shouldn't be here")) + .catch(err => { }); }); - it("allow for no browsers", (done) => { - let called = false; - cli(_testConfig({ - browserOptions: { - detectFromCLI: () => { - called = true; - return null; + it("allow worker error", () => { + return cli(_testConfig({ + WorkerAllocator: class InvalidWorkerAllocator { + constructor() { } - } - })).then(() => { - }).catch((e) => { - expect(called).to.be.true; - done(); - }); - }); - - it("allow for zero browsers", (done) => { - let called = false; - cli(_testConfig({ - browserOptions: { - detectFromCLI: () => { - called = true; - return []; + initialize(cb) { + cb("FAKE_ERROR"); } - } - })).then(() => { - }).catch((e) => { - expect(called).to.be.true; - done(); - }); - }); - - it("allow for just phantomjs", (done) => { - let called = false; - cli(_testConfig({ - browserOptions: { - detectFromCLI: () => { - called = true; - return ["phantomjs"]; + teardown(cb) { + cb(); } } - })).then(() => { - expect(called).to.be.true; - done(); - }).catch((e) => { - }); - }); - - it("allow for debug", (done) => { - cli(_testConfig({ - margs: { - argv: { - debug: true - } - }, - TestRunner: FakeTestRunner - })).then(() => { - done(); - }).catch((e) => { - }); - }); - - it("allow for no tests", (done) => { - let called = false; - cli(_testConfig({ - getTests: () => { - called = true; - return []; - } - })).then(() => { - }).catch((e) => { - expect(called).to.be.true; - done(); - }); - }); - - it("allow for failing worker", (done) => { - cli(_testConfig({ - TestRunner: FailingTestRunner - })).then(() => { - }).catch((e) => { - done(); - }); - }); - - it("allow for bad WorkerAllocator", (done) => { - cli(_testConfig({ - WorkerAllocator: FakeAllocator - })).then(() => { - done(); - }).catch((e) => { - }); - }); - - it("allow for bail_early", (done) => { - cli(_testConfig({ - margs: { - argv: { - bail_early: true - } - }, - TestRunner: FakeTestRunner - })).then(() => { - done(); - }).catch((e) => { - }); + })) + .then(() => assert(false, "shouldn't be here")) + .catch(err => { }); }); - it("allow for bail_fast", (done) => { - cli(_testConfig({ + it("executor teardownRunner error", () => { + return cli(_testConfig({ margs: { argv: { - bail_fast: true + executors: ["testarmada-magellan-local-executor"] } }, - TestRunner: FakeTestRunner - })).then(() => { - done(); - }).catch((e) => { - }); - }); - - it("allow for slack initialization", (done) => { - cli(_testConfig({ require: _fakeRequire((name) => { - if (name === "./reporters/slack/settings") { + if (name === "testarmada-magellan-local-executor") { return { - enabled: true - }; - } - if (name === "./reporters/slack/slack") { - return FakeReporter; + name: "testarmada-magellan-local-executor", + shortName: "local", + help: { + "local_list_browsers": { + "visible": true, + "type": "function", + "description": "List the available browsers configured." + }, + "local_list_fakes": { + "visible": true, + "type": "function", + "description": "List the available browsers configured." + } + }, + validateConfig() { }, + setupRunner() { + return new Promise((resolve) => { + resolve(); + }); + }, + teardownRunner() { + return new Promise((resolve, reject) => { + reject("FAKE_ERROR"); + }); + }, + listBrowsers(param, callback) { + callback(); + } + } } }) - })).then(() => { - done(); - }).catch((e) => { - }); - }); - - it("list browsers", (done) => { - const spy = sinon.spy(); - cli(_testConfig({ - margs: { - argv: { - list_browsers: true - } - }, - browsers: { - initialize: (a) => { - expect(a).to.be.true; - const defer = Q.defer(); - defer.resolve(); - return defer.promise; - }, - listBrowsers: spy - } - })).then(() => { - expect(spy.called).to.be.true; - done(); - }); + })) + .then(() => assert(false, "shouldn't be here")) + .catch(err => { }); }); - it("fail in list browsers", (done) => { - cli(_testConfig({ - margs: { - argv: { - list_browsers: true + it("runner on failure", () => { + return cli(_testConfig({ + TestRunner: class InvalidRunner { + constructor(tests, opts) { + this.tests = tests; + this.opts = opts; } - }, - browsers: { - listBrowsers: () => { - throw "Foo!"; + start() { + this.opts.onFailure(); } } })) - .then(() => { - }) - .catch(() => { - done(); - }); + .then(() => assert(false, "shouldn't be here")) + .catch(err => { }); }); - it("list browsers with device_additions", (done) => { - const spy = sinon.spy(); - cli(_testConfig({ - margs: { - argv: { - list_browsers: true, - device_additions: "hey" + it("executor teardownRunner error with onFailure", () => { + return cli(_testConfig({ + TestRunner: class InvalidRunner { + constructor(tests, opts) { + this.tests = tests; + this.opts = opts; } - }, - browsers: { - initialize: (a) => { - expect(a).to.be.true; - const defer = Q.defer(); - defer.resolve(); - return defer.promise; - }, - listBrowsers: spy, - addDevicesFromFile: (f) => { - expect(f).to.eql("hey"); + start() { + this.opts.onFailure(); } - } - })) - .then(() => { - expect(spy.called).to.be.true; - done(); - }) - .catch((err) => { - }); - }); - - it("deal with device_additions", (done) => { - let called = false; - cli(_testConfig({ + }, margs: { argv: { - device_additions: "hey" + executors: ["testarmada-magellan-local-executor"] } }, - browsers: { - addDevicesFromFile: (f) => { - called = true; - expect(f).to.eql("hey"); + require: _fakeRequire((name) => { + if (name === "testarmada-magellan-local-executor") { + return { + name: "testarmada-magellan-local-executor", + shortName: "local", + help: { + "local_list_browsers": { + "visible": true, + "type": "function", + "description": "List the available browsers configured." + }, + "local_list_fakes": { + "visible": true, + "type": "function", + "description": "List the available browsers configured." + } + }, + validateConfig() { }, + setupRunner() { + return new Promise((resolve) => { + resolve(); + }); + }, + teardownRunner() { + return new Promise((resolve, reject) => { + reject("FAKE_ERROR"); + }); + }, + listBrowsers(param, callback) { + callback(); + } + } } - } - })).then(() => { - expect(called).to.be.true; - done(); - }); + }) + })) + .then(() => assert(false, "shouldn't be here")) + .catch(err => { }); }); -}); +}); \ No newline at end of file diff --git a/test/cli_help.js b/test/cli_help.js new file mode 100644 index 0000000..fe1e056 --- /dev/null +++ b/test/cli_help.js @@ -0,0 +1,39 @@ +"use strict"; + +const chai = require("chai"); +const chaiAsPromise = require("chai-as-promised"); + +const help = require("../src/cli_help"); + +chai.use(chaiAsPromise); + +const expect = chai.expect; +const assert = chai.assert; + +const opts = { + settings: { + testExecutors: { + "sauce": { + name: "FAKE_EXE_NAME", + help: { + "visible-command": { + "category": "Usability", + "visible": true, + "description": "FAKE_VISIBLE_DES" + }, + "invisible-command": { + "category": "Usability", + "visible": false, + "description": "FAKE_INVISIBLE_DES" + } + } + } + } + } +}; + +describe("cli_help", () => { + it("print executors", () => { + help.help(opts); + }); +}); diff --git a/test/detect_browsers.js b/test/detect_browsers.js deleted file mode 100644 index a0da6fe..0000000 --- a/test/detect_browsers.js +++ /dev/null @@ -1,178 +0,0 @@ -/* eslint no-undef: 0, no-magic-numbers: 0 */ -"use strict"; -const expect = require("chai").expect; -const detectBrowsers = require("../src/detect_browsers"); - -describe("detectBrowsers", () => { - it("should detect from CLI", () => { - detectBrowsers.detectFromCLI({profile: "http://foo/#profile,profile"}, true, true, { - console: {log: () => {}}, - syncRequest: () => { - return { - getBody: () => { - return JSON.stringify({ - profiles: "foo,bar,baz" - }); - } - }; - } - }); - }); - - it("should detect from CLI with https", () => { - detectBrowsers.detectFromCLI({profile: "https://foo/#profile,profile"}, true, true, { - console: {log: () => {}}, - syncRequest: () => { - return { - getBody: () => { - return JSON.stringify({ - profiles: "foo,bar,baz" - }); - } - }; - } - }); - }); - - it("should detect from CLI with https but no profiles", () => { - detectBrowsers.detectFromCLI({profile: "https://foo/#profile,profile"}, true, true, { - console: {log: () => {}}, - syncRequest: () => { - return { - getBody: () => { - return JSON.stringify({ - profiles: [] - }); - } - }; - } - }); - }); - - it("should detect with profiles and profile", () => { - detectBrowsers.detectFromCLI({profile: "a,b,c", profiles: { - a: [1, 2, 3] - }}, true, true, { - console: {log: () => {}} - }); - }); - - it("should detect with profiles and profile", () => { - detectBrowsers.detectFromCLI({profile: "a", profiles: { - a: [1, 2, 3] - }}, true, true, { - console: {log: () => {}} - }); - }); - - it("should detect with a profile that doesnt match", () => { - detectBrowsers.detectFromCLI({profile: "z", profiles: { - a: [1, 2, 3] - }}, true, true, { - console: {log: () => {}} - }); - }); - - it("should detect with no profile", () => { - detectBrowsers.detectFromCLI({}, true, true, { - console: {log: () => {}} - }); - }); - - it("should detect with bad profile", () => { - detectBrowsers.detectFromCLI({profile: "bar"}, true, true, { - console: {log: () => {}} - }); - }); - - it("should detect with browser", () => { - detectBrowsers.detectFromCLI({browser: "bar,baz"}, true, true, { - console: {log: () => {}} - }); - }); - - it("should detect with browser singular", () => { - detectBrowsers.detectFromCLI({browser: "bar"}, true, true, { - console: {log: () => {}} - }); - }); - - it("should detect with browser info", () => { - detectBrowsers.detectFromCLI({ - browser: "bar", - resolution: "1024", - orientation: "left" - }, true, true, { - console: {log: () => {}} - }); - }); - - it("should detect with browser info and some profiles", () => { - detectBrowsers.detectFromCLI({ - browser: "bar", - resolution: "1024", - orientation: "left", - profiles: {bar: [1], baz: [1]}, - profile: "bar" - }, true, true, { - console: {log: () => {}} - }); - }); - - it("should detect without sauce", () => { - detectBrowsers.detectFromCLI({ - browser: "bar", - resolution: "1024", - orientation: "left", - profiles: {bar: [1], baz: [1]}, - profile: "bar" - }, false, true, { - console: {log: () => {}} - }); - }); - - it("should detect without sauce and not node", () => { - detectBrowsers.detectFromCLI({ - browser: "iphone_9_3_OS_X_10_11_iPhone_5", - resolution: "1024", - orientation: "left", - profiles: {baz: [1]}, - profile: "iphone_9_3_OS_X_10_11_iPhone_5" - }, false, false, { - console: {log: () => {}} - }); - }); - - it("should detect with profiles but without sauce and not node", () => { - detectBrowsers.detectFromCLI({ - profiles: {baz: [1]}, - profile: "iphone_9_3_OS_X_10_11_iPhone_5" - }, false, false, { - console: {log: () => {}} - }); - }); - - it("should create browsers", () => { - expect(detectBrowsers.createBrowser("a", "b", "c").slug()).to.eql("a_b_c"); - expect(detectBrowsers.createBrowser("a", "b", "c").toString()).to.eql("a @b orientation: c"); - expect(detectBrowsers.createBrowser("a", null, "c").slug()).to.eql("a_c"); - expect(detectBrowsers.createBrowser("a", null, "c").toString()).to.eql("a orientation: c"); - expect(detectBrowsers.createBrowser("a", "b", null).slug()).to.eql("a_b"); - expect(detectBrowsers.createBrowser("a", "b", null).toString()).to.eql("a @b"); - }); - - it("should detect from CLI without sauce", () => { - detectBrowsers.detectFromCLI({profile: "http://foo/#profile,profile"}, false, false, { - console: {log: () => {}}, - syncRequest: () => { - return { - getBody: () => { - return JSON.stringify({ - profiles: "foo,bar,baz" - }); - } - }; - } - }); - }); -}); diff --git a/test/profiles.js b/test/profiles.js new file mode 100644 index 0000000..65e35d5 --- /dev/null +++ b/test/profiles.js @@ -0,0 +1,290 @@ +"use strict"; + +const chai = require("chai"); +const chaiAsPromise = require("chai-as-promised"); +const _ = require("lodash"); + +const profile = require("../src/profiles"); +const logger = require("../src/logger"); + +chai.use(chaiAsPromise); + +const expect = chai.expect; +const assert = chai.assert; + +const opts = { + settings: { + testExecutors: { + "sauce": { + getProfiles: (opts) => { + return new Promise((resolve) => { + resolve(opts.profiles); + }); + }, + getCapabilities: (profile, opts) => { + return new Promise((resolve) => { + resolve(profile); + }); + } + } + } + }, + margs: { + argv: {} + }, + syncRequest: (method, url) => { + return { + getBody(encoding) { + return "{\"profiles\":{\"chrome\":{\"browser\":\"chrome\"},\"firefox\":{\"browser\":\"firefox\"}}}"; + } + }; + } +}; + +let runOpts = {}; + +describe("handleProfiles", () => { + beforeEach(() => { + runOpts = _.cloneDeep(opts); + }); + + describe("Read from --profile", () => { + it("one profile from http", () => { + runOpts.margs.argv.profile = "http://some_fake_url#chrome"; + + return profile + .detectFromCLI(runOpts) + .then((resolvedprofiles) => { + expect(resolvedprofiles.length).to.equal(1); + expect(resolvedprofiles[0].browser).to.equal("chrome"); + expect(resolvedprofiles[0].executor).to.equal("sauce"); + }); + }); + + it("one profile from https", () => { + runOpts.margs.argv.profile = "https://some_fake_url#chrome"; + + return profile + .detectFromCLI(runOpts) + .then((resolvedprofiles) => { + expect(resolvedprofiles.length).to.equal(1); + expect(resolvedprofiles[0].browser).to.equal("chrome"); + expect(resolvedprofiles[0].executor).to.equal("sauce"); + }); + }); + + it("one profile from https with given executor", () => { + runOpts.syncRequest = (method, url) => { + return { + getBody(encoding) { + return "{\"profiles\":{\"chrome\":{\"browser\":\"chrome\", \"executor\":\"local\"}}}"; + } + }; + }; + + runOpts.settings.testExecutors.local = { + getProfiles: (opts) => { + return new Promise((resolve) => { + resolve(opts.profiles); + }); + }, + getCapabilities: (profile, opts) => { + return new Promise((resolve) => { + resolve(profile); + }); + } + }; + runOpts.margs.argv.profile = "https://some_fake_url#chrome"; + + return profile + .detectFromCLI(runOpts) + .then((resolvedprofiles) => { + expect(resolvedprofiles.length).to.equal(1); + expect(resolvedprofiles[0].browser).to.equal("chrome"); + expect(resolvedprofiles[0].executor).to.equal("local"); + }); + }); + + it("multiple profiles from http", () => { + runOpts.margs.argv.profile = "http://some_fake_url#chrome,firefox"; + + return profile + .detectFromCLI(runOpts) + .then((resolvedprofiles) => { + expect(resolvedprofiles.length).to.equal(2); + expect(resolvedprofiles[0].browser).to.equal("chrome"); + expect(resolvedprofiles[0].executor).to.equal("sauce"); + expect(resolvedprofiles[1].browser).to.equal("firefox"); + expect(resolvedprofiles[1].executor).to.equal("sauce"); + }); + }); + + it("no profile from url", () => { + runOpts.syncRequest = (method, url) => { + return { + getBody(encoding) { + return "{\"profiles\":{}}"; + } + }; + }; + runOpts.margs.argv.profile = "https://some_fake_url#chrome"; + + return profile + .detectFromCLI(runOpts) + .then((resolvedprofiles) => { + assert(false, "shouldn't be here"); + }) + .catch((err) => { + expect(err).to.equal("Profile chrome not found!"); + }); + }); + + it("no profile matches from url", () => { + runOpts.margs.argv.profile = "https://some_fake_url#internet"; + + return profile + .detectFromCLI(runOpts) + .then((resolvedprofiles) => { + assert(false, "shouldn't be here"); + }) + .catch((err) => { + expect(err).to.equal("Profile internet not found!"); + }); + }); + + it("no executor found for profile", () => { + runOpts.syncRequest = (method, url) => { + return { + getBody(encoding) { + return "{\"profiles\":{\"firefox\":{\"browser\":\"firefox\", \"executor\":\"local\"}}}"; + } + }; + }; + runOpts.margs.argv.profile = "https://some_fake_url#firefox"; + + return profile + .detectFromCLI(runOpts) + .then((resolvedprofiles) => { + assert(false, "shouldn't be here"); + }) + .catch((err) => { + expect(err).to.equal("Executor local not found! You\'ll need to configure it in magellan.json"); + }); + }); + + it("getCapabilities failed", () => { + runOpts.settings.testExecutors.sauce.getCapabilities = () => { + return new Promise((resolve, reject) => { + reject(new Error("FAKE_ERROR")); + }); + }; + + runOpts.margs.argv.profile = "https://some_fake_url#chrome"; + + return profile + .detectFromCLI(runOpts) + .then((resolvedprofiles) => { + assert(false, "shouldn't be here"); + }) + .catch((err) => { + expect(err.message).to.equal("FAKE_ERROR"); + }); + }); + }); + + describe("Read from local", () => { + it("one profile", () => { + runOpts.profiles = [ + { browser: "chrome" } + ]; + + return profile + .detectFromCLI(runOpts) + .then((resolvedprofiles) => { + expect(resolvedprofiles.length).to.equal(1); + expect(resolvedprofiles[0].browser).to.equal("chrome"); + }); + }); + + it("multiple profiles", () => { + runOpts.profiles = [ + { browser: "chrome" }, + { browser: "firefox" }, + { browser: "internet explorer" } + ]; + + return profile + .detectFromCLI(runOpts) + .then((resolvedprofiles) => { + expect(resolvedprofiles.length).to.equal(3); + expect(resolvedprofiles[0].browser).to.equal("chrome"); + expect(resolvedprofiles[1].browser).to.equal("firefox"); + expect(resolvedprofiles[2].browser).to.equal("internet explorer"); + }); + }); + + it("multiple executors", () => { + runOpts.profiles = [ + { browser: "chrome" } + ]; + + runOpts.settings.testExecutors.local = { + getProfiles: (opts) => { + return new Promise((resolve) => { + resolve(opts.profiles); + }); + } + }; + + + return profile + .detectFromCLI(runOpts) + .then((resolvedprofiles) => { + expect(resolvedprofiles.length).to.equal(2); + expect(resolvedprofiles[0].browser).to.equal("chrome"); + expect(resolvedprofiles[1].browser).to.equal("chrome"); + }); + }); + + it("failed", () => { + runOpts.profiles = [ + { browser: "chrome" } + ]; + + runOpts.settings.testExecutors.sauce.getProfiles = () => { + return new Promise((resolve, reject) => { + reject(new Error("FAKE_ERROR")); + }); + }; + + return profile + .detectFromCLI(runOpts) + .then((resolvedprofiles) => { + assert(false, "shouldn't be here"); + }) + .catch((err) => { + expect(err.message).to.equal("FAKE_ERROR"); + }); + }); + }); + + it("profile.toString", () => { + runOpts.profiles = [ + { + browserName: "chrome", + version: 10, + resolution: "1x1", + orientation: "upright", + executor: "on mars" + } + ]; + + return profile + .detectFromCLI(runOpts) + .then((resolvedprofiles) => { + expect(resolvedprofiles.length).to.equal(1); + expect(resolvedprofiles[0].toString()) + .to.equal("chrome|version:10|resolution:1x1|orientation:upright|executor:on mars"); + }); + }); +}); diff --git a/test/sauce/browsers.js b/test/sauce/browsers.js deleted file mode 100644 index 00df98b..0000000 --- a/test/sauce/browsers.js +++ /dev/null @@ -1,46 +0,0 @@ -/* eslint no-undef: 0, no-magic-numbers: 0, max-nested-callbacks: 0, no-unused-expressions: 0, - no-invalid-this: 0 */ -"use strict"; -const expect = require("chai").expect; -const browsers = require("../../src/sauce/browsers"); -const sinon = require("sinon"); - -describe("browsers", () => { - before(function (done) { - this.timeout(10000); - browsers.initialize(false).then(() => { - browsers.initialize(true).then(() => { - done(); - }); - }); - }); - - it("should listBrowsers", () => { - const spy = sinon.spy(); - browsers.listBrowsers({ - console: {log: spy}, - listSauceCliBrowsers: (cb) => { - cb({options: {head: {}}}); - } - }); - expect(spy.called).to.be.true; - }); - - it("should get a browser", () => { - const b = browsers.browser("chrome_55_Windows_10_Desktop"); - expect(b.browserName).to.eql("chrome"); - }); - - it("should get no browser", () => { - const b = browsers.browser("whatever"); - expect(b).to.not.exist; - }); - - it("should add from file", () => { - try { - browsers.addDevicesFromFile("foo"); - } catch (e) { - expect(e).to.not.be.null; - } - }); -}); diff --git a/test/sauce/settings.js b/test/sauce/settings.js deleted file mode 100644 index f45dbae..0000000 --- a/test/sauce/settings.js +++ /dev/null @@ -1,134 +0,0 @@ -/* eslint no-undef: 0, no-magic-numbers: 0, camelcase: 0, no-unused-expressions: 0 */ -"use strict"; -const expect = require("chai").expect; -const settingsFunc = require("../../src/sauce/settings"); - -describe("sauce/settings", () => { - it("should handle no args or env", () => { - const st = settingsFunc({console: {log: () => {}}}); - expect(st.locksPollingInterval).to.eql(2500); - }); - - it("should handle locksServerLocation", () => { - const st = settingsFunc({ - console: {log: () => {}}, - argv: { - locks_server: "foo/", - debug: true - } - }); - expect(st.locksPollingInterval).to.eql(2500); - }); - - it("should handle invalid locksServerLocation", () => { - const st = settingsFunc({ - console: {log: () => {}}, - argv: { - locks_server: "foo" - } - }); - expect(st.locksPollingInterval).to.eql(2500); - }); - - it("should handle SAUCE_USERNAME", () => { - const st = settingsFunc({ - console: {log: () => {}}, - env: { - SAUCE_USERNAME: "foo" - } - }); - expect(st.locksPollingInterval).to.eql(2500); - }); - - it("should sauce argv", () => { - const st = settingsFunc({ - console: {log: () => {}}, - argv: { - sauce: true - }, - env: { - SAUCE_USERNAME: "jack", - SAUCE_ACCESS_KEY: "hoobie", - SAUCE_CONNECT_VERSION: "doobie" - } - }); - expect(st.username).to.eql("jack"); - }); - - it("should sauce argv without optional version", () => { - const st = settingsFunc({ - console: {log: () => {}}, - argv: { - sauce: true - }, - env: { - SAUCE_USERNAME: "jack", - SAUCE_ACCESS_KEY: "hoobie" - } - }); - expect(st.username).to.eql("jack"); - }); - - it("should sauce throw argv without user", () => { - let ex = null; - try { - settingsFunc({ - console: {log: () => {}}, - argv: { - sauce: true - }, - env: { - SAUCE_ACCESS_KEY: "hoobie", - SAUCE_CONNECT_VERSION: "doobie" - } - }); - } catch (e) { - ex = e; - } - expect(ex).to.not.be.null; - }); - - it("should throw on bad tunnel config", () => { - let ex = null; - try { - settingsFunc({ - console: {log: () => {}}, - argv: { - sauce: true, - sauce_tunnel_id: "foo", - create_tunnels: true - }, - env: { - SAUCE_USERNAME: "jack", - SAUCE_ACCESS_KEY: "hoobie", - SAUCE_CONNECT_VERSION: "doobie" - } - }); - } catch (e) { - ex = e; - } - expect(ex).to.not.be.null; - }); - - it("should throw on bad tunnel parent config", () => { - let ex = null; - try { - settingsFunc({ - console: {log: () => {}}, - argv: { - sauce: true, - shared_sauce_parent_account: "foo", - create_tunnels: true - }, - env: { - SAUCE_USERNAME: "jack", - SAUCE_ACCESS_KEY: "hoobie", - SAUCE_CONNECT_VERSION: "doobie" - } - }); - } catch (e) { - ex = e; - } - expect(ex).to.not.be.null; - }); -}); diff --git a/test/sauce/tunnel.js b/test/sauce/tunnel.js deleted file mode 100644 index 8ba29f1..0000000 --- a/test/sauce/tunnel.js +++ /dev/null @@ -1,255 +0,0 @@ -/* eslint no-undef: 0, no-magic-numbers: 0 */ -"use strict"; -const expect = require("chai").expect; -const tunnel = require("../../src/sauce/tunnel"); -const sinon = require("sinon"); - -describe("sauce/tunnel", () => { - it("should initialize without access key", () => { - const spy = sinon.spy(); - tunnel.initialize(spy, - { - env: { - SAUCE_USERNAME: "foo", - SAUCE_ACCESS_KEY: null - }, - console: {log: () => {}}, - analytics: { - push: () => {}, - mark: () => {} - }, - sauceConnectLauncher: { - download: (opts, cb) => { - cb(null); - } - } - }); - expect(spy.called).to.eql(true); - }); - - it("should initialize without username", () => { - const spy = sinon.spy(); - tunnel.initialize(spy, - { - env: { - SAUCE_USERNAME: null - }, - console: {log: () => {}}, - analytics: { - push: () => {}, - mark: () => {} - }, - sauceConnectLauncher: { - download: (opts, cb) => { - cb(null); - } - } - }); - expect(spy.called).to.eql(true); - }); - - it("should initialize", () => { - const spy = sinon.spy(); - tunnel.initialize(spy, - { - console: {log: () => {}}, - analytics: { - push: () => {}, - mark: () => {} - }, - sauceConnectLauncher: { - download: (opts, cb) => { - cb(null); - } - } - }); - expect(spy.called).to.eql(true); - }); - - it("should initialize", () => { - const spy = sinon.spy(); - tunnel.initialize(spy, - { - console: {log: () => {}}, - analytics: { - push: () => {}, - mark: () => {} - }, - sauceConnectLauncher: { - download: (opts, cb) => { - cb(null); - } - } - }); - expect(spy.called).to.eql(true); - }); - - it("should initialize with error", () => { - const spy = sinon.spy(); - tunnel.initialize(spy, - { - env: { - SAUCE_USERNAME: "foo", - SAUCE_ACCESS_KEY: "bar" - }, - console: {log: () => {}}, - analytics: { - push: () => {}, - mark: () => {} - }, - sauceConnectLauncher: { - download: (opts, cb) => { - cb(new Error("foo")); - } - } - }); - expect(spy.called).to.eql(true); - }); - - it("should initialize without", () => { - const spy = sinon.spy(); - tunnel.initialize(spy, - { - env: { - SAUCE_USERNAME: "foo", - SAUCE_ACCESS_KEY: "bar" - }, - console: {log: () => {}}, - analytics: { - push: () => {}, - mark: () => {} - }, - sauceConnectLauncher: { - download: (opts, cb) => { - cb(null); - } - } - }); - expect(spy.called).to.eql(true); - }); - - it("should open", () => { - const spy = sinon.spy(); - tunnel.open( - { - tunnelId: "foo", - callback: spy, - seleniumPort: 10 - }, - { - settings: { - fastFailRegexps: true, - debug: true - }, - env: { - SAUCE_USERNAME: "foo", - SAUCE_ACCESS_KEY: "bar" - }, - console: { - log: () => {}, - info: () => {} - }, - sauceConnectLauncher: (opts, cb) => { - cb(null, 15); - } - }); - expect(spy.called).to.eql(true); - }); - - it("should open with error", () => { - const spy = sinon.spy(); - tunnel.open( - { - tunnelId: "foo", - callback: spy - }, - { - settings: { - }, - env: { - SAUCE_USERNAME: "foo", - SAUCE_ACCESS_KEY: "bar" - }, - console: { - log: () => {}, - info: () => {}, - error: () => {} - }, - sauceConnectLauncher: (opts, cb) => { - cb({message: "bar"}); - } - }); - expect(spy.called).to.eql(true); - }); - - it("should open with error with debug", () => { - const spy = sinon.spy(); - tunnel.open( - { - tunnelId: "foo", - callback: spy - }, - { - settings: { - debug: true - }, - env: { - SAUCE_USERNAME: "foo", - SAUCE_ACCESS_KEY: "bar" - }, - console: { - log: () => {}, - info: () => {}, - error: () => {} - }, - sauceConnectLauncher: (opts, cb) => { - cb({message: "bar"}); - } - }); - expect(spy.called).to.eql(true); - }); - - it("should open with error with not connecting", () => { - const spy = sinon.spy(); - tunnel.open( - { - tunnelId: "foo", - callback: spy - }, - { - settings: { - debug: true - }, - env: { - SAUCE_USERNAME: "foo", - SAUCE_ACCESS_KEY: "bar" - }, - console: { - log: () => {}, - info: () => {}, - error: () => {} - }, - sauceConnectLauncher: (opts, cb) => { - cb({message: "Could not start Sauce Connect"}); - } - }); - expect(spy.called).to.eql(true); - }); - - it("should handle close", () => { - const spy = sinon.spy(); - tunnel.close( - {process: { - close: (cb) => { - cb(); - } - }}, - spy, - { - console: { - log: () => {} - } - }); - expect(spy.called).to.eql(true); - }); -}); diff --git a/test/sauce/worker_allocator.js b/test/sauce/worker_allocator.js deleted file mode 100644 index c6a33fc..0000000 --- a/test/sauce/worker_allocator.js +++ /dev/null @@ -1,507 +0,0 @@ -/* eslint no-undef: 0, no-magic-numbers: 0, no-unused-expressions: 0 */ -"use strict"; -const expect = require("chai").expect; -const WorkerAllocator = require("../../src/sauce/worker_allocator"); -const sinon = require("sinon"); -const _ = require("lodash"); - -const _settingsBuilder = (extra) => { - return _.merge({ - console: {log: () => {}}, - setTimeout: (cb) => {cb();}, - clearTimeout: () => {}, - tunnel: { - initialize: (cb) => { - cb(null); - }, - open: (opts) => { - opts.callback(null, {}); - } - }, - sauceSettings: { - useTunnels: true, - maxTunnels: 1, - locksServerLocation: "foo" - } - }, extra); -}; - -describe("sauce/worker_allocator", () => { - it("should create", () => { - const wa = new WorkerAllocator(10, { - console: { - log: () => {} - }, - sauceSettings: { - locksServerLocation: "foo/" - } - }); - expect(wa.tunnelPrefix).to.not.be.null; - }); - - it("should create without sauceSettings", () => { - const wa = new WorkerAllocator(10, { - console: { - log: () => {} - } - }); - expect(wa.tunnelPrefix).to.not.be.null; - }); - - it("should release", () => { - const wa = new WorkerAllocator(10, { - console: { - log: () => {} - }, - request: (opts, cb) => { - expect(opts).to.not.be.null; - cb(); - }, - sauceSettings: { - locksServerLocation: "foo/" - } - }); - wa.release(wa.workers[0]); - }); - - it("should release without server loc", () => { - const wa = new WorkerAllocator(10, { - console: { - log: () => {} - }, - request: (opts, cb) => { - expect(opts).to.not.be.null; - cb(); - }, - sauceSettings: { - } - }); - wa.release(wa.workers[0]); - }); - - it("should teardown", () => { - const spy = sinon.spy(); - const wa = new WorkerAllocator(10, { - console: { - log: () => {} - }, - request: (opts, cb) => { - expect(opts).to.not.be.null; - cb(); - }, - sauceSettings: { - locksServerLocation: "foo/" - } - }); - wa.teardown(spy); - expect(spy.called).to.be.true; - }); - - it("should teardown with tunnels", () => { - const spy = sinon.spy(); - const wa = new WorkerAllocator(10, { - console: { - log: () => {} - }, - request: (opts, cb) => { - expect(opts).to.not.be.null; - cb(); - }, - setTimeout: (cb) => { - cb(); - }, - clearTimeout: () => {}, - sauceSettings: { - useTunnels: true - }, - settings: { - debug: true - } - }); - wa.teardown(spy); - expect(spy.called).to.be.true; - }); - - it("should initialize with nothing", () => { - const spy = sinon.spy(); - const wa = new WorkerAllocator(10, { - console: { - log: () => {} - }, - request: (opts, cb) => { - expect(opts).to.not.be.null; - cb(); - }, - setTimeout: (cb) => { - cb(); - }, - clearTimeout: () => {}, - sauceSettings: { - }, - settings: { - debug: true - } - }); - wa.initialize(spy); - expect(spy.called).to.be.true; - }); - - it("should initialize with sauceTunnelId", () => { - const spy = sinon.spy(); - const wa = new WorkerAllocator(10, { - console: { - log: () => {} - }, - request: (opts, cb) => { - expect(opts).to.not.be.null; - cb(); - }, - setTimeout: (cb) => { - cb(); - }, - clearTimeout: () => {}, - sauceSettings: { - sauceTunnelId: 52 - } - }); - wa.initialize(spy); - expect(spy.called).to.be.true; - }); - - it("should initialize without sauceTunnelId", (done) => { - const spy = sinon.spy(); - const wa = new WorkerAllocator(10, { - console: { - log: () => {} - }, - request: (opts, cb) => { - expect(opts).to.not.be.null; - cb(); - }, - setTimeout: (cb) => { - cb(); - }, - clearTimeout: () => {}, - tunnel: { - initialize: (cb) => { - cb(null); - }, - open: (opts) => { - opts.callback(null, {}); - }, - close: (tunnel, cb) => { - cb(); - } - }, - sauceSettings: { - useTunnels: true, - maxTunnels: 2 - }, - delay: (cb) => { - cb(); - } - }); - wa.initialize(() => { - wa.teardownTunnels(spy); - done(); - }); - }); - - it("should initialize without sauceTunnelId with tunnel error", () => { - const spy = sinon.spy(); - const wa = new WorkerAllocator(10, { - console: { - log: () => {} - }, - request: (opts, cb) => { - expect(opts).to.not.be.null; - cb(); - }, - setTimeout: (cb) => { - cb(); - }, - clearTimeout: () => {}, - tunnel: { - initialize: (cb) => { - cb(null); - }, - open: (opts) => { - opts.callback(new Error("bad tunnel"), {}); - } - }, - sauceSettings: { - useTunnels: true, - maxTunnels: 1 - } - }); - wa.initialize(spy); - wa.teardownTunnels(spy); - }); - - it("should initialize without sauceTunnelId with error", () => { - const spy = sinon.spy(); - const wa = new WorkerAllocator(10, { - console: { - log: () => {} - }, - request: (opts, cb) => { - expect(opts).to.not.be.null; - cb(); - }, - setTimeout: (cb) => { - cb(); - }, - clearTimeout: () => {}, - tunnel: { - initialize: (cb) => { - cb(new Error("foo")); - } - }, - analytics: { - push: () => {} - }, - sauceSettings: { - useTunnels: true, - maxTunnels: 5 - } - }); - wa.initialize(spy); - }); - - it("should initialize without sauceTunnelId and get", () => { - const spy = sinon.spy(); - const wa = new WorkerAllocator(10, { - console: { - log: () => {} - }, - request: { - post: (opts, cb) => { - expect(opts).to.not.be.null; - cb(null, {}, JSON.stringify( - { - accepted: true, - token: "foo" - } - )); - } - }, - setTimeout: (cb) => { - cb(); - }, - clearTimeout: () => {}, - tunnel: { - initialize: (cb) => { - cb(null); - }, - open: (opts) => { - opts.callback(null, {}); - } - }, - sauceSettings: { - useTunnels: true, - maxTunnels: 1, - locksServerLocation: "foo" - }, - settings: { - debug: true - } - }); - wa.initialize(spy); - wa.get(spy); - wa.teardownTunnels(spy); - }); - - it("should initialize without sauceTunnelId and get", () => { - const spy = sinon.spy(); - const wa = new WorkerAllocator(10, _settingsBuilder({ - request: { - post: (opts, cb) => { - expect(opts).to.not.be.null; - cb(null, {}, JSON.stringify( - { - accepted: true, - token: "foo" - } - )); - } - } - })); - wa.initialize(spy); - wa.get(spy); - wa.teardownTunnels(spy); - }); - - it("should initialize without sauceTunnelId and get without locksServerLocation", () => { - const spy = sinon.spy(); - const wa = new WorkerAllocator(10, { - console: { - log: () => {} - }, - request: { - }, - setTimeout: (cb) => { - cb(); - }, - clearTimeout: () => {}, - tunnel: { - initialize: (cb) => { - cb(null); - }, - open: (opts) => { - opts.callback(null, {}); - } - }, - sauceSettings: { - useTunnels: true, - maxTunnels: 1 - } - }); - wa.initialize(spy); - wa.get(spy); - wa.teardownTunnels(spy); - }); - - it("should initialize without sauceTunnelId with bad payload", () => { - const spy = sinon.spy(); - const wa = new WorkerAllocator(10, _settingsBuilder({ - request: { - post: (opts, cb) => { - expect(opts).to.not.be.null; - cb(null, {}, "foo"); - } - }, - sauceSettings: { - useTunnels: true, - maxTunnels: 1, - locksServerLocation: "foo", - locksOutageTimeout: 0 - } - })); - wa.initialize(spy); - wa.get(spy); - }); - - it("should initialize without sauceTunnelId with err", () => { - const spy = sinon.spy(); - const wa = new WorkerAllocator(10, _settingsBuilder({ - request: { - post: (opts, cb) => { - expect(opts).to.not.be.null; - cb(new Error("Bad req"), {}, "foo"); - } - }, - sauceSettings: { - useTunnels: true, - maxTunnels: 1, - locksServerLocation: "foo", - locksOutageTimeout: 0 - }, - settings: { - debug: true - } - })); - wa.initialize(spy); - wa.get(spy); - }); - - it("should initialize without sauceTunnelId with null result", () => { - const spy = sinon.spy(); - const wa = new WorkerAllocator(10, _settingsBuilder({ - request: { - post: (opts, cb) => { - expect(opts).to.not.be.null; - cb(null, {}, "null"); - } - }, - sauceSettings: { - useTunnels: true, - maxTunnels: 1, - locksServerLocation: "foo", - locksOutageTimeout: 0 - } - })); - wa.initialize(spy); - wa.get(spy); - }); - - it("should initialize without sauceTunnelId with empty result", () => { - const spy = sinon.spy(); - const wa = new WorkerAllocator(10, _settingsBuilder({ - request: { - post: (opts, cb) => { - expect(opts).to.not.be.null; - cb(null, {}, "{}"); - } - }, - sauceSettings: { - useTunnels: true, - maxTunnels: 1, - locksServerLocation: "foo", - locksOutageTimeout: 0 - } - })); - wa.initialize(spy); - }); - - it("should initialize without sauceTunnelId with empty result and debug", () => { - const spy = sinon.spy(); - const wa = new WorkerAllocator(10, _settingsBuilder({ - request: { - post: (opts, cb) => { - expect(opts).to.not.be.null; - cb(null, {}, "{}"); - } - }, - sauceSettings: { - useTunnels: true, - maxTunnels: 1, - locksServerLocation: "foo", - locksOutageTimeout: 0 - }, - settings: { - debug: true - } - })); - wa.initialize(spy); - wa.get(spy); - }); - - it("should initialize without sauceTunnelId with one good, one bad", () => { - const spy = sinon.spy(); - let index = 0; - const wa = new WorkerAllocator(10, { - console: { - log: () => {} - }, - request: (opts, cb) => { - expect(opts).to.not.be.null; - cb(); - }, - setTimeout: (cb) => { - cb(); - }, - clearTimeout: () => {}, - tunnel: { - initialize: (cb) => { - cb(null); - }, - open: (opts) => { - opts.callback( - index === 1 ? new Error("bad tunnel") : null, - {} - ); - if (index === 0) { - index = 1; - } - } - }, - sauceSettings: { - useTunnels: true, - locksServerLocation: "foo", - maxTunnels: 2 - } - }); - wa.initialize(spy); - wa.teardownTunnels(spy); - }); -}); diff --git a/test/test.js b/test/test.js index dbda298..7437f78 100644 --- a/test/test.js +++ b/test/test.js @@ -26,7 +26,7 @@ describe("Test Class", () => { const browser = "myBrowser"; const myTest = new Test("", browser); myTest.getRuntime(); - expect(myTest.toString()).to.equal(" @undefined "); + expect(myTest.toString()).to.equal(" @myBrowser"); }); it("should use passed in browser", () => { @@ -34,13 +34,13 @@ describe("Test Class", () => { const myTest = new Test("", browser); myTest.toString(); myTest.getRuntime(); - expect(myTest.browser).to.equal(browser); + expect(myTest.browser).to.equal(undefined); }); it("should use passed in Sauce Browser Settings", () => { const sauceBrowserSettings = {1: "a"}; const myTest = new Test("", "", sauceBrowserSettings); - expect(myTest.sauceBrowserSettings).to.equal(sauceBrowserSettings); + expect(myTest.sauceBrowserSettings).to.equal(undefined); }); it("should use passed in max attempts", () => { diff --git a/test/test_runner.js b/test/test_runner.js index 1c6c7e9..c1e5acc 100644 --- a/test/test_runner.js +++ b/test/test_runner.js @@ -1,592 +1,566 @@ -/* eslint no-undef: 0, no-invalid-this: 0, no-magic-numbers: 0, no-unused-expressions: 0, - no-throw-literal: 0 */ "use strict"; -const expect = require("chai").expect; + +const chai = require("chai"); +const chaiAsPromise = require("chai-as-promised"); const _ = require("lodash"); -const EventEmitter = require("events").EventEmitter; -const sinon = require("sinon"); const TestRunner = require("../src/test_runner"); - -class BadTestRun { - getEnvironment() { - throw new Error("foo"); - } - getCommand() { - return "command"; - } - getArguments() { - return "args"; - } -} - -class FakeTestRun { - getEnvironment() { - return "foo"; - } - getCommand() { - return "command"; - } - getArguments() { - return "args"; - } -} - -class MockIO extends EventEmitter { - constructor() { - super(); - } - unpipe() { - } -} - -class MockChildProcess extends EventEmitter { - constructor() { - super(); - this.stdout = new MockIO(); - this.stdout.setMaxListeners(50); - this.stderr = new MockIO(); - this.stderr.setMaxListeners(50); - } - removeAllListeners() { - } -} - -const _testStruct = (moreOpts) => { - return _.merge({ - testLocator: "baz", - stopClock: () => {}, - startClock: () => {}, - getRuntime: () => {return 20;}, - fail: () => {}, - browser: { - browserId: "chrome" - }, - locator: "bar", - status: 3, - canRun: () => { - return true; - } - }, moreOpts); -}; - -const _tests = () => { - return [ - {test: "foo", locator: "bar"}, - {test: "bar", locator: "bar"}, - {test: "baz", locator: "bar"} - ]; -}; - -const _options = (moreOpts) => { - return _.merge({ - browsers: [{ - browserId: "chrome", - resolution: 2000, - orientation: "portrait" - }], - allocator: { - get: (cb) => { - cb(null, {index: 0, tunnelId: 50, token: "foo"}); - }, - release: () => {} - }, - listeners: [ - { - listenTo: () => {} - }, - { - listenTo: () => {}, - flush: () => {} - }, - { - listenTo: () => {}, - flush: () => { - return { - then: () => { - return { - catch: (cb) => { cb({}); } - }; - } - }; - } +const logger = require("../src/logger"); + +chai.use(chaiAsPromise); + +const expect = chai.expect; +const assert = chai.assert; + +const settings = { + bailTime: 1, + buildId: "FADSFASDF_ASDFSADF2", + bailTimeExplicitlySet: true, + gatherTrends: true, + testFramework: { + TestRun: function () { + return { + getEnvironment() { }, + enableExecutor() { } } - ], - sauceSettings: { - user: "Jack" } - }, moreOpts); + } }; -const _testOptions = (moreOpts) => { - return _.merge({ - console: { - log: () => {}, - error: () => {} +const tests = [ + { filename: 'tests/demo-app.js' }, + { filename: 'tests/demo-web.js' } +]; + +const executors = { + "sauce": { + getProfiles(opts) { + return new Promise((resolve) => { + resolve(opts.profiles); + }); }, - fs: { - readFileSync: () => { - return JSON.stringify({failures: { - foo: 1, - baz: 2 - }}); - }, - writeFileSync: () => {} + getCapabilities(profile, opts) { + return new Promise((resolve) => { + resolve(profile); + }); }, - mkdirSync: () => {}, - fork: () => { - const m = new MockChildProcess(); - m.setMaxListeners(50); - return m; + setupTest(callback) { + callback(null, "FAKE_EXECUTOR_TOKEN"); }, - sauceBrowsers: { - browser: () => { - return {foo: 1}; + teardownTest(token, callback) { + callback(); + }, + execute() { + return { + on(code, callback) { + if (code === "message") { + callback({ type: "test-meta-data", metadata: "FAKE_META" }) + } + else { + callback(0); + } + }, + send() { }, + removeAllListeners() { }, + stdout: { + on(type, callback) { callback() }, + removeAllListeners() { }, + unpipe() { } + }, + stderr: { + on(type, callback) { callback() }, + removeAllListeners() { }, + unpipe() { } + } } }, - settings: { - testFramework: { - TestRun: FakeTestRun - }, - tempDir: "foo", - buildId: "buildId-bar" + summerizeTest(buildid, metadat, callback) { callback(); }, + wrapup(callback) { callback(); } + } +}; + +const profiles = [ + { browser: "chrome", executor: "sauce" }, + { browser: "firefox", executor: "sauce" } +]; + +const allocator = { + get(callback) { callback(null, { token: "FAKE_WORKER_TOKEN" }); }, + release() { } +}; + +const options = { + debug: true, + bailFast: false, + bailOnThreshold: false, + maxWorkers: 1, + maxTestAttempts: 1, + serial: true, + onFailure() { }, + onSuccess() { }, + allocator: {}, + listeners: [{ + flush() { return new Promise((resolve) => { resolve() }); }, + listenTo() { } + }] +}; + +let optsMock = { + fs: { + readFileSync() { + return "{\"failures\":{\"a\":1}}"; }, - clearInterval: () => {}, - setTimeout: (cb) => { cb(); }, - setInterval: (cb) => { cb(); }, - prettyMs: () => {return "";} - }, moreOpts); + writeFileSync() { } + }, + setTimeout(callback) { callback(); }, + path: { + resolve() { return "FAKE_TEMP_PATH"; } + }, + mkdirSync() { }, + setInterval(callback) { callback(); } }; -describe("TestRunner Class", () => { - it("should initialize", () => { - const tr = new TestRunner(_tests(), _options({}), _testOptions({})); - expect(tr).to.be.not.be.null; +let optionsMock = {}; + +describe("test_runner", () => { + beforeEach(() => { + optsMock.settings = _.cloneDeep(settings); + optionsMock = _.cloneDeep(options); + optionsMock.profiles = _.cloneDeep(profiles); + optionsMock.executors = _.cloneDeep(executors); + optionsMock.allocator = _.cloneDeep(allocator); }); - it("should initialize with bail options", () => { - const tr1 = new TestRunner(_tests(), _options({bailFast: true}), _testOptions({})); - expect(tr1).to.be.not.be.null; - expect(tr1.strictness).to.eql(4); + describe("initialize", () => { + it("should pass", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + expect(tr.numTests).to.equal(4); + expect(tr.profiles.length).to.equal(2); + expect(tr.strictness).to.equal(2); + }); - const tr2 = new TestRunner(_tests(), _options({bailOnThreshold: 3}), _testOptions({})); - expect(tr2).to.be.not.be.null; - expect(tr2.strictness).to.eql(3); + it("should pass with bail fast", () => { + optionsMock.bailFast = true; + const tr = new TestRunner(tests, optionsMock, optsMock); + expect(tr.strictness).to.equal(4); + }); - const tr3 = new TestRunner(_tests(), _options(), _testOptions({ - settings: { - bailTimeExplicitlySet: 1000 - } - })); - expect(tr3).to.be.not.be.null; - expect(tr3.strictness).to.eql(2); - }); + it("should pass with bail early", () => { + optionsMock.bailOnThreshold = true; + const tr = new TestRunner(tests, optionsMock, optsMock); + expect(tr.strictness).to.equal(3); + }); - it("should initialize with trends", () => { - const tr = new TestRunner(_tests(), _options(), _testOptions({ - settings: { - gatherTrends: true - } - })); - expect(tr).to.be.not.be.null; - expect(tr.trends).to.eql({failures: {}}); + it("should pass with bail never", () => { + optsMock.settings.bailTimeExplicitlySet = false; + const tr = new TestRunner(tests, optionsMock, optsMock); + expect(tr.strictness).to.equal(1); + }); }); - it("should start", () => { - const tr1 = new TestRunner(_tests(), _options(), _testOptions({})); - tr1.start(); - expect(tr1.startTime).to.not.be.null; - - // Test serial - const tr2 = new TestRunner(_tests(), _options({ - serial: true - }), _testOptions({})); - tr2.start(); - expect(tr2.startTime).to.not.be.null; - - // Test no tests - const tr3 = new TestRunner([], _options({}), _testOptions({})); - tr3.start(); - expect(tr3.startTime).to.not.be.null; + it("notIdle", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.notIdle(); + expect(tr.busyCount).to.equal(1); }); - it("should idle", () => { - const tr1 = new TestRunner(_tests(), _options(), _testOptions({})); - tr1.notIdle(); - expect(tr1.busyCount).to.eql(1); - tr1.notIdle(); - expect(tr1.busyCount).to.eql(2); - tr1.notIdle(); - expect(tr1.busyCount).to.eql(3); - tr1.maybeIdle(); - expect(tr1.busyCount).to.eql(2); - tr1.maybeIdle(); - expect(tr1.busyCount).to.eql(1); - tr1.maybeIdle(); - expect(tr1.busyCount).to.eql(0); + it("maybeIdle", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.busyCount = 1; + tr.maybeIdle(); + expect(tr.busyCount).to.equal(0); }); - it("should run a test", () => { - const spy1 = sinon.spy(); - const tr1 = new TestRunner(_tests(), _options(), _testOptions({ - analytics: { - push: spy1, - mark: spy1 - } - })); - tr1.stageTest(_testStruct(), () => {}); - expect(spy1.called).to.be.true; + it("logFailedTest", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.failedTests = [{ + toString() { }, + attempts: 3, + stdout: "", + stderr: "" + }]; + + tr.logFailedTests(); }); - it("should fail on a bad worker allocation", (done) => { - // Uncle Owen! This one"s got a bad motivator! - const spy = sinon.spy(); - const test = { - fail: spy + it("gatherTrends", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.trends.failures = { + a: 1 }; - const tr = new TestRunner(_tests(), _options({ - allocator: { - get: (cb) => { - cb({bad: "stuff"}); - } - } - }), _testOptions({})); - tr.stageTest(_testStruct(test), () => { - expect(spy.called).to.be.true; - done(); - }); + tr.gatherTrends(); }); - it("should run through a passing test", (done) => { - const myMock = new MockChildProcess(); - myMock.setMaxListeners(50); - const spy = sinon.spy(); - const tr = new TestRunner(_tests(), _options({ - listeners: [ - {} - ], - debug: true - }), _testOptions({ - fork: () => { - return myMock; - }, - settings: { - gatherTrends: true - } - })); - tr.stageTest(_testStruct({ - pass: spy - }), () => { - expect(spy.called).to.be.true; - - tr.trends.failures = { - fooz: 1, - bar: 2, - baz: 3 - }; - tr.gatherTrends(); - tr.logFailedTests(); - tr.summarizeCompletedBuild(); - - done(); - }); - - myMock.emit("message", {sessionId: 52}); - myMock.emit("message", {type: "selenium-session-info", sessionId: 52}); - myMock.stdout.emit("data", ""); - myMock.stdout.emit("data", "Lotsa love"); - myMock.stdout.emit("data", "Lotsa love\n2"); - myMock.stderr.emit("data", ""); - myMock.stderr.emit("data", "Notso lotsa love"); - myMock.stderr.emit("data", "Notso lotsa love\n2"); - myMock.emit("close", 0); + describe("shouldBail", () => { + it("should not bail if bail never", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + expect(tr.shouldBail()).to.equal(false); + }); + + it("should not bail if bail time only", () => { + optsMock.settings.bailTimeExplicitlySet = false; + const tr = new TestRunner(tests, optionsMock, optsMock); + expect(tr.shouldBail()).to.equal(false); + }); + + it("bail fast with 1 failure", () => { + optionsMock.bailFast = true; + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.failedTests = [{}]; + expect(tr.shouldBail()).to.equal(true); + }); + + it("bail fast with 0 failure", () => { + optionsMock.bailFast = true; + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.failedTests = []; + expect(tr.shouldBail()).to.equal(false); + }); + + it("should not bail if unknown bail setup", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.failedTests = []; + tr.strictness = 10; + expect(tr.shouldBail()).to.equal(false); + }); + + it("should bail on threshold", () => { + optionsMock.bailOnThreshold = true; + const tr = new TestRunner(tests, optionsMock, optsMock); + + tr.passedTests = [{ attempts: 1 }, { attempts: 1 }, { attempts: 1 }, { attempts: 1 }]; + tr.failedTests = [{ attempts: 3 }, { attempts: 3 }, { attempts: 3 }]; + expect(tr.shouldBail()).to.equal(true); + }); + + it("should not bail if threshold isn't meet", () => { + optionsMock.bailOnThreshold = true; + const tr = new TestRunner(tests, optionsMock, optsMock); + + tr.passedTests = [{ attempts: 1 }, { attempts: 1 }, { attempts: 1 }, { attempts: 1 }]; + tr.failedTests = [{ attempts: 3 }]; + expect(tr.shouldBail()).to.equal(false); + }); }); - it("should run through a passing test w/o debugging", (done) => { - const myMock = new MockChildProcess(); - myMock.setMaxListeners(50); - const spy = sinon.spy(); - const tr = new TestRunner(_tests(), _options({ - listeners: [ - {} - ] - }), _testOptions({ - fork: () => { - return myMock; - }, - settings: { - gatherTrends: true - } - })); - tr.stageTest(_testStruct({ - pass: spy - }), () => { - expect(spy.called).to.be.true; + describe("summarizeCompletedBuild", () => { + it("no failed test", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.startTime = (new Date()).getTime() - 300000; + return tr.summarizeCompletedBuild(); + }); + + it("two failed tests, bail", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.hasBailed = true; + tr.tests[0].status = 3; + tr.tests[0].getRetries = () => 3; + tr.failedTests = [{ attempts: 3 }]; + tr.startTime = (new Date()).getTime() - 300000; + return tr.summarizeCompletedBuild(); + }); + + it("two failed tests, bail with existing retries", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.hasBailed = true; + tr.tests[0].status = 3; + tr.tests[0].getRetries = () => 3; + tr.tests[1].status = 3; + tr.tests[1].getRetries = () => 3; + tr.failedTests = [{ attempts: 3 }]; + tr.startTime = (new Date()).getTime() - 300000; + return tr.summarizeCompletedBuild(); + }); + + it("two failed tests, no bail", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.failedTests = [{ attempts: 3 }, { attempts: 3 }]; + tr.startTime = (new Date()).getTime() - 300000; + return tr.summarizeCompletedBuild(); + }); + + it("listener doesn't flush function", () => { + optionsMock.listeners = [{ flush: "asdf" }]; + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.startTime = (new Date()).getTime() - 300000; + return tr.summarizeCompletedBuild(); + }); - tr.gatherTrends(); - tr.logFailedTests(); - tr.summarizeCompletedBuild(); - done(); + it("listener doesn't flush promise", () => { + optionsMock.listeners = [{ flush() { } }]; + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.startTime = (new Date()).getTime() - 300000; + return tr.summarizeCompletedBuild(); }); - myMock.emit("message", {sessionId: 52}); - myMock.emit("message", {type: "selenium-session-info", sessionId: 52}); - myMock.emit("close", 0); + it("listener doesn't flush promise resolve", () => { + optionsMock.listeners = [{ flush() { return new Promise((resolve, reject) => { reject(); }) } }]; + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.startTime = (new Date()).getTime() - 300000; + return tr.summarizeCompletedBuild(); + }); }); - it("should report failed tests", () => { - const spy = sinon.spy(); - let text = ""; - const tr = new TestRunner(_tests(), _options(), _testOptions({ - settings: { - gatherTrends: true - }, - console: { - log: (t) => { - text += t; - } - }, - fs: { - writeFileSync: spy - } - })); - tr.failedTests = [ - { - stdout: "---FOOOZ---", - stderr: "---BAAAZ---" - } - ]; - tr.gatherTrends(); - tr.logFailedTests(); - tr.summarizeCompletedBuild(); + describe("buildFinished", () => { + it("should succeed", (done) => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.onFailure = () => assert(false, "shouldn't be here"); + tr.onSuccess = () => done(); + tr.startTime = (new Date()).getTime() - 300000; + tr.buildFinished(); + }); - expect(spy.called).to.be.true; - expect(text.match(/---FOOOZ---/).length).to.eql(1); - expect(text.match(/---BAAAZ---/).length).to.eql(1); + it("should fail", (done) => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.onFailure = () => done(); + tr.onSuccess = () => assert(false, "shouldn't be here"); + tr.failedTests = [{}]; + tr.startTime = (new Date()).getTime() - 300000; + tr.buildFinished(); + }); }); - it("should handle failed tests", (done) => { - const myMock = new MockChildProcess(); - myMock.setMaxListeners(50); - const spy = sinon.spy(); - const tr = new TestRunner(_tests(), _options(), _testOptions({ - fork: () => { - return myMock; - }, - settings: { - gatherTrends: true - } - })); - tr.stageTest(_testStruct({ - fail: spy - }), () => { - expect(spy.called).to.be.true; - tr.gatherTrends(); - tr.logFailedTests(); - tr.summarizeCompletedBuild(); - done(); - }); - myMock.emit("close", -1); + it("checkBuild", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.hasBailed = false; + tr.THRESHOLD_MIN_ATTEMPTS = 1; + tr.startTime = (new Date()).getTime() - 300000; + tr.checkBuild(); }); - it("should handle inability to get test environment", (done) => { - const spy = sinon.spy(); - const tr = new TestRunner(_tests(), _options(), _testOptions({ - settings: { - testFramework: { - TestRun: BadTestRun - } - } - })); - tr.stageTest(_testStruct({ - fail: spy - }), () => { - expect(spy.called).to.be.true; - done(); + describe("onTestComplete", () => { + const failedTest = { + locator: { filename: 'tests/demo-app.js' }, + maxAttempts: 3, + attempts: 0, + status: 2, + profile: { browser: 'chrome' }, + executor: undefined, + workerIndex: -1, + error: undefined, + stdout: '', + stderr: '', + getRetries() { }, + canRun() { return false }, + getRuntime() { } + }; + + const successfulTest = { + locator: { filename: 'tests/demo-app.js' }, + maxAttempts: 1, + attempts: 0, + status: 3, + profile: { browser: 'chrome' }, + executor: executors["sauce"], + workerIndex: -1, + error: undefined, + stdout: '', + stderr: '', + getRetries() { } + }; + + it("has bailed", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.hasBailed = true; + tr.onTestComplete(null, failedTest); }); - }); - it("should handle inability to fork", (done) => { - const spy = sinon.spy(); - const tr = new TestRunner(_tests(), _options(), _testOptions({ - fork: () => { - throw new Error("Nope!"); - } - })); - tr.stageTest(_testStruct({ - fail: spy - }), () => { - expect(spy.called).to.be.true; - done(); + it("successful test", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.onTestComplete(null, successfulTest); }); - }); - it("should handle throwing in listenTo", () => { - const spy = sinon.spy(); - const tr = new TestRunner(_tests(), _options({ - listeners: [ - { - listenTo: () => { - throw "Whoops!"; - } - } - ] - }), _testOptions()); - tr.stageTest(_testStruct({ - fail: spy - }), () => { - expect(spy.called).to.be.true; - // Not sure this is actually correct behavior. A throw in a listener is not a test failure. - done(); + it("successful test without serial", () => { + optionsMock.serial = false; + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.onTestComplete(null, successfulTest); + }); + + it("failed test", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.onTestComplete(null, failedTest); }); }); - it("should handle bailing", () => { - const myMock = new MockChildProcess(); - myMock.setMaxListeners(50); - const tr = new TestRunner(_tests(), _options({ - bailOnThreshold: 1 - }), _testOptions({ - fork: () => { - return myMock; - }, - settings: { - gatherTrends: true - } - })); - tr.stageTest(_testStruct(), () => {}); - myMock.emit("close", -1); - myMock.emit("close", -1); - myMock.emit("close", -1); - - expect(tr.shouldBail()).to.be.false; - tr.passedTests = [{attempts: 2}, {attempts: 1}]; - expect(tr.shouldBail()).to.be.false; - tr.failedTests = [{attempts: 2}, {attempts: 1}]; - expect(tr.shouldBail()).to.be.false; - tr.failedTests = [{attempts: 20}, {attempts: 25}]; - expect(tr.shouldBail()).to.be.true; + describe("start", () => { + it("no test", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.tests = []; + tr.start(); + }); - tr.gatherTrends(); - tr.logFailedTests(); - tr.summarizeCompletedBuild(); + it("multi tests without serial", () => { + optionsMock.serial = false; + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.start(); + }); }); - it("should have bail fast logic", () => { - const tr = new TestRunner(_tests(), _options({ - bailFast: true - }), _testOptions({})); + describe("runTest", () => { + const worker = { portOffset: 1 }; + + it("no bail", () => { + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.hasBailed = false; + return tr.runTest(tr.tests[0], worker).then(); + }); - expect(tr.shouldBail()).to.be.false; - tr.passedTests = [{attempts: 2}, {attempts: 1}]; - expect(tr.shouldBail()).to.be.false; - tr.failedTests = [{attempts: 2}, {attempts: 1}]; - expect(tr.shouldBail()).to.be.true; + it("throws error", () => { + optsMock.settings.testFramework.TestRun = function () { + throw new Error("FAKE_ERROR"); + }; + + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.hasBailed = false; + return tr.runTest(tr.tests[0], worker) + .then() + .catch(err => expect(err.message).to.equal("FAKE_ERROR")); + }); }); - it("should have bail fast logic", () => { - const tr = new TestRunner(_tests(), _options({ - bailFast: true - }), _testOptions({})); + describe("execute", () => { + const successfulTest = { + locator: { filename: 'tests/demo-app.js' }, + maxAttempts: 1, + attempts: 0, + status: 3, + profile: { browser: 'chrome', executor: "sauce" }, + executor: executors["sauce"], + workerIndex: -1, + error: undefined, + stdout: '', + stderr: '', + getRetries() { }, + startClock() { }, + getRuntime() { }, + stopClock() { } + }; - expect(tr.shouldBail()).to.be.false; - tr.passedTests = [{attempts: 2}, {attempts: 1}]; - expect(tr.shouldBail()).to.be.false; - tr.failedTests = [{attempts: 2}, {attempts: 1}]; - expect(tr.shouldBail()).to.be.true; + it("getEnvironment failed", () => { + const testRun = { + getEnvironment() { throw new Error("FAKE_ERROR") }, + enableExecutor() { } + }; - tr.hasBailed = false; - tr.checkBuild(); - }); + const tr = new TestRunner(tests, optionsMock, optsMock); + return tr.execute(testRun, successfulTest) + .then() + .catch(err => { + expect(err.message).to.equal("FAKE_ERROR"); + }) + }); - it("should have bail early logic", () => { - const tr = new TestRunner(_tests(), _options({ - bailOnThreshold: 1 - }), _testOptions({})); + it("bail fast", () => { + const testRun = { + getEnvironment() { }, + enableExecutor() { } + }; - expect(tr.shouldBail()).to.be.false; - tr.passedTests = [{attempts: 2}, {attempts: 1}]; - expect(tr.shouldBail()).to.be.false; - tr.failedTests = [{attempts: 25}, {attempts: 26}]; - expect(tr.shouldBail()).to.be.true; - }); - it("should summarize under constious circumnstances", () => { - let text = ""; - const tr = new TestRunner(_tests(), _options(), _testOptions({ - console: { - log: (t) => { - text += t; + optionsMock.executors["sauce"].execute = () => { + return { + on(code, callback) { + if (code === "message") { + callback({ type: "test-meta-data", metadata: "FAKE_META" }) + } + else { + callback(1); + } + }, + send() { }, + removeAllListeners() { }, + stdout: { + on() { }, + removeAllListeners() { }, + unpipe() { } + }, + stderr: { + on() { }, + removeAllListeners() { }, + unpipe() { } + } } } - })); - - tr.summarizeCompletedBuild(); - expect(text.match(/BAILED/)).to.be.null; - - text = ""; - tr.hasBailed = true; - tr.summarizeCompletedBuild(); - expect(text.match(/BAILED/).length).to.eql(1); - - text = ""; - tr.tests = [ - {status: 0}, - {status: 3, getRetries: () => { return 0; }}, - {status: 3, getRetries: () => { return 1; }}, - {status: 3, getRetries: () => { return 1; }} - ]; - tr.summarizeCompletedBuild(); - expect(text.match(/Skipped: 3/).length).to.eql(1); - }); - it("should handle constious forms of test completion", () => { - let text = ""; - const tr1 = new TestRunner(_tests(), _options(), _testOptions({ - settings: { - gatherTrends: true - }, - console: { - log: (t) => { - text += t; + + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.strictness = 4; + tr.hasBailed = true; + return tr.execute(testRun, successfulTest) + .then(result => expect(result.error).to.equal("Child test run process exited with code 1")); + }); + + it("no bail", () => { + const testRun = { + getEnvironment() { }, + enableExecutor() { } + }; + + optionsMock.executors["sauce"].execute = () => { + return { + on(code, callback) { + if (code === "message") { + callback({ type: "test-meta-data", metadata: "FAKE_META" }) + } + else { + callback(1); + } + }, + send() { }, + removeAllListeners() { }, + stdout: { + on() { }, + removeAllListeners() { }, + unpipe() { } + }, + stderr: { + on() { }, + removeAllListeners() { }, + unpipe() { } + } } - } - })); - - text = ""; - tr1.hasBailed = true; - tr1.onTestComplete(new Error("foo"), _testStruct()); - expect(text.match(/KILLED/).length).to.eql(1); - - tr1.hasBailed = false; - tr1.onTestComplete(new Error("foo"), _testStruct({ - status: 0 - })); - expect(tr1.passedTests.length).to.eql(0); - - tr1.hasBailed = false; - tr1.onTestComplete(new Error("foo"), _testStruct({ - status: 3 - })); - expect(tr1.passedTests.length).to.eql(1); - - text = ""; - tr1.serial = true; - tr1.onTestComplete(new Error("foo"), _testStruct({ - status: 3 - })); - expect(text.match(/worker/)).to.be.null; - - const tr2 = new TestRunner(_tests(), _options(), _testOptions()); - tr2.onTestComplete(new Error("foo"), _testStruct({ - status: 0 - })); - expect(tr2.failedTests.length).to.eql(1); - - tr2.onTestComplete(new Error("foo"), _testStruct({ - status: 0, - canRun: () => { return false; } - })); - expect(tr2.failedTests.length).to.eql(2); + }; + + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.strictness = 1; + tr.hasBailed = false; + return tr.execute(testRun, successfulTest) + .then(result => expect(result.error).to.equal("Child test run process exited with code 1")); + }); + }); + + describe("stageTest", () => { + it("executor stage error", (done) => { + const onTestComplete = () => done(); + optionsMock.executors["sauce"].setupTest = (callback) => callback("error"); + + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.stageTest(tr.tests[0], onTestComplete); + }); + + it("allocator get error", (done) => { + const onTestComplete = () => done(); + optionsMock.allocator.get = (callback) => callback("error"); + + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.stageTest(tr.tests[0], onTestComplete); + }); + + it("runTestError", () => { + optsMock.settings.testFramework.TestRun = function () { + throw new Error("FAKE_ERROR"); + }; + + const onTestComplete = () => done(); + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.stageTest(tr.tests[0], onTestComplete); + }); + + it("successful", (done) => { + const onTestComplete = () => done(); + + const tr = new TestRunner(tests, optionsMock, optsMock); + tr.stageTest(tr.tests[0], onTestComplete); + }); }); -}); +}); \ No newline at end of file diff --git a/test/utils/port_util.js b/test/utils/port_util.js index 8827416..5df79a8 100644 --- a/test/utils/port_util.js +++ b/test/utils/port_util.js @@ -6,8 +6,8 @@ const sinon = require("sinon"); describe("port_util", () => { it("should get the next port", () => { - expect(portUtil.getNextPort()).to.eql(12009); - expect(portUtil.getNextPort()).to.eql(12012); + expect(portUtil.getNextPort()).to.eql(12000); + expect(portUtil.getNextPort()).to.eql(12003); }); it("should acquire a port", () => {