diff --git a/README.md b/README.md index c4631f4..ab02395 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,8 @@ file values; see the result of `--help` for more information. * `landBase`: if specified, overrides the auto-detected base commit when running the `land` command. Generally this is specified on the command line using `--land-base` rather than in a config file. +* `numWorkers`: if specified, the number of parallel workers to use for parallel + operations like `decaffeinate` and `eslint --fix`. * `skipVerify`: set to `true` to skip the initial verification step when running the `convert` command. This makes bulk-decaffeinate take less time, but if any files fail to convert, it may leave the filesystem in a partially-converted diff --git a/src/check.js b/src/check.js index 7ffe032..0e37b67 100644 --- a/src/check.js +++ b/src/check.js @@ -9,6 +9,7 @@ import pluralize from './util/pluralize'; export default async function check(config) { let filesToProcess = await getFilesToProcess(config, COFFEE_FILE_RECOGNIZER); let decaffeinateResults = await runWithProgressBar( + config, `Doing a dry run of decaffeinate on ${pluralize(filesToProcess.length, 'file')}...`, filesToProcess, makeDecaffeinateVerifyFn(config), {allowFailures: true}); diff --git a/src/cli.js b/src/cli.js index 95fdc23..c3ca22a 100644 --- a/src/cli.js +++ b/src/cli.js @@ -49,6 +49,8 @@ export default function () { `The git revision to use as the base commit when running the "land" command. If none is specified, bulk-decaffeinate tries to use the first auto-generated commit in recent history.`) + .option('--num-workers [number]', + `The number of workers to use for parallel operations.`) .option('--skip-verify', `If specified, skips the initial verification step when running the "convert" command.`) diff --git a/src/config/resolveConfig.js b/src/config/resolveConfig.js index 516f3d8..852305a 100644 --- a/src/config/resolveConfig.js +++ b/src/config/resolveConfig.js @@ -40,6 +40,7 @@ export default async function resolveConfig(commander) { mochaEnvFilePattern: config.mochaEnvFilePattern, codePrefix: config.codePrefix, landBase: config.landBase, + numWorkers: config.numWorkers || 8, skipVerify: config.skipVerify, decaffeinatePath: await resolveDecaffeinatePath(config), jscodeshiftPath: await resolveJscodeshiftPath(config), @@ -99,6 +100,7 @@ function getCLIParamsConfig(config, commander) { dir, useJsModules, landBase, + numWorkers, skipVerify, decaffeinatePath, jscodeshiftPath, @@ -127,6 +129,9 @@ function getCLIParamsConfig(config, commander) { if (landBase) { config.landBase = landBase; } + if (numWorkers) { + config.numWorkers = numWorkers; + } if (skipVerify) { config.skipVerify = true; } diff --git a/src/convert.js b/src/convert.js index bad78ae..4f1076b 100644 --- a/src/convert.js +++ b/src/convert.js @@ -36,6 +36,7 @@ export default async function convert(config) { if (!config.skipVerify) { try { await runWithProgressBar( + config, 'Verifying that decaffeinate can successfully convert these files...', coffeeFiles, makeDecaffeinateVerifyFn(config)); } catch (e) { @@ -46,6 +47,7 @@ Re-run with the "check" command for more details.`); } await runWithProgressBar( + config, 'Backing up files to .original.coffee...', coffeeFiles, async function(coffeePath) { @@ -53,6 +55,7 @@ Re-run with the "check" command for more details.`); }); await runWithProgressBar( + config, `Renaming files from .coffee to .${config.outputFileExtension}...`, movingCoffeeFiles, async function(coffeePath) { @@ -71,6 +74,7 @@ Re-run with the "check" command for more details.`); } await runWithProgressBar( + config, 'Moving files back...', movingCoffeeFiles, async function(coffeePath) { @@ -78,12 +82,14 @@ Re-run with the "check" command for more details.`); }); await runWithProgressBar( + config, 'Running decaffeinate on all files...', coffeeFiles, makeCLIFn(path => `${decaffeinatePath} ${decaffeinateArgs.join(' ')} ${path}`) ); await runWithProgressBar( + config, 'Deleting old files...', coffeeFiles, async function(coffeePath) { @@ -91,6 +97,7 @@ Re-run with the "check" command for more details.`); }); await runWithProgressBar( + config, 'Setting proper extension for all files...', coffeeFiles, async function(coffeePath) { @@ -112,7 +119,7 @@ Re-run with the "check" command for more details.`); await runJscodeshiftScripts(jsFiles, config); } if (config.mochaEnvFilePattern) { - await prependMochaEnv(jsFiles, config.mochaEnvFilePattern); + await prependMochaEnv(config, jsFiles, config.mochaEnvFilePattern); } let thirdCommitModifiedFiles = jsFiles.slice(); if (config.fixImportsConfig) { @@ -120,7 +127,7 @@ Re-run with the "check" command for more details.`); } await runEslintFix(jsFiles, config, {isUpdate: false}); if (config.codePrefix) { - await prependCodePrefix(jsFiles, config.codePrefix); + await prependCodePrefix(config, jsFiles, config.codePrefix); } let postProcessCommitMsg = diff --git a/src/modernize/prependCodePrefix.js b/src/modernize/prependCodePrefix.js index f789e09..c888b11 100644 --- a/src/modernize/prependCodePrefix.js +++ b/src/modernize/prependCodePrefix.js @@ -1,8 +1,9 @@ import prependToFile from '../util/prependToFile'; import runWithProgressBar from '../runner/runWithProgressBar'; -export default async function prependCodePrefix(jsFiles, codePrefix) { +export default async function prependCodePrefix(config, jsFiles, codePrefix) { await runWithProgressBar( + config, 'Adding code prefix to converted files...', jsFiles, async function(path) { await prependToFile(path, codePrefix); return {error: null}; diff --git a/src/modernize/prependMochaEnv.js b/src/modernize/prependMochaEnv.js index 3bed4ba..e668498 100644 --- a/src/modernize/prependMochaEnv.js +++ b/src/modernize/prependMochaEnv.js @@ -1,10 +1,11 @@ import runWithProgressBar from '../runner/runWithProgressBar'; import prependToFile from '../util/prependToFile'; -export default async function prependMochaEnv(jsFiles, mochaEnvFilePattern) { +export default async function prependMochaEnv(config, jsFiles, mochaEnvFilePattern) { let regex = new RegExp(mochaEnvFilePattern); let testFiles = jsFiles.filter(f => regex.test(f)); await runWithProgressBar( + config, 'Adding /* eslint-env mocha */ to test files...', testFiles, async function(path) { await prependToFile(path, '/* eslint-env mocha */\n'); return {error: null}; diff --git a/src/modernize/removeAutogeneratedHeader.js b/src/modernize/removeAutogeneratedHeader.js index e718bef..3922462 100644 --- a/src/modernize/removeAutogeneratedHeader.js +++ b/src/modernize/removeAutogeneratedHeader.js @@ -3,9 +3,12 @@ import { readFile, writeFile } from 'fs-promise'; import runWithProgressBar from '../runner/runWithProgressBar'; import { HEADER_COMMENT_LINES } from './runEslintFix'; -export default async function removeAutogeneratedHeader(jsFiles) { +export default async function removeAutogeneratedHeader(config, jsFiles) { await runWithProgressBar( - 'Removing any existing autogenerated headers...', jsFiles, removeHeadersFromFile); + config, + 'Removing any existing autogenerated headers...', + jsFiles, + removeHeadersFromFile); } async function removeHeadersFromFile(path) { diff --git a/src/modernize/runEslintFix.js b/src/modernize/runEslintFix.js index 13c9b6e..9d9f24d 100644 --- a/src/modernize/runEslintFix.js +++ b/src/modernize/runEslintFix.js @@ -6,6 +6,7 @@ import prependToFile from '../util/prependToFile'; export default async function runEslintFix(jsFiles, config, {isUpdate}) { let eslintResults = await runWithProgressBar( + config, 'Running eslint --fix on all files...', jsFiles, makeEslintFixFn(config, {isUpdate})); for (let result of eslintResults) { for (let message of result.messages) { diff --git a/src/modernize/runFixImports.js b/src/modernize/runFixImports.js index 8b0b601..2c4a474 100644 --- a/src/modernize/runFixImports.js +++ b/src/modernize/runFixImports.js @@ -21,7 +21,8 @@ export default async function runFixImports(jsFiles, config) { convertedFiles: jsFiles.map(p => resolve(p)), absoluteImportPaths: absoluteImportPaths.map(p => resolve(p)), }; - let eligibleFixImportsFiles = await getEligibleFixImportsFiles(searchPath, jsFiles); + let eligibleFixImportsFiles = await getEligibleFixImportsFiles( + config, searchPath, jsFiles); console.log('Fixing any imports across the whole codebase...'); if (eligibleFixImportsFiles.length > 0) { // Note that the args can get really long, so we take reasonable steps to @@ -36,11 +37,12 @@ export default async function runFixImports(jsFiles, config) { return eligibleFixImportsFiles; } -async function getEligibleFixImportsFiles(searchPath, jsFiles) { +async function getEligibleFixImportsFiles(config, searchPath, jsFiles) { let jsBasenames = jsFiles.map(p => basename(p, '.js')); let resolvedPaths = jsFiles.map(p => resolve(p)); let allJsFiles = await getFilesUnderPath(searchPath, p => p.endsWith('.js')); await runWithProgressBar( + config, 'Searching for files that may need to have updated imports...', allJsFiles, async function(p) { diff --git a/src/modernizeJS.js b/src/modernizeJS.js index 2e55073..c9314f3 100644 --- a/src/modernizeJS.js +++ b/src/modernizeJS.js @@ -17,8 +17,9 @@ export default async function modernizeJS(config) { return; } - await removeAutogeneratedHeader(jsFiles); + await removeAutogeneratedHeader(config, jsFiles); await runWithProgressBar( + config, 'Running decaffeinate --modernize-js on all files...', jsFiles, makeCLIFn(path => `${decaffeinatePath} --modernize-js ${decaffeinateArgs.join(' ')} ${path}`) diff --git a/src/runner/runWithProgressBar.js b/src/runner/runWithProgressBar.js index 6996ac7..dc487ce 100644 --- a/src/runner/runWithProgressBar.js +++ b/src/runner/runWithProgressBar.js @@ -5,8 +5,6 @@ import runInParallel from './runInParallel'; import CLIError from '../util/CLIError'; import pluralize from '../util/pluralize'; -const NUM_CONCURRENT_PROCESSES = 8; - /** * Run the given command in parallel, showing a progress bar of results. * @@ -15,13 +13,13 @@ const NUM_CONCURRENT_PROCESSES = 8; * any other fields. */ export default async function runWithProgressBar( - description, files, asyncFn, {runInSeries, allowFailures}={}) { + config, description, files, asyncFn, {runInSeries, allowFailures}={}) { let numProcessed = 0; let numFailures = 0; let numTotal = files.length; let startTime = moment(); - console.log(description); - let numConcurrentProcesses = runInSeries ? 1 : NUM_CONCURRENT_PROCESSES; + let numConcurrentProcesses = runInSeries ? 1 : config.numWorkers; + console.log(`${description} (${pluralize(numConcurrentProcesses, 'worker')})`); let results; try { results = await runInParallel(files, asyncFn, numConcurrentProcesses, ({result}) => { diff --git a/test/bulk-decaffeinate-test.js b/test/bulk-decaffeinate-test.js index 1c4d731..5086cf5 100644 --- a/test/bulk-decaffeinate-test.js +++ b/test/bulk-decaffeinate-test.js @@ -141,4 +141,12 @@ describe('check', () => { assertIncludes(stdout, 'All checks succeeded'); }); }); + + it('allows specifying the number of parallel workers', async function() { + await runWithTemplateDir('simple-success', async function() { + let {stdout, stderr} = await runCli('check --num-workers 3'); + assert.equal(stderr, ''); + assertIncludes(stdout, '(3 workers)'); + }); + }); });