-
Notifications
You must be signed in to change notification settings - Fork 111
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #687 from DaNish808/more-memory-#677
Prevent out-of-memory error
- Loading branch information
Showing
15 changed files
with
218 additions
and
179 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,25 +1,20 @@ | ||
| const readFile = require('./readFile') | ||
| const Issue = require('../issues/issue') | ||
|
|
||
| function createIssueForEmpty(file) { | ||
| const size = typeof window !== 'undefined' ? file.size : file.stats.size | ||
| return size <= 0 && new Issue({ code: 99, file: file }) | ||
| } | ||
| function clearNonIssues(x) { | ||
| return x instanceof Issue | ||
| } | ||
|
|
||
| /** | ||
| * validateMisc | ||
| * | ||
| * takes a list of files and returns an issue for each file | ||
| */ | ||
| module.exports = function validateMisc(miscFiles) { | ||
| const issuePromises = miscFiles.reduce( | ||
| (issues, file) => [ | ||
| ...issues, | ||
| readFile(file, false, null).catch(err => { | ||
| if (err instanceof Issue) return err | ||
| throw err | ||
| }), | ||
| ], | ||
| [], | ||
| ) | ||
| return ( | ||
| Promise.all(issuePromises) | ||
| // remove non-issues | ||
| .then(res => res.filter(o => o instanceof Issue)) | ||
| return Promise.resolve( | ||
| miscFiles.map(createIssueForEmpty).filter(clearNonIssues), | ||
| ) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,60 +1,61 @@ | ||
| var Issue = require('./issues').Issue | ||
| var JSHINT = require('jshint').JSHINT | ||
| const Issue = require('./issues').Issue | ||
| const { JSHINT } = require('jshint') | ||
|
|
||
| module.exports = { | ||
| /** | ||
| * Parse | ||
| * | ||
| * Similar to native JSON.parse but uses | ||
| * a callback structure, jshint for more | ||
| * thorough error reporting and error formatting | ||
| * of the rest of the validator. | ||
| */ | ||
| parse: function(file, contents, callback) { | ||
| var jsObj = null | ||
| var err = null | ||
| /** | ||
| * Similar to native JSON.parse but returns a promise and | ||
| * runs jshint for more thorough error reporting | ||
| */ | ||
| function parse(file, contents) { | ||
| return new Promise(resolve => { | ||
| let jsObj | ||
| let err | ||
| try { | ||
| jsObj = JSON.parse(contents) | ||
| } catch (exception) { | ||
| err = exception | ||
| } finally { | ||
| if (err) { | ||
| this.jshint(file, contents, function(issues) { | ||
| callback(issues, null) | ||
| jshint(file, contents, function(issues) { | ||
| resolve({ issues, parsed: null }) | ||
| }) | ||
| } else { | ||
| callback([], jsObj) | ||
| resolve({ issues: [], parsed: jsObj }) | ||
| } | ||
| } | ||
| }, | ||
| }) | ||
| } | ||
|
|
||
| /** | ||
| * JSHint | ||
| * | ||
| * Checks known invalid JSON file | ||
| * content in order to produce a | ||
| * verbose error message. | ||
| */ | ||
| jshint: function(file, contents, callback) { | ||
| var issues = [] | ||
| if (!JSHINT(contents)) { | ||
| var out = JSHINT.data() | ||
| for (var i = 0; out.errors.length > i; ++i) { | ||
| var error = out.errors[i] | ||
| if (error) { | ||
| issues.push( | ||
| new Issue({ | ||
| code: 27, | ||
| file: file, | ||
| line: error.line ? error.line : null, | ||
| character: error.character ? error.character : null, | ||
| reason: error.reason ? error.reason : null, | ||
| evidence: error.evidence ? error.evidence : null, | ||
| }), | ||
| ) | ||
| } | ||
| /** | ||
| * JSHint | ||
| * | ||
| * Checks known invalid JSON file | ||
| * content in order to produce a | ||
| * verbose error message. | ||
| */ | ||
| function jshint(file, contents, callback) { | ||
| var issues = [] | ||
| if (!JSHINT(contents)) { | ||
| var out = JSHINT.data() | ||
| for (var i = 0; out.errors.length > i; ++i) { | ||
| var error = out.errors[i] | ||
| if (error) { | ||
| issues.push( | ||
| new Issue({ | ||
| code: 27, | ||
| file: file, | ||
| line: error.line ? error.line : null, | ||
| character: error.character ? error.character : null, | ||
| reason: error.reason ? error.reason : null, | ||
| evidence: error.evidence ? error.evidence : null, | ||
| }), | ||
| ) | ||
| } | ||
| } | ||
| callback(issues) | ||
| }, | ||
| } | ||
| callback(issues) | ||
| } | ||
|
|
||
| module.exports = { | ||
| parse, | ||
| jshint, | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| /* limits promises to LIMIT to prevent memory overuse */ | ||
|
|
||
| const pLimit = require('p-limit') | ||
| const LIMIT = 200 | ||
|
|
||
| module.exports = pLimit(LIMIT) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,36 +1,52 @@ | ||
| const utils = require('../../utils') | ||
|
|
||
| class JSONParseError extends Error { | ||
| constructor(message) { | ||
| super(message) | ||
| this.name = 'JSONParseError' | ||
| } | ||
| } | ||
|
|
||
| const load = (files, jsonFiles, jsonContentsDict, annexed, dir) => { | ||
| let issues = [] | ||
| const jsonPromises = files.map(function(file) { | ||
| return new Promise((resolve, reject) => { | ||
| utils.files | ||
| .readFile(file, annexed, dir) | ||
| .then(contents => { | ||
| utils.json.parse(file, contents, function(parseIssues, jsObj) { | ||
| issues = issues.concat(parseIssues) | ||
|
|
||
| // abort further tests if schema test does not pass | ||
| if (issues.some(issue => issue.severity === 'error')) { | ||
| return reject() | ||
| } | ||
|
|
||
| jsonContentsDict[file.relativePath] = jsObj | ||
| jsonFiles.push(file) | ||
| resolve() | ||
| }) | ||
| }) | ||
| .catch(issue => { | ||
| issues.push(issue) | ||
| resolve() | ||
| }) | ||
| }) | ||
| }) | ||
| return new Promise(resolve => | ||
| Promise.all(jsonPromises) | ||
| .then(() => resolve(issues)) | ||
| .catch(() => resolve(issues)), | ||
|
|
||
| // Read JSON file contents and parse for issues | ||
| const readJsonFile = (file, annexed, dir) => | ||
| utils.files | ||
| .readFile(file, annexed, dir) | ||
| .then(contents => utils.json.parse(file, contents)) | ||
| .then(({ issues: parseIssues, parsed }) => { | ||
| // Append any parse issues to returned issues | ||
| Array.prototype.push.apply(issues, parseIssues) | ||
|
|
||
| // Abort further tests if an error is found | ||
| if ( | ||
| parseIssues && | ||
| parseIssues.some(issue => issue.severity === 'error') | ||
| ) { | ||
| throw new JSONParseError('Aborted due to parse error') | ||
| } | ||
|
|
||
| jsonContentsDict[file.relativePath] = parsed | ||
| jsonFiles.push(file) | ||
| }) | ||
|
|
||
| // Start concurrent read/parses | ||
| const fileReads = files.map(file => | ||
| utils.limit(() => readJsonFile(file, annexed, dir)), | ||
| ) | ||
|
|
||
| // After all reads/parses complete, return any found issues | ||
| return Promise.all(fileReads) | ||
| .then(() => issues) | ||
| .catch(err => { | ||
| // Handle early exit | ||
| if (err instanceof JSONParseError) { | ||
| return issues | ||
| } else { | ||
| throw err | ||
| } | ||
| }) | ||
| } | ||
|
|
||
| module.exports = load |
Oops, something went wrong.