From 7f2af5609d92a4fb6efa6bb9829b125e95170050 Mon Sep 17 00:00:00 2001 From: Mo Kamioner Date: Wed, 28 Feb 2018 11:16:20 +0200 Subject: [PATCH 1/6] Allow writing json contents to file with tests --- .gitignore | 1 + docs/index.md | 2 + lib/reporters/json.js | 32 ++++- lib/reporters/xunit.js | 1 - test/reporters/json.spec.js | 235 ++++++++++++++++++++++++++++++------ 5 files changed, 231 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index cf748d8fa3..428af5598a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ yarn.lock *_REMOTE_* docs/_site docs/_dist +output.json.tmp diff --git a/docs/index.md b/docs/index.md index 3e62f0d6f3..6645c7e0fc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1100,6 +1100,8 @@ The "JSON" reporter outputs a single large JSON object when the tests have compl ![json reporter](images/reporter-json.png?withoutEnlargement&resize=920,9999){:class="screenshot"} +By default, it will output to the console. To write directly to a file, use `--reporter-options output=filename.json`. + ### JSON Stream The "JSON stream" reporter outputs newline-delimited JSON "events" as they occur, beginning with a "start" event, followed by test passes or failures, and then the final "end" event. diff --git a/lib/reporters/json.js b/lib/reporters/json.js index 259a782121..8b89f2f920 100644 --- a/lib/reporters/json.js +++ b/lib/reporters/json.js @@ -5,6 +5,9 @@ */ var Base = require('./base'); +var fs = require('fs'); +var mkdirp = require('mkdirp'); +var path = require('path'); /** * Expose `JSON`. @@ -18,7 +21,7 @@ exports = module.exports = JSONReporter; * @api public * @param {Runner} runner */ -function JSONReporter (runner) { +function JSONReporter (runner, options) { Base.call(this, runner); var self = this; @@ -27,6 +30,16 @@ function JSONReporter (runner) { var failures = []; var passes = []; + if (options && options.reporterOptions) { + if (options.reporterOptions.output) { + if (!fs.writeFileSync) { + throw new Error('file output not supported in browser'); + } + mkdirp.sync(path.dirname(options.reporterOptions.output)); + self.fileName = path.resolve(options.reporterOptions.output); + } + } + runner.on('test end', function (test) { tests.push(test); }); @@ -54,7 +67,7 @@ function JSONReporter (runner) { runner.testResults = obj; - process.stdout.write(JSON.stringify(obj, null, 2)); + self.write(JSON.stringify(obj, null, 2)); }); } @@ -90,3 +103,18 @@ function errorJSON (err) { }, err); return res; } + +/** + * Write out the given data. + * + * @param {string} data + */ +JSONReporter.prototype.write = function (data) { + if (this.fileName) { + fs.writeFileSync(this.fileName, data); + } else if (typeof process === 'object' && process.stdout) { + process.stdout.write(data); + } else { + console.log(data); + } +}; diff --git a/lib/reporters/xunit.js b/lib/reporters/xunit.js index 7d98815ca2..0043c72bfc 100644 --- a/lib/reporters/xunit.js +++ b/lib/reporters/xunit.js @@ -38,7 +38,6 @@ exports = module.exports = XUnit; */ function XUnit (runner, options) { Base.call(this, runner); - var stats = this.stats; var tests = []; var self = this; diff --git a/test/reporters/json.spec.js b/test/reporters/json.spec.js index 0b0866ad1e..046ca58e3b 100644 --- a/test/reporters/json.spec.js +++ b/test/reporters/json.spec.js @@ -1,63 +1,224 @@ 'use strict'; +var fs = require('fs'); +var path = require('path'); +var assert = require('assert'); var Mocha = require('../../'); var Suite = Mocha.Suite; var Runner = Mocha.Runner; var Test = Mocha.Test; +var TEMP_FILE_PATH = path.resolve('output.json.tmp'); + describe('json reporter', function () { var suite, runner; - beforeEach(function () { - var mocha = new Mocha({ - reporter: 'json' - }); - suite = new Suite('JSON suite', 'root'); - runner = new Runner(suite); - /* eslint no-unused-vars: off */ - var mochaReporter = new mocha._reporter(runner); + after(function () { + if (fs.existsSync(TEMP_FILE_PATH)) { + fs.unlinkSync(TEMP_FILE_PATH); + } }); - it('should have 1 test failure', function (done) { - var testTitle = 'json test 1'; - var error = { message: 'oh shit' }; + describe('with no output specified', function () { + beforeEach(function () { + var mocha = new Mocha({ + reporter: 'json' + }); + suite = new Suite('JSON suite', 'root'); + runner = new Runner(suite); + /* eslint no-unused-vars: off */ + var mochaReporter = new mocha._reporter(runner); + }); + it('should have 1 test failure', function (done) { + var testTitle = 'json test 1'; + var error = { message: 'oh shit' }; - suite.addTest(new Test(testTitle, function (done) { - done(new Error(error.message)); - })); + suite.addTest(new Test(testTitle, function (done) { + done(new Error(error.message)); + })); - runner.run(function (failureCount) { - expect(failureCount).to.be(1); - expect(runner).to.have.property('testResults'); - expect(runner.testResults).to.have.property('failures'); - expect(runner.testResults.failures).to.be.an('array'); - expect(runner.testResults.failures).to.have.length(1); + runner.run(function (failureCount) { + expect(failureCount).to.be(1); + expect(runner).to.have.property('testResults'); + expect(runner.testResults).to.have.property('failures'); + expect(runner.testResults.failures).to.be.an('array'); + expect(runner.testResults.failures).to.have.length(1); - var failure = runner.testResults.failures[0]; - expect(failure).to.have.property('title', testTitle); - expect(failure.err.message).to.equal(error.message); - expect(failure).to.have.property('err'); + var failure = runner.testResults.failures[0]; + expect(failure).to.have.property('title', testTitle); + expect(failure.err.message).to.equal(error.message); + expect(failure).to.have.property('err'); - done(); + done(); + }); + }); + + it('should have 1 test pending', function (done) { + var testTitle = 'json test 1'; + + suite.addTest(new Test(testTitle)); + + runner.run(function (failureCount) { + expect(failureCount).to.be(0); + expect(runner).to.have.property('testResults'); + expect(runner.testResults).to.have.property('pending'); + expect(runner.testResults.pending).to.be.an('array'); + expect(runner.testResults.pending).to.have.length(1); + + var pending = runner.testResults.pending[0]; + expect(pending).to.have.property('title', testTitle); + + done(); + }); }); }); - it('should have 1 test pending', function (done) { - var testTitle = 'json test 1'; + describe('with output specified', function () { + beforeEach(function () { + var mocha = new Mocha({ + reporter: 'json' + }); + suite = new Suite('JSON suite', 'root'); + runner = new Runner(suite); + /* eslint no-unused-vars: off */ + var mochaReporter = new mocha._reporter(runner, { reporterOptions: { output: TEMP_FILE_PATH } }); + }); + + it('should have 1 test failure', function (done) { + var testTitle = 'json test 1'; + var error = { message: 'oh shit' }; + + suite.addTest(new Test(testTitle, function (done) { + done(new Error(error.message)); + })); + + runner.run(function (failureCount) { + expect(failureCount).to.be(1); + expect(runner).to.have.property('testResults'); + expect(runner.testResults).to.have.property('failures'); + expect(runner.testResults.failures).to.be.an('array'); + expect(runner.testResults.failures).to.have.length(1); + + var failure = runner.testResults.failures[0]; + expect(failure).to.have.property('title', testTitle); + expect(failure.err.message).to.equal(error.message); + expect(failure).to.have.property('err'); + + var fileContents; + try { + fileContents = JSON.parse(fs.readFileSync(TEMP_FILE_PATH, 'utf-8')); + } catch (e) { + done(e); + return; + } + + assert.deepEqual(fileContents, { + stats: { + suites: 1, + tests: 1, + passes: 0, + pending: 0, + failures: 1, + start: fileContents.stats.start, + end: fileContents.stats.end, + duration: fileContents.stats.duration + }, + tests: [{ + title: 'json test 1', + fullTitle: 'JSON suite json test 1', + duration: fileContents.tests[0].duration, + currentRetry: 0, + err: { + stack: fileContents.tests[0].err.stack, + message: 'oh shit' + } + }], + pending: [], + failures: [{ + title: 'json test 1', + fullTitle: 'JSON suite json test 1', + duration: fileContents.failures[0].duration, + currentRetry: 0, + err: { + stack: fileContents.failures[0].err.stack, + message: 'oh shit' + } + }], + passes: [] + }); + assert.equal(Date.now() - new Date(fileContents.stats.start).getTime() < 10000, true); + assert.equal(Date.now() - new Date(fileContents.stats.end).getTime() < 10000, true); + assert.equal(typeof fileContents.stats.duration, 'number'); + assert.equal(fileContents.stats.duration <= 10, true); + assert.equal(typeof fileContents.tests[0].duration, 'number'); + assert.equal(fileContents.tests[0].duration <= 10, true); + assert.equal(typeof fileContents.failures[0].duration, 'number'); + assert.equal(fileContents.failures[0].duration <= 10, true); + var errLines = fileContents.tests[0].err.stack.split('\n'); + assert.equal(errLines[0], 'Error: oh shit'); + assert.equal(errLines[1], ' at Object. (test/reporters/json.spec.js:92:14)'); + errLines = fileContents.failures[0].err.stack.split('\n'); + assert.equal(errLines[0], 'Error: oh shit'); + assert.equal(errLines[1], ' at Object. (test/reporters/json.spec.js:92:14)'); + + done(); + }); + }); + + it('should have 1 test pending', function (done) { + var testTitle = 'json test 1'; + + suite.addTest(new Test(testTitle)); + + runner.run(function (failureCount) { + expect(failureCount).to.be(0); + expect(runner).to.have.property('testResults'); + expect(runner.testResults).to.have.property('pending'); + expect(runner.testResults.pending).to.be.an('array'); + expect(runner.testResults.pending).to.have.length(1); - suite.addTest(new Test(testTitle)); + var pending = runner.testResults.pending[0]; + expect(pending).to.have.property('title', testTitle); - runner.run(function (failureCount) { - expect(failureCount).to.be(0); - expect(runner).to.have.property('testResults'); - expect(runner.testResults).to.have.property('pending'); - expect(runner.testResults.pending).to.be.an('array'); - expect(runner.testResults.pending).to.have.length(1); + var fileContents; + try { + fileContents = JSON.parse(fs.readFileSync(TEMP_FILE_PATH, 'utf-8')); + } catch (e) { + done(e); + return; + } - var pending = runner.testResults.pending[0]; - expect(pending).to.have.property('title', testTitle); + assert.deepEqual(fileContents, { + stats: { + suites: 1, + tests: 1, + passes: 0, + pending: 1, + failures: 0, + start: fileContents.stats.start, + end: fileContents.stats.end, + duration: fileContents.stats.duration + }, + tests: [{ + title: 'json test 1', + fullTitle: 'JSON suite json test 1', + currentRetry: 0, + err: {} + }], + pending: [{ + title: 'json test 1', + fullTitle: 'JSON suite json test 1', + currentRetry: 0, + err: {} + }], + failures: [], + passes: [] + }); + assert.equal(Date.now() - new Date(fileContents.stats.start).getTime() < 10000, true); + assert.equal(Date.now() - new Date(fileContents.stats.end).getTime() < 10000, true); - done(); + done(); + }); }); }); }); From e4ce8076c77f8c8b527629bed909c5cb75baa508 Mon Sep 17 00:00:00 2001 From: Mo Kamioner Date: Wed, 28 Feb 2018 14:47:13 +0200 Subject: [PATCH 2/6] Fix tests --- test/reporters/json.spec.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/reporters/json.spec.js b/test/reporters/json.spec.js index 046ca58e3b..41e4f4787c 100644 --- a/test/reporters/json.spec.js +++ b/test/reporters/json.spec.js @@ -156,10 +156,10 @@ describe('json reporter', function () { assert.equal(fileContents.failures[0].duration <= 10, true); var errLines = fileContents.tests[0].err.stack.split('\n'); assert.equal(errLines[0], 'Error: oh shit'); - assert.equal(errLines[1], ' at Object. (test/reporters/json.spec.js:92:14)'); + assert.equal(!!/^ {4}at (Object|Context)\. \(test\/reporters\/json\.spec\.js:92:14\)$/g.exec(errLines[1]), true); errLines = fileContents.failures[0].err.stack.split('\n'); assert.equal(errLines[0], 'Error: oh shit'); - assert.equal(errLines[1], ' at Object. (test/reporters/json.spec.js:92:14)'); + assert.equal(!!/^ {4}at (Object|Context)\. \(test\/reporters\/json\.spec\.js:92:14\)$/g.exec(errLines[1]), true); done(); }); From bb83b197d1358da5b579b7532b4238b0972efce6 Mon Sep 17 00:00:00 2001 From: Mo Kamioner Date: Wed, 28 Feb 2018 15:02:16 +0200 Subject: [PATCH 3/6] Add some logs to understand test failures --- test/reporters/json.spec.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/reporters/json.spec.js b/test/reporters/json.spec.js index 41e4f4787c..a1e7b7de2c 100644 --- a/test/reporters/json.spec.js +++ b/test/reporters/json.spec.js @@ -156,10 +156,12 @@ describe('json reporter', function () { assert.equal(fileContents.failures[0].duration <= 10, true); var errLines = fileContents.tests[0].err.stack.split('\n'); assert.equal(errLines[0], 'Error: oh shit'); - assert.equal(!!/^ {4}at (Object|Context)\. \(test\/reporters\/json\.spec\.js:92:14\)$/g.exec(errLines[1]), true); + var regexResult = /^ {4}at (Object|Context)\. \(test\/reporters\/json\.spec\.js:92:14\)$/g.exec(errLines[1]); + assert.equal(!!regexResult, true, JSON.stringify({ regexResult, line: errLines[1] }, null, 2)); errLines = fileContents.failures[0].err.stack.split('\n'); assert.equal(errLines[0], 'Error: oh shit'); - assert.equal(!!/^ {4}at (Object|Context)\. \(test\/reporters\/json\.spec\.js:92:14\)$/g.exec(errLines[1]), true); + regexResult = /^ {4}at (Object|Context)\. \(test\/reporters\/json\.spec\.js:92:14\)$/g.exec(errLines[1]); + assert.equal(!!regexResult, true, JSON.stringify({ regexResult, line: errLines[1] }, null, 2)); done(); }); From d02600214162996fe474fb787dc145d0dbf645a8 Mon Sep 17 00:00:00 2001 From: Mo Kamioner Date: Wed, 28 Feb 2018 15:09:51 +0200 Subject: [PATCH 4/6] More logs for test failing --- test/reporters/json.spec.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/reporters/json.spec.js b/test/reporters/json.spec.js index a1e7b7de2c..ddca627f12 100644 --- a/test/reporters/json.spec.js +++ b/test/reporters/json.spec.js @@ -148,20 +148,20 @@ describe('json reporter', function () { }); assert.equal(Date.now() - new Date(fileContents.stats.start).getTime() < 10000, true); assert.equal(Date.now() - new Date(fileContents.stats.end).getTime() < 10000, true); - assert.equal(typeof fileContents.stats.duration, 'number'); - assert.equal(fileContents.stats.duration <= 10, true); - assert.equal(typeof fileContents.tests[0].duration, 'number'); - assert.equal(fileContents.tests[0].duration <= 10, true); - assert.equal(typeof fileContents.failures[0].duration, 'number'); - assert.equal(fileContents.failures[0].duration <= 10, true); + assert.equal(typeof fileContents.stats.duration, 'number', JSON.stringify({ fileContents }, null, 2)); + assert.equal(fileContents.stats.duration <= 10, true, JSON.stringify({ fileContents }, null, 2)); + assert.equal(typeof fileContents.tests[0].duration, 'number', JSON.stringify({ fileContents }, null, 2)); + assert.equal(fileContents.tests[0].duration <= 10, true, JSON.stringify({ fileContents }, null, 2)); + assert.equal(typeof fileContents.failures[0].duration, 'number', JSON.stringify({ fileContents }, null, 2)); + assert.equal(fileContents.failures[0].duration <= 10, true, JSON.stringify({ fileContents }, null, 2)); var errLines = fileContents.tests[0].err.stack.split('\n'); - assert.equal(errLines[0], 'Error: oh shit'); + assert.equal(errLines[0], 'Error: oh shit', JSON.stringify({ fileContents }, null, 2)); var regexResult = /^ {4}at (Object|Context)\. \(test\/reporters\/json\.spec\.js:92:14\)$/g.exec(errLines[1]); - assert.equal(!!regexResult, true, JSON.stringify({ regexResult, line: errLines[1] }, null, 2)); + assert.equal(!!regexResult, true, JSON.stringify({ regexResult, fileContents, line: errLines[1] }, null, 2)); errLines = fileContents.failures[0].err.stack.split('\n'); - assert.equal(errLines[0], 'Error: oh shit'); + assert.equal(errLines[0], 'Error: oh shit', JSON.stringify({ fileContents }, null, 2)); regexResult = /^ {4}at (Object|Context)\. \(test\/reporters\/json\.spec\.js:92:14\)$/g.exec(errLines[1]); - assert.equal(!!regexResult, true, JSON.stringify({ regexResult, line: errLines[1] }, null, 2)); + assert.equal(!!regexResult, true, JSON.stringify({ regexResult, fileContents, line: errLines[1] }, null, 2)); done(); }); From b60044abc91f176267d16dbf847c6fc0f9682902 Mon Sep 17 00:00:00 2001 From: Mo Kamioner Date: Wed, 28 Feb 2018 15:20:47 +0200 Subject: [PATCH 5/6] Fixed regex test to account for windows like slashes --- test/reporters/json.spec.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/reporters/json.spec.js b/test/reporters/json.spec.js index ddca627f12..9ff2882c64 100644 --- a/test/reporters/json.spec.js +++ b/test/reporters/json.spec.js @@ -156,11 +156,11 @@ describe('json reporter', function () { assert.equal(fileContents.failures[0].duration <= 10, true, JSON.stringify({ fileContents }, null, 2)); var errLines = fileContents.tests[0].err.stack.split('\n'); assert.equal(errLines[0], 'Error: oh shit', JSON.stringify({ fileContents }, null, 2)); - var regexResult = /^ {4}at (Object|Context)\. \(test\/reporters\/json\.spec\.js:92:14\)$/g.exec(errLines[1]); + var regexResult = /^ {4}at ((Object)|(Context))\. \(test(\/|\\)reporters(\/|\\)json\.spec\.js:92:14\)$/g.exec(errLines[1]); assert.equal(!!regexResult, true, JSON.stringify({ regexResult, fileContents, line: errLines[1] }, null, 2)); errLines = fileContents.failures[0].err.stack.split('\n'); assert.equal(errLines[0], 'Error: oh shit', JSON.stringify({ fileContents }, null, 2)); - regexResult = /^ {4}at (Object|Context)\. \(test\/reporters\/json\.spec\.js:92:14\)$/g.exec(errLines[1]); + regexResult = /^ {4}at ((Object)|(Context))\. \(test(\/|\\)reporters(\/|\\)json\.spec\.js:92:14\)$/g.exec(errLines[1]); assert.equal(!!regexResult, true, JSON.stringify({ regexResult, fileContents, line: errLines[1] }, null, 2)); done(); From 55b1f82e2c1b4d91dec4690c574314dfcc9f766d Mon Sep 17 00:00:00 2001 From: Mo Kamioner Date: Wed, 28 Feb 2018 15:41:53 +0200 Subject: [PATCH 6/6] Don't pay attention to length of running --- test/reporters/json.spec.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/reporters/json.spec.js b/test/reporters/json.spec.js index 9ff2882c64..9d94892104 100644 --- a/test/reporters/json.spec.js +++ b/test/reporters/json.spec.js @@ -149,11 +149,11 @@ describe('json reporter', function () { assert.equal(Date.now() - new Date(fileContents.stats.start).getTime() < 10000, true); assert.equal(Date.now() - new Date(fileContents.stats.end).getTime() < 10000, true); assert.equal(typeof fileContents.stats.duration, 'number', JSON.stringify({ fileContents }, null, 2)); - assert.equal(fileContents.stats.duration <= 10, true, JSON.stringify({ fileContents }, null, 2)); + assert.equal(fileContents.stats.duration <= 30, true, JSON.stringify({ fileContents }, null, 2)); assert.equal(typeof fileContents.tests[0].duration, 'number', JSON.stringify({ fileContents }, null, 2)); - assert.equal(fileContents.tests[0].duration <= 10, true, JSON.stringify({ fileContents }, null, 2)); + assert.equal(fileContents.tests[0].duration <= 30, true, JSON.stringify({ fileContents }, null, 2)); assert.equal(typeof fileContents.failures[0].duration, 'number', JSON.stringify({ fileContents }, null, 2)); - assert.equal(fileContents.failures[0].duration <= 10, true, JSON.stringify({ fileContents }, null, 2)); + assert.equal(fileContents.failures[0].duration <= 30, true, JSON.stringify({ fileContents }, null, 2)); var errLines = fileContents.tests[0].err.stack.split('\n'); assert.equal(errLines[0], 'Error: oh shit', JSON.stringify({ fileContents }, null, 2)); var regexResult = /^ {4}at ((Object)|(Context))\. \(test(\/|\\)reporters(\/|\\)json\.spec\.js:92:14\)$/g.exec(errLines[1]);