Skip to content

Commit

Permalink
fix(expect, jest-snapshot, jest-circus, jest-types): Pass test.failin…
Browse files Browse the repository at this point in the history
…g tests when containing failing snapshot matchers
  • Loading branch information
KhaledElmorsy committed Jul 11, 2023
1 parent 596422e commit 90bb2e2
Show file tree
Hide file tree
Showing 20 changed files with 231 additions and 5 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

### Fixes

- `[expect, jest-snapshot]` Pass `test.failing` tests when containing failing snapshot matchers.
- `[jest-circus]` Prevent false test failures caused by promise rejections handled asynchronously ([#14110](https://github.com/jestjs/jest/pull/14110))
- `[jest-config]` Handle frozen config object ([#14054](https://github.com/facebook/jest/pull/14054))
- `[jest-config]` Allow `coverageDirectory` and `collectCoverageFrom` in project config ([#14180](https://github.com/jestjs/jest/pull/14180))
Expand Down
33 changes: 33 additions & 0 deletions e2e/__tests__/__snapshots__/testFailingSnapshot.test.js.snap
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`test.failing doesnt update or remove snapshots 1`] = `
"// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[\`snapshots not updated nor removed 1\`] = \`"1"\`;
exports[\`snapshots not updated nor removed 2\`] = \`"1"\`;
exports[\`snapshots not updated nor removed 3\`] = \`"1"\`;
"
`;

exports[`test.failing doesnt update or remove snapshots 2`] = `
"/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
test.failing('inline snapshot not updated', () => {
// eslint-disable-next-line quotes
expect('0').toMatchInlineSnapshot(\`'1'\`);
});
"
`;
exports[`test.failing should pass when at least one snapshot fails 1`] = `"1"`;
exports[`test.failing should pass when at least one snapshot fails 2`] = `"1"`;
exports[`test.failing should pass when snapshot matchers fails 1`] = `"1"`;
61 changes: 61 additions & 0 deletions e2e/__tests__/testFailingSnapshot.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/

import * as path from 'path';
import * as fs from 'graceful-fs';
import runJest from '../runJest';

describe('test.failing', () => {
describe('should pass when', () => {
test.failing('snapshot matchers fails', () => {
expect('0').toMatchSnapshot();
});

test.failing('snapshot doesnt exist', () => {
expect('0').toMatchSnapshot();
});

test.failing('inline snapshot matchers fails', () => {
expect('0').toMatchInlineSnapshot('0');
});

test.failing('at least one snapshot fails', () => {
expect('1').toMatchSnapshot();
expect('0').toMatchSnapshot();
});
});

describe('should fail when', () => {
test.each([
['snapshot', 'snapshot'],
['inline snapshot', 'inlineSnapshot'],
])('%s matchers pass', (_, fileName) => {
const dir = path.resolve(__dirname, '../test-failing-snapshot-all-pass');
const result = runJest(dir, [`./__tests__/${fileName}.test.js`]);
expect(result.exitCode).toBe(1);
});
});

it('doesnt update or remove snapshots', () => {
const dir = path.resolve(__dirname, '../test-failing-snapshot');
const result = runJest(dir, ['-u']);
expect(result.exitCode).toBe(0);
expect(result.stdout).not.toMatch(/snapshots? (written|removed|obsolete)/);

const snapshot = fs
.readFileSync(
path.resolve(dir, './__tests__/__snapshots__/snapshot.test.js.snap'),
)
.toString();
expect(snapshot).toMatchSnapshot();

const inlineSnapshot = fs
.readFileSync(path.resolve(dir, './__tests__/inlineSnapshot.test.js'))
.toString();
expect(inlineSnapshot).toMatchSnapshot();
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`snapshots not updated 1`] = `"1"`;

exports[`snapshots not updated 2`] = `"1"`;
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/

test.failing('inline snapshot not updated', () => {
// eslint-disable-next-line quotes
expect('1').toMatchInlineSnapshot(`"1"`);
});
11 changes: 11 additions & 0 deletions e2e/test-failing-snapshot-all-pass/__tests__/snapshot.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/

test.failing('snapshots not updated', () => {
expect('1').toMatchSnapshot();
expect('1').toMatchSnapshot();
});
5 changes: 5 additions & 0 deletions e2e/test-failing-snapshot-all-pass/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"jest": {
"testEnvironment": "node"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`snapshots not updated nor removed 1`] = `"1"`;

exports[`snapshots not updated nor removed 2`] = `"1"`;

exports[`snapshots not updated nor removed 3`] = `"1"`;
11 changes: 11 additions & 0 deletions e2e/test-failing-snapshot/__tests__/inlineSnapshot.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/

test.failing('inline snapshot not updated', () => {
// eslint-disable-next-line quotes
expect('0').toMatchInlineSnapshot(`'1'`);
});
12 changes: 12 additions & 0 deletions e2e/test-failing-snapshot/__tests__/snapshot.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/

test.failing('snapshots not updated nor removed', () => {
expect('1').toMatchSnapshot();
expect('0').toMatchSnapshot();
expect('0').toMatchSnapshot();
});
5 changes: 5 additions & 0 deletions e2e/test-failing-snapshot/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"jest": {
"testEnvironment": "node"
}
}
1 change: 1 addition & 0 deletions packages/expect/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ export interface MatcherState {
promise?: string;
suppressedErrors: Array<Error>;
testPath?: string;
testFailing?: boolean;
}

export type MatcherContext = MatcherUtils & Readonly<MatcherState>;
Expand Down
12 changes: 9 additions & 3 deletions packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,10 +109,16 @@ const _addSnapshotData = (
results: TestResult,
snapshotState: SnapshotState,
) => {
results.testResults.forEach(({fullName, status}) => {
if (status === 'pending' || status === 'failed') {
// if test is skipped or failed, we don't want to mark
results.testResults.forEach(({fullName, status, failing}) => {
if (
status === 'pending' ||
status === 'failed' ||
(failing && status === 'passed')
) {
// If test is skipped or failed, we don't want to mark
// its snapshots as obsolete.
// When tests called with test.failing pass, they've thrown an exception,
// so maintain any snapshots after the error.
snapshotState.markSnapshotsAsCheckedForTest(fullName);
}
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ export const runAndTransformResultsToJestFormat = async ({
return {
ancestorTitles,
duration: testResult.duration,
failing: testResult.failing,
failureDetails: testResult.errorsDetailed,
failureMessages: testResult.errors,
fullName: title
Expand Down Expand Up @@ -243,7 +244,10 @@ const handleSnapshotStateAfterRetry =
const eventHandler = async (event: Circus.Event) => {
switch (event.name) {
case 'test_start': {
jestExpect.setState({currentTestName: getTestID(event.test)});
jestExpect.setState({
currentTestName: getTestID(event.test),
testFailing: event.test.failing,
});
break;
}
case 'test_done': {
Expand Down
2 changes: 2 additions & 0 deletions packages/jest-circus/src/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -376,6 +376,7 @@ export const makeSingleTestResult = (
duration: test.duration,
errors: errorsDetailed.map(getErrorStack),
errorsDetailed,
failing: test.failing,
invocations: test.invocations,
location,
numPassingAsserts: test.numPassingAsserts,
Expand Down Expand Up @@ -510,6 +511,7 @@ export const parseSingleTestResult = (
return {
ancestorTitles,
duration: testResult.duration,
failing: testResult.failing,
failureDetails: testResult.errorsDetailed,
failureMessages: Array.from(testResult.errors),
fullName,
Expand Down
19 changes: 19 additions & 0 deletions packages/jest-snapshot/src/State.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ export type SnapshotMatchOptions = {
readonly inlineSnapshot?: string;
readonly isInline: boolean;
readonly error?: Error;
readonly pure?: boolean;
};

type SnapshotReturnOptions = {
Expand Down Expand Up @@ -197,6 +198,7 @@ export default class SnapshotState {
inlineSnapshot,
isInline,
error,
pure = false,
}: SnapshotMatchOptions): SnapshotReturnOptions {
this._counters.set(testName, (this._counters.get(testName) || 0) + 1);
const count = Number(this._counters.get(testName));
Expand Down Expand Up @@ -230,6 +232,23 @@ export default class SnapshotState {
this._snapshotData[key] = receivedSerialized;
}

// In pure runs, return the match result while skipping any updates
// reports.
if (pure) {
if (hasSnapshot && !isInline) {
// Retain current snapshot values.
this._addSnapshot(key, expected, {error, isInline});
}
return {
actual: removeExtraLineBreaks(receivedSerialized),
count,
expected:
expected !== undefined ? removeExtraLineBreaks(expected) : undefined,
key,
pass,
};
}

// These are the conditions on when to write snapshots:
// * There's no snapshot file in a non-CI environment.
// * There is a snapshot file and we decided to update the snapshot.
Expand Down
19 changes: 18 additions & 1 deletion packages/jest-snapshot/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,23 @@ const _toMatchSnapshot = (config: MatchSnapshotConfig) => {
config;
let {received} = config;

context.dontThrow && context.dontThrow();
/** If a test was ran with `test.failing`. Passed by Jest Circus. */
const {testFailing = false} = context;

/**
* Run the snapshot matcher as a normal throwing matcher with
* no side effects and no error supression.
*
* Skipped side effects occur in {@link snapshotState.match}
* and include updating the snapshot and recording passed/failed
* snapshots in the snapshot state.
*/
const pure = testFailing;

if (!pure && context.dontThrow) {
// Supress errors while running tests
context.dontThrow();
}

const {currentConcurrentTestName, isNot, snapshotState} = context;
const currentTestName =
Expand Down Expand Up @@ -358,6 +374,7 @@ const _toMatchSnapshot = (config: MatchSnapshotConfig) => {
error: context.error,
inlineSnapshot,
isInline,
pure,
received,
testName: fullTestName,
});
Expand Down
5 changes: 5 additions & 0 deletions packages/jest-test-result/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,11 @@ export type TestResult = {
testFilePath: string;
testResults: Array<AssertionResult>;
v8Coverage?: V8CoverageResult;
/**
* Whether [`test.failing()`](https://jestjs.io/docs/api#testfailingname-fn-timeout)
* was used.
*/
failing?: boolean;
};

export type FormattedTestResult = {
Expand Down
5 changes: 5 additions & 0 deletions packages/jest-types/src/Circus.ts
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,11 @@ export type TestResult = {
numPassingAsserts: number;
retryReasons: Array<FormattedError>;
testPath: TestNamesPath;
/**
* Whether [`test.failing()`](https://jestjs.io/docs/api#testfailingname-fn-timeout)
* was used.
*/
failing?: boolean;
};

export type RunResult = {
Expand Down
5 changes: 5 additions & 0 deletions packages/jest-types/src/TestResult.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ export type AssertionResult = {
retryReasons?: Array<string>;
status: Status;
title: string;
/**
* Whether [`test.failing()`](https://jestjs.io/docs/api#testfailingname-fn-timeout)
* was used.
*/
failing?: boolean;
};

export type SerializableError = {
Expand Down

0 comments on commit 90bb2e2

Please sign in to comment.