feat: add test.failing method

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### Features
 
+- `[jest-circus]` Add `failing` test modifier that inverts the behaviour of tests ([#12610](https://github.com/facebook/jest/pull/12610))
 - `[jest-environment-node, jest-environment-jsdom]` Allow specifying `customExportConditions` ([#12774](https://github.com/facebook/jest/pull/12774))
 
 ### Fixes

docs/GlobalAPI.md

@@ -731,6 +731,62 @@ test.each`
 });
 ```
 
+### `test.failing(name, fn, timeout)`
+
+Also under the alias: `it.failing(name, fn, timeout)`
+
+:::note
+
+This is only available with the default [jest-circus](https://github.com/facebook/jest/tree/main/packages/jest-circus) runner.
+
+:::
+
+Use `test.failing` when you are writing a test and expecting it to fail. These tests will behave the other way normal tests do. If `failing` test will throw any errors then it will pass. If it does not throw it will fail.
+
+:::tip
+
+You can use this type of tests i.e. when writing code in a BDD way. In that case the tests will not show up as failing until they pass. Then you can just remove the `failing` modifier to make them pass.
+
+It can also be a nice way to contribute failing tests to a project, even if you don't know how to fix the bug.
+
+:::
+
+Example:
+
+```js
+test.failing('it is not equal', () => {
+  expect(5).toBe(6); // this test will pass
+});
+
+test.failing('it is equal', () => {
+  expect(10).toBe(10); // this test will fail
+});
+```
+
+### `test.only.failing(name, fn, timeout)`
+
+Also under the aliases: `it.only.failing(name, fn, timeout)`, `fit.failing(name, fn, timeout)`
+
+:::note
+
+This is only available with the default [jest-circus](https://github.com/facebook/jest/tree/main/packages/jest-circus) runner.
+
+:::
+
+Use `test.only.failing` if you want to only run a specific failing test.
+
+### `test.skip.failing(name, fn, timeout)`
+
+Also under the aliases: `it.skip.failing(name, fn, timeout)`, `xit.failing(name, fn, timeout)`, `xtest.failing(name, fn, timeout)`
+
+:::note
+
+This is only available with the default [jest-circus](https://github.com/facebook/jest/tree/main/packages/jest-circus) runner.
+
+:::
+
+Use `test.skip.failing` if you want to skip running a specific failing test.
+
 ### `test.only(name, fn, timeout)`
 
 Also under the aliases: `it.only(name, fn, timeout)`, and `fit(name, fn, timeout)`

e2e/__tests__/__snapshots__/circusDeclarationErrors.test.ts.snap

@@ -16,7 +16,7 @@ exports[`defining tests and hooks asynchronously throws 1`] = `
       14 |   });
       15 | });
 
-      at eventHandler (../../packages/jest-circus/build/eventHandler.js:145:11)
+      at eventHandler (../../packages/jest-circus/build/eventHandler.js:153:11)
       at test (__tests__/asyncDefinition.test.js:12:5)
 
   ● Test suite failed to run
@@ -46,7 +46,7 @@ exports[`defining tests and hooks asynchronously throws 1`] = `
       20 | });
       21 |
 
-      at eventHandler (../../packages/jest-circus/build/eventHandler.js:145:11)
+      at eventHandler (../../packages/jest-circus/build/eventHandler.js:153:11)
       at test (__tests__/asyncDefinition.test.js:18:3)
 
   ● Test suite failed to run

e2e/__tests__/__snapshots__/testFailing.test.ts.snap

@@ -0,0 +1,165 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`works with all statuses 1`] = `
+"FAIL __tests__/statuses.test.js
+  ✓ passes
+  ✕ fails
+  ✓ failing fails = passes
+  ✓ failing fails = passes with test syntax
+  ✕ failing passes = fails
+  ○ skipped skips
+  ○ skipped skipped failing 1
+  ○ skipped skipped failing 2
+  ○ skipped skipped failing with different syntax
+  ○ skipped skipped failing with another different syntax
+  ✎ todo todo
+
+  ● fails
+
+    expect(received).toBe(expected) // Object.is equality
+
+    Expected: 101
+    Received: 10
+
+      11 |
+      12 | it('fails', () => {
+    > 13 |   expect(10).toBe(101);
+         |              ^
+      14 | });
+      15 |
+      16 | it.skip('skips', () => {
+
+      at Object.toBe (__tests__/statuses.test.js:13:14)
+
+  ● failing passes = fails
+
+    Failing test passed even though it was supposed to fail. Remove \`.failing\` to remove error."
+`;
+
+exports[`works with concurrent and only mode 1`] = `
+"FAIL __tests__/worksWithConcurrentOnlyMode.test.js
+  block with concurrent
+    ✕ failing passes = fails
+    ✓ failing fails = passes
+    ○ skipped skipped failing test
+    ○ skipped skipped failing fails
+
+  ● block with concurrent › failing passes = fails
+
+    Failing test passed even though it was supposed to fail. Remove \`.failing\` to remove error."
+`;
+
+exports[`works with concurrent mode 1`] = `
+"FAIL __tests__/worksWithConcurrentMode.test.js
+  block with concurrent
+    ✕ failing test
+    ✕ failing passes = fails
+    ✓ failing fails = passes
+    ○ skipped skipped failing fails
+
+  ● block with concurrent › failing test
+
+    expect(received).toBe(expected) // Object.is equality
+
+    Expected: 101
+    Received: 10
+
+       8 | describe('block with concurrent', () => {
+       9 |   it('failing test', () => {
+    > 10 |     expect(10).toBe(101);
+         |                ^
+      11 |   });
+      12 |
+      13 |   it.concurrent.failing('failing passes = fails', () => {
+
+      at Object.toBe (__tests__/worksWithConcurrentMode.test.js:10:16)
+
+  ● block with concurrent › failing passes = fails
+
+    Failing test passed even though it was supposed to fail. Remove \`.failing\` to remove error."
+`;
+
+exports[`works with only mode 1`] = `
+"FAIL __tests__/worksWithOnlyMode.test.js
+  block with only, should pass
+    ✓ failing fails = passes, should pass
+    ○ skipped failing test but skipped
+    ○ skipped passing test but skipped
+  block with only, should fail
+    ✕ failing passes = fails, should fail
+    ○ skipped failing test but skipped
+    ○ skipped passing test but skipped
+  block with only in other it, should skip
+    ✕ failing test
+    ○ skipped failing passes = fails, should fail but skipped
+    ○ skipped passing test but skipped
+  block with only with different syntax, should fail
+    ✕ failing passes = fails, should fail 1
+    ✕ failing passes = fails, should fail 2
+    ○ skipped failing test but skipped
+    ○ skipped passing test but skipped
+
+  ● block with only, should fail › failing passes = fails, should fail
+
+    Failing test passed even though it was supposed to fail. Remove \`.failing\` to remove error.
+
+
+
+  ● block with only in other it, should skip › failing test
+
+    expect(received).toBe(expected) // Object.is equality
+
+    Expected: 101
+    Received: 10
+
+      41 |   // eslint-disable-next-line jest/no-focused-tests
+      42 |   it.only('failing test', () => {
+    > 43 |     expect(10).toBe(101);
+         |                ^
+      44 |   });
+      45 |
+      46 |   it('passing test but skipped', () => {
+
+      at Object.toBe (__tests__/worksWithOnlyMode.test.js:43:16)
+
+  ● block with only with different syntax, should fail › failing passes = fails, should fail 1
+
+    Failing test passed even though it was supposed to fail. Remove \`.failing\` to remove error.
+
+
+
+  ● block with only with different syntax, should fail › failing passes = fails, should fail 2
+
+    Failing test passed even though it was supposed to fail. Remove \`.failing\` to remove error."
+`;
+
+exports[`works with skip mode 1`] = `
+"FAIL __tests__/worksWithSkipMode.test.js
+  block with only, should pass
+    ✕ failing test
+    ✓ failing fails = passes
+    ○ skipped skipped failing fails = passes, should pass
+    ○ skipped passing test
+  block with only, should fail
+    ✓ passing test
+    ✓ failing passes = fails
+    ○ skipped failing passes = fails, should fail
+    ○ skipped failing test
+
+  ● block with only, should pass › failing test
+
+    expect(received).toBe(expected) // Object.is equality
+
+    Expected: 101
+    Received: 10
+
+      12 |
+      13 |   it('failing test', () => {
+    > 14 |     expect(10).toBe(101);
+         |                ^
+      15 |   });
+      16 |
+      17 |   it.skip('passing test', () => {
+
+      at Object.toBe (__tests__/worksWithSkipMode.test.js:14:16)"
+`;

e2e/__tests__/testFailing.test.ts

@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import * as path from 'path';
+import {skipSuiteOnJasmine} from '@jest/test-utils';
+import {extractSummary} from '../Utils';
+import runJest, {json as runWithJson} from '../runJest';
+
+skipSuiteOnJasmine();
+
+const dir = path.resolve(__dirname, '../test-failing');
+
+test('works with all statuses', () => {
+  const result = runJest(dir, ['statuses.test.js']);
+  expect(result.exitCode).toBe(1);
+  const {rest} = extractSummary(result.stderr);
+  expect(rest).toMatchSnapshot();
+});
+
+test('works with only mode', () => {
+  const result = runJest(dir, ['worksWithOnlyMode.test.js']);
+  expect(result.exitCode).toBe(1);
+  const {rest} = extractSummary(result.stderr);
+  expect(rest).toMatchSnapshot();
+});
+
+test('works with skip mode', () => {
+  const result = runJest(dir, ['worksWithSkipMode.test.js']);
+  expect(result.exitCode).toBe(1);
+  const {rest} = extractSummary(result.stderr);
+  expect(rest).toMatchSnapshot();
+});
+
+test('works with concurrent mode', () => {
+  const result = runWithJson(dir, ['worksWithConcurrentMode.test.js']);
+  expect(result.exitCode).toBe(1);
+  const {rest} = extractSummary(result.stderr);
+  expect(rest).toMatchSnapshot();
+});
+
+test('works with concurrent and only mode', () => {
+  const result = runWithJson(dir, ['worksWithConcurrentOnlyMode.test.js']);
+  expect(result.exitCode).toBe(1);
+  const {rest} = extractSummary(result.stderr);
+  expect(rest).toMatchSnapshot();
+});

e2e/test-failing/__tests__/statuses.test.js

@@ -0,0 +1,48 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+it('passes', () => {
+  expect(10).toBe(10);
+});
+
+it('fails', () => {
+  expect(10).toBe(101);
+});
+
+it.skip('skips', () => {
+  expect(10).toBe(101);
+});
+
+it.todo('todo');
+
+it.failing('failing fails = passes', () => {
+  expect(10).toBe(101);
+});
+
+test.failing('failing fails = passes with test syntax', () => {
+  expect(10).toBe(101);
+});
+
+it.skip.failing('skipped failing 1', () => {
+  expect(10).toBe(10);
+});
+
+test.skip.failing('skipped failing 2', () => {
+  expect(10).toBe(101);
+});
+
+it.failing('failing passes = fails', () => {
+  expect(10).toBe(10);
+});
+
+xit.failing('skipped failing with different syntax', () => {
+  expect(10).toBe(10);
+});
+
+xtest.failing('skipped failing with another different syntax', () => {
+  expect(10).toBe(10);
+});

e2e/test-failing/__tests__/worksWithConcurrentMode.test.js

@@ -0,0 +1,24 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+describe('block with concurrent', () => {
+  it('failing test', () => {
+    expect(10).toBe(101);
+  });
+
+  it.concurrent.failing('failing passes = fails', () => {
+    expect(10).toBe(10);
+  });
+
+  it.concurrent.failing('failing fails = passes', () => {
+    expect(10).toBe(101);
+  });
+
+  it.concurrent.skip.failing('skipped failing fails', () => {
+    expect(10).toBe(101);
+  });
+});