diff --git a/docs/api-tutorials/custom-reporter.md b/docs/api-tutorials/custom-reporter.md index b537fe1ee2..991163fa41 100644 --- a/docs/api-tutorials/custom-reporter.md +++ b/docs/api-tutorials/custom-reporter.md @@ -48,6 +48,8 @@ The event names are exported from the `constants` property of `Mocha.Runner`: | `EVENT_TEST_PENDING` | `pending` | `Test` | A {@link Test} was skipped. | | `EVENT_TEST_RETRY` | `retry` | `Test`, `Error` | A {@link Test} failed, but is about to be retried; only emitted if the `retry` option is nonzero. | +| `EVENT_TEST_REPEAT` | `repeat` | `Test`, `Error` | A {@link Test} succeeded, but is about to be repeated; only emitted if the `repeat` option is nonzero. | + **Please use these constants** instead of the event names in your own reporter! This will ensure compatibility with future versions of Mocha. > It's important to understand that all `Suite` callbacks will be run _before_ the {@link Runner} emits `EVENT_RUN_BEGIN`. Hooks and tests won't run until _after_ the {@link Runner} emits `EVENT_RUN_BEGIN`. diff --git a/docs/index.md b/docs/index.md index 0e0f8c4ed0..c4bd8cee16 100644 --- a/docs/index.md +++ b/docs/index.md @@ -707,6 +707,31 @@ describe("retries", function () { }); ``` +## Repeat Tests + +Tests can also be repeated when they pass. This feature can be used to test for leaks and proper tear-down procedures. In this case a test is considered to be successful only if all the runs are successful. + +This feature does re-run a passed test and its corresponding `beforeEach/afterEach` hooks, but not `before/after` hooks. + +If using both `repeat` and `retries`, the test will be run `repeat` times tolerating up to `retries` failures in total. + +```js +describe('repeat', function () { + // Repeat all tests in this suite 4 times + this.repeats(4); + + beforeEach(function () { + browser.get('http://www.yahoo.com'); + }); + + it('should use proper tear-down', function () { + // Specify that only this test is to be repeated twice (and not 4x2=8 times) + this.repeats(2); + expect($('.foo').isDisplayed()).to.eventually.be.true; + }); +}); +``` + ## Dynamically Generating Tests Given Mocha's use of function expressions to define suites and test cases, it's straightforward to generate your tests dynamically. No special syntax is required — plain ol' JavaScript can be used to achieve functionality similar to "parameterized" tests, which you may have seen in other frameworks. @@ -2218,6 +2243,7 @@ mocha.setup({ forbidPending: true, global: ['MyLib'], retries: 3, + repeats: 1, rootHooks: { beforeEach(done) { ... done();} }, slow: '100', timeout: '2000', diff --git a/example/config/.mocharc.js b/example/config/.mocharc.js index a97a793efc..bb2396a973 100644 --- a/example/config/.mocharc.js +++ b/example/config/.mocharc.js @@ -36,6 +36,7 @@ module.exports = { "reporter-option": ["foo=bar", "baz=quux"], // array, not object require: "@babel/register", retries: 1, + repeats: 1, slow: "75", sort: false, spec: ["test/**/*.spec.js"], // the positional arguments! diff --git a/example/config/.mocharc.yml b/example/config/.mocharc.yml index 167098148e..310265f7da 100644 --- a/example/config/.mocharc.yml +++ b/example/config/.mocharc.yml @@ -40,6 +40,7 @@ reporter-option: # array, not object - "baz=quux" require: "@babel/register" retries: 1 +repeats: 1 slow: "75" sort: false spec: diff --git a/lib/cli/run-option-metadata.js b/lib/cli/run-option-metadata.js index a43531080a..216fe5439d 100644 --- a/lib/cli/run-option-metadata.js +++ b/lib/cli/run-option-metadata.js @@ -51,7 +51,7 @@ const TYPES = (exports.types = { "sort", "watch", ], - number: ["retries", "jobs"], + number: ["retries", "repeats", "jobs"], string: [ "config", "fgrep", diff --git a/lib/cli/run.js b/lib/cli/run.js index 111b40ded1..c009959c2d 100644 --- a/lib/cli/run.js +++ b/lib/cli/run.js @@ -247,6 +247,10 @@ exports.builder = (yargs) => description: "Retry failed tests this many times", group: GROUPS.RULES, }, + repeats: { + description: 'Repeat passed tests this many times', + group: GROUPS.RULES + }, slow: { default: defaults.slow, description: 'Specify "slow" test threshold (in milliseconds)', diff --git a/lib/context.js b/lib/context.js index 900a0f4dd8..f03d521c4b 100644 --- a/lib/context.js +++ b/lib/context.js @@ -89,3 +89,18 @@ Context.prototype.retries = function (n) { this.runnable().retries(n); return this; }; + +/** + * Set or get a number of repeats on passed tests + * + * @private + * @param {number} n + * @return {Context} self + */ +Context.prototype.repeats = function (n) { + if (!arguments.length) { + return this.runnable().repeats(); + } + this.runnable().repeats(n); + return this; +}; diff --git a/lib/hook.js b/lib/hook.js index fb50d825f2..19c930c2ed 100644 --- a/lib/hook.js +++ b/lib/hook.js @@ -63,6 +63,7 @@ Hook.prototype.error = function (err) { Hook.prototype.serialize = function serialize() { return { $$currentRetry: this.currentRetry(), + $$currentRepeat: this.currentRepeat(), $$fullTitle: this.fullTitle(), $$isPending: Boolean(this.isPending()), $$titlePath: this.titlePath(), diff --git a/lib/mocha.js b/lib/mocha.js index df250ec2b7..0265e76425 100644 --- a/lib/mocha.js +++ b/lib/mocha.js @@ -188,6 +188,10 @@ function Mocha(options = {}) { this.retries(options.retries); } + if ("repeats" in options) { + this.repeats(options.repeats); + } + [ "allowUncaught", "asyncOnly", @@ -745,6 +749,25 @@ Mocha.prototype.retries = function (retry) { return this; }; +/** + * Sets the number of times to repeat passed tests. + * + * @public + * @see [CLI option](../#-repeats-n) + * @see [Repeat Tests](../#repeat-tests) + * @param {number} repeats - Number of times to repeat passed tests. + * @return {Mocha} this + * @chainable + * @example + * + * // Allow any passed test to be repeated multiple times + * mocha.repeats(1); + */ +Mocha.prototype.repeats = function (repeats) { + this.suite.repeats(repeats); + return this; +}; + /** * Sets slowness threshold value. * diff --git a/lib/nodejs/reporters/parallel-buffered.js b/lib/nodejs/reporters/parallel-buffered.js index 085e09bdc2..d51a9f2475 100644 --- a/lib/nodejs/reporters/parallel-buffered.js +++ b/lib/nodejs/reporters/parallel-buffered.js @@ -24,6 +24,7 @@ const { EVENT_TEST_BEGIN, EVENT_TEST_END, EVENT_TEST_RETRY, + EVENT_TEST_REPEAT, EVENT_DELAY_BEGIN, EVENT_DELAY_END, EVENT_HOOK_BEGIN, @@ -49,6 +50,7 @@ const EVENT_NAMES = [ EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_RETRY, + EVENT_TEST_REPEAT, EVENT_TEST_END, EVENT_HOOK_BEGIN, EVENT_HOOK_END, diff --git a/lib/reporters/json-stream.js b/lib/reporters/json-stream.js index 9d157e1eb4..90a89899e7 100644 --- a/lib/reporters/json-stream.js +++ b/lib/reporters/json-stream.js @@ -86,6 +86,7 @@ function clean(test) { file: test.file, duration: test.duration, currentRetry: test.currentRetry(), + currentRepeat: test.currentRepeat(), speed: test.speed, }; } diff --git a/lib/reporters/json.js b/lib/reporters/json.js index 8ceef6bd80..fc6d4bbe39 100644 --- a/lib/reporters/json.js +++ b/lib/reporters/json.js @@ -120,6 +120,7 @@ function clean(test) { file: test.file, duration: test.duration, currentRetry: test.currentRetry(), + currentRepeat: test.currentRepeat(), speed: test.speed, err: cleanCycles(err), }; diff --git a/lib/runnable.js b/lib/runnable.js index 97097dda5e..f705e3cdee 100644 --- a/lib/runnable.js +++ b/lib/runnable.js @@ -44,6 +44,7 @@ function Runnable(title, fn) { this._timeout = 2000; this._slow = 75; this._retries = -1; + this._repeats = 1; utils.assignNewMochaID(this); Object.defineProperty(this, "id", { get() { @@ -64,6 +65,7 @@ utils.inherits(Runnable, EventEmitter); Runnable.prototype.reset = function () { this.timedOut = false; this._currentRetry = 0; + this._currentRepeat = 1; this.pending = false; delete this.state; delete this.err; @@ -185,6 +187,18 @@ Runnable.prototype.retries = function (n) { this._retries = n; }; +/** + * Set or get number of repeats. + * + * @private + */ +Runnable.prototype.repeats = function (n) { + if (!arguments.length) { + return this._repeats; + } + this._repeats = n; +}; + /** * Set or get current retry * @@ -197,6 +211,18 @@ Runnable.prototype.currentRetry = function (n) { this._currentRetry = n; }; +/** + * Set or get current repeat + * + * @private + */ +Runnable.prototype.currentRepeat = function (n) { + if (!arguments.length) { + return this._currentRepeat; + } + this._currentRepeat = n; +}; + /** * Return the full title generated by recursively concatenating the parent's * full title. diff --git a/lib/runner.js b/lib/runner.js index ebce7523af..a682ae61cd 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -154,6 +154,10 @@ var constants = utils.defineConstants( * Emitted when {@link Test} execution has failed, but will retry */ EVENT_TEST_RETRY: "retry", + /** + * Emitted when {@link Test} execution has succeeded, but will repeat + */ + EVENT_TEST_REPEAT: "repeat", /** * Initial state of Runner */ @@ -867,6 +871,14 @@ Runner.prototype.runTests = function (suite, fn) { self.fail(test, err); } self.emit(constants.EVENT_TEST_END, test); + return self.hookUp(HOOK_TYPE_AFTER_EACH, next); + } else if (test.currentRepeat() < test.repeats()) { + var repeatedTest = test.clone(); + repeatedTest.currentRepeat(test.currentRepeat() + 1); + tests.unshift(repeatedTest); + + self.emit(constants.EVENT_TEST_REPEAT, test, null); + return self.hookUp(HOOK_TYPE_AFTER_EACH, next); } diff --git a/lib/suite.js b/lib/suite.js index 20e1a93e17..61e6f38229 100644 --- a/lib/suite.js +++ b/lib/suite.js @@ -77,6 +77,7 @@ function Suite(title, parentContext, isRoot) { this.root = isRoot === true; this.pending = false; this._retries = -1; + this._repeats = 1; this._beforeEach = []; this._beforeAll = []; this._afterEach = []; @@ -131,6 +132,7 @@ Suite.prototype.clone = function () { suite.root = this.root; suite.timeout(this.timeout()); suite.retries(this.retries()); + suite.repeats(this.repeats()); suite.slow(this.slow()); suite.bail(this.bail()); return suite; @@ -178,6 +180,22 @@ Suite.prototype.retries = function (n) { return this; }; +/** + * Set or get number of times to repeat a passed test. + * + * @private + * @param {number|string} n + * @return {Suite|number} for chaining + */ +Suite.prototype.repeats = function (n) { + if (!arguments.length) { + return this._repeats; + } + debug('repeats %d', n); + this._repeats = parseInt(n, 10) || 0; + return this; +}; + /** * Set or get slow `ms` or short-hand such as "2s". * @@ -234,6 +252,7 @@ Suite.prototype._createHook = function (title, fn) { hook.parent = this; hook.timeout(this.timeout()); hook.retries(this.retries()); + hook.repeats(this.repeats()); hook.slow(this.slow()); hook.ctx = this.ctx; hook.file = this.file; @@ -348,6 +367,7 @@ Suite.prototype.addSuite = function (suite) { suite.root = false; suite.timeout(this.timeout()); suite.retries(this.retries()); + suite.repeats(this.repeats()); suite.slow(this.slow()); suite.bail(this.bail()); this.suites.push(suite); @@ -366,6 +386,7 @@ Suite.prototype.addTest = function (test) { test.parent = this; test.timeout(this.timeout()); test.retries(this.retries()); + test.repeats(this.repeats()); test.slow(this.slow()); test.ctx = this.ctx; this.tests.push(test); diff --git a/lib/test.js b/lib/test.js index 8997b48b54..205638832b 100644 --- a/lib/test.js +++ b/lib/test.js @@ -73,7 +73,9 @@ Test.prototype.clone = function () { test.timeout(this.timeout()); test.slow(this.slow()); test.retries(this.retries()); + test.repeats(this.repeats()); test.currentRetry(this.currentRetry()); + test.currentRepeat(this.currentRepeat()); test.retriedTest(this.retriedTest() || this); test.globals(this.globals()); test.parent = this.parent; @@ -91,6 +93,7 @@ Test.prototype.clone = function () { Test.prototype.serialize = function serialize() { return { $$currentRetry: this._currentRetry, + $$currentRepeat: this._currentRepeat, $$fullTitle: this.fullTitle(), $$isPending: Boolean(this.pending), $$retriedTest: this._retriedTest || null, diff --git a/lib/types.d.ts b/lib/types.d.ts index e50bbe3e8e..eddf80f8ba 100644 --- a/lib/types.d.ts +++ b/lib/types.d.ts @@ -80,6 +80,9 @@ export interface MochaOptions { /** Number of times to retry failed tests. */ retries?: number; + /** Number of times to repeat each test. */ + repeats?: number; + /** Slow threshold value, in milliseconds. */ slow?: number; diff --git a/test/assertions.js b/test/assertions.js index b5e5a41644..a93571082a 100644 --- a/test/assertions.js +++ b/test/assertions.js @@ -306,6 +306,42 @@ module.exports = { }); }, ) + .addAssertion( + " [not] to have repeated test ", + (expect, result, title) => { + expect(result.tests, "[not] to have an item satisfying", { + title, + currentRepeat: expect.it("to be positive") + }); + }, + ) + .addAssertion( + " [not] to have repeated test ", + (expect, result, title, count) => { + expect(result.tests, "[not] to have an item satisfying", { + title, + currentRepeat: count, + }); + }, + ) + .addAssertion( + " [not] to have repeated test ", + (expect, result, title) => { + expect(result.tests, "[not] to have an item satisfying", { + title, + currentRepeat: expect.it("to be positive"), + }); + } + ) + .addAssertion( + " [not] to have repeated test ", + (expect, result, title, count) => { + expect(result.tests, "[not] to have an item satisfying", { + title, + currentRepeat: count, + }); + } + ) .addAssertion( " [not] to have failed with (error|errors) ", function (expect, result, ...errors) { diff --git a/test/integration/events.spec.js b/test/integration/events.spec.js index 57264e36a8..fcb5c9aa13 100644 --- a/test/integration/events.spec.js +++ b/test/integration/events.spec.js @@ -58,6 +58,44 @@ describe("event order", function () { }); }); + describe("--repeats test case", function () { + it("should assert --repeats event order", function (done) { + runMochaJSON( + "runner/events-repeats.fixture.js", + ["--repeats", "2"], + function (err, res) { + if (err) { + done(err); + return; + } + expect(res, "to have passed") + .and("to have failed test count", 0) + .and("to have passed test count", 1); + done(); + }, + ); + }); + }); + + describe("--repeats test case", function () { + it("should assert --repeats event order", function (done) { + runMochaJSON( + "runner/events-repeats.fixture.js", + ["--repeats", "2"], + function (err, res) { + if (err) { + done(err); + return; + } + expect(res, "to have passed") + .and("to have failed test count", 0) + .and("to have passed test count", 1); + done(); + }, + ); + }); + }); + describe("--delay test case", function () { it("should assert --delay event order", function (done) { runMochaJSON( diff --git a/test/integration/fixtures/options/parallel/repeats.fixture.js b/test/integration/fixtures/options/parallel/repeats.fixture.js new file mode 100644 index 0000000000..26d571c3bf --- /dev/null +++ b/test/integration/fixtures/options/parallel/repeats.fixture.js @@ -0,0 +1,14 @@ +describe('repeats suite', function() { + let calls = 0; + this.repeats(3); + + it('should pass', function() { + + }); + + it('should fail on the second call', function () { + calls++; + console.log(`RUN: ${calls}`); + if (calls > 1) throw new Error(); + }); +}); diff --git a/test/integration/fixtures/options/repeats.fixture.js b/test/integration/fixtures/options/repeats.fixture.js new file mode 100644 index 0000000000..1d7ae07807 --- /dev/null +++ b/test/integration/fixtures/options/repeats.fixture.js @@ -0,0 +1,5 @@ +'use strict'; + +describe('repeats', function () { + it('should pass', () => undefined); +}); diff --git a/test/integration/fixtures/repeats/async.fixture.js b/test/integration/fixtures/repeats/async.fixture.js new file mode 100644 index 0000000000..03a6076fc4 --- /dev/null +++ b/test/integration/fixtures/repeats/async.fixture.js @@ -0,0 +1,30 @@ +'use strict'; + +describe('repeats', function () { + var times = 0; + before(function () { + console.log('before'); + }); + + after(function () { + console.log('after'); + }); + + beforeEach(function () { + console.log('before each', times); + }); + + afterEach(function () { + console.log('after each', times); + }); + + it('should allow override and run appropriate hooks', function (done) { + this.timeout(100); + this.repeats(5); + console.log('TEST', times); + if (times++ > 2) { + return setTimeout(done, 300); + } + setTimeout(done, 50); + }); +}); diff --git a/test/integration/fixtures/repeats/early-fail.fixture.js b/test/integration/fixtures/repeats/early-fail.fixture.js new file mode 100644 index 0000000000..d04dcb1cc8 --- /dev/null +++ b/test/integration/fixtures/repeats/early-fail.fixture.js @@ -0,0 +1,9 @@ +'use strict'; +describe('repeats', function () { + this.repeats(2); + var times = 0; + + it('should fail on the second attempt', function () { + if (times++ > 0) throw new Error; + }); +}); diff --git a/test/integration/fixtures/repeats/hooks.fixture.js b/test/integration/fixtures/repeats/hooks.fixture.js new file mode 100644 index 0000000000..b4fc081aaa --- /dev/null +++ b/test/integration/fixtures/repeats/hooks.fixture.js @@ -0,0 +1,27 @@ +'use strict'; + +describe('retries', function () { + var times = 0; + before(function () { + console.log('before'); + }); + + after(function () { + console.log('after'); + }); + + beforeEach(function () { + console.log('before each', times); + }); + + afterEach(function () { + console.log('after each', times); + }); + + it('should allow override and run appropriate hooks', function () { + this.retries(4); + console.log('TEST', times); + times++; + throw new Error('retry error'); + }); +}); diff --git a/test/integration/fixtures/repeats/nested.fixture.js b/test/integration/fixtures/repeats/nested.fixture.js new file mode 100644 index 0000000000..4c7f0b1bd2 --- /dev/null +++ b/test/integration/fixtures/repeats/nested.fixture.js @@ -0,0 +1,15 @@ +'use strict'; + +describe('repeats', function () { + this.repeats(3); + describe('nested', function () { + let count = 0; + + it('should be executed only once', function () { + this.repeats(1); + count++; + if (count > 1) + throw new Error('repeat error'); + }); + }); +}); diff --git a/test/integration/fixtures/runner/events-repeats.fixture.js b/test/integration/fixtures/runner/events-repeats.fixture.js new file mode 100644 index 0000000000..78759ef38f --- /dev/null +++ b/test/integration/fixtures/runner/events-repeats.fixture.js @@ -0,0 +1,55 @@ +'use strict'; +var Runner = require('../../../../lib/runner.js'); +var assert = require('assert'); +var constants = Runner.constants; +var EVENT_HOOK_BEGIN = constants.EVENT_HOOK_BEGIN; +var EVENT_HOOK_END = constants.EVENT_HOOK_END; +var EVENT_RUN_BEGIN = constants.EVENT_RUN_BEGIN; +var EVENT_RUN_END = constants.EVENT_RUN_END; +var EVENT_SUITE_BEGIN = constants.EVENT_SUITE_BEGIN; +var EVENT_SUITE_END = constants.EVENT_SUITE_END; +var EVENT_TEST_BEGIN = constants.EVENT_TEST_BEGIN; +var EVENT_TEST_END = constants.EVENT_TEST_END; +var EVENT_TEST_PASS = constants.EVENT_TEST_PASS; +var EVENT_TEST_RETRY = constants.EVENT_TEST_RETRY; +var EVENT_TEST_REPEAT = constants.EVENT_TEST_REPEAT; + +var emitOrder = [ + EVENT_RUN_BEGIN, + EVENT_SUITE_BEGIN, + EVENT_SUITE_BEGIN, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_TEST_BEGIN, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_TEST_REPEAT, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_TEST_BEGIN, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_TEST_PASS, + EVENT_TEST_END, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_SUITE_END, + EVENT_SUITE_END, + EVENT_RUN_END +]; + +var realEmit = Runner.prototype.emit; +Runner.prototype.emit = function(event, ...args) { + assert.strictEqual(event, emitOrder.shift()); + return realEmit.call(this, event, ...args); +}; + +describe('suite A', function() { + before('before', function() {}); + beforeEach('beforeEach', function() {}); + it('test A', () => undefined); + afterEach('afterEach', function() {}); + after('after', function() {}); +}); diff --git a/test/integration/options/parallel.spec.js b/test/integration/options/parallel.spec.js index a038903051..fbd4ba9153 100644 --- a/test/integration/options/parallel.spec.js +++ b/test/integration/options/parallel.spec.js @@ -177,6 +177,38 @@ describe("--parallel", function () { }); }); + describe('when used with --repeats', function () { + it('should repeat tests appropriately', async function () { + return expect( + runMochaAsync('options/parallel/repeats*', ['--parallel']), + 'when fulfilled', + 'to satisfy', + expect + .it('to have failed') + .and('to have passed test count', 1) + .and('to have pending test count', 0) + .and('to have failed test count', 1) + .and('to contain output', /RUN: 2/) + ); + }); + }); + + describe('when used with --repeats', function () { + it('should repeat tests appropriately', async function () { + return expect( + runMochaAsync('options/parallel/repeats*', ['--parallel']), + 'when fulfilled', + 'to satisfy', + expect + .it('to have failed') + .and('to have passed test count', 1) + .and('to have pending test count', 0) + .and('to have failed test count', 1) + .and('to contain output', /RUN: 2/) + ); + }); + }); + describe("when used with --allow-uncaught", function () { it("should bubble up an exception", async function () { return expect( diff --git a/test/integration/options/repeats.spec.js b/test/integration/options/repeats.spec.js new file mode 100644 index 0000000000..84c6fb930e --- /dev/null +++ b/test/integration/options/repeats.spec.js @@ -0,0 +1,23 @@ +'use strict'; + +var path = require('node:path').posix; +var helpers = require('../helpers'); +var runMochaJSON = helpers.runMochaJSON; + +describe('--repeats', function () { + var args = []; + + it('should repeat tests', function (done) { + args = ['--repeats', '3']; + var fixture = path.join('options', 'repeats'); + runMochaJSON(fixture, args, function (err, res) { + if (err) { + return done(err); + } + expect(res, 'to have passed') + .and('not to have pending tests') + .and('to have repeated test', 'should pass', 3); + done(); + }); + }); +}); diff --git a/test/integration/repeats.spec.js b/test/integration/repeats.spec.js new file mode 100644 index 0000000000..70fc870ba4 --- /dev/null +++ b/test/integration/repeats.spec.js @@ -0,0 +1,141 @@ +'use strict'; + +var assert = require('node:assert'); +var helpers = require('./helpers'); +var runJSON = helpers.runMochaJSON; +var args = []; +var bang = require('../../lib/reporters/base').symbols.bang; + +describe('repeats', function () { + it('are ran in correct order', function (done) { + helpers.runMocha( + 'repeats/hooks.fixture.js', + ['--reporter', 'dot'], + function (err, res) { + var lines, expected; + + if (err) { + done(err); + return; + } + + lines = res.output + .split(helpers.SPLIT_DOT_REPORTER_REGEXP) + .map(function (line) { + return line.trim(); + }) + .filter(function (line) { + return line.length; + }) + .slice(0, -1); + + expected = [ + 'before', + 'before each 0', + 'TEST 0', + 'after each 1', + 'before each 1', + 'TEST 1', + 'after each 2', + 'before each 2', + 'TEST 2', + 'after each 3', + 'before each 3', + 'TEST 3', + 'after each 4', + 'before each 4', + 'TEST 4', + bang + 'after each 5', + 'after' + ]; + + expected.forEach(function (line, i) { + assert.strictEqual(lines[i], line); + }); + + assert.strictEqual(res.code, 1); + done(); + } + ); + }); + + it('should exit early if test fails', function (done) { + runJSON('repeats/early-fail.fixture.js', args, function (err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have failed') + .and('to have passed test count', 0) + .and('to have failed test count', 1) + .and('to have repeated test', 'should fail on the second attempt', 2); + + done(); + }); + }); + + it('should let test override', function (done) { + runJSON('repeats/nested.fixture.js', args, function (err, res) { + if (err) { + done(err); + return; + } + assert.strictEqual(res.stats.passes, 1); + assert.strictEqual(res.stats.failures, 0); + assert.strictEqual(res.stats.tests, 1); + assert.strictEqual(res.tests[0].currentRepeat, 1); + assert.strictEqual(res.code, 0); + done(); + }); + }); + + it('should not hang w/ async test', function (done) { + this.timeout(2500); + helpers.runMocha( + 'repeats/async.fixture.js', + ['--reporter', 'dot'], + function (err, res) { + var lines, expected; + + if (err) { + done(err); + return; + } + + lines = res.output + .split(helpers.SPLIT_DOT_REPORTER_REGEXP) + .map(function (line) { + return line.trim(); + }) + .filter(function (line) { + return line.length; + }) + .slice(0, -1); + + expected = [ + 'before', + 'before each 0', + 'TEST 0', + 'after each 1', + 'before each 1', + 'TEST 1', + 'after each 2', + 'before each 2', + 'TEST 2', + 'after each 3', + 'before each 3', + 'TEST 3', + bang + 'after each 4', + 'after' + ]; + + expected.forEach(function (line, i) { + assert.strictEqual(lines[i], line); + }); + + assert.strictEqual(res.code, 1); + done(); + } + ); + }); +}); diff --git a/test/node-unit/reporters/parallel-buffered.spec.js b/test/node-unit/reporters/parallel-buffered.spec.js index d03c10eb28..762414264c 100644 --- a/test/node-unit/reporters/parallel-buffered.spec.js +++ b/test/node-unit/reporters/parallel-buffered.spec.js @@ -12,6 +12,7 @@ const { EVENT_TEST_BEGIN, EVENT_TEST_END, EVENT_TEST_RETRY, + EVENT_TEST_REPEAT, EVENT_DELAY_BEGIN, EVENT_DELAY_END, EVENT_HOOK_BEGIN, @@ -76,6 +77,7 @@ describe("ParallelBuffered", function () { [EVENT_TEST_FAIL, expect.it("to be a function")], [EVENT_TEST_PASS, expect.it("to be a function")], [EVENT_TEST_RETRY, expect.it("to be a function")], + [EVENT_TEST_REPEAT, expect.it("to be a function")], [EVENT_TEST_END, expect.it("to be a function")], [EVENT_HOOK_BEGIN, expect.it("to be a function")], [EVENT_HOOK_END, expect.it("to be a function")], diff --git a/test/reporters/helpers.js b/test/reporters/helpers.js index e7eabf3045..166b10cf47 100644 --- a/test/reporters/helpers.js +++ b/test/reporters/helpers.js @@ -166,6 +166,7 @@ function makeExpectedTest( expectedFile, expectedDuration, currentRetry, + currentRepeat, ) { return { title: expectedTitle, @@ -177,6 +178,9 @@ function makeExpectedTest( currentRetry: function () { return currentRetry; }, + currentRepeat: function () { + return currentRepeat; + }, slow: function () {}, }; } diff --git a/test/reporters/json-stream.spec.js b/test/reporters/json-stream.spec.js index 679548652a..09e772cd0b 100644 --- a/test/reporters/json-stream.spec.js +++ b/test/reporters/json-stream.spec.js @@ -21,6 +21,7 @@ describe("JSON Stream reporter", function () { var expectedFile = "someTest.spec.js"; var expectedDuration = 1000; var currentRetry = 1; + var currentRepeat = 1; var expectedSpeed = "fast"; var expectedTest = makeExpectedTest( expectedTitle, @@ -28,6 +29,7 @@ describe("JSON Stream reporter", function () { expectedFile, expectedDuration, currentRetry, + currentRepeat, expectedSpeed, ); var expectedErrorMessage = "error message"; @@ -78,6 +80,8 @@ describe("JSON Stream reporter", function () { expectedDuration + ',"currentRetry":' + currentRetry + + ',"currentRepeat":' + + currentRepeat + ',"speed":' + `"${expectedSpeed}"` + "}]\n", @@ -113,6 +117,8 @@ describe("JSON Stream reporter", function () { expectedDuration + ',"currentRetry":' + currentRetry + + ',"currentRepeat":' + + currentRepeat + ',"speed":' + `"${expectedSpeed}"` + ',"err":' + @@ -151,6 +157,8 @@ describe("JSON Stream reporter", function () { expectedDuration + ',"currentRetry":' + currentRetry + + ',"currentRepeat":' + + currentRepeat + ',"speed":' + `"${expectedSpeed}"` + ',"err":' + diff --git a/test/reporters/markdown.spec.js b/test/reporters/markdown.spec.js index c46a563fdf..b6a5e2b62a 100644 --- a/test/reporters/markdown.spec.js +++ b/test/reporters/markdown.spec.js @@ -78,6 +78,7 @@ describe("Markdown reporter", function () { }; var expectedDuration = 1000; var currentRetry = 1; + var currentRepeat = 1; var expectedBody = "some body"; var expectedTest = { title: expectedTitle, @@ -88,6 +89,9 @@ describe("Markdown reporter", function () { currentRetry: function () { return currentRetry; }, + currentRepeat: function () { + return currentRepeat; + }, slow: noop, body: expectedBody, }; diff --git a/test/unit/context.spec.js b/test/unit/context.spec.js index 819dff8f9e..33bf5c149d 100644 --- a/test/unit/context.spec.js +++ b/test/unit/context.spec.js @@ -95,4 +95,10 @@ describe("methods", function () { expect(this.retries(), "to be", -1); }); }); + + describe('repeats', function () { + it('should return the number of repeats', function () { + expect(this.repeats(), 'to be', 1); + }); + }); }); diff --git a/test/unit/mocha.spec.js b/test/unit/mocha.spec.js index cd985a8300..f4197ce2c4 100644 --- a/test/unit/mocha.spec.js +++ b/test/unit/mocha.spec.js @@ -94,9 +94,11 @@ describe("Mocha", function () { mocha = sinon.createStubInstance(Mocha); mocha.timeout.returnsThis(); mocha.retries.returnsThis(); + mocha.repeats.returnsThis(); sinon.stub(Mocha.prototype, "timeout").returnsThis(); sinon.stub(Mocha.prototype, "global").returnsThis(); sinon.stub(Mocha.prototype, "retries").returnsThis(); + sinon.stub(Mocha.prototype, 'repeats').returnsThis(); sinon.stub(Mocha.prototype, "rootHooks").returnsThis(); sinon.stub(Mocha.prototype, "parallelMode").returnsThis(); sinon.stub(Mocha.prototype, "globalSetup").returnsThis(); @@ -155,6 +157,42 @@ describe("Mocha", function () { }); }); + describe('when `repeats` option is present', function () { + it('should attempt to set repeats`', function () { + // eslint-disable-next-line no-new + new Mocha({repeats: 1}); + expect(Mocha.prototype.repeats, 'to have a call satisfying', [1]).and( + 'was called once' + ); + }); + }); + + describe('when `repeats` option is not present', function () { + it('should not attempt to set repeats', function () { + // eslint-disable-next-line no-new + new Mocha({}); + expect(Mocha.prototype.repeats, 'was not called'); + }); + }); + + describe('when `repeats` option is present', function () { + it('should attempt to set repeats`', function () { + // eslint-disable-next-line no-new + new Mocha({repeats: 1}); + expect(Mocha.prototype.repeats, 'to have a call satisfying', [1]).and( + 'was called once' + ); + }); + }); + + describe('when `repeats` option is not present', function () { + it('should not attempt to set repeats', function () { + // eslint-disable-next-line no-new + new Mocha({}); + expect(Mocha.prototype.repeats, 'was not called'); + }); + }); + describe("when `rootHooks` option is truthy", function () { it("shouid attempt to set root hooks", function () { // eslint-disable-next-line no-new diff --git a/test/unit/runnable.spec.js b/test/unit/runnable.spec.js index 44b800aba8..191496da0e 100644 --- a/test/unit/runnable.spec.js +++ b/test/unit/runnable.spec.js @@ -145,6 +145,7 @@ describe("Runnable(title, fn)", function () { run.reset(); expect(run.timedOut, "to be false"); expect(run._currentRetry, "to be", 0); + expect(run._currentRepeat, 'to be', 1); expect(run.pending, "to be false"); expect(run.err, "to be undefined"); expect(run.state, "to be undefined"); @@ -219,6 +220,22 @@ describe("Runnable(title, fn)", function () { }); }); + describe('#repeats(n)', function () { + it('should set the number of repeats', function () { + var run = new Runnable(); + run.repeats(3); + expect(run.repeats(), 'to be', 3); + }); + }); + + describe('#repeats(n)', function () { + it('should set the number of repeats', function () { + var run = new Runnable(); + run.repeats(3); + expect(run.repeats(), 'to be', 3); + }); + }); + describe(".run(fn)", function () { describe("when .pending", function () { it("should not invoke the callback", function (done) { diff --git a/test/unit/runner.spec.js b/test/unit/runner.spec.js index c9cb4a4a39..72f780104d 100644 --- a/test/unit/runner.spec.js +++ b/test/unit/runner.spec.js @@ -15,6 +15,7 @@ const { EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_RETRY, + EVENT_TEST_REPEAT, EVENT_TEST_END, EVENT_RUN_END, EVENT_SUITE_END, @@ -528,6 +529,34 @@ describe("Runner", function () { }); }); + it('should emit "repeat" when a repeatable test passes', function (done) { + var repeats = 2; + var runs = 0; + var retries = 0; + + var test = new Test('i do nothing', () => { + runs++; + }); + + suite.repeats(repeats); + suite.addTest(test); + + runner.on(EVENT_TEST_REPEAT, function (testClone, testErr) { + retries++; + expect(testClone.currentRepeat(), 'to be', runs); + expect(testErr, 'to be', null); + expect(testClone.title, 'to be', test.title); + }); + + runner.run(function (failures) { + expect(failures, 'to be', 0); + expect(runs, 'to be', repeats); + expect(retries, 'to be', repeats - 1); + + done(); + }); + }); + // karma-mocha is inexplicably doing this with a Hook it("should not throw an exception if something emits EVENT_TEST_END with a non-Test object", function () { expect(function () { diff --git a/test/unit/test.spec.js b/test/unit/test.spec.js index c6a677084d..633dea4eb4 100644 --- a/test/unit/test.spec.js +++ b/test/unit/test.spec.js @@ -17,6 +17,8 @@ describe("Test", function () { this._test._slow = 101; this._test._retries = 3; this._test._currentRetry = 1; + this._test._repeats = 3; + this._test._currentRepeat = 1; this._test._allowedGlobals = ["foo"]; this._test.parent = "foo"; this._test.file = "bar"; @@ -48,6 +50,22 @@ describe("Test", function () { expect(clone1.clone().retriedTest(), "to be", this._test); }); + it('should copy the repeats value', function () { + expect(this._test.clone().repeats(), 'to be', 3); + }); + + it('should copy the repeats value', function () { + expect(this._test.clone().repeats(), 'to be', 3); + }); + + it('should copy the currentRepeat value', function () { + expect(this._test.clone().currentRepeat(), 'to be', 1); + }); + + it('should copy the currentRepeat value', function () { + expect(this._test.clone().currentRepeat(), 'to be', 1); + }); + it("should copy the globals value", function () { expect(this._test.clone().globals(), "not to be empty"); });