diff --git a/.eslintignore b/.eslintignore deleted file mode 100644 index f39debef2..000000000 --- a/.eslintignore +++ /dev/null @@ -1,2 +0,0 @@ -test/data/output -lib/css2xpath/* diff --git a/.eslintrc.js b/.eslintrc.js new file mode 100644 index 000000000..8b8c9405f --- /dev/null +++ b/.eslintrc.js @@ -0,0 +1,48 @@ +module.exports = { + extends: 'airbnb-base', + env: { + node: true, + }, + parserOptions: { + ecmaVersion: 2020, + }, + rules: { + 'func-names': 0, + 'no-use-before-define': 0, + 'no-unused-vars': 0, + 'no-underscore-dangle': 0, + 'no-undef': 0, + 'prefer-destructuring': 0, + 'no-param-reassign': 0, + 'max-len': 0, + camelcase: 0, + 'no-shadow': 0, + 'consistent-return': 0, + 'no-console': 0, + 'global-require': 0, + 'class-methods-use-this': 0, + 'no-plusplus': 0, + 'no-return-assign': 0, + 'prefer-rest-params': 0, + 'no-useless-escape': 0, + 'no-restricted-syntax': 0, + 'no-unused-expressions': 0, + 'guard-for-in': 0, + 'no-multi-assign': 0, + 'require-yield': 0, + 'prefer-spread': 0, + 'import/no-dynamic-require': 0, + 'no-continue': 0, + 'no-mixed-operators': 0, + 'default-case': 0, + 'import/no-extraneous-dependencies': 0, + 'no-cond-assign': 0, + 'import/no-unresolved': 0, + 'no-await-in-loop': 0, + 'arrow-body-style': 0, + 'no-loop-func': 0, + 'arrow-parens': 0, + 'default-param-last': 0, + }, + ignorePatterns: ['test/data/output', 'lib/css2xpath/*'], +}; diff --git a/.eslintrc.json b/.eslintrc.json deleted file mode 100644 index ee991d23a..000000000 --- a/.eslintrc.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "extends": "airbnb-base", - "env": { - "node": true - }, - "parserOptions": { - "ecmaVersion": 2020 - }, - "rules": { - "func-names": 0, - "no-use-before-define": 0, - "no-unused-vars": 0, - "no-underscore-dangle": 0, - "no-undef": 0, - "prefer-destructuring": 0, - "no-param-reassign": 0, - "max-len": 0, - "camelcase": 0, - "no-shadow": 0, - "consistent-return": 0, - "no-console": 0, - "global-require": 0, - "class-methods-use-this": 0, - "no-plusplus": 0, - "no-return-assign": 0, - "prefer-rest-params": 0, - "no-useless-escape": 0, - "no-restricted-syntax": 0, - "no-unused-expressions": 0, - "guard-for-in": 0, - "no-multi-assign": 0, - "require-yield": 0, - "prefer-spread": 0, - "import/no-dynamic-require": 0, - "no-continue": 0, - "no-mixed-operators": 0, - "default-case": 0, - "import/no-extraneous-dependencies": 0, - "no-cond-assign": 0, - "import/no-unresolved": 0, - "no-await-in-loop": 0, - "arrow-body-style": 0, - "no-loop-func": 0, - "arrow-parens": 0, - "default-param-last": 0 - } -} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 82f20f55b..7d43abc66 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -17,4 +17,6 @@ updates: versions: [">=5.0"] - dependency-name: "apollo-server-express" versions: [">=3.0"] - + - dependency-name: "eslint" + versions: [ ">8.57.0" ] + diff --git a/.hound.yml b/.hound.yml index 6728a4cba..5133f32e0 100644 --- a/.hound.yml +++ b/.hound.yml @@ -1,3 +1,3 @@ eslint: enabled: true - config_file: .eslintrc.json \ No newline at end of file + config_file: .eslintrc.js diff --git a/CHANGELOG.md b/CHANGELOG.md index b1e71d9c0..b28ac45fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +## 3.6.3 + +❤ī¸ Thanks all to those who contributed to make this release! ❤ī¸ + +🛩ī¸ *Features* +* feat(plugin): coverage with WebDriver - devtools (#4349) - by @KobeNguyent + ![Screenshot 2024-05-16 at 16 49 20](https://github.com/codeceptjs/CodeceptJS/assets/7845001/a02f0f99-ac78-4d3f-9774-2cb51c688025) + +🐛 *Bug Fixes* +* fix(cli): stale process (#4367) - by @Horsty80 @kobenguyent +* fix(runner): screenshot error in beforeSuite/AfterSuite (#4385) - by @kobenguyent +* fix(cli): gherkin command init with TypeScript (#4366) - by @andonary +* fix(webApi): error message of dontSeeCookie (#4357) - by @a-stankevich + +📖 *Documentation* +* fix(doc): Expect helper is not described correctly (#4370) - by @kobenguyent +* fix(docs): some strange characters (#4387) - by @kobenguyent +* fix: Puppeteer helper doc typo (#4369) - by @yoannfleurydev + ## 3.6.2 ❤ī¸ Thanks all to those who contributed to make this release! ❤ī¸ diff --git a/README.md b/README.md index 3417207aa..b483e3c80 100644 --- a/README.md +++ b/README.md @@ -315,8 +315,8 @@ Thanks all to those who are and will have contributing to this awesome project! - + diff --git a/docs/data.md b/docs/data.md index 8ac5f0583..91d8f4a22 100644 --- a/docs/data.md +++ b/docs/data.md @@ -92,11 +92,8 @@ I.sendPostRequest('/update-status', {}, { http_x_requested_with: 'xmlhttprequest ## GraphQL [GraphQL helper](https://codecept.io/helpers/GraphQL/) allows sending GraphQL queries and mutations to application, over Http. -<<<<<<< HEAD -This is a tool to make shortcuts and create your data pragmatically via GraphQL endpoint. However, it doesn't provide tools for testing the endpoint, so it should be paired with WebDriver helper for browser testing. -======= -This is a tool to make shortcuts and create your data pragmatically via GraphQL endpoint. However, it doesn't provide tools for testing the endpoint, so it should be paired with WebDriver helpers for browser testing. ->>>>>>> 3.x + +This tool allows you to create shortcuts and manage your data pragmatically via a GraphQL endpoint. However, it does not include tools for testing the endpoint, so it should be used in conjunction with WebDriver helpers for browser testing. Enable GraphQL helper in the config. It is recommended to set `endpoint`, the URL to which the requests go to. If you need some authorization you can optionally set default headers too. diff --git a/docs/helpers/Expect.md b/docs/helpers/ExpectHelper.md similarity index 98% rename from docs/helpers/Expect.md rename to docs/helpers/ExpectHelper.md index 4628ea41c..7ffe415ab 100644 --- a/docs/helpers/Expect.md +++ b/docs/helpers/ExpectHelper.md @@ -1,8 +1,8 @@ --- -permalink: /helpers/Expect +permalink: /helpers/ExpectHelper editLink: false sidebar: auto -title: Expect +title: ExpectHelper --- @@ -20,7 +20,7 @@ Zero-configuration when paired with other helpers like REST, Playwright: { helpers: { Playwright: {...}, - Expect: {}, + ExpectHelper: {}, } } ``` diff --git a/docs/helpers/Puppeteer.md b/docs/helpers/Puppeteer.md index 3bf4cf9a0..55fba6b8c 100644 --- a/docs/helpers/Puppeteer.md +++ b/docs/helpers/Puppeteer.md @@ -2476,7 +2476,7 @@ Returns **void** automatically synchronized promise through #recorder [18]: https://codecept.io/helpers/FileSystem -[19]: https://pptr.dev/next/guides/request-interception +[19]: https://pptr.dev/guides/network-interception [20]: https://github.com/GoogleChrome/puppeteer/issues/1313 diff --git a/lib/cli.js b/lib/cli.js index 4312a5078..269430287 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -186,20 +186,24 @@ class Cli extends Base { } } - let stack = err.stack ? err.stack.split('\n') : []; - if (stack[0] && stack[0].includes(err.message)) { - stack.shift(); - } + try { + let stack = err.stack ? err.stack.split('\n') : []; + if (stack[0] && stack[0].includes(err.message)) { + stack.shift(); + } - if (output.level() < 3) { - stack = stack.slice(0, 3); - } + if (output.level() < 3) { + stack = stack.slice(0, 3); + } - err.stack = `${stack.join('\n')}\n\n${output.colors.blue(log)}`; + err.stack = `${stack.join('\n')}\n\n${output.colors.blue(log)}`; - // clone err object so stack trace adjustments won't affect test other reports - test.err = err; - return test; + // clone err object so stack trace adjustments won't affect test other reports + test.err = err; + return test; + } catch (e) { + throw Error(e); + } }); const originalLog = Base.consoleLog; diff --git a/lib/command/gherkin/init.js b/lib/command/gherkin/init.js index b0aabe4a8..a909c27bb 100644 --- a/lib/command/gherkin/init.js +++ b/lib/command/gherkin/init.js @@ -4,7 +4,7 @@ const mkdirp = require('mkdirp'); const output = require('../../output'); const { fileExists } = require('../../utils'); const { - getConfig, getTestRoot, updateConfig, safeFileWrite, + getConfig, getTestRoot, updateConfig, safeFileWrite, findConfigFile, } = require('../utils'); const featureFile = `Feature: Business rules @@ -26,7 +26,17 @@ Given('I have a defined step', () => { module.exports = function (genPath) { const testsPath = getTestRoot(genPath); + const configFile = findConfigFile(testsPath); + + if (!configFile) { + output.error( + "Can't initialize Gherkin. This command must be run in an already initialized project.", + ); + process.exit(1); + } + const config = getConfig(testsPath); + const extension = path.extname(configFile).substring(1); output.print('Initializing Gherkin (Cucumber BDD) for CodeceptJS'); output.print('--------------------------'); @@ -53,18 +63,18 @@ module.exports = function (genPath) { output.success(`Created ${dir}, place step definitions into it`); } - if (safeFileWrite(path.join(dir, 'steps.js'), stepsFile)) { - output.success('Created sample steps file: step_definitions/steps.js'); + if (safeFileWrite(path.join(dir, `steps.${extension}`), stepsFile)) { + output.success( + `Created sample steps file: step_definitions/steps.${extension}`, + ); } config.gherkin = { features: './features/*.feature', - steps: [ - './step_definitions/steps.js', - ], + steps: [`./step_definitions/steps.${extension}`], }; - updateConfig(testsPath, config); + updateConfig(testsPath, config, extension); output.success('Gherkin setup is done.'); output.success('Start writing feature files and implement corresponding steps.'); diff --git a/lib/command/utils.js b/lib/command/utils.js index ac3cf7fd6..60ceee8a1 100644 --- a/lib/command/utils.js +++ b/lib/command/utils.js @@ -41,15 +41,15 @@ function fail(msg) { module.exports.fail = fail; -function updateConfig(testsPath, config, key, extension = 'js') { +function updateConfig(testsPath, config, extension) { const configFile = path.join(testsPath, `codecept.conf.${extension}`); if (!fileExists(configFile)) { - console.log(); const msg = `codecept.conf.${extension} config can\'t be updated automatically`; + console.log(); console.log(`${output.colors.bold.red(msg)}`); - console.log('Please update it manually:'); + console.log(`${output.colors.bold.red('Please update it manually:')}`); console.log(); - console.log(`${key}: ${config[key]}`); + console.log(config); console.log(); return; } @@ -104,3 +104,14 @@ module.exports.createOutputDir = (config, testRoot) => { mkdirp.sync(outputDir); } }; + +module.exports.findConfigFile = (testsPath) => { + const extensions = ['js', 'ts']; + for (const ext of extensions) { + const configFile = path.join(testsPath, `codecept.conf.${ext}`); + if (fileExists(configFile)) { + return configFile; + } + } + return null; +}; diff --git a/lib/helper/Expect.js b/lib/helper/ExpectHelper.js similarity index 99% rename from lib/helper/Expect.js rename to lib/helper/ExpectHelper.js index c9de3693c..c6a9dd27d 100644 --- a/lib/helper/Expect.js +++ b/lib/helper/ExpectHelper.js @@ -23,7 +23,7 @@ import('chai').then(chai => { *{ * helpers: { * Playwright: {...}, - * Expect: {}, + * ExpectHelper: {}, * } *} * ``` diff --git a/lib/helper/Playwright.js b/lib/helper/Playwright.js index 697e27561..d03e3af2b 100644 --- a/lib/helper/Playwright.js +++ b/lib/helper/Playwright.js @@ -1981,7 +1981,7 @@ class Playwright extends Helper { */ async dontSeeCookie(name) { const cookies = await this.browserContext.cookies(); - empty(`cookie ${name} to be set`).assert(cookies.filter(c => c.name === name)); + empty(`cookie ${name} not to be set`).assert(cookies.filter(c => c.name === name)); } /** diff --git a/lib/helper/Puppeteer.js b/lib/helper/Puppeteer.js index 2bb8edac8..2ae747dea 100644 --- a/lib/helper/Puppeteer.js +++ b/lib/helper/Puppeteer.js @@ -1627,7 +1627,7 @@ class Puppeteer extends Helper { */ async dontSeeCookie(name) { const cookies = await this.page.cookies(); - empty(`cookie ${name} to be set`).assert(cookies.filter(c => c.name === name)); + empty(`cookie ${name} not to be set`).assert(cookies.filter(c => c.name === name)); } /** @@ -2472,12 +2472,12 @@ class Puppeteer extends Helper { } /** - * Mocks network request using [`Request Interception`](https://pptr.dev/next/guides/request-interception) + * Mocks network request using [`Request Interception`](https://pptr.dev/guides/network-interception) * * ```js * I.mockRoute(/(\.png$)|(\.jpg$)/, route => route.abort()); * ``` - * This method allows intercepting and mocking requests & responses. [Learn more about it](https://pptr.dev/next/guides/request-interception) + * This method allows intercepting and mocking requests & responses. [Learn more about it](https://pptr.dev/guides/network-interception) * * @param {string|RegExp} [url] URL, regex or pattern for to match URL * @param {function} [handler] a function to process request diff --git a/lib/helper/WebDriver.js b/lib/helper/WebDriver.js index 82182b3a7..703f9a2be 100644 --- a/lib/helper/WebDriver.js +++ b/lib/helper/WebDriver.js @@ -642,6 +642,7 @@ class WebDriver extends Helper { if (this.options.automationProtocol) { this.puppeteerBrowser = await this.browser.getPuppeteer(); + this.page = (await this.puppeteerBrowser.pages())[0]; } return this.browser; @@ -2731,7 +2732,6 @@ class WebDriver extends Helper { this.recording = true; this.recordedAtLeastOnce = true; - this.page = (await this.puppeteerBrowser.pages())[0]; await this.page.setRequestInterception(true); this.page.on('request', (request) => { diff --git a/lib/plugin/coverage.js b/lib/plugin/coverage.js index 65a6bc22e..176f8cf1d 100644 --- a/lib/plugin/coverage.js +++ b/lib/plugin/coverage.js @@ -11,7 +11,7 @@ const defaultConfig = { outputDir: 'output/coverage', }; -const supportedHelpers = ['Puppeteer', 'Playwright']; +const supportedHelpers = ['Puppeteer', 'Playwright', 'WebDriver']; const v8CoverageHelpers = { Playwright: { @@ -61,6 +61,33 @@ const v8CoverageHelpers = { await coverageReport.add(coverageList); }, }, + WebDriver: { + startCoverage: async (page) => { + await Promise.all([ + page.coverage.startJSCoverage({ + resetOnNavigation: false, + includeRawScriptCoverage: true, + }), + page.coverage.startCSSCoverage({ + resetOnNavigation: false, + }), + ]); + }, + takeCoverage: async (page, coverageReport) => { + const [jsCoverage, cssCoverage] = await Promise.all([ + page.coverage.stopJSCoverage(), + page.coverage.stopCSSCoverage(), + ]); + // to raw V8 script coverage + const coverageList = [...jsCoverage.map((it) => { + return { + source: it.text, + ...it.rawScriptCoverage, + }; + }), ...cssCoverage]; + await coverageReport.add(coverageList); + }, + }, }; /** @@ -109,11 +136,17 @@ module.exports = function (config) { const debug = debugModule(`codeceptjs:plugin:${helperName.toLowerCase()}Coverage`); const helper = helpers[helperName]; + + if (helperName === 'WebDriver' && !helper.config.devtoolsProtocol) throw Error('Coverage is currently supporting the WebDriver running with Devtools protocol.'); + const v8Helper = v8CoverageHelpers[helperName]; const coverageOptions = { ...config, }; + + if (helperName === 'WebDriver') coverageOptions.coverageProvider = 'v8'; + const coverageReport = new CoverageReport(coverageOptions); coverageReport.cleanCache(); diff --git a/lib/plugin/retryTo.js b/lib/plugin/retryTo.js index 9929944f1..4c93b56c9 100644 --- a/lib/plugin/retryTo.js +++ b/lib/plugin/retryTo.js @@ -1,5 +1,4 @@ const recorder = require('../recorder'); -const store = require('../store'); const { debug } = require('../output'); const defaultConfig = { @@ -73,49 +72,56 @@ const defaultConfig = { * const retryTo = codeceptjs.container.plugins('retryTo'); * ``` * -*/ + */ module.exports = function (config) { config = Object.assign(defaultConfig, config); + function retryTo(callback, maxTries, pollInterval = config.pollInterval) { + return new Promise((done, reject) => { + let tries = 1; - if (config.registerGlobal) { - global.retryTo = retryTo; - } - return retryTo; + function handleRetryException(err) { + recorder.throw(err); + reject(err); + } - function retryTo(callback, maxTries, pollInterval = undefined) { - let tries = 1; - if (!pollInterval) pollInterval = config.pollInterval; - - let err = null; - - return new Promise((done) => { const tryBlock = async () => { + tries++; recorder.session.start(`retryTo ${tries}`); - await callback(tries); + try { + await callback(tries); + } catch (err) { + handleRetryException(err); + } + + // Call done if no errors recorder.add(() => { recorder.session.restore(`retryTo ${tries}`); done(null); }); - recorder.session.catch((e) => { - err = e; + + // Catch errors and retry + recorder.session.catch((err) => { recorder.session.restore(`retryTo ${tries}`); - tries++; if (tries <= maxTries) { debug(`Error ${err}... Retrying`); - err = null; - recorder.add(`retryTo ${tries}`, () => setTimeout(tryBlock, pollInterval)); } else { - done(null); + // if maxTries reached + handleRetryException(err); } }); }; - recorder.add('retryTo', async () => { - tryBlock(); + recorder.add('retryTo', tryBlock).catch(err => { + console.error('An error occurred:', err); + done(null); }); - }).then(() => { - if (err) recorder.throw(err); }); } + + if (config.registerGlobal) { + global.retryTo = retryTo; + } + + return retryTo; }; diff --git a/lib/plugin/screenshotOnFail.js b/lib/plugin/screenshotOnFail.js index 58ef1948a..501ce2777 100644 --- a/lib/plugin/screenshotOnFail.js +++ b/lib/plugin/screenshotOnFail.js @@ -73,6 +73,10 @@ module.exports = function (config) { } event.dispatcher.on(event.test.failed, (test) => { + if (test.ctx?._runnable.title.includes('hook: ')) { + output.plugin('screenshotOnFail', 'BeforeSuite/AfterSuite do not have any access to the browser, hence it could not take screenshot.'); + return; + } recorder.add('screenshot of failed test', async () => { let fileName = clearString(test.title); const dataType = 'image/png'; diff --git a/lib/plugin/stepByStepReport.js b/lib/plugin/stepByStepReport.js index 52fad61e5..9f22dc576 100644 --- a/lib/plugin/stepByStepReport.js +++ b/lib/plugin/stepByStepReport.js @@ -99,12 +99,12 @@ module.exports = function (config) { currentTest = test; }); - event.dispatcher.on(event.step.failed, persistStep); - - event.dispatcher.on(event.step.after, (step) => { + event.dispatcher.on(event.step.failed, (step) => { recorder.add('screenshot of failed test', async () => persistStep(step), true); }); + event.dispatcher.on(event.step.after, persistStep); + event.dispatcher.on(event.test.passed, (test) => { if (!config.deleteSuccessful) return persist(test); // cleanup @@ -112,8 +112,10 @@ module.exports = function (config) { }); event.dispatcher.on(event.test.failed, (test, err) => { - // BeforeSuite/AfterSuite don't have any access to the browser, hence it could not take screenshot. - if (test.ctx._runnable.title.includes('hook: BeforeSuite')) return; + if (test.ctx._runnable.title.includes('hook: ')) { + output.plugin('stepByStepReport', 'BeforeSuite/AfterSuite do not have any access to the browser, hence it could not take screenshot.'); + return; + } persist(test, err); }); diff --git a/lib/scenario.js b/lib/scenario.js index 40f5759a1..ab816a1ab 100644 --- a/lib/scenario.js +++ b/lib/scenario.js @@ -18,6 +18,16 @@ const injectHook = function (inject, suite) { return recorder.promise(); }; +function makeDoneCallableOnce(done) { + let called = false; + return function (err) { + if (called) { + return; + } + called = true; + return done(err); + }; +} /** * Wraps test function, injects support objects from container, * starts promise chain with recorder, performs before/after hooks @@ -34,15 +44,17 @@ module.exports.test = (test) => { test.async = true; test.fn = function (done) { + const doneFn = makeDoneCallableOnce(done); recorder.errHandler((err) => { recorder.session.start('teardown'); recorder.cleanAsyncErr(); - if (test.throws) { // check that test should actually fail + if (test.throws) { + // check that test should actually fail try { assertThrown(err, test.throws); event.emit(event.test.passed, test); event.emit(event.test.finished, test); - recorder.add(() => done()); + recorder.add(doneFn); return; } catch (newErr) { err = newErr; @@ -50,40 +62,26 @@ module.exports.test = (test) => { } event.emit(event.test.failed, test, err); event.emit(event.test.finished, test); - recorder.add(() => done(err)); + recorder.add(() => doneFn(err)); }); if (isAsyncFunction(testFn)) { event.emit(event.test.started, test); - - const catchError = e => { - recorder.throw(e); - recorder.catch((e) => { - const err = (recorder.getAsyncErr() === null) ? e : recorder.getAsyncErr(); - recorder.session.start('teardown'); - recorder.cleanAsyncErr(); - event.emit(event.test.failed, test, err); - event.emit(event.test.finished, test); - recorder.add(() => done(err)); + testFn + .call(test, getInjectedArguments(testFn, test)) + .then(() => { + recorder.add('fire test.passed', () => { + event.emit(event.test.passed, test); + event.emit(event.test.finished, test); + }); + recorder.add('finish test', doneFn); + }) + .catch((err) => { + recorder.throw(err); + }) + .finally(() => { + recorder.catch(); }); - }; - - let injectedArguments; - try { - injectedArguments = getInjectedArguments(testFn, test); - } catch (e) { - catchError(e); - return; - } - - testFn.call(test, injectedArguments).then(() => { - recorder.add('fire test.passed', () => { - event.emit(event.test.passed, test); - event.emit(event.test.finished, test); - }); - recorder.add('finish test', () => done()); - recorder.catch(); - }).catch(catchError); return; } @@ -97,7 +95,7 @@ module.exports.test = (test) => { event.emit(event.test.passed, test); event.emit(event.test.finished, test); }); - recorder.add('finish test', () => done()); + recorder.add('finish test', doneFn); recorder.catch(); } }; @@ -109,13 +107,14 @@ module.exports.test = (test) => { */ module.exports.injected = function (fn, suite, hookName) { return function (done) { + const doneFn = makeDoneCallableOnce(done); const errHandler = (err) => { recorder.session.start('teardown'); recorder.cleanAsyncErr(); event.emit(event.test.failed, suite, err); if (hookName === 'after') event.emit(event.test.after, suite); if (hookName === 'afterSuite') event.emit(event.suite.after, suite); - recorder.add(() => done(err)); + recorder.add(() => doneFn(err)); }; recorder.errHandler((err) => { @@ -137,28 +136,32 @@ module.exports.injected = function (fn, suite, hookName) { const opts = suite.opts || {}; const retries = opts[`retry${ucfirst(hookName)}`] || 0; - promiseRetry(async (retry, number) => { - try { - recorder.startUnlessRunning(); - await fn.call(this, getInjectedArguments(fn)); - await recorder.promise().catch(err => retry(err)); - } catch (err) { - retry(err); - } finally { - if (number < retries) { - recorder.stop(); - recorder.start(); + promiseRetry( + async (retry, number) => { + try { + recorder.startUnlessRunning(); + await fn.call(this, getInjectedArguments(fn)); + await recorder.promise().catch((err) => retry(err)); + } catch (err) { + retry(err); + } finally { + if (number < retries) { + recorder.stop(); + recorder.start(); + } } - } - }, { retries }) + }, + { retries }, + ) .then(() => { recorder.add('fire hook.passed', () => event.emit(event.hook.passed, suite)); - recorder.add(`finish ${hookName} hook`, () => done()); + recorder.add(`finish ${hookName} hook`, doneFn); recorder.catch(); - }).catch((e) => { + }) + .catch((e) => { recorder.throw(e); recorder.catch((e) => { - const err = (recorder.getAsyncErr() === null) ? e : recorder.getAsyncErr(); + const err = recorder.getAsyncErr() === null ? e : recorder.getAsyncErr(); errHandler(err); }); recorder.add('fire hook.failed', () => event.emit(event.hook.failed, suite, e)); diff --git a/package.json b/package.json index 12966061d..f07fed24a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codeceptjs", - "version": "3.6.2", + "version": "3.6.3", "description": "Supercharged End 2 End Testing Framework for NodeJS", "keywords": [ "acceptance", @@ -76,8 +76,8 @@ "@xmldom/xmldom": "0.8.10", "acorn": "8.11.3", "arrify": "2.0.1", - "axios": "1.6.7", - "chai": "5.1.0", + "axios": "1.7.2", + "chai": "5.1.1", "chai-deep-match": "1.2.1", "chai-exclude": "2.1.0", "chai-json-schema": "1.5.1", @@ -89,7 +89,7 @@ "cross-spawn": "7.0.3", "css-to-xpath": "0.1.0", "csstoxpath": "1.6.0", - "devtools": "8.33.1", + "devtools": "8.38.2", "envinfo": "7.11.1", "escape-string-regexp": "4.0.0", "figures": "3.2.0", @@ -104,10 +104,10 @@ "lodash.merge": "4.6.2", "mkdirp": "1.0.4", "mocha": "10.4.0", - "monocart-coverage-reports": "2.7.4", + "monocart-coverage-reports": "2.8.3", "ms": "2.1.3", "ora-classic": "5.4.2", - "pactum": "3.6.7", + "pactum": "3.6.9", "parse-function": "5.6.10", "parse5": "7.1.2", "promise-retry": "1.1.1", @@ -116,7 +116,7 @@ "uuid": "9.0" }, "optionalDependencies": { - "@codeceptjs/detox-helper": "1.0.5" + "@codeceptjs/detox-helper": "1.0.7" }, "devDependencies": { "@codeceptjs/mock-request": "0.3.1", @@ -131,40 +131,40 @@ "@wdio/utils": "8.36.1", "@xmldom/xmldom": "0.8.10", "apollo-server-express": "2.25.3", - "chai-as-promised": "7.1.1", + "chai-as-promised": "7.1.2", "chai-subset": "1.6.0", "contributor-faces": "1.1.0", "documentation": "12.3.0", - "electron": "30.0.1", - "eslint": "8.56.0", + "electron": "30.1.0", + "eslint": "8.57.0", "eslint-config-airbnb-base": "15.0.0", "eslint-plugin-import": "2.29.1", - "eslint-plugin-mocha": "6.3.0", + "eslint-plugin-mocha": "10.4.3", "expect": "29.7.0", "express": "4.19.2", "graphql": "14.6.0", "husky": "8.0.3", "inquirer-test": "2.0.1", - "jsdoc": "3.6.11", + "jsdoc": "4.0.3", "jsdoc-typeof-plugin": "1.0.0", "json-server": "0.10.1", - "playwright": "1.43.1", - "puppeteer": "22.6.3", + "playwright": "1.44.1", + "puppeteer": "22.10.0", "qrcode-terminal": "0.12.0", "rosie": "2.1.1", "runok": "0.9.3", - "sinon": "17.0.1", + "sinon": "18.0.0", "sinon-chai": "3.7.0", "testcafe": "3.5.0", "ts-morph": "21.0.1", "ts-node": "10.9.2", "tsd": "^0.31.0", "tsd-jsdoc": "2.5.0", - "typedoc": "0.25.12", - "typedoc-plugin-markdown": "3.17.1", - "typescript": "5.3.3", + "typedoc": "0.25.13", + "typedoc-plugin-markdown": "4.0.3", + "typescript": "5.4.5", "wdio-docker-service": "1.5.0", - "webdriverio": "8.36.1", + "webdriverio": "8.38.2", "xml2js": "0.6.2", "xpath": "0.0.34" }, diff --git a/test/acceptance/codecept.Playwright.coverage.js b/test/acceptance/codecept.Playwright.coverage.js index f1c57c214..11017f986 100644 --- a/test/acceptance/codecept.Playwright.coverage.js +++ b/test/acceptance/codecept.Playwright.coverage.js @@ -23,7 +23,7 @@ module.exports.config = { require: '../support/ScreenshotSessionHelper.js', outputPath: 'test/acceptance/output', }, - Expect: {}, + ExpectHelper: {}, }, include: {}, bootstrap: false, diff --git a/test/acceptance/codecept.Playwright.js b/test/acceptance/codecept.Playwright.js index d45a49f50..e4dca9ded 100644 --- a/test/acceptance/codecept.Playwright.js +++ b/test/acceptance/codecept.Playwright.js @@ -23,7 +23,7 @@ module.exports.config = { require: '../support/ScreenshotSessionHelper.js', outputPath: 'test/acceptance/output', }, - Expect: {}, + ExpectHelper: {}, }, include: {}, bootstrap: false, diff --git a/test/acceptance/codecept.Playwright.retryTo.js b/test/acceptance/codecept.Playwright.retryTo.js index d45a49f50..e4dca9ded 100644 --- a/test/acceptance/codecept.Playwright.retryTo.js +++ b/test/acceptance/codecept.Playwright.retryTo.js @@ -23,7 +23,7 @@ module.exports.config = { require: '../support/ScreenshotSessionHelper.js', outputPath: 'test/acceptance/output', }, - Expect: {}, + ExpectHelper: {}, }, include: {}, bootstrap: false, diff --git a/test/acceptance/codecept.Puppeteer.js b/test/acceptance/codecept.Puppeteer.js index d18087620..8c2a8e987 100644 --- a/test/acceptance/codecept.Puppeteer.js +++ b/test/acceptance/codecept.Puppeteer.js @@ -19,7 +19,7 @@ module.exports.config = { require: '../support/ScreenshotSessionHelper.js', outputPath: './output', }, - Expect: {}, + ExpectHelper: {}, }, include: {}, bootstrap: false, diff --git a/test/acceptance/codecept.Testcafe.js b/test/acceptance/codecept.Testcafe.js index 1191bb18a..13b529103 100644 --- a/test/acceptance/codecept.Testcafe.js +++ b/test/acceptance/codecept.Testcafe.js @@ -9,7 +9,7 @@ module.exports.config = { url: TestHelper.siteUrl(), show: true, }, - Expect: {}, + ExpectHelper: {}, }, include: {}, bootstrap: false, diff --git a/test/acceptance/codecept.WebDriver.devtools.coverage.js b/test/acceptance/codecept.WebDriver.devtools.coverage.js new file mode 100644 index 000000000..6fb8f5aaa --- /dev/null +++ b/test/acceptance/codecept.WebDriver.devtools.coverage.js @@ -0,0 +1,49 @@ +const TestHelper = require('../support/TestHelper'); + +module.exports.config = { + tests: './*_test.js', + timeout: 10000, + output: './output', + helpers: { + WebDriver: { + url: TestHelper.siteUrl(), + browser: 'Chromium', + windowSize: '500x700', + devtoolsProtocol: true, + waitForTimeout: 5000, + capabilities: { + chromeOptions: { + args: ['--headless', '--disable-gpu', '--window-size=500,700'], + }, + }, + }, + ScreenshotSessionHelper: { + require: '../support/ScreenshotSessionHelper.js', + outputPath: './output', + }, + ExpectHelper: {}, + }, + include: {}, + mocha: {}, + name: 'acceptance', + plugins: { + screenshotOnFail: { + enabled: true, + }, + coverage: { + enabled: true, + debug: true, + name: 'CodeceptJS Coverage Report', + sourceFilter: '**/src/**', + sourcePath: { + 'todomvc-react/': '', + 'todomvc.com/examples/react/': '', + }, + outputDir: 'output/coverage', + }, + }, + gherkin: { + features: './gherkin/*.feature', + steps: ['./gherkin/steps.js'], + }, +}; diff --git a/test/acceptance/codecept.WebDriver.devtools.js b/test/acceptance/codecept.WebDriver.devtools.js index fc749d22e..f86312789 100644 --- a/test/acceptance/codecept.WebDriver.devtools.js +++ b/test/acceptance/codecept.WebDriver.devtools.js @@ -21,7 +21,7 @@ module.exports.config = { require: '../support/ScreenshotSessionHelper.js', outputPath: './output', }, - Expect: {}, + ExpectHelper: {}, }, include: {}, bootstrap: async () => new Promise(done => { diff --git a/test/acceptance/codecept.WebDriver.js b/test/acceptance/codecept.WebDriver.js index fd7af8b22..54dd4e3e4 100644 --- a/test/acceptance/codecept.WebDriver.js +++ b/test/acceptance/codecept.WebDriver.js @@ -21,7 +21,7 @@ module.exports.config = { require: '../support/ScreenshotSessionHelper.js', outputPath: './output', }, - Expect: {}, + ExpectHelper: {}, }, include: {}, bootstrap: async () => new Promise(done => { diff --git a/test/acceptance/retryTo_test.js b/test/acceptance/retryTo_test.js index b568e8607..6253fdb80 100644 --- a/test/acceptance/retryTo_test.js +++ b/test/acceptance/retryTo_test.js @@ -14,3 +14,23 @@ Scenario('retryTo works with non await steps @plugin', async () => { if (tryNum < 3) I.waitForVisible('.nothing', 1); }, 4); }); + +Scenario('Should be succeed', async ({ I }) => { + I.amOnPage('http://example.org'); + I.waitForVisible('.nothing', 1); // should fail here but it won't terminate + await retryTo((tryNum) => { + I.see('.doesNotMatter'); + }, 10); +}); + +Scenario('Should fail after reached max retries', async () => { + await retryTo(() => { + throw new Error('Custom pluginRetryTo Error'); + }, 3); +}); + +Scenario('Should succeed at the third attempt @plugin', async () => { + await retryTo(async (tryNum) => { + if (tryNum < 2) throw new Error('Custom pluginRetryTo Error'); + }, 3); +}); diff --git a/test/data/sandbox/codecept.scenario-stale.js b/test/data/sandbox/codecept.scenario-stale.js new file mode 100644 index 000000000..f07399b38 --- /dev/null +++ b/test/data/sandbox/codecept.scenario-stale.js @@ -0,0 +1,10 @@ +exports.config = { + tests: './test.scenario-stale.js', + timeout: 10000, + retry: 2, + output: './output', + include: {}, + bootstrap: false, + mocha: {}, + name: 'sandbox', +}; diff --git a/test/data/sandbox/configs/gherkin/config_js/codecept.conf.init.js b/test/data/sandbox/configs/gherkin/config_js/codecept.conf.init.js new file mode 100644 index 000000000..4e93e8435 --- /dev/null +++ b/test/data/sandbox/configs/gherkin/config_js/codecept.conf.init.js @@ -0,0 +1,16 @@ +/** @type {CodeceptJS.MainConfig} */ +exports.config = { + tests: './*_test.js', + output: './output', + helpers: { + Playwright: { + browser: 'chromium', + url: 'http://localhost', + show: true, + }, + }, + include: { + I: './steps_file.js', + }, + name: 'CodeceptJS', +}; diff --git a/test/data/sandbox/configs/gherkin/config_ts/codecept.conf.init.ts b/test/data/sandbox/configs/gherkin/config_ts/codecept.conf.init.ts new file mode 100644 index 000000000..86fe45f4f --- /dev/null +++ b/test/data/sandbox/configs/gherkin/config_ts/codecept.conf.init.ts @@ -0,0 +1,15 @@ +export const config: CodeceptJS.MainConfig = { + tests: "./*_test.ts", + output: "./output", + helpers: { + Playwright: { + browser: "chromium", + url: "http://localhost", + show: true + } + }, + include: { + I: "./steps_file" + }, + name: "CodeceptJS" +} diff --git a/test/data/sandbox/test.scenario-stale.js b/test/data/sandbox/test.scenario-stale.js new file mode 100644 index 000000000..7dfdf435b --- /dev/null +++ b/test/data/sandbox/test.scenario-stale.js @@ -0,0 +1,22 @@ +Feature('Scenario should not be staling'); + +const SHOULD_NOT_STALE = 'should not stale scenario error'; + +Scenario('Rejected promise should not stale the process', async () => { + await new Promise((_resolve, reject) => { setTimeout(reject(new Error(SHOULD_NOT_STALE)), 500); }); +}); + +Scenario('Should handle throw inside synchronous and terminate gracefully', () => { + throw new Error(SHOULD_NOT_STALE); +}); +Scenario('Should handle throw inside async and terminate gracefully', async () => { + throw new Error(SHOULD_NOT_STALE); +}); + +Scenario('Should throw, retry and keep failing', async () => { + setTimeout(() => { + throw new Error(SHOULD_NOT_STALE); + }, 500); + await new Promise((resolve) => { setTimeout(resolve, 300); }); + throw new Error(SHOULD_NOT_STALE); +}).retry(2); diff --git a/test/helper/Expect_test.js b/test/helper/Expect_test.js index 9567943c0..7db470e01 100644 --- a/test/helper/Expect_test.js +++ b/test/helper/Expect_test.js @@ -5,7 +5,7 @@ import('chai').then(chai => { expect = chai.expect; }); -const ExpectHelper = require('../../lib/helper/Expect'); +const ExpectHelper = require('../../lib/helper/ExpectHelper'); global.codeceptjs = require('../../lib'); diff --git a/test/plugin/plugin_test.js b/test/plugin/plugin_test.js index 4255aeda8..4a4f49f66 100644 --- a/test/plugin/plugin_test.js +++ b/test/plugin/plugin_test.js @@ -32,17 +32,53 @@ describe('CodeceptJS plugin', function () { }); }); + it('should failed before the retryTo instruction', (done) => { + exec(`${config_run_config('codecept.Playwright.retryTo.js', 'Should be succeed')} --verbose`, (err, stdout) => { + expect(stdout).toContain('locator.waitFor: Timeout 1000ms exceeded.'); + expect(stdout).toContain('[1] Error | Error: element (.nothing) still not visible after 1 sec'); + expect(err).toBeTruthy(); + done(); + }); + }); + it('should generate the coverage report', (done) => { exec(`${config_run_config('codecept.Playwright.coverage.js', '@coverage')} --debug`, (err, stdout) => { const lines = stdout.split('\n'); expect(lines).toEqual( expect.arrayContaining([ expect.stringContaining('writing output/coverage'), - expect.stringContaining('generated coverage reports: output/coverage/index.html'), + expect.stringContaining('generated coverage reports:'), + expect.stringContaining('output/coverage/index.html'), + ]), + ); + expect(err).toBeFalsy(); + done(); + }); + }); + + it('should generate the coverage report - WebDriver - Devtools protocol', (done) => { + exec(`${config_run_config('codecept.WebDriver.devtools.coverage.js', '@coverage')} --debug`, (err, stdout) => { + const lines = stdout.split('\n'); + expect(lines).toEqual( + expect.arrayContaining([ + expect.stringContaining('writing output/coverage'), + expect.stringContaining('generated coverage reports:'), + expect.stringContaining('output/coverage/index.html'), ]), ); expect(err).toBeFalsy(); done(); }); }); + + it('should retry to failure', (done) => { + exec(`${config_run_config('codecept.Playwright.retryTo.js', 'Should fail after reached max retries')} --verbose`, (err, stdout) => { + const lines = stdout.split('\n'); + expect(lines).toEqual( + expect.arrayContaining([expect.stringContaining('Custom pluginRetryTo Error')]), + ); + expect(err).toBeTruthy(); + done(); + }); + }); }); diff --git a/test/runner/gherkin_test.js b/test/runner/gherkin_test.js new file mode 100644 index 000000000..77e7070d2 --- /dev/null +++ b/test/runner/gherkin_test.js @@ -0,0 +1,97 @@ +const assert = require('assert'); +const path = require('path'); +const fs = require('fs'); +const exec = require('child_process').exec; + +const runner = path.join(__dirname, '/../../bin/codecept.js'); +const codecept_dir = path.join(__dirname, '/../data/sandbox/configs/gherkin/'); + +describe('gherkin bdd commands', () => { + describe('bdd:init', () => { + const codecept_dir_js = path.join(codecept_dir, 'config_js'); + const codecept_dir_ts = path.join(codecept_dir, 'config_ts'); + + beforeEach(() => { + fs.copyFileSync( + path.join(codecept_dir_js, 'codecept.conf.init.js'), + path.join(codecept_dir_js, 'codecept.conf.js'), + ); + fs.copyFileSync( + path.join(codecept_dir_ts, 'codecept.conf.init.ts'), + path.join(codecept_dir_ts, 'codecept.conf.ts'), + ); + }); + + afterEach(() => { + try { + fs.rmSync(path.join(codecept_dir_js, 'codecept.conf.js')); + fs.rmSync(path.join(codecept_dir_js, 'features'), { + recursive: true, + }); + fs.rmSync(path.join(codecept_dir_js, 'step_definitions'), { + recursive: true, + }); + } catch (e) { + // catch some error + } + try { + fs.rmSync(path.join(codecept_dir_ts, 'codecept.conf.ts')); + fs.rmSync(path.join(codecept_dir_ts, 'features'), { + recursive: true, + }); + fs.rmSync(path.join(codecept_dir_ts, 'step_definitions'), { + recursive: true, + }); + } catch (e) { + // catch some error + } + }); + + [ + { + codecept_dir_test: codecept_dir_js, + extension: 'js', + }, + { + codecept_dir_test: codecept_dir_ts, + extension: 'ts', + }, + ].forEach(({ codecept_dir_test, extension }) => { + it(`prepare CodeceptJS to run feature files (codecept.conf.${extension})`, (done) => { + exec(`${runner} gherkin:init ${codecept_dir_test}`, (err, stdout) => { + let dir = path.join(codecept_dir_test, 'features'); + + stdout.should.include( + 'Initializing Gherkin (Cucumber BDD) for CodeceptJS', + ); + stdout.should.include( + `Created ${dir}, place your *.feature files in it`, + ); + stdout.should.include( + 'Created sample feature file: features/basic.feature', + ); + + dir = path.join(codecept_dir_test, 'step_definitions'); + stdout.should.include( + `Created ${dir}, place step definitions into it`, + ); + stdout.should.include( + `Created sample steps file: step_definitions/steps.${extension}`, + ); + assert(!err); + + const configResult = fs + .readFileSync( + path.join(codecept_dir_test, `codecept.conf.${extension}`), + ) + .toString(); + configResult.should.contain('features: \'./features/*.feature\''); + configResult.should.contain( + `steps: ['./step_definitions/steps.${extension}']`, + ); + done(); + }); + }); + }); + }); +}); diff --git a/test/runner/run_workers_test.js b/test/runner/run_workers_test.js index 0cd0f3b46..ccd1afe99 100644 --- a/test/runner/run_workers_test.js +++ b/test/runner/run_workers_test.js @@ -19,7 +19,6 @@ describe('CodeceptJS Workers Runner', function () { if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version'); console.log(`${codecept_run} 3 --debug`); exec(`${codecept_run} 3 --debug`, (err, stdout) => { - console.log('aaaaaaaaaaaaa', stdout); expect(stdout).toContain('CodeceptJS'); // feature expect(stdout).toContain('glob current dir'); expect(stdout).toContain('From worker @1_grep print message 1'); diff --git a/test/runner/scenario_stale_test.js b/test/runner/scenario_stale_test.js new file mode 100644 index 000000000..927d92acd --- /dev/null +++ b/test/runner/scenario_stale_test.js @@ -0,0 +1,22 @@ +const { expect } = require('expect'); +const path = require('path'); +const { exec } = require('child_process'); + +const runner = path.join(__dirname, '/../../bin/codecept.js'); +const codecept_dir = path.join(__dirname, '/../data/sandbox'); +const codecept_run = `${runner} run`; +const config_run_config = config => `${codecept_run} --config ${codecept_dir}/${config}`; + +describe('Scenario termination check', () => { + before(() => { + process.chdir(codecept_dir); + }); + + it('Should always fail and terminate', (done) => { + exec(config_run_config('codecept.scenario-stale.js'), (err, stdout) => { + expect(stdout).toContain('should not stale scenario error'); // feature + expect(err).toBeTruthy(); + done(); + }); + }); +}); diff --git a/test/runner/timeout_test.js b/test/runner/timeout_test.js index 79d74f054..8e14efc09 100644 --- a/test/runner/timeout_test.js +++ b/test/runner/timeout_test.js @@ -11,7 +11,7 @@ describe('CodeceptJS Timeouts', function () { it('should stop test when timeout exceeded', (done) => { exec(config_run_config('codecept.conf.js', 'timed out'), (err, stdout) => { - console.log(stdout); + debug_this_test && console.log(stdout); expect(stdout).toContain('Timeout 2s exceeded'); expect(stdout).toContain('Timeout 1s exceeded'); expect(err).toBeTruthy(); diff --git a/test/unit/plugin/retryto_test.js b/test/unit/plugin/retryto_test.js index 28187e02c..293adf1d7 100644 --- a/test/unit/plugin/retryto_test.js +++ b/test/unit/plugin/retryto_test.js @@ -20,11 +20,11 @@ describe('retryTo plugin', () => { it('should execute few times command on fail', async () => { let counter = 0; let errorCaught = false; - await retryTo(() => { - recorder.add(() => counter++); - recorder.add(() => { throw new Error('Ups'); }); - }, 5, 10); try { + await retryTo(() => { + recorder.add(() => counter++); + recorder.add(() => { throw new Error('Ups'); }); + }, 5, 10); await recorder.promise(); } catch (err) { errorCaught = true; diff --git a/test/unit/plugin/screenshotOnFail_test.js b/test/unit/plugin/screenshotOnFail_test.js index b2f24a627..fc6b7295f 100644 --- a/test/unit/plugin/screenshotOnFail_test.js +++ b/test/unit/plugin/screenshotOnFail_test.js @@ -64,5 +64,19 @@ describe('screenshotOnFail', () => { const regexpFileName = /test1_[0-9]{10}.failed.png/; expect(fileName.match(regexpFileName).length).is.equal(1); }); + + it('should not save screenshot in BeforeSuite', async () => { + screenshotOnFail({ uniqueScreenshotNames: true }); + event.dispatcher.emit(event.test.failed, { title: 'test1', ctx: { _runnable: { title: 'hook: BeforeSuite' } } }); + await recorder.promise(); + expect(!screenshotSaved.called).is.ok; + }); + + it('should not save screenshot in AfterSuite', async () => { + screenshotOnFail({ uniqueScreenshotNames: true }); + event.dispatcher.emit(event.test.failed, { title: 'test1', ctx: { _runnable: { title: 'hook: AfterSuite' } } }); + await recorder.promise(); + expect(!screenshotSaved.called).is.ok; + }); // TODO: write more tests for different options });