diff --git a/docs/plugins/junitReporter.md b/docs/plugins/junitReporter.md index cfeb34d5e..d1bab1ff5 100644 --- a/docs/plugins/junitReporter.md +++ b/docs/plugins/junitReporter.md @@ -33,7 +33,7 @@ The produced file is consumable by Jenkins, GitLab CI, CircleCI, GitHub Actions Possible config options: * `outputName`: file name for the report. Default: `report.xml`. -* `output`: directory where the report is stored, relative to the project root. Default: the `output` directory. +* `output`: directory where the report is stored, relative to the project root. Default: the `output` directory. With `run-workers` and `multiple` runs, each run writes its own report into its run folder (e.g. `output/dev_chromium1`) and this option is ignored. * `testGroupName`: value of the `name` attribute on the root `` element. Default: `CodeceptJS`. * `attachMeta`: add the test's meta information (`meta` keys, `tags`, `retries`) as ``. Default: true. * `attachSteps`: add the step/substep log as ``. Default: true. diff --git a/lib/codecept.js b/lib/codecept.js index 3166c317e..c6d153cce 100644 --- a/lib/codecept.js +++ b/lib/codecept.js @@ -279,7 +279,7 @@ class Codecept { */ filterLastFailedTests(mocha) { if (!this.opts.lastFailed) return - const failedTestUids = new Set(Result.loadFailedTests().map(test => test.uid)) + const failedTestUids = new Set(Result.loadFailedTests().map(test => test.uid?.split('@')[0])) const filterSuite = suite => { suite.tests = suite.tests.filter(test => failedTestUids.has(test.uid)) suite.suites.forEach(filterSuite) diff --git a/lib/plugin/junitReporter.js b/lib/plugin/junitReporter.js index 2e84d2076..c79768edc 100644 --- a/lib/plugin/junitReporter.js +++ b/lib/plugin/junitReporter.js @@ -1,6 +1,7 @@ import fs from 'fs' import path from 'path' import os from 'os' +import { isMainThread } from 'worker_threads' import { mkdirp } from 'mkdirp' import { DOMImplementation, XMLSerializer } from '@xmldom/xmldom' @@ -45,7 +46,7 @@ const INVALID_XML_CHARS = new RegExp('[\\u0000-\\u0008\\u000B\\u000C\\u000E-\\u0 * Possible config options: * * * `outputName`: file name for the report. Default: `report.xml`. - * * `output`: directory where the report is stored, relative to the project root. Default: the `output` directory. + * * `output`: directory where the report is stored, relative to the project root. Default: the `output` directory. With `run-workers` and `multiple` runs, each run writes its own report into its run folder (e.g. `output/dev_chromium1`) and this option is ignored. * * `testGroupName`: value of the `name` attribute on the root `` element. Default: `CodeceptJS`. * * `attachMeta`: add the test's meta information (`meta` keys, `tags`, `retries`) as ``. Default: true. * * `attachSteps`: add the step/substep log as ``. Default: true. @@ -100,19 +101,35 @@ export default function (config = {}) { meta: {}, steps: [], duration: (runnable && runnable.duration) || 0, + outputDir: hook.outputDir, }) }) const writeReport = result => { if (written) return if (!result || !Array.isArray(result.tests)) return + if (!isMainThread) return written = true + const runDirs = [...new Set(result.tests.map(test => test.outputDir).filter(Boolean))] + if (runDirs.length) { + for (const dir of runDirs) { + const runResult = { tests: result.tests.filter(test => test.outputDir === dir), stats: result.stats } + const runHookFailures = hookFailures.filter(hf => hf.outputDir === dir) + saveXml(dir, buildXml(runResult, config, runHookFailures)) + } + return + } + const dir = config.output ? path.resolve(store.codeceptDir || process.cwd(), config.output) : store.outputDir || process.cwd() + saveXml(dir, buildXml(result, config, hookFailures)) + } + + const saveXml = (dir, xml) => { mkdirp.sync(dir) const file = path.join(dir, config.outputName) - fs.writeFileSync(file, buildXml(result, config, hookFailures)) + fs.writeFileSync(file, xml) output.plugin('junitReporter', `JUnit report saved to ${file}`) } diff --git a/lib/workers.js b/lib/workers.js index 25119512d..7bf6034b9 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -104,9 +104,22 @@ const createWorker = (workerObject, isPoolMode = false, skipTests = []) => { }) WorkerStorage.addWorker(worker) + worker.runOutputDir = workerObject.runOutputDir return worker } +const tagWithRun = (message, runOutputDir) => { + if (!message.event || !message.data || typeof message.data !== 'object') return + const runName = path.basename(runOutputDir) + const tag = obj => { + if (!obj || typeof obj !== 'object' || obj.outputDir) return + obj.outputDir = runOutputDir + if (obj.uid) obj.uid = `${obj.uid}@${runName}` + } + tag(message.data) + if (Array.isArray(message.data.tests)) message.data.tests.forEach(tag) +} + const simplifyObject = object => { return Object.keys(object) .filter(k => k.indexOf('_') !== 0) @@ -169,6 +182,7 @@ const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns workerObj.addTests(tests) workerObj.setTestRoot(testRoot) workerObj.addOptions(options) + workerObj.runOutputDir = path.resolve(testRoot, finalConfig.output) testWorkerArray.push(workerObj) }) workers.push(...testWorkerArray) @@ -659,6 +673,7 @@ class Workers extends EventEmitter { worker.on('message', message => { lastActivity = Date.now() // Update activity timestamp + if (worker.runOutputDir) tagWithRun(message, worker.runOutputDir) // Track current test if (message.event === event.test.started && message.data) { diff --git a/test/data/sandbox/codecept.workers-junit-multiple.conf.js b/test/data/sandbox/codecept.workers-junit-multiple.conf.js new file mode 100644 index 000000000..cea2bbdfe --- /dev/null +++ b/test/data/sandbox/codecept.workers-junit-multiple.conf.js @@ -0,0 +1,24 @@ +export const config = { + tests: './workers-multiple-browsers/*.js', + timeout: 10000, + output: './output/junit-multiple', + helpers: { + FakeDriver: { + require: '../fake_driver', + browser: 'dummy', + }, + }, + include: {}, + mocha: {}, + multiple: { + dev: { + browsers: ['chromium', 'webkit'], + }, + }, + plugins: { + junitReporter: { + enabled: true, + }, + }, + name: 'sandbox', +} diff --git a/test/data/sandbox/codecept.workers-testomatio-multiple.conf.js b/test/data/sandbox/codecept.workers-testomatio-multiple.conf.js new file mode 100644 index 000000000..78df8cecd --- /dev/null +++ b/test/data/sandbox/codecept.workers-testomatio-multiple.conf.js @@ -0,0 +1,25 @@ +export const config = { + tests: './workers-multiple-browsers/*.js', + timeout: 10000, + output: './output/testomatio-multiple', + helpers: { + FakeDriver: { + require: '../fake_driver', + browser: 'dummy', + }, + }, + include: {}, + mocha: {}, + multiple: { + dev: { + browsers: ['chromium', 'webkit'], + }, + }, + plugins: { + testomatio: { + enabled: true, + require: '@testomatio/reporter/codecept', + }, + }, + name: 'sandbox', +} diff --git a/test/data/sandbox/configs/last-failed/codecept.conf.js b/test/data/sandbox/configs/last-failed/codecept.conf.js index 5df0d76ac..cfff2bf86 100644 --- a/test/data/sandbox/configs/last-failed/codecept.conf.js +++ b/test/data/sandbox/configs/last-failed/codecept.conf.js @@ -5,5 +5,10 @@ export const config = { include: {}, bootstrap: null, mocha: {}, + multiple: { + dev: { + browsers: ['chromium', 'webkit'], + }, + }, name: 'last-failed', } diff --git a/test/data/sandbox/workers-multiple-browsers/suite_a.worker.js b/test/data/sandbox/workers-multiple-browsers/suite_a.worker.js new file mode 100644 index 000000000..17c430c4d --- /dev/null +++ b/test/data/sandbox/workers-multiple-browsers/suite_a.worker.js @@ -0,0 +1,9 @@ +Feature('MultipleBrowsersSuiteA') + +Scenario('scenario 1', ({ I }) => { + I.printBrowser() +}) + +Scenario('scenario 2', ({ I }) => { + I.printBrowser() +}) diff --git a/test/data/sandbox/workers-multiple-browsers/suite_b.worker.js b/test/data/sandbox/workers-multiple-browsers/suite_b.worker.js new file mode 100644 index 000000000..2c0f545dd --- /dev/null +++ b/test/data/sandbox/workers-multiple-browsers/suite_b.worker.js @@ -0,0 +1,9 @@ +Feature('MultipleBrowsersSuiteB') + +Scenario('scenario 1', ({ I }) => { + I.printBrowser() +}) + +Scenario('scenario 2', ({ I }) => { + I.printBrowser() +}) diff --git a/test/runner/last_failed_test.js b/test/runner/last_failed_test.js index d9dd1f386..99467af62 100644 --- a/test/runner/last_failed_test.js +++ b/test/runner/last_failed_test.js @@ -119,6 +119,17 @@ describe('--last-failed', function () { expect(stdout).not.toContain('multiplies numbers') }) + it('should rerun only failed tests in workers with multiple browsers', async () => { + const fullRun = await run('run-workers 2 dev') + expect(fullRun.stdout).toContain('4 passed, 6 failed') + + const { code, stdout } = await run('run-workers 2 dev --last-failed') + expect(code).toEqual(1) + expect(stdout).toContain('0 passed, 6 failed') + expect(stdout).not.toContain('adds numbers') + expect(stdout).not.toContain('concats strings') + }) + it('should rerun only failed tests in workers by pool', async () => { await run('run') const { stdout } = await run('run-workers 2 --by pool --last-failed') diff --git a/test/runner/run_workers_test.js b/test/runner/run_workers_test.js index 45930d8ad..51cd6a132 100644 --- a/test/runner/run_workers_test.js +++ b/test/runner/run_workers_test.js @@ -3,6 +3,7 @@ chai.should() import { expect } from 'expect' import path from 'path' import fs from 'fs' +import os from 'os' import semver from 'semver' import { exec } from 'child_process' import { fileURLToPath } from 'url' @@ -573,6 +574,60 @@ describe('CodeceptJS Workers Runner', function () { }) }) + it('should write one JUnit report per browser with all suites of that browser in multi-run --by suite', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + const outputDir = path.join(codecept_dir, 'output', 'junit-multiple') + fs.rmSync(outputDir, { recursive: true, force: true }) + + exec(`${codecept_run_glob('codecept.workers-junit-multiple.conf.js')} 2 dev --by suite`, (err, stdout) => { + ;(async () => { + expect(stdout).toContain('Worker 4: 2 tests') + expect(stdout).toMatch(/OK\s+\|\s+8 passed/) + expect(err).toEqual(null) + + expect(fs.existsSync(path.join(outputDir, 'report.xml'))).toEqual(false) + for (const run of ['dev_chromium1', 'dev_webkit1']) { + const reportFile = path.join(outputDir, run, 'report.xml') + expect(fs.existsSync(reportFile)).toEqual(true) + const parsed = await new xml2js.Parser().parseStringPromise(fs.readFileSync(reportFile, 'utf8')) + expect(parsed.testsuites.$.tests).toEqual('4') + const testcases = parsed.testsuites.testsuite.flatMap(s => s.testcase.map(tc => `${s.$.name}: ${tc.$.name}`)).sort() + expect(testcases).toEqual(['MultipleBrowsersSuiteA: scenario 1', 'MultipleBrowsersSuiteA: scenario 2', 'MultipleBrowsersSuiteB: scenario 1', 'MultipleBrowsersSuiteB: scenario 2']) + } + })().then(done, done) + }) + }) + + it('should keep results of all browsers and suites in the Testomatio HTML report in multi-run --by suite', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + const outputDir = path.join(codecept_dir, 'output', 'testomatio-multiple') + fs.rmSync(outputDir, { recursive: true, force: true }) + const htmlDir = path.join(outputDir, 'html') + const nodePath = fs.mkdtempSync(path.join(os.tmpdir(), 'cjs-node-path-')) + fs.symlinkSync(path.join(__dirname, '../..'), path.join(nodePath, 'codeceptjs'), 'dir') + const env = { ...process.env, NODE_PATH: nodePath, TESTOMATIO_HTML_REPORT_SAVE: '1', TESTOMATIO_HTML_REPORT_FOLDER: htmlDir } + delete env.TESTOMATIO + + exec(`${codecept_run_glob('codecept.workers-testomatio-multiple.conf.js')} 2 dev --by suite`, { env }, (err, stdout) => { + fs.rmSync(nodePath, { recursive: true, force: true }) + expect(stdout).toMatch(/OK\s+\|\s+8 passed/) + expect(err).toEqual(null) + + const reports = fs.readdirSync(outputDir, { recursive: true }).filter(file => file.endsWith('testomatio-report.html')) + expect(reports).toHaveLength(1) + const html = fs.readFileSync(path.join(outputDir, reports[0]), 'utf8') + const entries = new Set([...html.matchAll(/"rid":"[^"]*@(dev_\w+)"[^{}]*?"title":"([^"]*)","suite_title":"([^"]*)"/g)].map(([, run, title, suite]) => `${run} ${suite}: ${title}`)) + const expected = [] + for (const run of ['dev_chromium1', 'dev_webkit1']) { + for (const suite of ['MultipleBrowsersSuiteA', 'MultipleBrowsersSuiteB']) { + expected.push(`${run} ${suite}: scenario 1`, `${run} ${suite}: scenario 2`) + } + } + expect([...entries].sort()).toEqual(expected) + done() + }) + }) + it('should handle large worker count without inflating statistics', function (done) { if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') // Test with more workers than tests to ensure no inflation diff --git a/test/unit/junitReporter_test.js b/test/unit/junitReporter_test.js index feef9efbc..ff94476d8 100644 --- a/test/unit/junitReporter_test.js +++ b/test/unit/junitReporter_test.js @@ -253,4 +253,40 @@ describe('JUnit Reporter Plugin', () => { expect(suiteEl.testcase[0].$.name).to.contain('"after all" hook: AfterSuite') expect(suiteEl.testcase[0].failure[0].$.message).to.contain('AfterSuite failed') }) + + it('writes one report per run output folder when tests carry outputDir', async () => { + const chromiumDir = path.join(tmpDir, 'dev_chromium1') + const webkitDir = path.join(tmpDir, 'dev_webkit1') + const result = makeResult() + const [passed, failed, skipped] = result.tests + result.tests = [ + { ...passed, uid: 'pass-1@dev_chromium1', outputDir: chromiumDir }, + { ...failed, uid: 'fail-1@dev_chromium1', outputDir: chromiumDir }, + { ...passed, uid: 'pass-1@dev_webkit1', outputDir: webkitDir }, + { ...skipped, uid: 'skip-1@dev_webkit1', outputDir: webkitDir }, + ] + + junitReporter({}) + event.dispatcher.emit(event.hook.failed, { + hookName: 'AfterSuite', + title: '"after all" hook: AfterSuite for "logs in successfully"', + error: { message: 'AfterSuite failed', stack: 'Error: AfterSuite failed' }, + outputDir: webkitDir, + }) + event.dispatcher.emit(event.workers.result, result) + + assert.isFalse(fs.existsSync(path.join(tmpDir, 'report.xml')), 'no combined report should be written') + + const chromium = await parseReport(chromiumDir) + expect(chromium.testsuites.$.tests).to.equal('2') + expect(chromium.testsuites.$.failures).to.equal('1') + expect(chromium.testsuites.$.skipped).to.equal('0') + + const webkit = await parseReport(webkitDir) + expect(webkit.testsuites.$.tests).to.equal('3') + expect(webkit.testsuites.$.failures).to.equal('1') + expect(webkit.testsuites.$.skipped).to.equal('1') + const names = webkit.testsuites.testsuite.flatMap(s => s.testcase.map(tc => tc.$.name)) + expect(names.some(n => n.includes('AfterSuite'))).to.equal(true) + }) })