From 50829bb0c75ccb9b671f52731c39821b384acd29 Mon Sep 17 00:00:00 2001 From: Jaromir Obr Date: Wed, 21 Oct 2026 14:14:17 +0200 Subject: [PATCH 1/2] fix(workers): keep results of each browser apart in multi-browser run-workers (#5411) In `run-workers --by suite` with several browsers, the same test had the same uid in every browser, so the main process merged their results, and every worker wrote its partial report.xml into the shared run folder. - Tag every message from a multi-run worker with the run's output folder and make the test uid unique per run - junitReporter: write reports only in the main process, one per run folder - --last-failed: ignore the run suffix when matching saved uids Co-Authored-By: Claude Opus 5.5 --- docs/plugins/junitReporter.md | 2 +- lib/codecept.js | 2 +- lib/plugin/junitReporter.js | 21 +++++++- lib/workers.js | 15 ++++++ .../codecept.workers-junit-multiple.conf.js | 24 +++++++++ ...decept.workers-testomatio-multiple.conf.js | 25 +++++++++ .../configs/last-failed/codecept.conf.js | 5 ++ .../suite_a.worker.js | 9 ++++ .../suite_b.worker.js | 9 ++++ test/runner/last_failed_test.js | 11 ++++ test/runner/run_workers_test.js | 53 +++++++++++++++++++ test/unit/junitReporter_test.js | 36 +++++++++++++ 12 files changed, 208 insertions(+), 4 deletions(-) create mode 100644 test/data/sandbox/codecept.workers-junit-multiple.conf.js create mode 100644 test/data/sandbox/codecept.workers-testomatio-multiple.conf.js create mode 100644 test/data/sandbox/workers-multiple-browsers/suite_a.worker.js create mode 100644 test/data/sandbox/workers-multiple-browsers/suite_b.worker.js diff --git a/docs/plugins/junitReporter.md b/docs/plugins/junitReporter.md index cfeb34d5e..d1bab1ff5 100644 --- a/docs/plugins/junitReporter.md +++ b/docs/plugins/junitReporter.md @@ -33,7 +33,7 @@ The produced file is consumable by Jenkins, GitLab CI, CircleCI, GitHub Actions Possible config options: * `outputName`: file name for the report. Default: `report.xml`. -* `output`: directory where the report is stored, relative to the project root. Default: the `output` directory. +* `output`: directory where the report is stored, relative to the project root. Default: the `output` directory. With `run-workers` and `multiple` runs, each run writes its own report into its run folder (e.g. `output/dev_chromium1`) and this option is ignored. * `testGroupName`: value of the `name` attribute on the root `` element. Default: `CodeceptJS`. * `attachMeta`: add the test's meta information (`meta` keys, `tags`, `retries`) as ``. Default: true. * `attachSteps`: add the step/substep log as ``. Default: true. diff --git a/lib/codecept.js b/lib/codecept.js index 3166c317e..c6d153cce 100644 --- a/lib/codecept.js +++ b/lib/codecept.js @@ -279,7 +279,7 @@ class Codecept { */ filterLastFailedTests(mocha) { if (!this.opts.lastFailed) return - const failedTestUids = new Set(Result.loadFailedTests().map(test => test.uid)) + const failedTestUids = new Set(Result.loadFailedTests().map(test => test.uid?.split('@')[0])) const filterSuite = suite => { suite.tests = suite.tests.filter(test => failedTestUids.has(test.uid)) suite.suites.forEach(filterSuite) diff --git a/lib/plugin/junitReporter.js b/lib/plugin/junitReporter.js index 2e84d2076..c79768edc 100644 --- a/lib/plugin/junitReporter.js +++ b/lib/plugin/junitReporter.js @@ -1,6 +1,7 @@ import fs from 'fs' import path from 'path' import os from 'os' +import { isMainThread } from 'worker_threads' import { mkdirp } from 'mkdirp' import { DOMImplementation, XMLSerializer } from '@xmldom/xmldom' @@ -45,7 +46,7 @@ const INVALID_XML_CHARS = new RegExp('[\\u0000-\\u0008\\u000B\\u000C\\u000E-\\u0 * Possible config options: * * * `outputName`: file name for the report. Default: `report.xml`. - * * `output`: directory where the report is stored, relative to the project root. Default: the `output` directory. + * * `output`: directory where the report is stored, relative to the project root. Default: the `output` directory. With `run-workers` and `multiple` runs, each run writes its own report into its run folder (e.g. `output/dev_chromium1`) and this option is ignored. * * `testGroupName`: value of the `name` attribute on the root `` element. Default: `CodeceptJS`. * * `attachMeta`: add the test's meta information (`meta` keys, `tags`, `retries`) as ``. Default: true. * * `attachSteps`: add the step/substep log as ``. Default: true. @@ -100,19 +101,35 @@ export default function (config = {}) { meta: {}, steps: [], duration: (runnable && runnable.duration) || 0, + outputDir: hook.outputDir, }) }) const writeReport = result => { if (written) return if (!result || !Array.isArray(result.tests)) return + if (!isMainThread) return written = true + const runDirs = [...new Set(result.tests.map(test => test.outputDir).filter(Boolean))] + if (runDirs.length) { + for (const dir of runDirs) { + const runResult = { tests: result.tests.filter(test => test.outputDir === dir), stats: result.stats } + const runHookFailures = hookFailures.filter(hf => hf.outputDir === dir) + saveXml(dir, buildXml(runResult, config, runHookFailures)) + } + return + } + const dir = config.output ? path.resolve(store.codeceptDir || process.cwd(), config.output) : store.outputDir || process.cwd() + saveXml(dir, buildXml(result, config, hookFailures)) + } + + const saveXml = (dir, xml) => { mkdirp.sync(dir) const file = path.join(dir, config.outputName) - fs.writeFileSync(file, buildXml(result, config, hookFailures)) + fs.writeFileSync(file, xml) output.plugin('junitReporter', `JUnit report saved to ${file}`) } diff --git a/lib/workers.js b/lib/workers.js index 25119512d..7bf6034b9 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -104,9 +104,22 @@ const createWorker = (workerObject, isPoolMode = false, skipTests = []) => { }) WorkerStorage.addWorker(worker) + worker.runOutputDir = workerObject.runOutputDir return worker } +const tagWithRun = (message, runOutputDir) => { + if (!message.event || !message.data || typeof message.data !== 'object') return + const runName = path.basename(runOutputDir) + const tag = obj => { + if (!obj || typeof obj !== 'object' || obj.outputDir) return + obj.outputDir = runOutputDir + if (obj.uid) obj.uid = `${obj.uid}@${runName}` + } + tag(message.data) + if (Array.isArray(message.data.tests)) message.data.tests.forEach(tag) +} + const simplifyObject = object => { return Object.keys(object) .filter(k => k.indexOf('_') !== 0) @@ -169,6 +182,7 @@ const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns workerObj.addTests(tests) workerObj.setTestRoot(testRoot) workerObj.addOptions(options) + workerObj.runOutputDir = path.resolve(testRoot, finalConfig.output) testWorkerArray.push(workerObj) }) workers.push(...testWorkerArray) @@ -659,6 +673,7 @@ class Workers extends EventEmitter { worker.on('message', message => { lastActivity = Date.now() // Update activity timestamp + if (worker.runOutputDir) tagWithRun(message, worker.runOutputDir) // Track current test if (message.event === event.test.started && message.data) { diff --git a/test/data/sandbox/codecept.workers-junit-multiple.conf.js b/test/data/sandbox/codecept.workers-junit-multiple.conf.js new file mode 100644 index 000000000..cea2bbdfe --- /dev/null +++ b/test/data/sandbox/codecept.workers-junit-multiple.conf.js @@ -0,0 +1,24 @@ +export const config = { + tests: './workers-multiple-browsers/*.js', + timeout: 10000, + output: './output/junit-multiple', + helpers: { + FakeDriver: { + require: '../fake_driver', + browser: 'dummy', + }, + }, + include: {}, + mocha: {}, + multiple: { + dev: { + browsers: ['chromium', 'webkit'], + }, + }, + plugins: { + junitReporter: { + enabled: true, + }, + }, + name: 'sandbox', +} diff --git a/test/data/sandbox/codecept.workers-testomatio-multiple.conf.js b/test/data/sandbox/codecept.workers-testomatio-multiple.conf.js new file mode 100644 index 000000000..78df8cecd --- /dev/null +++ b/test/data/sandbox/codecept.workers-testomatio-multiple.conf.js @@ -0,0 +1,25 @@ +export const config = { + tests: './workers-multiple-browsers/*.js', + timeout: 10000, + output: './output/testomatio-multiple', + helpers: { + FakeDriver: { + require: '../fake_driver', + browser: 'dummy', + }, + }, + include: {}, + mocha: {}, + multiple: { + dev: { + browsers: ['chromium', 'webkit'], + }, + }, + plugins: { + testomatio: { + enabled: true, + require: '@testomatio/reporter/codecept', + }, + }, + name: 'sandbox', +} diff --git a/test/data/sandbox/configs/last-failed/codecept.conf.js b/test/data/sandbox/configs/last-failed/codecept.conf.js index 5df0d76ac..cfff2bf86 100644 --- a/test/data/sandbox/configs/last-failed/codecept.conf.js +++ b/test/data/sandbox/configs/last-failed/codecept.conf.js @@ -5,5 +5,10 @@ export const config = { include: {}, bootstrap: null, mocha: {}, + multiple: { + dev: { + browsers: ['chromium', 'webkit'], + }, + }, name: 'last-failed', } diff --git a/test/data/sandbox/workers-multiple-browsers/suite_a.worker.js b/test/data/sandbox/workers-multiple-browsers/suite_a.worker.js new file mode 100644 index 000000000..17c430c4d --- /dev/null +++ b/test/data/sandbox/workers-multiple-browsers/suite_a.worker.js @@ -0,0 +1,9 @@ +Feature('MultipleBrowsersSuiteA') + +Scenario('scenario 1', ({ I }) => { + I.printBrowser() +}) + +Scenario('scenario 2', ({ I }) => { + I.printBrowser() +}) diff --git a/test/data/sandbox/workers-multiple-browsers/suite_b.worker.js b/test/data/sandbox/workers-multiple-browsers/suite_b.worker.js new file mode 100644 index 000000000..2c0f545dd --- /dev/null +++ b/test/data/sandbox/workers-multiple-browsers/suite_b.worker.js @@ -0,0 +1,9 @@ +Feature('MultipleBrowsersSuiteB') + +Scenario('scenario 1', ({ I }) => { + I.printBrowser() +}) + +Scenario('scenario 2', ({ I }) => { + I.printBrowser() +}) diff --git a/test/runner/last_failed_test.js b/test/runner/last_failed_test.js index d9dd1f386..99467af62 100644 --- a/test/runner/last_failed_test.js +++ b/test/runner/last_failed_test.js @@ -119,6 +119,17 @@ describe('--last-failed', function () { expect(stdout).not.toContain('multiplies numbers') }) + it('should rerun only failed tests in workers with multiple browsers', async () => { + const fullRun = await run('run-workers 2 dev') + expect(fullRun.stdout).toContain('4 passed, 6 failed') + + const { code, stdout } = await run('run-workers 2 dev --last-failed') + expect(code).toEqual(1) + expect(stdout).toContain('0 passed, 6 failed') + expect(stdout).not.toContain('adds numbers') + expect(stdout).not.toContain('concats strings') + }) + it('should rerun only failed tests in workers by pool', async () => { await run('run') const { stdout } = await run('run-workers 2 --by pool --last-failed') diff --git a/test/runner/run_workers_test.js b/test/runner/run_workers_test.js index 45930d8ad..0d49a4138 100644 --- a/test/runner/run_workers_test.js +++ b/test/runner/run_workers_test.js @@ -3,6 +3,7 @@ chai.should() import { expect } from 'expect' import path from 'path' import fs from 'fs' +import os from 'os' import semver from 'semver' import { exec } from 'child_process' import { fileURLToPath } from 'url' @@ -573,6 +574,58 @@ describe('CodeceptJS Workers Runner', function () { }) }) + it('should write one JUnit report per browser with all suites of that browser in multi-run --by suite', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + const outputDir = path.join(codecept_dir, 'output', 'junit-multiple') + fs.rmSync(outputDir, { recursive: true, force: true }) + + exec(`${codecept_run_glob('codecept.workers-junit-multiple.conf.js')} 2 dev --by suite`, (err, stdout) => { + ;(async () => { + expect(stdout).toContain('Worker 4: 2 tests') + expect(stdout).toMatch(/OK\s+\|\s+8 passed/) + expect(err).toEqual(null) + + expect(fs.existsSync(path.join(outputDir, 'report.xml'))).toEqual(false) + for (const run of ['dev_chromium1', 'dev_webkit1']) { + const reportFile = path.join(outputDir, run, 'report.xml') + expect(fs.existsSync(reportFile)).toEqual(true) + const parsed = await new xml2js.Parser().parseStringPromise(fs.readFileSync(reportFile, 'utf8')) + expect(parsed.testsuites.$.tests).toEqual('4') + const testcases = parsed.testsuites.testsuite.flatMap(s => s.testcase.map(tc => `${s.$.name}: ${tc.$.name}`)).sort() + expect(testcases).toEqual(['MultipleBrowsersSuiteA: scenario 1', 'MultipleBrowsersSuiteA: scenario 2', 'MultipleBrowsersSuiteB: scenario 1', 'MultipleBrowsersSuiteB: scenario 2']) + } + })().then(done, done) + }) + }) + + it('should keep results of all browsers and suites in the Testomatio HTML report in multi-run --by suite', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + const outputDir = path.join(codecept_dir, 'output', 'testomatio-multiple') + fs.rmSync(outputDir, { recursive: true, force: true }) + const htmlDir = path.join(outputDir, 'html') + const nodePath = fs.mkdtempSync(path.join(os.tmpdir(), 'cjs-node-path-')) + fs.symlinkSync(path.join(__dirname, '../..'), path.join(nodePath, 'codeceptjs'), 'dir') + const env = { ...process.env, NODE_PATH: nodePath, TESTOMATIO_HTML_REPORT_SAVE: '1', TESTOMATIO_HTML_REPORT_FOLDER: htmlDir } + delete env.TESTOMATIO + + exec(`${codecept_run_glob('codecept.workers-testomatio-multiple.conf.js')} 2 dev --by suite`, { env }, (err, stdout) => { + fs.rmSync(nodePath, { recursive: true, force: true }) + expect(stdout).toMatch(/OK\s+\|\s+8 passed/) + expect(err).toEqual(null) + + const html = fs.readFileSync(path.join(htmlDir, 'testomatio-report.html'), 'utf8') + const entries = new Set([...html.matchAll(/"rid":"[^"]*@(dev_\w+)"[^{}]*?"title":"([^"]*)","suite_title":"([^"]*)"/g)].map(([, run, title, suite]) => `${run} ${suite}: ${title}`)) + const expected = [] + for (const run of ['dev_chromium1', 'dev_webkit1']) { + for (const suite of ['MultipleBrowsersSuiteA', 'MultipleBrowsersSuiteB']) { + expected.push(`${run} ${suite}: scenario 1`, `${run} ${suite}: scenario 2`) + } + } + expect([...entries].sort()).toEqual(expected) + done() + }) + }) + it('should handle large worker count without inflating statistics', function (done) { if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') // Test with more workers than tests to ensure no inflation diff --git a/test/unit/junitReporter_test.js b/test/unit/junitReporter_test.js index feef9efbc..ff94476d8 100644 --- a/test/unit/junitReporter_test.js +++ b/test/unit/junitReporter_test.js @@ -253,4 +253,40 @@ describe('JUnit Reporter Plugin', () => { expect(suiteEl.testcase[0].$.name).to.contain('"after all" hook: AfterSuite') expect(suiteEl.testcase[0].failure[0].$.message).to.contain('AfterSuite failed') }) + + it('writes one report per run output folder when tests carry outputDir', async () => { + const chromiumDir = path.join(tmpDir, 'dev_chromium1') + const webkitDir = path.join(tmpDir, 'dev_webkit1') + const result = makeResult() + const [passed, failed, skipped] = result.tests + result.tests = [ + { ...passed, uid: 'pass-1@dev_chromium1', outputDir: chromiumDir }, + { ...failed, uid: 'fail-1@dev_chromium1', outputDir: chromiumDir }, + { ...passed, uid: 'pass-1@dev_webkit1', outputDir: webkitDir }, + { ...skipped, uid: 'skip-1@dev_webkit1', outputDir: webkitDir }, + ] + + junitReporter({}) + event.dispatcher.emit(event.hook.failed, { + hookName: 'AfterSuite', + title: '"after all" hook: AfterSuite for "logs in successfully"', + error: { message: 'AfterSuite failed', stack: 'Error: AfterSuite failed' }, + outputDir: webkitDir, + }) + event.dispatcher.emit(event.workers.result, result) + + assert.isFalse(fs.existsSync(path.join(tmpDir, 'report.xml')), 'no combined report should be written') + + const chromium = await parseReport(chromiumDir) + expect(chromium.testsuites.$.tests).to.equal('2') + expect(chromium.testsuites.$.failures).to.equal('1') + expect(chromium.testsuites.$.skipped).to.equal('0') + + const webkit = await parseReport(webkitDir) + expect(webkit.testsuites.$.tests).to.equal('3') + expect(webkit.testsuites.$.failures).to.equal('1') + expect(webkit.testsuites.$.skipped).to.equal('1') + const names = webkit.testsuites.testsuite.flatMap(s => s.testcase.map(tc => tc.$.name)) + expect(names.some(n => n.includes('AfterSuite'))).to.equal(true) + }) }) From 4d40c0ef9c4568e35f047d76ce8279fc5ebd2354 Mon Sep 17 00:00:00 2001 From: Jaromir Obr Date: Wed, 21 Oct 2026 14:27:30 +0200 Subject: [PATCH 2/2] test(workers): find the Testomatio HTML report wherever the reporter writes it Newer @testomatio/reporter versions write the report to /report and ignore TESTOMATIO_HTML_REPORT_FOLDER. Co-Authored-By: Claude Opus 5.5 --- test/runner/run_workers_test.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/runner/run_workers_test.js b/test/runner/run_workers_test.js index 0d49a4138..51cd6a132 100644 --- a/test/runner/run_workers_test.js +++ b/test/runner/run_workers_test.js @@ -613,7 +613,9 @@ describe('CodeceptJS Workers Runner', function () { expect(stdout).toMatch(/OK\s+\|\s+8 passed/) expect(err).toEqual(null) - const html = fs.readFileSync(path.join(htmlDir, 'testomatio-report.html'), 'utf8') + const reports = fs.readdirSync(outputDir, { recursive: true }).filter(file => file.endsWith('testomatio-report.html')) + expect(reports).toHaveLength(1) + const html = fs.readFileSync(path.join(outputDir, reports[0]), 'utf8') const entries = new Set([...html.matchAll(/"rid":"[^"]*@(dev_\w+)"[^{}]*?"title":"([^"]*)","suite_title":"([^"]*)"/g)].map(([, run, title, suite]) => `${run} ${suite}: ${title}`)) const expected = [] for (const run of ['dev_chromium1', 'dev_webkit1']) {