From 86fb9c5c0162f16fcebbc06b781432ab7b949f81 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Wed, 21 Oct 2026 16:16:39 +0530 Subject: [PATCH 1/2] End a dashboard run whose process exits before reporting A run started from the dashboard leaves its "Running" state only when the worker streams its final suites or when the backend sends testStopped. A child that exits before a worker connects, such as a config preflight refusing to start or a missing driver, sends neither. The row then showed only a Stop button, so the test could not be rerun. The backend now sends testStopped with a reason when a spawned run closes before any worker connected. The app marks the tests the run never reached as failed, with that reason as their error. The same close handler also cleared the rerun-child flag only on stop(), so the next unrelated worker after an early exit was taken for a rerun child, and it ran for a stopped child after its replacement had started, clearing the new run. Co-Authored-By: Claude Opus 5.5 --- .changeset/rerun-early-exit.md | 6 ++ packages/app/src/controller/DataManager.ts | 11 ++- packages/app/src/controller/mark-running.ts | 31 ++++--- packages/app/tests/data-manager.test.ts | 26 ++++++ packages/backend/src/index.ts | 22 +++-- packages/backend/src/runner.ts | 18 +++- .../tests/run-early-exit-broadcast.test.ts | 90 +++++++++++++++++++ packages/backend/tests/runner.test.ts | 74 +++++++++++++++ .../changes/test-stopped-payload.md | 5 ++ .../src/selenium_devtools/_wire_types.py | 11 ++- packages/shared/src/ws.ts | 11 ++- packages/shared/wire-schema.json | 23 ++++- 12 files changed, 301 insertions(+), 27 deletions(-) create mode 100644 .changeset/rerun-early-exit.md create mode 100644 packages/backend/tests/run-early-exit-broadcast.test.ts create mode 100644 packages/selenium-devtools-py/changes/test-stopped-payload.md diff --git a/.changeset/rerun-early-exit.md b/.changeset/rerun-early-exit.md new file mode 100644 index 00000000..9486211a --- /dev/null +++ b/.changeset/rerun-early-exit.md @@ -0,0 +1,6 @@ +--- +'@wdio/devtools-backend': patch +'@wdio/devtools-app': patch +--- + +A rerun whose process exited before it reported anything, such as a config preflight refusing to start or a missing driver, left the dashboard showing "Running" with only a Stop button, so it could not be rerun. The backend now tells the dashboard when that happens, and the tests the run never reached are marked failed with the exit code or signal as their error. diff --git a/packages/app/src/controller/DataManager.ts b/packages/app/src/controller/DataManager.ts index 3748b64f..d5f9fa32 100644 --- a/packages/app/src/controller/DataManager.ts +++ b/packages/app/src/controller/DataManager.ts @@ -7,7 +7,8 @@ import type { NetworkRequest, TraceLog, PreservedAttempt, - TracePlayerData + TracePlayerData, + WsPayloadFor } from '@wdio/devtools-shared' import { @@ -285,7 +286,7 @@ export class DataManagerController implements ReactiveController { // dispatch should be skipped. Caller is responsible for requestUpdate(). #handleControlScope(scope: string, data: unknown): boolean { if (scope === WS_SCOPE.testStopped) { - this.#handleTestStopped() + this.#handleTestStopped(data as WsPayloadFor<'testStopped'> | undefined) return true } if (scope === 'screencast') { @@ -396,11 +397,13 @@ export class DataManagerController implements ReactiveController { this.#host.requestUpdate() } - #handleTestStopped() { + #handleTestStopped(data?: WsPayloadFor<'testStopped'>) { this.#activeRerunTestUid = undefined rerunState.activeRerunSuiteUid = undefined const suites = this.suitesContextProvider.value || [] - this.suitesContextProvider.setValue(markRunningAsStopped(suites)) + this.suitesContextProvider.setValue( + markRunningAsStopped(suites, data?.reason) + ) } #handleMutationsUpdate(data: TraceMutation[]) { diff --git a/packages/app/src/controller/mark-running.ts b/packages/app/src/controller/mark-running.ts index 7e350889..98c66d8c 100644 --- a/packages/app/src/controller/mark-running.ts +++ b/packages/app/src/controller/mark-running.ts @@ -152,19 +152,21 @@ export function markSpecificRunning( /** A test the run never settled is cancelled by the stop; one that already * reported an outcome keeps it. */ -const stopTest = (test: TestStatsFragment): TestStatsFragment => - test && isInFlight(test) - ? { - ...test, - end: new Date(), - state: TEST_STATE.FAILED, - error: { message: 'Test execution stopped', name: 'TestStoppedError' } - } - : test +const stopTest = + (message: string) => + (test: TestStatsFragment): TestStatsFragment => + test && isInFlight(test) + ? { + ...test, + end: new Date(), + state: TEST_STATE.FAILED, + error: { message, name: 'TestStoppedError' } + } + : test /** - * Mark every test the run never settled as failed. Used when the user manually - * stops the run from the dashboard — without this, suites with + * Mark every test the run never settled as failed. Used when the run is + * stopped from the dashboard or exits before reporting — without this, suites with * `state: 'running'` would keep showing their spinner indefinitely. A test that * already reported an outcome keeps it: a skipped test carries no `end` stamp, * so keying the flip off `end` used to relabel every skipped test as failed. @@ -174,9 +176,12 @@ const stopTest = (test: TestStatsFragment): TestStatsFragment => * ends up failed — a stopped run is not a passing one. Otherwise the existing * state is preserved. */ -export function markRunningAsStopped(suites: SuiteChunks): SuiteChunks { +export function markRunningAsStopped( + suites: SuiteChunks, + reason = 'Test execution stopped' +): SuiteChunks { const updateSuite = (s: SuiteStatsFragment): SuiteStatsFragment => { - const updatedTests = s.tests?.map(stopTest) + const updatedTests = s.tests?.map(stopTest(reason)) const updatedNestedSuites = s.suites?.map(updateSuite) const childStates = tallyOutcomes([ diff --git a/packages/app/tests/data-manager.test.ts b/packages/app/tests/data-manager.test.ts index 585dfba2..c0c1063d 100644 --- a/packages/app/tests/data-manager.test.ts +++ b/packages/app/tests/data-manager.test.ts @@ -899,6 +899,32 @@ describe('DataManagerController', () => { expect(first.tests?.[0].state).toBe('failed') expect(first.tests?.[0].end).toBeDefined() expect(first.state).toBe('failed') + expect(first.tests?.[0].error?.message).toBe('Test execution stopped') + }) + + it('ends a run that exited before reporting, naming why', async () => { + const { manager, deliver } = await boot() + deliver( + 'suites', + suitesFrame( + suite('login-suite', { + state: 'running', + tests: [test('t-1', { state: 'pending', end: undefined })] + }) + ) + ) + + const reason = 'Run exited before reporting (exit code 1)' + deliver(WS_SCOPE.testStopped, { + stopped: true, + timestamp: RUN_START, + reason + }) + + const [first] = publishedSuites(manager) + expect(first.tests?.[0].state).toBe('failed') + expect(first.tests?.[0].error?.message).toBe(reason) + expect(first.state).toBe('failed') }) }) diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index e6f433b9..2adbb8e6 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -37,6 +37,7 @@ import { type BaselineGetQuery, type BaselineSavedWsPayload, type BaselineClearedWsPayload, + type SocketMessage, type TracePlayerData } from '@wdio/devtools-shared' import type { RunnerRequestBody } from './types.js' @@ -192,14 +193,23 @@ function registerTestRoutes( s.post('/api/tests/stop', async (_request, reply) => { testRunner.stop() - broadcastToClients( - JSON.stringify({ - scope: WS_SCOPE.testStopped, - data: { stopped: true, timestamp: Date.now() } - }) - ) + broadcastTestStopped() reply.send({ ok: true }) }) + + testRunner.onEarlyExit((reason) => broadcastTestStopped(reason)) +} + +function broadcastTestStopped(reason?: string): void { + const message: SocketMessage<'testStopped'> = { + scope: WS_SCOPE.testStopped, + data: { + stopped: true, + timestamp: Date.now(), + ...(reason ? { reason } : {}) + } + } + broadcastToClients(JSON.stringify(message)) } async function handleBaselinePreserve( diff --git a/packages/backend/src/runner.ts b/packages/backend/src/runner.ts index e1231048..258b8718 100644 --- a/packages/backend/src/runner.ts +++ b/packages/backend/src/runner.ts @@ -50,6 +50,11 @@ class TestRunner { // Set on rerun spawn; consumed once by the next worker handshake so the // accumulated suite tree is preserved instead of wiped. #expectingRerunChild = false + #onEarlyExit?: (reason: string) => void + /** Fires when a spawned run exits before any worker connected. */ + onEarlyExit(listener: (reason: string) => void) { + this.#onEarlyExit = listener + } consumeRerunChildFlag(): boolean { const v = this.#expectingRerunChild this.#expectingRerunChild = false @@ -180,10 +185,21 @@ class TestRunner { this.#child = child this.#lastPayload = payload - child.once('close', () => { + child.once('close', (code, signal) => { + // stop() detached this child already, and a later run() may own the slot. + if (this.#child !== child) { + return + } + const neverConnected = this.#expectingRerunChild this.#child = undefined this.#lastPayload = undefined + this.#expectingRerunChild = false this.#baseDir = process.cwd() + if (neverConnected) { + const reason = `Run exited before reporting (${signal ? `signal ${signal}` : `exit code ${code}`})` + log.warn(reason) + this.#onEarlyExit?.(reason) + } }) await this.#waitForSpawn(child) } diff --git a/packages/backend/tests/run-early-exit-broadcast.test.ts b/packages/backend/tests/run-early-exit-broadcast.test.ts new file mode 100644 index 00000000..fce40de2 --- /dev/null +++ b/packages/backend/tests/run-early-exit-broadcast.test.ts @@ -0,0 +1,90 @@ +/** + * A rerun whose process exits before any worker connects (a config preflight + * refusing to start, a missing driver) used to leave the dashboard on + * "Running" for good: the app leaves that state only on the worker's own + * suites frames or on `testStopped`, and neither ever came. This drives a real + * child through the run route and asserts the frame a real client receives. + */ + +import os from 'node:os' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { WebSocket } from 'ws' +import { quote as shellQuote } from 'shell-quote' +import type { FastifyInstance } from 'fastify' +import { WS_PATHS, WS_SCOPE } from '@wdio/devtools-shared' +import { start } from '../src/index.js' +import * as utils from '../src/utils.js' + +vi.mock('../src/utils.js', () => ({ + getDevtoolsApp: vi.fn(), + getCollectorSource: vi.fn() +})) + +const WAIT_TIMEOUT_MS = 5000 + +let server: FastifyInstance | undefined + +afterEach(async () => { + await server?.close() + server = undefined + vi.restoreAllMocks() +}) + +type Frame = { scope: string; data: Record } + +async function bootWithClient(): Promise<{ + server: FastifyInstance + frames: Frame[] +}> { + vi.mocked(utils.getDevtoolsApp).mockResolvedValue(os.tmpdir()) + vi.mocked(utils.getCollectorSource).mockResolvedValue('// collector') + const started = await start({ port: 0 }) + server = started.server + const socket = new WebSocket( + `ws://localhost:${started.port}${WS_PATHS.client}` + ) + const frames: Frame[] = [] + socket.on('message', (raw) => frames.push(JSON.parse(String(raw)))) + await new Promise((resolve, reject) => { + socket.once('open', () => resolve()) + socket.once('error', reject) + }) + return { server: started.server, frames } +} + +async function frameFor(frames: Frame[], scope: string): Promise { + const deadline = Date.now() + WAIT_TIMEOUT_MS + while (Date.now() < deadline) { + const found = frames.find((frame) => frame.scope === scope) + if (found) { + return found + } + await new Promise((resolve) => setTimeout(resolve, 10)) + } + throw new Error(`timed out waiting for a ${scope} frame`) +} + +describe('a run that exits before reporting', () => { + it('tells the dashboard the run ended, and why', async () => { + const { server: app, frames } = await bootWithClient() + const exitsAtOnce = shellQuote([process.execPath, '-e', 'process.exit(3)']) + + const response = await app.inject({ + method: 'POST', + url: '/api/tests/run', + payload: { + uid: 'examples/test_login.py::test_one', + entryType: 'test', + launchCommand: exitsAtOnce, + rerunCommand: exitsAtOnce + } + }) + + expect(response.statusCode).toBe(200) + const stopped = await frameFor(frames, WS_SCOPE.testStopped) + expect(stopped.data).toMatchObject({ + stopped: true, + reason: 'Run exited before reporting (exit code 3)' + }) + }) +}) diff --git a/packages/backend/tests/runner.test.ts b/packages/backend/tests/runner.test.ts index f9415e6e..2590a3c0 100644 --- a/packages/backend/tests/runner.test.ts +++ b/packages/backend/tests/runner.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' import { spawn } from 'node:child_process' +import { EventEmitter } from 'node:events' import fs from 'node:fs' import path from 'node:path' import { RERUN_SLOT, REUSE_ENV } from '@wdio/devtools-shared' @@ -165,6 +166,79 @@ describe('TestRunner', () => { }) }) + describe('early exit', () => { + const rerun: RunnerRequestBody = { + uid: 'test-1', + entryType: 'test', + configFile: mockConfigPath, + devtoolsHost: 'localhost', + devtoolsPort: 3000 + } + const spawnedChildren: EventEmitter[] = [] + + beforeEach(() => { + spawnedChildren.length = 0 + vi.mocked(spawn).mockImplementation(() => { + const child = Object.assign(new EventEmitter(), { pid: 12345 }) + spawnedChildren.push(child) + setTimeout(() => child.emit('spawn'), 0) + return child as any + }) + }) + + it('reports a run that exits before its worker connects', async () => { + const onEarlyExit = vi.fn() + testRunner.onEarlyExit(onEarlyExit) + await testRunner.run(rerun) + + spawnedChildren[0].emit('close', 1, null) + + expect(onEarlyExit).toHaveBeenCalledWith( + 'Run exited before reporting (exit code 1)' + ) + // An unconsumed flag would make the next unrelated worker read as a rerun child. + expect(testRunner.consumeRerunChildFlag()).toBe(false) + }) + + it('names the signal that ended it', async () => { + const onEarlyExit = vi.fn() + testRunner.onEarlyExit(onEarlyExit) + await testRunner.run(rerun) + + spawnedChildren[0].emit('close', null, 'SIGKILL') + + expect(onEarlyExit).toHaveBeenCalledWith( + 'Run exited before reporting (signal SIGKILL)' + ) + }) + + it('stays silent for a run whose worker connected', async () => { + const onEarlyExit = vi.fn() + testRunner.onEarlyExit(onEarlyExit) + await testRunner.run(rerun) + testRunner.consumeRerunChildFlag() + + spawnedChildren[0].emit('close', 0, null) + + expect(onEarlyExit).not.toHaveBeenCalled() + }) + + it('ignores a stopped child closing after its replacement started', async () => { + const onEarlyExit = vi.fn() + testRunner.onEarlyExit(onEarlyExit) + await testRunner.run(rerun) + testRunner.stop() + await testRunner.run(rerun) + + spawnedChildren[0].emit('close', null, 'SIGTERM') + expect(onEarlyExit).not.toHaveBeenCalled() + + // The replacement still owns the slot, so its own early exit is reported. + spawnedChildren[1].emit('close', 1, null) + expect(onEarlyExit).toHaveBeenCalledTimes(1) + }) + }) + describe('configuration', () => { it('should find config and use environment variables', async () => { const specFile = '/project/test/specs/test.spec.ts' diff --git a/packages/selenium-devtools-py/changes/test-stopped-payload.md b/packages/selenium-devtools-py/changes/test-stopped-payload.md new file mode 100644 index 00000000..25163c53 --- /dev/null +++ b/packages/selenium-devtools-py/changes/test-stopped-payload.md @@ -0,0 +1,5 @@ +--- +patch +--- + +The generated wire types include the payload the dashboard now receives when a run ends: whether it was stopped, when, and why, for a run that exited before it reported anything. diff --git a/packages/selenium-devtools-py/src/selenium_devtools/_wire_types.py b/packages/selenium-devtools-py/src/selenium_devtools/_wire_types.py index a008db4b..272c9b82 100644 --- a/packages/selenium-devtools-py/src/selenium_devtools/_wire_types.py +++ b/packages/selenium-devtools-py/src/selenium_devtools/_wire_types.py @@ -4,7 +4,7 @@ from typing import Any, Dict, List, Literal, Optional, TypedDict, Union -__all__ = ['AccessibilityNode', 'ActionSnapshot', 'BaselineClearedWsPayload', 'BaselineSavedWsPayload', 'ClearExecutionDataWsPayload', 'CommandLog', 'CommandLogPoint', 'ConsoleLog', 'DeviceInfo', 'DocumentInfo', 'DocumentInfoDocumentInfo', 'DocumentInfoHeaders', 'ElementScriptsResponse', 'Metadata', 'NetworkRequest', 'NetworkRequestResponse', 'NetworkTimings', 'PerformanceData', 'PerformanceDataNavigation', 'PerformanceDataNavigationTiming', 'PerformanceDataResources', 'PreservedAttempt', 'PreservedAttemptTest', 'PreservedAttemptWindow', 'PreservedStep', 'ReplaceCommandWsPayload', 'ScreencastFrame', 'ScreencastInfo', 'SerializedError', 'SuiteStats', 'TestError', 'TestErrorMatcherResult', 'TestStats', 'TraceExportRequest', 'TraceExportResult', 'TraceMutation', 'Viewport'] +__all__ = ['AccessibilityNode', 'ActionSnapshot', 'BaselineClearedWsPayload', 'BaselineSavedWsPayload', 'ClearExecutionDataWsPayload', 'CommandLog', 'CommandLogPoint', 'ConsoleLog', 'DeviceInfo', 'DocumentInfo', 'DocumentInfoDocumentInfo', 'DocumentInfoHeaders', 'ElementScriptsResponse', 'Metadata', 'NetworkRequest', 'NetworkRequestResponse', 'NetworkTimings', 'PerformanceData', 'PerformanceDataNavigation', 'PerformanceDataNavigationTiming', 'PerformanceDataResources', 'PreservedAttempt', 'PreservedAttemptTest', 'PreservedAttemptWindow', 'PreservedStep', 'ReplaceCommandWsPayload', 'ScreencastFrame', 'ScreencastInfo', 'SerializedError', 'SuiteStats', 'TestError', 'TestErrorMatcherResult', 'TestStats', 'TestStoppedWsPayload', 'TraceExportRequest', 'TraceExportResult', 'TraceMutation', 'Viewport'] class _AccessibilityNodeRequired(TypedDict): @@ -358,6 +358,15 @@ class TestStats(_TestStatsRequired, total=False): order: float +class _TestStoppedWsPayloadRequired(TypedDict): + stopped: Literal[True] + timestamp: float + + +class TestStoppedWsPayload(_TestStoppedWsPayloadRequired, total=False): + reason: str + + class _TraceExportRequestRequired(TypedDict): requestId: str outputDir: str diff --git a/packages/shared/src/ws.ts b/packages/shared/src/ws.ts index 5f8a6ce1..23a3a895 100644 --- a/packages/shared/src/ws.ts +++ b/packages/shared/src/ws.ts @@ -41,6 +41,13 @@ export interface ClearExecutionDataWsPayload { runStart?: boolean } +/** `reason` is set only when the run exited before any worker connected. */ +export interface TestStoppedWsPayload { + stopped: true + timestamp: number + reason?: string +} + /** Discriminated-union envelope for every message that crosses the WS. */ export interface SocketMessage { scope: T @@ -54,7 +61,9 @@ export interface SocketMessage { ? BaselineSavedWsPayload : T extends 'baseline:cleared' ? BaselineClearedWsPayload - : unknown + : T extends 'testStopped' + ? TestStoppedWsPayload + : unknown } /** Payload type for a given WS scope. Inverse of `SocketMessage['data']` — diff --git a/packages/shared/wire-schema.json b/packages/shared/wire-schema.json index d418fac7..102ff087 100644 --- a/packages/shared/wire-schema.json +++ b/packages/shared/wire-schema.json @@ -87,7 +87,9 @@ } } }, - "testStopped": {}, + "testStopped": { + "$ref": "#/$defs/TestStoppedWsPayload" + }, "traceExport": { "$ref": "#/$defs/TraceExportRequest" }, @@ -1222,6 +1224,25 @@ ], "additionalProperties": false }, + "TestStoppedWsPayload": { + "type": "object", + "properties": { + "stopped": { + "const": true + }, + "timestamp": { + "type": "number" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "stopped", + "timestamp" + ], + "additionalProperties": false + }, "TraceExportRequest": { "type": "object", "properties": { From 9e29fc15cb5bdf501d2f538ab27521cd19fab42b Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Wed, 21 Oct 2026 16:31:21 +0530 Subject: [PATCH 2/2] Show why a suite rerun ended when no test is left to say so A suite rerun or Run All clears each suite's tests before the child starts, so when that child exits before reporting, the stop reason landed on no test: the suites turned failed and the Errors tab stayed empty. The reason now goes on the topmost suite the stop alone failed, which the Errors tab lists, and is cleared when the suite runs again. A test-level rerun still carries it on the test, so the suite does not repeat it. Co-Authored-By: Claude Opus 5.5 --- .changeset/rerun-early-exit.md | 2 +- .../components/workbench/errors/collect.ts | 29 +++- packages/app/src/controller/mark-running.ts | 126 ++++++++++++------ packages/app/src/controller/types.ts | 2 + packages/app/tests/data-manager.test.ts | 30 +++++ packages/app/tests/errors-collect.test.ts | 24 ++++ packages/app/tests/mark-running.test.ts | 43 ++++++ 7 files changed, 210 insertions(+), 46 deletions(-) diff --git a/.changeset/rerun-early-exit.md b/.changeset/rerun-early-exit.md index 9486211a..cf37d54e 100644 --- a/.changeset/rerun-early-exit.md +++ b/.changeset/rerun-early-exit.md @@ -3,4 +3,4 @@ '@wdio/devtools-app': patch --- -A rerun whose process exited before it reported anything, such as a config preflight refusing to start or a missing driver, left the dashboard showing "Running" with only a Stop button, so it could not be rerun. The backend now tells the dashboard when that happens, and the tests the run never reached are marked failed with the exit code or signal as their error. +A rerun whose process exited before it reported anything, such as a config preflight refusing to start or a missing driver, left the dashboard showing "Running" with only a Stop button, so it could not be rerun. The backend now tells the dashboard when that happens. The tests the run never reached are marked failed with the exit code or signal as their error, and a suite or Run All rerun, which has no test left to mark, shows the reason on its suite and in the Errors tab. diff --git a/packages/app/src/components/workbench/errors/collect.ts b/packages/app/src/components/workbench/errors/collect.ts index fa7e1bb9..9cf894d6 100644 --- a/packages/app/src/components/workbench/errors/collect.ts +++ b/packages/app/src/components/workbench/errors/collect.ts @@ -197,6 +197,33 @@ function collectFailedTests( return [...byUid.values()] } +/** Suites carrying their own error: a stopped rerun that cleared their tests. */ +function suiteErrors( + suites: Record[] | undefined +): CollectedError[] { + const byUid = new Map() + const visit = (suite: SuiteStatsFragment) => { + const read = readError(suite.error) + if (read) { + byUid.set(suite.uid, { + title: suite.fullTitle || suite.title || suite.uid, + message: read.message, + stack: read.stack, + callSource: suite.callSource + }) + } + for (const child of suite.suites ?? []) { + visit(child) + } + } + for (const map of suites ?? []) { + for (const suite of Object.values(map)) { + visit(suite) + } + } + return [...byUid.values()] +} + function commandErrors(commands: CommandLog[] | undefined): CollectedError[] { return (commands ?? []) .flatMap((command) => { @@ -293,5 +320,5 @@ export function collectErrors( ] }) - return [...fromCommands, ...fromTests] + return [...fromCommands, ...fromTests, ...suiteErrors(suites)] } diff --git a/packages/app/src/controller/mark-running.ts b/packages/app/src/controller/mark-running.ts index 98c66d8c..2cdda4b5 100644 --- a/packages/app/src/controller/mark-running.ts +++ b/packages/app/src/controller/mark-running.ts @@ -29,6 +29,7 @@ export function markAllRunning(suites: SuiteChunks): SuiteChunks { state: 'running', start: new Date(), end: undefined, + error: undefined, tests: [] as TestStatsFragment[], suites: s.suites?.map(markAllAsRunning) || [] }) @@ -67,6 +68,7 @@ function markSuiteTreeAsRunning( state: 'running', start: runStart, end: undefined, + error: undefined, tests: [] as TestStatsFragment[], suites: suiteNode.suites?.map((s) => markSuiteTreeAsRunning(s, runStart)) || [] @@ -110,6 +112,7 @@ function markSuiteWithUid( ...(matched ? { state: 'running' as const, + error: undefined, // Preserve parent's start/end if already running — subsequent // child-scenario marks would otherwise reset the feature's // original run timestamp. @@ -150,71 +153,106 @@ export function markSpecificRunning( }) } +type StopError = NonNullable + /** A test the run never settled is cancelled by the stop; one that already * reported an outcome keeps it. */ const stopTest = - (message: string) => + (error: StopError) => (test: TestStatsFragment): TestStatsFragment => test && isInFlight(test) - ? { - ...test, - end: new Date(), - state: TEST_STATE.FAILED, - error: { message, name: 'TestStoppedError' } - } + ? { ...test, end: new Date(), state: TEST_STATE.FAILED, error } : test /** - * Mark every test the run never settled as failed. Used when the run is - * stopped from the dashboard or exits before reporting — without this, suites with - * `state: 'running'` would keep showing their spinner indefinitely. A test that - * already reported an outcome keeps it: a skipped test carries no `end` stamp, - * so keying the flip off `end` used to relabel every skipped test as failed. - * * The suite's state is derived from its updated children: if any child failed * (or the suite itself was 'running' with no live children left), the suite * ends up failed — a stopped run is not a passing one. Otherwise the existing - * state is preserved. + * state is preserved. `failedByStopAlone` collects the suites whose failure + * has no cause but the stop: no failed test, and only such suites failed below. */ -export function markRunningAsStopped( - suites: SuiteChunks, - reason = 'Test execution stopped' -): SuiteChunks { - const updateSuite = (s: SuiteStatsFragment): SuiteStatsFragment => { - const updatedTests = s.tests?.map(stopTest(reason)) - const updatedNestedSuites = s.suites?.map(updateSuite) +function stopSuite( + s: SuiteStatsFragment, + error: StopError, + failedByStopAlone: WeakSet +): SuiteStatsFragment { + const updatedTests = s.tests?.map(stopTest(error)) || [] + const updatedNestedSuites = + s.suites?.map((child) => stopSuite(child, error, failedByStopAlone)) || [] - const childStates = tallyOutcomes([ - ...(updatedTests || []), - ...(updatedNestedSuites || []) - ]) - const stillRunning = hasInFlight(childStates) - const derivedState: SuiteStatsFragment['state'] = stillRunning - ? s.state - : hasFailure(childStates) + const childStates = tallyOutcomes([...updatedTests, ...updatedNestedSuites]) + const stillRunning = hasInFlight(childStates) + const derivedState: SuiteStatsFragment['state'] = stillRunning + ? s.state + : hasFailure(childStates) + ? TEST_STATE.FAILED + : s.state === TEST_STATE.RUNNING ? TEST_STATE.FAILED - : s.state === TEST_STATE.RUNNING - ? TEST_STATE.FAILED - : s.state + : s.state - return { - ...s, - state: derivedState, - ...(!stillRunning && !s.end ? { end: new Date() } : {}), - tests: updatedTests || [], - suites: updatedNestedSuites || [] - } + const stopped: SuiteStatsFragment = { + ...s, + state: derivedState, + ...(!stillRunning && !s.end ? { end: new Date() } : {}), + tests: updatedTests, + suites: updatedNestedSuites + } + if ( + s.state === TEST_STATE.RUNNING && + derivedState === TEST_STATE.FAILED && + !updatedTests.some((test) => test.state === TEST_STATE.FAILED) && + updatedNestedSuites.every( + (child) => + child.state !== TEST_STATE.FAILED || failedByStopAlone.has(child) + ) + ) { + failedByStopAlone.add(stopped) + } + return stopped +} + +/** A suite rerun leaves no test to show why, so the topmost suite the stop alone failed carries it. */ +function carryReason( + s: SuiteStatsFragment, + error: StopError, + failedByStopAlone: WeakSet +): SuiteStatsFragment { + if (failedByStopAlone.has(s)) { + return { ...s, error } } + return { + ...s, + suites: s.suites?.map((child) => + carryReason(child, error, failedByStopAlone) + ) + } +} +/** + * Mark every test the run never settled as failed. Used when the run is + * stopped from the dashboard or exits before reporting — without this, suites + * with `state: 'running'` would keep showing their spinner indefinitely. A test + * that already reported an outcome keeps it: a skipped test carries no `end` + * stamp, so keying the flip off `end` used to relabel every skipped test as + * failed. + */ +export function markRunningAsStopped( + suites: SuiteChunks, + reason = 'Test execution stopped' +): SuiteChunks { + const error: StopError = { message: reason, name: 'TestStoppedError' } + const failedByStopAlone = new WeakSet() return suites.map((chunk) => { const updatedChunk: Record = {} Object.entries(chunk as Record).forEach( ([uid, suite]) => { - if (!suite) { - updatedChunk[uid] = suite - return - } - updatedChunk[uid] = updateSuite(suite) + updatedChunk[uid] = suite + ? carryReason( + stopSuite(suite, error, failedByStopAlone), + error, + failedByStopAlone + ) + : suite } ) return updatedChunk diff --git a/packages/app/src/controller/types.ts b/packages/app/src/controller/types.ts index 1ecbdb52..1b99c5c5 100644 --- a/packages/app/src/controller/types.ts +++ b/packages/app/src/controller/types.ts @@ -35,4 +35,6 @@ export type SuiteStatsFragment = Omit< featureLine?: number type?: string file?: string + /** Set only by a stop that failed the suite with no test left to say why. */ + error?: TestStatsFragment['error'] } diff --git a/packages/app/tests/data-manager.test.ts b/packages/app/tests/data-manager.test.ts index c0c1063d..354cc655 100644 --- a/packages/app/tests/data-manager.test.ts +++ b/packages/app/tests/data-manager.test.ts @@ -926,6 +926,36 @@ describe('DataManagerController', () => { expect(first.tests?.[0].error?.message).toBe(reason) expect(first.state).toBe('failed') }) + + it('keeps the reason when a suite rerun exits before reporting', async () => { + const { manager, deliver } = await boot() + deliver( + 'suites', + suitesFrame( + suite('login-suite', { + state: 'passed', + tests: [test('t-1', { state: 'passed' })] + }) + ) + ) + deliver(WS_SCOPE.clearExecutionData, { + uid: 'login-suite', + entryType: 'suite', + runStart: true + }) + + const reason = 'Run exited before reporting (exit code 1)' + deliver(WS_SCOPE.testStopped, { + stopped: true, + timestamp: RUN_START, + reason + }) + + const [first] = publishedSuites(manager) + expect(first.tests).toEqual([]) + expect(first.state).toBe('failed') + expect(first.error?.message).toBe(reason) + }) }) describe('screencast handoff', () => { diff --git a/packages/app/tests/errors-collect.test.ts b/packages/app/tests/errors-collect.test.ts index b42b91b4..063cbaf0 100644 --- a/packages/app/tests/errors-collect.test.ts +++ b/packages/app/tests/errors-collect.test.ts @@ -189,6 +189,30 @@ describe('collectErrors', () => { expect(errors[0].timestamp).toBeUndefined() }) + it('collects a suite carrying its own error after a stopped rerun', () => { + const errors = collectErrors( + [], + suiteMap({ + uid: 'login', + fullTitle: 'login suite', + state: 'failed', + tests: [], + error: { + name: 'TestStoppedError', + message: 'Run exited before reporting (exit code 1)' + } + }) + ) + expect(errors).toEqual([ + { + title: 'login suite', + message: 'Run exited before reporting (exit code 1)', + stack: undefined, + callSource: undefined + } + ]) + }) + it('reads the first entry of errors[] when error is absent', () => { const [error] = collectErrors( [], diff --git a/packages/app/tests/mark-running.test.ts b/packages/app/tests/mark-running.test.ts index badfb80f..eecdcd71 100644 --- a/packages/app/tests/mark-running.test.ts +++ b/packages/app/tests/mark-running.test.ts @@ -192,4 +192,47 @@ describe('markRunningAsStopped', () => { expect(out[0].root.suites?.[0]?.state).toBe('failed') expect(out[0].root.suites?.[0]?.tests?.[0]?.state).toBe('failed') }) + it('puts the reason on the topmost suite when a rerun left no test to carry it', () => { + const input = markAllRunning( + chunks(suite('root', { suites: [suite('a'), suite('b')] })) + ) + const out = markRunningAsStopped( + input, + 'Run exited before reporting (exit code 1)' + ) + expect(out[0].root.state).toBe('failed') + expect(out[0].root.error?.message).toBe( + 'Run exited before reporting (exit code 1)' + ) + expect(out[0].root.suites?.map((child) => child.error)).toEqual([ + undefined, + undefined + ]) + }) + + it('leaves the reason on the tests when a test carries it', () => { + const input = chunks( + suite('root', { + state: 'running', + end: null, + tests: [test('t1', { state: 'pending', end: undefined })] + }) + ) + const out = markRunningAsStopped(input, 'Run exited before reporting') + expect(out[0].root.tests?.[0]?.error?.message).toBe( + 'Run exited before reporting' + ) + expect(out[0].root.error).toBeUndefined() + }) + + it('clears a stop reason once the suite runs again', () => { + const stopped = markRunningAsStopped( + markAllRunning(chunks(suite('root'))), + 'Run exited before reporting' + ) + expect(markAllRunning(stopped)[0].root.error).toBeUndefined() + expect( + markSpecificRunning(stopped, 'root', 'suite')[0].root.error + ).toBeUndefined() + }) })