diff --git a/changelog.d/timeline-outcomes.added.md b/changelog.d/timeline-outcomes.added.md new file mode 100644 index 00000000..427a5078 --- /dev/null +++ b/changelog.d/timeline-outcomes.added.md @@ -0,0 +1 @@ +The Timeline window now reports how each Bot's turns ended — completed, aborted, error or lost — and lists its recent failed or aborted turns with their error code, so the panel or a tool can see a Bot's reliability at a glance. It is read-only and part of the existing `timeline/window` answer, so nothing new needs configuring. diff --git a/docs/subsystems/timeline.md b/docs/subsystems/timeline.md index 761731f5..d7987478 100644 --- a/docs/subsystems/timeline.md +++ b/docs/subsystems/timeline.md @@ -26,7 +26,7 @@ Everything in `plugins/timeline/`: the recorder, the trace file, the read model | `plugins/timeline/src/adapter.js` | The only file that knows the `ctx.flybot` shape. | | `plugins/timeline/src/recorder.js` | Feed events to compact trace records; rebuilds every field from a strict alphabet. | | `plugins/timeline/src/store.js` | The trace: single writer, write-tmp + fsync + rename, capped segments. | -| `plugins/timeline/src/window.js` | Pure read model: records to lanes, turns, tools, arrows, edges, utilization, and the token spend per Bot (#357). | +| `plugins/timeline/src/window.js` | Pure read model: records to lanes, turns, tools, arrows, edges, utilization, the token spend per Bot (#357) and per-Bot outcomes and recent failures (#471). | | `plugins/timeline/client.js` | Browser entry (`@flybot/timeline`): the 2 s poller exposed as `ctx.flybot.timeline`, the geometry functions (`layoutSwimlanes`, `layoutMap`, `waitRows`, `utilizationRows`) and the panel (swimlanes, who waits on whom, utilization), opened from its `flybot.sidebar.section` row, the only entry (not in the + menu, #269). | | `plugins/timeline/src/sanitize.js` | The cleaners for ids, labels, codes, times, counts. | | `plugins/timeline/src/otel.js` | OTLP export: maps trace records to OpenTelemetry GenAI spans (`spansFrom`, `otlpPayload`) and a cursor exporter (`createExporter`). Off unless `config.exporter` is set. | diff --git a/plugins/timeline/src/window.js b/plugins/timeline/src/window.js index f8f0e5fa..154dbe28 100644 --- a/plugins/timeline/src/window.js +++ b/plugins/timeline/src/window.js @@ -274,12 +274,25 @@ export function buildWindow(records, { now = Date.now(), to, windowMs, bots = [] } const edges = [...edgeMap.values()].map(({ waits, ...edge }) => ({ ...edge, ...(waits.length ? { medianMs: median(waits), maxMs: Math.max(...waits), totalWaitMs: waits.reduce((sum, value) => sum + value, 0) } : {}) })) + // ---- how each Bot's turns ended, and its recent failures (#357 quality / #471 failed turns) + const FAILED_KEPT = 20 + const outcomes = lanes.filter(lane => lane.kind === 'bot').map((lane) => { + const own = turnList.filter(turn => turn.bot === lane.id && turn.place !== 'room') + const count = kind => own.filter(turn => (turn.lost === true ? 'lost' : turn.outcome) === kind).length + return { lane: lane.id, completed: count('completed'), aborted: count('aborted'), error: count('error'), lost: count('lost') } + }) + const failures = turnList + .filter(turn => turn.place !== 'room' && typeof turn.bot === 'string' && turn.t1 !== undefined && (turn.outcome === 'error' || turn.outcome === 'aborted')) + .sort((a, b) => b.t1 - a.t1) + .slice(0, FAILED_KEPT) + .map(turn => ({ id: turn.id, bot: turn.bot, at: turn.t1, outcome: turn.outcome, ...(turn.errorCode ? { code: cleanCode(turn.errorCode) } : {}) })) + return { version: 1, now, from: lo, to: hi, windowMs: windowMsActual, live, coverage: { oldest: oldest ?? null, newest: last || null, complete: oldest !== undefined && oldest <= lo }, trace: stats ?? null, truncated, - lanes, turns: shapedTurns, rounds, exchanges, utilization, edges, usage, + lanes, turns: shapedTurns, rounds, exchanges, utilization, edges, usage, outcomes, failures, } } diff --git a/plugins/timeline/test/window.test.js b/plugins/timeline/test/window.test.js index dd38cf99..a245f02f 100644 --- a/plugins/timeline/test/window.test.js +++ b/plugins/timeline/test/window.test.js @@ -258,3 +258,19 @@ test('usage: token spend per Bot (#357/#401)', () => { assert.deepEqual([a.calls, a.input, a.output, a.cacheRead], [2, 150, 30, 5]) assert.equal(win.usage.find(entry => entry.lane === 'b').calls, 0) }) + +test('outcomes: per-Bot turn endings and the recent failures (#357/#471)', () => { + const recs = [ + { r: 'ts', t: at(1000), id: 'a#1', pl: 'chat', bot: 'a' }, + { r: 'te', t: at(2000), id: 'a#1', o: 'completed' }, + { r: 'ts', t: at(3000), id: 'a#2', pl: 'chat', bot: 'a' }, + { r: 'te', t: at(4000), id: 'a#2', o: 'error', ec: 'RATE_LIMIT' }, + { r: 'ts', t: at(5000), id: 'b#1', pl: 'chat', bot: 'b' }, + { r: 'te', t: at(6000), id: 'b#1', o: 'aborted' }, + ] + const win = buildWindow(recs, { now: at(10_000), bots, rooms }) + const a = win.outcomes.find(entry => entry.lane === 'a') + assert.deepEqual([a.completed, a.error, a.aborted, a.lost], [1, 1, 0, 0]) + assert.deepEqual(win.outcomes.find(entry => entry.lane === 'b').aborted, 1) + assert.deepEqual(win.failures.map(f => [f.id, f.bot, f.outcome, f.code]), [['b#1', 'b', 'aborted', undefined], ['a#2', 'a', 'error', 'RATE_LIMIT']]) +})