diff --git a/changelog.d/timeline-tooluse.added.md b/changelog.d/timeline-tooluse.added.md new file mode 100644 index 00000000..48d36e3d --- /dev/null +++ b/changelog.d/timeline-tooluse.added.md @@ -0,0 +1 @@ +The Timeline window now shows which tools each Bot used, with how many times and how many of those failed, so a flaky tool or connector is visible where the run is drawn. Read-only, part of the existing `timeline/window` answer, tool names cleaned and capped. diff --git a/docs/subsystems/timeline.md b/docs/subsystems/timeline.md index d7987478..982620b9 100644 --- a/docs/subsystems/timeline.md +++ b/docs/subsystems/timeline.md @@ -26,7 +26,7 @@ Everything in `plugins/timeline/`: the recorder, the trace file, the read model | `plugins/timeline/src/adapter.js` | The only file that knows the `ctx.flybot` shape. | | `plugins/timeline/src/recorder.js` | Feed events to compact trace records; rebuilds every field from a strict alphabet. | | `plugins/timeline/src/store.js` | The trace: single writer, write-tmp + fsync + rename, capped segments. | -| `plugins/timeline/src/window.js` | Pure read model: records to lanes, turns, tools, arrows, edges, utilization, the token spend per Bot (#357) and per-Bot outcomes and recent failures (#471). | +| `plugins/timeline/src/window.js` | Pure read model: records to lanes, turns, tools, arrows, edges, utilization, the token spend per Bot (#357), per-Bot outcomes and recent failures (#471) and the tools each Bot used (#359). | | `plugins/timeline/client.js` | Browser entry (`@flybot/timeline`): the 2 s poller exposed as `ctx.flybot.timeline`, the geometry functions (`layoutSwimlanes`, `layoutMap`, `waitRows`, `utilizationRows`) and the panel (swimlanes, who waits on whom, utilization), opened from its `flybot.sidebar.section` row, the only entry (not in the + menu, #269). | | `plugins/timeline/src/sanitize.js` | The cleaners for ids, labels, codes, times, counts. | | `plugins/timeline/src/otel.js` | OTLP export: maps trace records to OpenTelemetry GenAI spans (`spansFrom`, `otlpPayload`) and a cursor exporter (`createExporter`). Off unless `config.exporter` is set. | diff --git a/plugins/timeline/src/window.js b/plugins/timeline/src/window.js index 154dbe28..5dd26f5b 100644 --- a/plugins/timeline/src/window.js +++ b/plugins/timeline/src/window.js @@ -260,6 +260,23 @@ export function buildWindow(records, { now = Date.now(), to, windowMs, bots = [] return { lane: lane.id, calls: calls.length, input: sum('in'), output: sum('out'), cacheRead: sum('cr'), cacheWrite: sum('cw') } }) + // ---- which tools each Bot used, and how often they failed (#357/#359 connectors) + const toolUse = lanes.filter(lane => lane.kind === 'bot').map((lane) => { + const counts = new Map() + for (const turn of turnList) { + if (turn.bot !== lane.id || turn.place === 'room') continue + for (const tool of toolsOf.get(turn.id) ?? []) { + const name = cleanLabel(tool.name, 60) || 'tool' + const entry = counts.get(name) ?? { name, calls: 0, failed: 0 } + entry.calls += 1 + if (tool.ok === false) entry.failed += 1 + counts.set(name, entry) + } + } + const tools = [...counts.values()].sort((a, b) => b.calls - a.calls || a.name.localeCompare(b.name)).slice(0, 20) + return { lane: lane.id, tools } + }) + // ---- communication map: directed edges, "A waits on B" from request -> reply latencies const edgeMap = new Map() for (const arrow of exchanges) { @@ -293,6 +310,6 @@ export function buildWindow(records, { now = Date.now(), to, windowMs, bots = [] coverage: { oldest: oldest ?? null, newest: last || null, complete: oldest !== undefined && oldest <= lo }, trace: stats ?? null, truncated, - lanes, turns: shapedTurns, rounds, exchanges, utilization, edges, usage, outcomes, failures, + lanes, turns: shapedTurns, rounds, exchanges, utilization, edges, usage, outcomes, failures, toolUse, } } diff --git a/plugins/timeline/test/window.test.js b/plugins/timeline/test/window.test.js index a245f02f..bce8a8ae 100644 --- a/plugins/timeline/test/window.test.js +++ b/plugins/timeline/test/window.test.js @@ -274,3 +274,19 @@ test('outcomes: per-Bot turn endings and the recent failures (#357/#471)', () => assert.deepEqual(win.outcomes.find(entry => entry.lane === 'b').aborted, 1) assert.deepEqual(win.failures.map(f => [f.id, f.bot, f.outcome, f.code]), [['b#1', 'b', 'aborted', undefined], ['a#2', 'a', 'error', 'RATE_LIMIT']]) }) + +test('toolUse: which tools a Bot used, and how often they failed (#357/#359)', () => { + const recs = [ + { r: 'ts', t: at(1000), id: 'a#1', pl: 'chat', bot: 'a' }, + { r: 'ks', t: at(1100), id: 'c1', tid: 'a#1', nm: 'web_search', bot: 'a' }, + { r: 'ke', t: at(1200), id: 'c1', tid: 'a#1', ok: true }, + { r: 'ks', t: at(1300), id: 'c2', tid: 'a#1', nm: 'mcp__github__search', bot: 'a' }, + { r: 'ke', t: at(1400), id: 'c2', tid: 'a#1', ok: false }, + { r: 'te', t: at(2000), id: 'a#1', o: 'completed' }, + ] + const win = buildWindow(recs, { now: at(10_000), bots, rooms }) + const a = win.toolUse.find(entry => entry.lane === 'a') + const byName = Object.fromEntries(a.tools.map(tool => [tool.name, [tool.calls, tool.failed]])) + assert.deepEqual(byName.web_search, [1, 0]) + assert.deepEqual(byName['mcp__github__search'], [1, 1]) +})