- This run recorded no tool-result tokens (no tool calls, or a
- run predating per-tool result sizing), so the tool-output
- lever has nothing to scale.
+ This run recorded no tool-result tokens (no tool calls), so
+ the tool-output lever has nothing to scale.
)}
diff --git a/evalboard/app/runs/[id]/__tests__/run-identity.test.tsx b/evalboard/app/runs/[id]/__tests__/run-identity.test.tsx
index a2951def1..9a17aee7d 100644
--- a/evalboard/app/runs/[id]/__tests__/run-identity.test.tsx
+++ b/evalboard/app/runs/[id]/__tests__/run-identity.test.tsx
@@ -21,14 +21,13 @@ describe("RunIdentity", () => {
test("uses the vendor logo, not just the raw harness id", () => {
render(
diff --git a/evalboard/app/scribe/page.tsx b/evalboard/app/scribe/page.tsx
index 7560a2f52..5cb383c4d 100644
--- a/evalboard/app/scribe/page.tsx
+++ b/evalboard/app/scribe/page.tsx
@@ -40,7 +40,7 @@ export default async function ScribePage({
// Everything on this page reads SCRIBE_SOURCE's container (aria-runs), not
// the skills nightly's. No harness selector: the aria suite runs on a single
- // harness (delegate-sdk), so a scope control would only ever have one entry.
+ // harness (delegate), so a scope control would only ever have one entry.
const [overview, listing, adhoc] = await Promise.all([
getOverview(WINDOW, null, null, null, SCRIBE_SOURCE),
getRunListing(null, null, limit, null, SCRIBE_SOURCE),
diff --git a/evalboard/lib/__tests__/harness.test.ts b/evalboard/lib/__tests__/harness.test.ts
index c8a07a99b..08fd814a9 100644
--- a/evalboard/lib/__tests__/harness.test.ts
+++ b/evalboard/lib/__tests__/harness.test.ts
@@ -4,6 +4,7 @@ import {
DEFAULT_HARNESS,
KNOWN_HARNESSES,
harnessColor,
+ normalizeHarness,
orderHarnesses,
parseHarnessParam,
parseHarnessScope,
@@ -101,6 +102,25 @@ describe("parseHarnessScope", () => {
});
});
+describe("normalizeHarness", () => {
+ test("the retired delegate-sdk id folds into delegate", () => {
+ // One Delegate series across the 2026-10-07 switch, and old ?h= links
+ // still land on it.
+ expect(normalizeHarness("delegate-sdk")).toBe("delegate");
+ expect(parseHarnessParam("delegate-sdk")).toBe("delegate");
+ expect(parseHarnessScope("delegate-sdk")).toBe("delegate");
+ });
+
+ test("an Object.prototype key in ?h= stays a plain string", () => {
+ for (const key of ["constructor", "__proto__", "toString", "valueOf"]) {
+ expect(normalizeHarness(key)).toBe(key);
+ expect(parseHarnessParam(key)).toBe(key);
+ expect(parseHarnessScope(key)).toBe(key);
+ expect(typeof harnessColor(key)).toBe("string");
+ }
+ });
+});
+
// Color is bound to the harness, never to its index in the series list — a
// filter that removes one line must not repaint the survivors.
describe("harnessColor", () => {
diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts
index 113816506..b6e959c67 100644
--- a/evalboard/lib/__tests__/overview.test.ts
+++ b/evalboard/lib/__tests__/overview.test.ts
@@ -536,7 +536,7 @@ describe("collectPipelineRuns", () => {
});
describe("normalizeHarness", () => {
- test("null/undefined fold to claude-code (legacy pre-stamp runs)", () => {
+ test("null/undefined fold to claude-code", () => {
expect(normalizeHarness(null)).toBe("claude-code");
expect(normalizeHarness(undefined)).toBe("claude-code");
});
diff --git a/evalboard/lib/__tests__/parseMessages.test.ts b/evalboard/lib/__tests__/parseMessages.test.ts
index 3bffe27b9..79130f6ec 100644
--- a/evalboard/lib/__tests__/parseMessages.test.ts
+++ b/evalboard/lib/__tests__/parseMessages.test.ts
@@ -13,6 +13,7 @@ function msg(
text?: string;
thinking?: string;
toolUseId?: string;
+ messageId?: string;
},
) {
const block: Record
= { block_type: kind };
@@ -24,6 +25,7 @@ function msg(
started_at: opts.startedAt,
completed_at: opts.completedAt,
generation_duration_ms: opts.genMs,
+ message_id: opts.messageId ?? null,
content_blocks: [block],
};
}
@@ -38,7 +40,6 @@ describe("parseMessages — per-block-kind generation attribution", () => {
completedAt: "2026-01-01T00:00:05.000Z",
genMs: 5000,
}),
- // separate emission > 100ms gap so it doesn't collapse
msg("text", {
startedAt: "2026-01-01T00:00:06.000Z",
completedAt: "2026-01-01T00:00:07.000Z",
@@ -58,7 +59,7 @@ describe("parseMessages — per-block-kind generation attribution", () => {
});
test("collapsed mixed emission attributes each raw's gen to its own kind", () => {
- // Same emission (gap < 100ms) → one MessageEvent with two block kinds.
+ // Same message_id → one MessageEvent with two block kinds.
const turns: TurnEntry[] = [
{
messages: [
@@ -66,12 +67,14 @@ describe("parseMessages — per-block-kind generation attribution", () => {
startedAt: "2026-01-01T00:00:00.000Z",
completedAt: "2026-01-01T00:00:08.000Z",
genMs: 8000,
+ messageId: "msg_1",
}),
msg("tool_use", {
- startedAt: "2026-01-01T00:00:08.010Z", // 10ms gap
+ startedAt: "2026-01-01T00:00:08.010Z",
completedAt: "2026-01-01T00:00:08.500Z",
genMs: 500,
toolUseId: "tu_1",
+ messageId: "msg_1",
}),
],
commands: [
@@ -185,9 +188,8 @@ describe("parseMessages — per-block-kind generation attribution", () => {
describe("parseMessages — message_id collapsing", () => {
test("collapses splits sharing a message_id even when wall-clock gap is large", () => {
- // The CLI emits per-block-kind events back-to-back, but if a slow tool
- // result lands between them the gap heuristic would (incorrectly) split
- // them. With a shared message_id we collapse anyway.
+ // A slow tool result can land between the per-block-kind events of one
+ // response; the shared message_id still collapses them.
const turns: TurnEntry[] = [
{
messages: [
@@ -201,7 +203,6 @@ describe("parseMessages — message_id collapsing", () => {
},
{
role: "assistant",
- // Far apart in wall-clock — gap heuristic would split.
started_at: "2026-01-01T00:00:30.000Z",
completed_at: "2026-01-01T00:00:30.500Z",
generation_duration_ms: 500,
@@ -257,7 +258,7 @@ describe("parseMessages — message_id collapsing", () => {
expect(events).toHaveLength(2);
});
- test("falls back to gap heuristic when message_id absent (legacy runs)", () => {
+ test("entries without a message_id never collapse, however close", () => {
const turns: TurnEntry[] = [
{
messages: [
@@ -270,7 +271,7 @@ describe("parseMessages — message_id collapsing", () => {
},
{
role: "assistant",
- started_at: "2026-01-01T00:00:01.050Z", // 50ms gap — under threshold
+ started_at: "2026-01-01T00:00:01.000Z",
completed_at: "2026-01-01T00:00:01.200Z",
generation_duration_ms: 150,
content_blocks: [{ block_type: "text", text: "hi" }],
@@ -279,8 +280,7 @@ describe("parseMessages — message_id collapsing", () => {
},
];
const events = parseMessages(turns);
- expect(events).toHaveLength(1);
- expect(events[0].blockTypes).toEqual(["thinking", "text"]);
+ expect(events).toHaveLength(2);
});
});
diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts
index 5629f6f54..5a868a10b 100644
--- a/evalboard/lib/__tests__/runs.test.ts
+++ b/evalboard/lib/__tests__/runs.test.ts
@@ -39,12 +39,16 @@ describe("parseCriterionResults", () => {
description: "artifact exists",
score: 1,
evaluation_status: "evaluated",
+ pass_threshold: 0.9,
+ gating: true,
},
{
criterion_type: "run_command",
description: "validator runs",
score: 0,
evaluation_status: "not_evaluated",
+ pass_threshold: 0.9,
+ gating: true,
},
]);
@@ -54,7 +58,7 @@ describe("parseCriterionResults", () => {
]);
});
- test("legacy results default to evaluated", () => {
+ test("a row without the fields keeps the Python model defaults", () => {
const [result] = parseCriterionResults([
{ criterion_type: "file_exists", score: 0 },
]);
@@ -568,7 +572,7 @@ describe("toTaskRow grading split (#212)", () => {
describe("extractComponentShas", () => {
const baseEnv = {
- git_commit: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2",
+ coder_eval: "0.12.14",
skills_git_commit: "b2c3d4e",
cli_version: "1.2.0-alpha.20260603.7393",
};
@@ -595,7 +599,7 @@ describe("extractComponentShas", () => {
);
});
- test("legacy runs without tool_plugins are unchanged", () => {
+ test("a run without tool_plugins gets only the core chips", () => {
const out = extractComponentShas(baseEnv);
expect(out.map((c) => c.name)).toEqual(["coder_eval", "skills", "cli"]);
});
@@ -610,15 +614,16 @@ describe("extractComponentShas", () => {
test("drops components whose value is 'unknown' (in-container git SHAs)", () => {
// Per-task env_info captured in the sandbox can't `git rev-parse` the
- // coder_eval / skills checkouts, so those come back "unknown"; only the
- // npm-resolved cli + tool plugins survive.
+ // skills checkout, so it comes back "unknown".
const out = extractComponentShas({
- git_commit: "unknown",
skills_git_commit: "unknown",
cli_version: "1.2.0-alpha.20260604.7394",
tool_plugins: { "maestro-tool": "1.2.0-alpha.20260604.7394" },
});
expect(out.map((c) => c.name)).toEqual(["cli", "maestro-tool"]);
+ expect(
+ extractComponentShas({ coder_eval: "unknown" }).map((c) => c.name),
+ ).toEqual([]);
});
// The framework is pinned and consumed by released version (env_info
@@ -635,37 +640,6 @@ describe("extractComponentShas", () => {
"https://github.com/UiPath/coder_eval/releases/tag/v0.9.1",
);
});
-
- test("the version wins over a git_commit present in the same run", () => {
- // Both keys are captured on every run; the SHA must not shadow the
- // version just because it comes first in env_info.
- const out = extractComponentShas({
- git_commit: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2",
- coder_eval: "0.10.0",
- });
- expect(out.find((c) => c.name === "coder_eval")?.sha).toBe("0.10.0");
- });
-
- test("falls back to the git SHA when no version was captured", () => {
- // Legacy runs, and editable checkouts between releases, have only the
- // SHA — better a tree link than no chip at all.
- const out = extractComponentShas(baseEnv);
- const framework = out.find((c) => c.name === "coder_eval");
- expect(framework?.sha).toBe(baseEnv.git_commit);
- expect(framework?.url).toBe(
- `https://github.com/UiPath/coder_eval/tree/${baseEnv.git_commit}`,
- );
- });
-
- test("an 'unknown' version still falls through to the SHA", () => {
- const out = extractComponentShas({
- ...baseEnv,
- coder_eval: "unknown",
- });
- expect(out.find((c) => c.name === "coder_eval")?.sha).toBe(
- baseEnv.git_commit,
- );
- });
});
// The run header claims a single harness and model for the whole run; tallyModels
diff --git a/evalboard/lib/__tests__/selectTokenTotals.test.ts b/evalboard/lib/__tests__/selectTokenTotals.test.ts
index 4e5df8293..48fdaf2a3 100644
--- a/evalboard/lib/__tests__/selectTokenTotals.test.ts
+++ b/evalboard/lib/__tests__/selectTokenTotals.test.ts
@@ -46,7 +46,7 @@ describe("selectTokenTotals", () => {
const turns: TurnEntry[] = [
{
token_usage: {
- input_tokens: 3,
+ uncached_input_tokens: 3,
output_tokens: 240,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 60_000, // snapshot — far below the 120k sum
@@ -62,13 +62,13 @@ describe("selectTokenTotals", () => {
});
test("falls back to iteration token_usage when the stream has no token data", () => {
- // Legacy runs predating per-message tokens: every MessageEvent token
- // field is 0, so the stream total is 0 and we use the iteration aggregate.
+ // Every MessageEvent token field is 0, so the stream total is 0 and the
+ // iteration aggregate wins.
const messages = [msg({}), msg({})];
const turns: TurnEntry[] = [
{
token_usage: {
- input_tokens: 1000,
+ uncached_input_tokens: 1000,
output_tokens: 500,
cache_creation_input_tokens: 200,
cache_read_input_tokens: 100,
@@ -99,7 +99,7 @@ describe("selectTokenTotals", () => {
const turns: TurnEntry[] = [
{
token_usage: {
- input_tokens: 834,
+ uncached_input_tokens: 834,
output_tokens: 1834,
cache_creation_input_tokens: 51339, // model_usage cumulative
cache_read_input_tokens: 41844,
@@ -123,7 +123,7 @@ describe("selectTokenTotals", () => {
const turns: TurnEntry[] = [
{
token_usage: {
- input_tokens: 999999, // deliberately wrong/divergent — must be ignored
+ uncached_input_tokens: 999999, // deliberately wrong/divergent — must be ignored
output_tokens: 999999,
cache_creation_input_tokens: 999999,
cache_read_input_tokens: 999999,
diff --git a/evalboard/lib/__tests__/thinkingSim.test.ts b/evalboard/lib/__tests__/thinkingSim.test.ts
index 996d60fc1..61f0dfedf 100644
--- a/evalboard/lib/__tests__/thinkingSim.test.ts
+++ b/evalboard/lib/__tests__/thinkingSim.test.ts
@@ -54,8 +54,7 @@ function call(opts: {
thinkingOutputTokens?: number | null;
model?: string;
toolUses?: MessageToolUse[];
- // null = main thread (default), string = sub-agent branch, undefined =
- // run didn't record branch info (legacy → simulator disabled).
+ // null = main thread (default), string = sub-agent branch.
parentToolUseId?: string | null;
}): MessageEvent {
return {
@@ -77,8 +76,7 @@ function call(opts: {
cacheWriteTokens:
opts.cacheWriteTokens === undefined ? 0 : opts.cacheWriteTokens,
cacheReadTokens: 0,
- parentToolUseId:
- "parentToolUseId" in opts ? opts.parentToolUseId : null,
+ parentToolUseId: opts.parentToolUseId ?? null,
reasoningTokens: null,
thinkingOutputTokens:
opts.thinkingOutputTokens === undefined
@@ -501,22 +499,6 @@ describe("buildThinkingModel — tree-structured (sub-agent) cascade", () => {
expect(m.toolResultTokens).toBe(0);
expect(m.coeffToolWrite).toBe(0);
});
-
- test("legacy run without branch info → no model (simulator hidden)", () => {
- const m = buildThinkingModel(
- [
- call({
- generationMs: 10,
- thinkingMs: 10,
- outputTokens: 100,
- parentToolUseId: undefined, // run never recorded the field
- }),
- ],
- TOTALS,
- 0.5,
- );
- expect(m).toBeNull();
- });
});
// ---------------------------------------------------------------------------
diff --git a/evalboard/lib/blob.ts b/evalboard/lib/blob.ts
index 9e9624fe3..df6152d43 100644
--- a/evalboard/lib/blob.ts
+++ b/evalboard/lib/blob.ts
@@ -327,11 +327,8 @@ export async function ensureTaskDir(
else await ensureRunSummary(container, runId, destRoot);
const c = await getContainer(container);
const ops: Promise[] = [];
- // `listBlobsFlat` recurses, so both the flat legacy layout
- // (`//task.json`) and the nested replicate layout
- // (`//00/task.json`) download unchanged — the prefix
- // scope is the task subtree either way. `resolveTaskContentDir` in
- // runs.ts then picks the right shape at render time.
+ // `listBlobsFlat` recurses, so the prefix pulls every replicate
+ // (`///`) of the task in one listing.
//
// The activation sub-run is single-variant by construction (it is a
// nested run of its own), so it keeps the literal `default` segment.
diff --git a/evalboard/lib/harness.ts b/evalboard/lib/harness.ts
index b7603eb6b..6505aeebf 100644
--- a/evalboard/lib/harness.ts
+++ b/evalboard/lib/harness.ts
@@ -12,28 +12,34 @@ export const KNOWN_HARNESSES = [
"claude-code",
"codex",
"antigravity",
- "delegate-sdk",
+ "delegate",
] as const;
export type KnownHarness = (typeof KNOWN_HARNESSES)[number];
-// A run with no RunConfig harness predates the stamp; every such run was
-// claude-code (the only nightly harness before codex/antigravity), so
-// null/undefined folds to the default. Mirrors HarnessBadge's default.
+// A run whose rows name no harness folds to the default.
export const DEFAULT_HARNESS = "claude-code";
+
+// coder_eval's built-in `delegate` agent replaced coder_eval_uipath's
+// `delegate-sdk` adapter on 2026-10-07. Same agent under test, so one series.
+const HARNESS_ALIASES: Record = { "delegate-sdk": "delegate" };
+
export function normalizeHarness(harness: string | null | undefined): string {
- return harness ?? DEFAULT_HARNESS;
+ const h = harness ?? DEFAULT_HARNESS;
+ return Object.hasOwn(HARNESS_ALIASES, h) ? HARNESS_ALIASES[h] : h;
}
// Accept any syntactically-plausible harness id, defaulting to claude-code when
// absent or malformed. Deliberately NOT whitelisted against KNOWN_HARNESSES so
-// a newly-added harness (e.g. "delegate") is selectable the moment its runs
+// a newly-added harness is selectable the moment its runs
// exist — the value is only ever compared for equality against a run's stamped
// harness, never used in a path or query, so a bounded charset is enough.
export function parseHarnessParam(raw: string | string[] | undefined): string {
const v = Array.isArray(raw) ? raw[0] : raw;
if (typeof v !== "string") return DEFAULT_HARNESS;
const trimmed = v.trim();
- return /^[\w.-]{1,64}$/.test(trimmed) ? trimmed : DEFAULT_HARNESS;
+ return /^[\w.-]{1,64}$/.test(trimmed)
+ ? normalizeHarness(trimmed)
+ : DEFAULT_HARNESS;
}
// URL value for the unscoped view. Reserved: a real harness id could never
@@ -53,7 +59,7 @@ export function parseHarnessScope(
if (typeof v !== "string") return null;
const trimmed = v.trim();
if (!trimmed || trimmed === ALL_HARNESSES) return null;
- return /^[\w.-]{1,64}$/.test(trimmed) ? trimmed : null;
+ return /^[\w.-]{1,64}$/.test(trimmed) ? normalizeHarness(trimmed) : null;
}
// Series color per harness, for the multi-harness overview charts and their
@@ -72,7 +78,7 @@ export function parseHarnessScope(
// contrast of the four, and no surface identifies it by color alone (chart
// legend and tooltip both pair the color with the vendor logo and the
// harness name).
-// - delegate-sdk is NOT UiPath orange (#FA4616), even though that is the
+// - delegate is NOT UiPath orange (#FA4616), even though that is the
// brand color. Against Anthropic's coral it scores ΔE 9.5 normal-vision /
// 5.1 deutan — below the hard floor, i.e. two lines a full-color reader
// cannot tell apart. It takes the nearest deep red that separates instead.
@@ -83,7 +89,7 @@ const HARNESS_COLORS: Record = {
"claude-code": "#d97757", // Anthropic coral
codex: "#171717", // OpenAI near-black
antigravity: "#4285f4", // Google blue
- "delegate-sdk": "#9b1c1c", // deep red (UiPath orange collides with Anthropic)
+ delegate: "#9b1c1c", // deep red (UiPath orange collides with Anthropic)
};
// Neutral for any harness with no reserved slot. Deliberately NOT a generated
@@ -92,7 +98,9 @@ const HARNESS_COLORS: Record = {
const HARNESS_COLOR_FALLBACK = "#6b7280";
export function harnessColor(harness: string): string {
- return HARNESS_COLORS[harness] ?? HARNESS_COLOR_FALLBACK;
+ return Object.hasOwn(HARNESS_COLORS, harness)
+ ? HARNESS_COLORS[harness]
+ : HARNESS_COLOR_FALLBACK;
}
// Stable series order for charts and legends: known harnesses in display order,
diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts
index 841b7f434..f6dbbdec2 100644
--- a/evalboard/lib/overview.ts
+++ b/evalboard/lib/overview.ts
@@ -25,7 +25,7 @@ import type { Window } from "./reviews-types";
export interface RunPoint {
runId: string;
timestamp: number; // ms since epoch (UTC); used as the chart x-coordinate
- // The harness this run ran on, normalized (legacy runs fold to claude-code).
+ // The harness this run ran on, normalized (see normalizeHarness).
// Every run belongs to exactly one harness, so the charts split the points
// into one series per harness and color each by identity.
harness: string;
@@ -196,8 +196,8 @@ export interface RunListingRow {
taskDurationSeconds: number | null;
// Optional so existing test factories stay valid.
agentSeconds?: number | null;
- // Run-level harness (coder-eval AgentKind) for the Harness column; null on
- // legacy runs that predate the RunConfig stamp / carry no agent_config.type.
+ // Run-level harness (coder-eval AgentKind) for the Harness column; null when
+ // no row names one.
// Optional so existing test factories stay valid.
harness?: string | null;
}
diff --git a/evalboard/lib/pricing.ts b/evalboard/lib/pricing.ts
index 6099dad65..876f131d5 100644
--- a/evalboard/lib/pricing.ts
+++ b/evalboard/lib/pricing.ts
@@ -119,8 +119,8 @@ export function tokenBucketUsd(
// SDK only reports a cumulative per-turn `total_cost_usd`, never per message.
//
// Returns null when the model isn't in the rate table (can't price it) or when
-// no token figure was recorded for the message (older runs predating
-// per-message tokens) — null renders as "—" rather than a misleading $0.00.
+// no token figure was recorded for the message — null renders as "—" rather
+// than a misleading $0.00.
export function messageCostUsd(usage: {
model: string | null;
inputTokens: number | null;
diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts
index ccd27bdba..ba91e3164 100644
--- a/evalboard/lib/runs.ts
+++ b/evalboard/lib/runs.ts
@@ -76,7 +76,7 @@ export interface RunSummary {
componentShas: ComponentSha[];
// What actually produced this run, for the run header. `harness` is the
// coder-eval AgentKind from the RunConfig stamp (falling back to the per-task
- // agent_config vote); null only on legacy runs that identify neither.
+ // agent_config vote); null only when no row names one.
harness: string | null;
// Dominant per-task `model_used`, with how many distinct models the run
// touched. Normally 1 — a run is one model — but an A/B experiment fans
@@ -91,13 +91,14 @@ export interface TaskResultSummary {
// Experiment arm this row belongs to. A run with variants writes one row per
// (task, variant, replicate) and stores each arm's content under its own
// // subtree, so this — not taskId alone — is what tells
- // two arms of the same task apart. Null on runs whose run.json predates the
- // field; every reader falls back to DEFAULT_VARIANT_ID, which is the segment
- // a single-arm run writes anyway.
+ // two arms of the same task apart. Null on a mature carry-forward row; every
+ // reader falls back to DEFAULT_VARIANT_ID, which is the segment a single-arm
+ // run writes anyway.
variantId: string | null;
- // Replicate index of this row, or null when repeats are disabled / on legacy
- // runs. Repeated runs share taskId, so this disambiguates sibling rows and is
- // carried in the per-task link (?r=NN) so each replicate opens its own detail.
+ // Replicate index of this row, or null when repeats are disabled or on a
+ // mature carry-forward row. Repeated runs share taskId, so this
+ // disambiguates sibling rows and is carried in the per-task link (?r=NN) so
+ // each replicate opens its own detail.
replicateIndex: number | null;
status: string | null;
weightedScore: number | null;
@@ -119,10 +120,10 @@ export interface TaskResultSummary {
expectedSeconds: number | null;
// True when the agent's final iteration emitted a text reply
// (i.e. ResultMessage.result was non-empty). Lets grid/trends
- // Turns cells inflate by +1 on legacy runs that lack total_turns.
+ // Turns cells inflate by +1 on a row that lacks total_turns.
hasFinalReply: boolean;
- // Per-task token totals from run.json. Null on legacy runs that
- // don't record per-task token counts. `inputTokens` is the disjoint
+ // Per-task token totals from run.json. Null on a row that records
+ // none (a mature carry-forward row). `inputTokens` is the disjoint
// uncached slice (run.json `input_tokens` is serialized from
// TokenUsage.uncached_input_tokens — see run_record.py), so it
// sits alongside the cache columns without overlap.
@@ -131,11 +132,11 @@ export interface TaskResultSummary {
cacheCreationTokens: number | null;
cacheReadTokens: number | null;
// Model the task ran on (run.json `model_used`). Used to price token
- // buckets as USD for the Tokens↔USD column toggle. Null on legacy runs.
+ // buckets as USD for the Tokens↔USD column toggle. Null on a mature
+ // carry-forward row.
model: string | null;
tags: string[];
- // Derived primary group. See deriveSkill below for the resolution chain
- // (new runs use task_path; older runs fall back to a tag heuristic).
+ // Derived primary group. See deriveSkill below for the resolution chain.
skill: string | null;
// True when this row is a "mature" task the nightly skipped this run and
// carried forward as a pass (run.json `mature_skipped`). The task wasn't
@@ -152,8 +153,7 @@ export interface CriterionResult {
evaluationStatus: "evaluated" | "not_evaluated";
// Mirrors the Python CriterionResult fields. `gating: false` (weight: 0) means
// the criterion is informational — measured, but excluded from the score and
- // the pass/fail gate, so it must not render as PASS/FAIL. Both default the way
- // pre-existing task.json files behave: gating, threshold 0.9.
+ // the pass/fail gate, so it must not render as PASS/FAIL.
passThreshold: number;
gating: boolean;
}
@@ -246,17 +246,16 @@ export interface MessageEvent {
// Per-message token usage from the Anthropic API. Summed across grouped
// raws (one logical emission may span multiple MessageEntry rows when
// the CLI splits content blocks). null when no raw in the group carried
- // a usage figure — older runs predating per-message tokens.
+ // a usage figure.
inputTokens: number | null;
outputTokens: number | null;
cacheWriteTokens: number | null;
cacheReadTokens: number | null;
// Branch identity for the prompt-cache cascade tree: the Task tool_use_id
- // that spawned this message's sub-agent, null for the main thread, or
- // `undefined` when the run never recorded it (legacy). Calls grouped by this
- // value form a branch whose context is re-read only within itself — a
- // sub-agent's tokens don't cascade into the main thread or siblings.
- parentToolUseId: string | null | undefined;
+ // that spawned this message's sub-agent, null for the main thread. Calls
+ // grouped by this value form a branch whose context is re-read only within
+ // itself — a sub-agent's tokens don't cascade into the main thread or siblings.
+ parentToolUseId: string | null;
// Anthropic's `reasoning_tokens`. ~Always 0 from the SDK, so it is NOT
// used to attribute thinking output — see thinkingOutputTokens. Kept for
// completeness. null when no raw in the group recorded it.
@@ -279,7 +278,7 @@ export interface MessageEvent {
// is the rate-accurate per-message attribution. Actual per-call cost (LiteLLM
// backend) is NOT distributed here — it lives in the separate providerCalls
// per-call table. null when the model is unpriced or no token figure was
- // recorded (older runs).
+ // recorded.
costUsd: number | null;
// Only set on a `reconciliation` row: the human-readable explanation of why
// these tokens are unattributed (e.g. fixed prompt overhead + sub-agent
@@ -307,8 +306,8 @@ export interface MessageToolUse {
// UNIONED rather than summed. `durationMs` alone cannot do that: two
// concurrent 5s calls sum to 10s of wall clock that only took 5, and a
// residual computed against the sum goes negative for a healthy run.
- // Null on a call the harness never timed, or on a run predating the
- // fields — such a call falls back to `durationMs` (see busyMs callers).
+ // Null on a call the harness never timed or never bounded (the retired
+ // `delegate-sdk` adapter recorded only `durationMs`).
execStartMs: number | null;
execEndMs: number | null;
isError: boolean;
@@ -320,8 +319,7 @@ export interface MessageToolUse {
outputTokens: number | null;
// Approx token size of THIS tool's result content (measured from the
// untruncated result, cache-independent). Drives the cost simulator's
- // tool-output lever — works whether or not prompt caching was enabled. null
- // on runs predating the field.
+ // tool-output lever — works whether or not prompt caching was enabled.
resultTokens: number | null;
}
@@ -339,8 +337,7 @@ export interface TaskDetail extends TaskResultSummary {
// environment_info (cli + @uipath/*-tool plugins) — the versions that
// actually ran for this task, which can diverge from the run-level
// aggregate when tool resolution drifts mid-run (a task that invokes
- // `uip ` can pull a different alpha than its siblings). Empty for
- // legacy task.json files captured before environment_info existed.
+ // `uip ` can pull a different alpha than its siblings).
componentShas: ComponentSha[];
finalStatus: string | null;
errorMessage: string | null;
@@ -354,12 +351,10 @@ export interface TaskDetail extends TaskResultSummary {
// last turn's ResultMessage.result. Renders as the trailing entry in
// the Turn timeline so the (5) header reconciles with the 4 tool calls.
finalAssistantText: string | null;
- // Per-assistant-message timeline data. Empty array for legacy task.json
- // files that don't have iterations[].messages; the message-timeline
- // section then doesn't render.
+ // Per-assistant-message timeline data. Empty when the task produced no
+ // messages; the message-timeline section then doesn't render.
messages: MessageEvent[];
- // Per-task token totals summed across iterations. All zeros for legacy
- // runs that don't record per-iteration token_usage.
+ // Per-task token totals summed across iterations.
tokens: TokenTotals;
// Per-sub-agent token breakdown, derived by grouping the parsed messages on
// `parentToolUseId` (the spawning Agent call's tool_use_id). Each sub-agent's
@@ -499,12 +494,12 @@ export function aggregateSubAgentUsage(
export interface RawTaskResult {
task_id?: string;
// Experiment arm that produced this row (the sub-dir). Written by
- // run_record.py on every run; absent on runs that predate it, which
- // read as DEFAULT_VARIANT_ID.
+ // run_record.py on every executed row; absent on a mature carry-forward row,
+ // which reads as DEFAULT_VARIANT_ID.
variant_id?: string | null;
// Replicate index of this row (the // sub-dir). Repeated
// runs of one task share a task_id; this is what tells the rows apart. Null
- // on runs that didn't track replicates (repeats disabled / legacy run.json).
+ // when repeats are disabled.
replicate_index?: number | null;
status?: string;
weighted_score?: number;
@@ -533,23 +528,22 @@ export interface RawTaskResult {
expected_seconds?: number | null;
// Documented visible-turn count (tool calls + final reply) — the canonical
// metric the "within expected turns" chart compares against expected_turns.
- // Absent on runs predating this field; visibleTurnsFromRaw() then reconstructs
- // it from actual_commands + has_final_reply (the identical formula) so the
- // metric still populates for historical runs.
+ // Absent on a salvaged run.json (eval_runner `salvage-run`);
+ // visibleTurnsFromRaw() then reconstructs it from actual_commands +
+ // has_final_reply (the identical formula).
visible_turns?: number | null;
// True iff the final iteration's ResultMessage.result was non-empty.
- // Absent on legacy runs predating the field — treated as false.
+ // Absent on a salvaged run.json — treated as false.
has_final_reply?: boolean;
tags?: string[];
- // Source YAML path. Persisted by coder-eval starting with the
- // task_path PR; absent on older runs (deriveSkill falls back to tags).
+ // Source YAML path.
task_path?: string | null;
// Model the task ran on (e.g. "claude-sonnet-4-6"). Used to price token
- // buckets as USD. Absent on legacy runs.
+ // buckets as USD.
model_used?: string | null;
// Per-task agent config; `type` is the harness (coder-eval AgentKind, e.g.
// "claude-code" | "codex" | "antigravity"). Used to derive the run's harness
- // when the run-level RunConfig stamp is absent (direct coder-eval / legacy runs).
+ // when the run-level RunConfig stamp is absent (a direct coder-eval run).
agent_config?: { type?: string | null } | null;
// Activation enrichment the dashboard folds onto each case row in the nested
// activation run.json (see cli.py _finalize_activation_run). Absent on skills
@@ -679,13 +673,9 @@ function mapActivation(
};
}
-// Components captured in run env_info. Each entry may accept multiple keys —
-// the CLI shipped as `cli_git_commit` (when built from source) until
-// coder_eval #276 switched the runner to consume @uipath/cli@alpha from
-// GitHub Packages, after which it's `cli_version` (output of `uip --version`,
-// e.g. "0.1.21-alpha.234"). New keys come first so they win when both
-// happen to be present. `nonShaUrl` is the link target used when the env
-// value isn't a SHA (e.g. an npm package version) — null means no link.
+// Components captured in run env_info, one key each. `nonShaUrl` is the link
+// target used when the env value isn't a SHA (e.g. an npm package version) —
+// null means no link.
const COMPONENTS: {
display: string;
repo: string | null;
@@ -694,39 +684,28 @@ const COMPONENTS: {
// precedence over `nonShaUrl` when it returns a URL. This is how a released
// component links at its own version rather than at a moving list page.
versionUrl?: (version: string) => string | null;
- keys: string[];
+ key: string;
}[] = [
{
display: "coder_eval",
repo: "UiPath/coder_eval",
nonShaUrl: null,
- // Releases are tagged `v` by the Release workflow, so the
- // version resolves to an exact release page. Runs from before tagging
- // began (v0.8.2) recorded a hardcoded `0.1.0` that was never released;
- // render those as plain text rather than a link that 404s.
- versionUrl: (v) =>
- v === "0.1.0"
- ? null
- : `https://github.com/UiPath/coder_eval/releases/tag/v${v}`,
- // The framework identifies itself by the released package version
- // (env_info `coder_eval`, e.g. "0.9.1"), which is what the suite
- // actually pins — so that is the chip, and it links to the release.
- // `git_commit` is a fallback for two cases the version can't cover: an
- // editable checkout between releases, and legacy runs written before the
- // version was captured. It is a SHA, so it links to the tree.
- keys: ["coder_eval", "git_commit"],
+ // The released package version (e.g. "0.12.14"), which is what the
+ // suite pins. The Release workflow tags `v`.
+ versionUrl: (v) => `https://github.com/UiPath/coder_eval/releases/tag/v${v}`,
+ key: "coder_eval",
},
{
display: "skills",
repo: "UiPath/skills",
nonShaUrl: null,
- keys: ["skills_git_commit"],
+ key: "skills_git_commit",
},
{
display: "cli",
repo: "UiPath/cli",
nonShaUrl: "https://github.com/UiPath/cli/pkgs/npm/cli/versions",
- keys: ["cli_version", "cli_git_commit"],
+ key: "cli_version",
},
];
@@ -736,20 +715,12 @@ export function extractComponentShas(
if (!env) return [];
const out: ComponentSha[] = [];
for (const comp of COMPONENTS) {
- let value: string | null = null;
- for (const k of comp.keys) {
- const v = env[k];
- // Skip "unknown": the git-SHA components (coder_eval / skills)
- // resolve to that string when env_info is captured in-container
- // (the task sandbox has no repo checkout to `git rev-parse`), and
- // an unknown SHA is never worth a chip. npm-based versions
- // (cli / *-tool) still resolve in-container, so those survive.
- if (typeof v === "string" && v && v.toLowerCase() !== "unknown") {
- value = v;
- break;
- }
+ const value = env[comp.key];
+ // Skip "unknown": what env_info records for a value it could not
+ // resolve (in-container there is no repo checkout to `git rev-parse`).
+ if (typeof value !== "string" || !value || value.toLowerCase() === "unknown") {
+ continue;
}
- if (value == null) continue;
let url: string | null = null;
if (comp.repo && SHA_RE.test(value)) {
url = `https://github.com/${comp.repo}/tree/${value}`;
@@ -871,10 +842,10 @@ function taskContentBase(
}
// Resolve the skill (primary grouping axis) for a task. Two-stage fallback:
-// 1. New runs: parse "tasks//..." from task_path — the folder a task
-// lives under in the skills repo is the authoritative skill name.
-// 2. Older runs (or any task without a recognisable task_path): pick the
-// first skill-shaped tag. Convention in the skills repo: every task
+// 1. Parse "tasks//..." from task_path — the folder a task lives
+// under in the skills repo is the authoritative skill name.
+// 2. A task_path outside a tasks// folder: pick the first
+// skill-shaped tag. Convention in the skills repo: every task
// lists its folder name as the first tag, and skill folders are
// either "activation" or "uipath-*". The prefix check filters out
// generic tags like "smoke"/"e2e" that may appear if a task is
@@ -1211,8 +1182,8 @@ export interface RunOverview {
// present, else reconstructed as actual_commands + (1 if final reply). That
// reconstruction is the documented turn rule (tool calls + final reply) and is
// provably identical to the persisted field — so it backfills the "within
-// expected turns" metric for runs written before `visible_turns` existed. null
-// when neither signal is present.
+// expected turns" metric for a salvaged run.json. null when neither signal is
+// present.
export function visibleTurnsFromRaw(t: {
visible_turns?: number | null;
actual_commands?: number | null;
@@ -1227,8 +1198,8 @@ export function visibleTurnsFromRaw(t: {
// The run's harness + environment. Prefers the run-level RunConfig stamped by
// eval_runner into environment_info.run_config; falls back to the most common
-// per-task agent_config.type (direct coder-eval / legacy runs carry no stamp).
-// Returns null harness when nothing identifies it (legacy pre-harness runs).
+// per-task agent_config.type (a direct coder-eval run carries no stamp).
+// Returns null harness when no row names one (nothing executed).
export function extractRunConfig(data: RawRunJson): {
harness: string | null;
environment: string | null;
@@ -1585,20 +1556,14 @@ interface MessageEntry {
completed_at?: string | null;
generation_duration_ms?: number | null;
content_blocks?: ContentBlockEntry[];
- // Anthropic API message_id. The Claude Code CLI splits one API response
- // into per-block-kind events that share this id; we collapse by id when
- // present, falling back to a wall-clock gap heuristic for older runs that
- // didn't record it.
+ // API message id. The CLI splits one API response into per-block-kind
+ // events that share this id, and parseMessages collapses by it.
message_id?: string | null;
// The Task tool_use_id that spawned this message's sub-agent, or null for
- // the main thread. KEY ABSENT (undefined) on runs that predate branch
- // capture — distinct from an explicit null. Used to model the cache cascade
- // as a tree; its absence disables the cost simulator (we can't tell whether
- // sub-agents ran, so a flat cascade would be wrong).
+ // the main thread. Used to model the cache cascade as a tree.
parent_tool_use_id?: string | null;
- // Per-message token usage (recorded since coder_eval #336). Absent on
- // legacy runs. cache_* keys here are the message-record names, distinct
- // from the iteration token_usage's cache_*_input_tokens.
+ // Per-message token usage. cache_* keys here are the message-record names,
+ // distinct from the iteration token_usage's cache_*_input_tokens.
input_tokens?: number | null;
output_tokens?: number | null;
cache_creation_tokens?: number | null;
@@ -1649,12 +1614,8 @@ export interface TurnEntry {
}
interface TokenUsageEntry {
- // On current runs `input_tokens` is the TOTAL prompt input (uncached +
- // cache-creation + cache-read); `uncached_input_tokens` is the disjoint
- // fresh slice we actually want to display next to the cache columns. Legacy
- // runs predate the split and carry only `input_tokens`, which back then
- // meant the disjoint uncached slice — so it's the correct fallback.
- input_tokens?: number | null;
+ // The disjoint fresh slice shown next to the cache columns. (`input_tokens`,
+ // also serialized, is the TOTAL prompt input and would double-count them.)
uncached_input_tokens?: number | null;
output_tokens?: number | null;
cache_creation_input_tokens?: number | null;
@@ -1662,8 +1623,7 @@ interface TokenUsageEntry {
total_cost_usd?: number | null;
}
-// Token counts summed across all iterations of a task. Optional so legacy
-// task.json files without per-iteration token_usage just expose zeros.
+// Token counts summed across all iterations of a task.
export interface TokenTotals {
input: number;
output: number;
@@ -1759,22 +1719,6 @@ function previewString(value: unknown, cap: number): string | null {
return s.length > 0 ? s : null;
}
-// Gap between one assistant message's completed_at and the next message's
-// started_at that we treat as "same model emission". coder_eval records each
-// content-block kind (thinking / tool_use / text) as its own MessageEntry,
-// so a single conceptual turn often spans 2-3 entries with gap≈0. Tool
-// execution sits *between* turns and adds 100ms+, so 100ms is a safe split
-// threshold. Anything below is grouped into one logical turn; this is also
-// how parallel tool_use blocks (whether one MessageEntry with N tool_use
-// blocks, or N MessageEntries emitted instantaneously) get folded together.
-const SAME_EMISSION_GAP_MS = 100;
-
-function tsMs(s: string | null | undefined): number | null {
- if (!s) return null;
- const t = Date.parse(s);
- return Number.isNaN(t) ? null : t;
-}
-
// ~4 chars/token is the common rough estimate for English/code. The exact
// constant doesn't matter for *splitting* time across parallel tools (it
// cancels in the ratio), but using a real-ish number keeps the helper useful
@@ -1876,13 +1820,10 @@ export function parseMessages(turns: TurnEntry[]): MessageEvent[] {
type Raw = {
startedAt: string | null;
completedAt: string | null;
- startMs: number | null;
- endMs: number | null;
generationMs: number | null;
messageId: string | null;
- // null = main thread, string = sub-agent branch, undefined = run
- // didn't record branch info (legacy).
- parentToolUseId: string | null | undefined;
+ // null = main thread, string = sub-agent branch.
+ parentToolUseId: string | null;
blockTypes: ("thinking" | "tool_use" | "text")[];
thinkingParts: string[];
textParts: string[];
@@ -1906,21 +1847,15 @@ export function parseMessages(turns: TurnEntry[]): MessageEvent[] {
const r: Raw = {
startedAt: msg.started_at ?? null,
completedAt: msg.completed_at ?? null,
- startMs: tsMs(msg.started_at),
- endMs: tsMs(msg.completed_at),
generationMs:
typeof msg.generation_duration_ms === "number"
? msg.generation_duration_ms
: null,
messageId: typeof msg.message_id === "string" ? msg.message_id : null,
- // Preserve the absent/null distinction: undefined when the run
- // never wrote the key (legacy), null when explicitly main thread.
parentToolUseId:
- msg.parent_tool_use_id === undefined
- ? undefined
- : typeof msg.parent_tool_use_id === "string"
- ? msg.parent_tool_use_id
- : null,
+ typeof msg.parent_tool_use_id === "string"
+ ? msg.parent_tool_use_id
+ : null,
blockTypes: [],
thinkingParts: [],
textParts: [],
@@ -2069,9 +2004,10 @@ export function parseMessages(turns: TurnEntry[]): MessageEvent[] {
raws.push(r);
}
- // Second pass: collapse consecutive raws that share a model emission
- // (tiny gap between prev.end and next.start). Parallel tool_use lives
- // entirely inside one emission, so it ends up in one MessageEvent.
+ // Second pass: collapse consecutive raws that share a message_id. The
+ // CLI splits one API response into one entry per content-block kind,
+ // and parallel tool_use lives inside one response, so each response
+ // ends up as one MessageEvent.
let group: Raw[] = [];
const flush = () => {
if (group.length === 0) return;
@@ -2083,9 +2019,8 @@ export function parseMessages(turns: TurnEntry[]): MessageEvent[] {
const toolUses: MessageToolUse[] = [];
let genSum = 0;
let haveGen = false;
- // Each raw is one content-block emission (see SAME_EMISSION_GAP_MS
- // comment), so its generation_duration_ms attaches to whichever
- // block kind it carries.
+ // Each raw is one content-block emission, so its
+ // generation_duration_ms attaches to whichever block kind it carries.
let thinkSum = 0;
let haveThink = false;
let textSum = 0;
@@ -2277,21 +2212,7 @@ export function parseMessages(turns: TurnEntry[]): MessageEvent[] {
continue;
}
const prev = group[group.length - 1];
- // Prefer message_id when both raws have one — that's the
- // authoritative signal that the CLI split a single API response.
- // Fall back to the wall-clock gap heuristic for older runs that
- // didn't record message_id.
- let sameEmission: boolean;
- if (prev.messageId != null && r.messageId != null) {
- sameEmission = prev.messageId === r.messageId;
- } else {
- const gap =
- prev.endMs != null && r.startMs != null
- ? r.startMs - prev.endMs
- : Number.POSITIVE_INFINITY;
- sameEmission = gap <= SAME_EMISSION_GAP_MS;
- }
- if (sameEmission) {
+ if (prev.messageId != null && prev.messageId === r.messageId) {
group.push(r);
} else {
flush();
@@ -2464,22 +2385,8 @@ export function replicateDirName(replicate: number): string {
return String(replicate).padStart(2, "0");
}
-// Runs uploaded before the replicate-index layout (/task.json) and
-// after (//task.json) coexist in the blob store. Prefer the
-// nested replicate shape; fall back to flat ONLY for replicate 0 (the implicit
-// single result a legacy run had) so legacy runs keep rendering. A missing
-// non-zero replicate resolves to its (absent) nested dir → read fails → 404.
-async function resolveTaskContentDir(
- taskDir: string,
- replicate: number,
-): Promise {
- const nested = path.join(taskDir, replicateDirName(replicate));
- try {
- await fs.access(path.join(nested, "task.json"));
- return nested;
- } catch {
- return replicate === 0 ? taskDir : nested;
- }
+function replicateContentDir(taskDir: string, replicate: number): string {
+ return path.join(taskDir, replicateDirName(replicate));
}
// Row matcher shared by every per-task reader: a row belongs to (taskId,
@@ -2526,9 +2433,9 @@ export async function resolveVariantId(
}
// Replicate indices present for a task in this run, ascending (e.g. [0, 1, 2]
-// for a task run 3×). Drives the task page's run selector. A non-repeated or
-// legacy run yields [0] (rows carry no replicate_index → treated as 0); an
-// unknown task yields []. Reads the same run.json readTaskDetail matches against.
+// for a task run 3×). Drives the task page's run selector. A non-repeated run
+// yields [0] (rows carry no replicate_index → treated as 0); an unknown task
+// yields []. Reads the same run.json readTaskDetail matches against.
export async function readTaskReplicates(
runId: string,
taskId: string,
@@ -2562,20 +2469,17 @@ export async function readTaskDetail(
: await readRunJson(runId, source);
// A run with variants repeats a task_id once per arm and a repeated run
// repeats it once per replicate, so the row is only identified by all three
- // of (task_id, variant_id, replicate_index). Legacy rows carry neither
- // variant_id nor replicate_index (null) → treated as the default arm at
- // replicate 0, so an old single-result run still resolves.
+ // of (task_id, variant_id, replicate_index). A null replicate_index
+ // (repeats disabled) reads as replicate 0.
const matches = (data?.task_results ?? []).filter((t) =>
rowMatches(t, taskId, variantId),
);
- const rawTask =
- matches.find((t) => (t.replicate_index ?? 0) === replicate) ??
- (replicate === 0 ? matches[0] : undefined);
+ const rawTask = matches.find((t) => (t.replicate_index ?? 0) === replicate);
if (!rawTask) return null;
const row = toTaskRow(rawTask);
const taskDir = taskContentBase(runId, taskId, variantId, source);
- const contentDir = await resolveTaskContentDir(taskDir, replicate);
+ const contentDir = replicateContentDir(taskDir, replicate);
const task = await readJson<{
final_status?: string;
error_message?: string;
@@ -2602,11 +2506,9 @@ export async function readTaskDetail(
);
const artifactRoot = path.join(contentDir, "artifacts");
- // relPath is stored relative to the run root so the /api/file route can
- // resolve it against the source's / without needing to know the task
- // subdir. New layout yields `default//00/artifacts/...`; flat
- // layout yields `default//artifacts/...`. `resolveSafePath`
- // validates parts[0] and parts[1], so both shapes pass the check.
+ // relPath is stored relative to the run root (`///
+ // artifacts/...`) so the /api/file route can resolve it against the
+ // source's / without needing to know the task subdir.
const artifactPrefix = path.relative(
path.join(dir, runId),
artifactRoot,
@@ -2666,7 +2568,7 @@ export async function readTaskDetail(
return {
...row,
// task.json is the richer source: it carries the trailing text
- // even on legacy run.json files that predate has_final_reply.
+ // even when the run.json row has no has_final_reply (a salvaged run).
// Overriding row.hasFinalReply here keeps detail TURNS, Turn
// timeline header, and the reply row in lockstep.
hasFinalReply: finalAssistantText != null,
@@ -2693,19 +2595,13 @@ export async function readTaskDetail(
};
}
-// Pick the run's authoritative cumulative token totals. The per-message stream
-// run-level token totals, picking the more complete of two sources:
-// - iteration token_usage: on current runs this is built from the SDK
-// ResultMessage `model_usage` (cumulative per-model billing, reconciles to
-// total_cost_usd, and INCLUDES sub-agent cache-creation/input). On legacy
-// runs it is the old ResultMessage `usage` snapshot, which under-reports the
-// cache-read cascade ~2-3x.
-// - per-message stream sum: deduped per message_id; captures the cache-read
-// cascade but UNDER-reports sub-agent tokens (those emissions aren't fully
-// bubbled into the stream).
-// Neither dominates across both run vintages, so pick the larger total — the
-// more complete figure. New runs → iteration (model_usage) wins; legacy runs →
-// per-message wins over the stale snapshot.
+// Pick the run's authoritative cumulative token totals. When the harness writes a
+// reconciliation entry per turn (as claude-code does), the message stream sums
+// exactly to the billed total, so the stream wins outright. Otherwise pick the
+// more complete of two sources:
+// - iteration token_usage: the harness's own cumulative per-turn figure.
+// - per-message stream sum: deduped per message_id; can UNDER-report
+// sub-agent tokens and fixed prompt overhead.
export function selectTokenTotals(
messages: MessageEvent[],
turns: TurnEntry[],
@@ -2718,15 +2614,13 @@ export function selectTokenTotals(
if (messages.some((m) => m.role === "reconciliation")) {
return fromMessages;
}
- // Legacy / pre-reconciliation runs: the stream may under-report (sub-agent
- // tokens, fixed prompt overhead). Fall back to the more-complete of the two
- // sources, as before.
+ // No reconciliation entry: the stream may under-report (sub-agent tokens,
+ // fixed prompt overhead), so take the more complete of the two sources.
const fromIterations = sumTokenTotals(turns);
return fromIterations.total >= fromMessages.total ? fromIterations : fromMessages;
}
-// Token totals from the iteration token_usage aggregate (model_usage-derived on
-// current runs; the ResultMessage snapshot on legacy runs).
+// Token totals from the iteration token_usage aggregate.
function sumTokenTotals(turns: TurnEntry[]): TokenTotals {
let input = 0;
let output = 0;
@@ -2735,9 +2629,7 @@ function sumTokenTotals(turns: TurnEntry[]): TokenTotals {
for (const t of turns) {
const tu = t.token_usage;
if (!tu) continue;
- // Disjoint uncached slice: prefer the explicit field on current runs,
- // fall back to legacy `input_tokens` (which meant uncached back then).
- input += tu.uncached_input_tokens ?? tu.input_tokens ?? 0;
+ input += tu.uncached_input_tokens ?? 0;
output += tu.output_tokens ?? 0;
cacheCreation += tu.cache_creation_input_tokens ?? 0;
cacheRead += tu.cache_read_input_tokens ?? 0;
@@ -2835,7 +2727,7 @@ export async function readLogTail(
variantId,
);
const taskDir = taskContentBase(runId, taskId, variantId, source);
- const contentDir = await resolveTaskContentDir(taskDir, replicate);
+ const contentDir = replicateContentDir(taskDir, replicate);
const logPath = path.join(contentDir, "task.log");
const raw = await fs.readFile(logPath, "utf-8").catch(() => "");
if (raw.length <= maxBytes) return raw;
@@ -2868,7 +2760,7 @@ export async function readConversationLog(
variantId,
);
const taskDir = taskContentBase(runId, taskId, variantId, source);
- const contentDir = await resolveTaskContentDir(taskDir, replicate);
+ const contentDir = replicateContentDir(taskDir, replicate);
const logPath = path.join(contentDir, "conversation.log");
const raw = await fs.readFile(logPath, "utf-8").catch(() => "");
if (raw.length <= maxBytes) return raw;
diff --git a/evalboard/lib/thinkingSim.ts b/evalboard/lib/thinkingSim.ts
index 1ca0f272c..78e51b982 100644
--- a/evalboard/lib/thinkingSim.ts
+++ b/evalboard/lib/thinkingSim.ts
@@ -14,12 +14,8 @@
// • Tool results are *injected* (not model output), so scaling them touches
// the cache write/read cascade only — never the output bill.
//
-// Tool-result token volume is not recorded directly (only a 280-char preview),
-// so it is ESTIMATED as cache growth unexplained by the model's own output: a
-// result returned after call j is the new cache-written content on call j+1, so
-// toolResult_j ≈ cacheWrite_{j+1} − output_j (clamped ≥ 0).
-// This needs per-message tokens (coder_eval #336+); on older runs the tool lever
-// has nothing to estimate and is disabled.
+// Tool-result token volume is MEASURED from each call's recorded result_tokens
+// (cache-independent, so it holds with prompt caching on or off).
//
// Model (all per-call quantities indexed by emission j = 0..M-1, in order):
//
@@ -44,7 +40,6 @@
// the first of those L_j is the cache write, the rest are cache reads →
// writes_j = (L_j ≥ 1 ? 1 : 0), reads_j = max(0, L_j - 1).
// (On a run with no sub-agents, L_j = M-1-j and this is the flat cascade.)
-// Requires per-call branch ids; runs lacking them disable the model entirely.
//
// Held fixed ("assuming everything else is fine", per the feature request): the
// trajectory itself — number of calls, tool calls, tool-result sizes, and the
@@ -221,7 +216,7 @@ export interface ThinkingModel {
thinkTokens: number;
// Total tool-result tokens injected across the run, MEASURED from the recorded
// result content (cache-independent — see header). 0 only when the run made no
- // tool calls (or predates per-tool resultTokens), which disables the lever.
+ // tool calls, which disables the lever.
toolResultTokens: number;
// Recorded cost (sum of iteration total_cost_usd), for reference. May be
// null on runs that didn't record it. The projection's own baseline is
@@ -278,17 +273,6 @@ export function buildThinkingModel(
const calls = messages.length;
if (calls === 0) return null;
- // The prompt-cache cascade is a TREE, not a flat list. Sub-agent (Task tool)
- // calls bubble up into the same message stream, but each sub-agent has its
- // own context that is re-read only within its own branch — never by the main
- // thread or sibling sub-agents (you can see it as the per-call cache_read
- // collapsing back to a small base each time a sub-agent spawns). Modeling the
- // cascade therefore requires each call's branch id (parent_tool_use_id).
- // Runs that predate branch capture don't carry it; rather than fall back to a
- // flat-list cascade that overstates re-read cost on any run with sub-agents,
- // we decline to build the model — callers then hide the simulator.
- const branchKnown = messages.every((m) => m.parentToolUseId !== undefined);
- if (!branchKnown) return null;
const perMessageOutputSum = messages.reduce(
(s, m) => s + (m.outputTokens ?? 0),
0,
@@ -372,7 +356,6 @@ export function buildThinkingModel(
// same whether or not prompt caching was enabled, so the tool-output lever
// works in both modes (cache-growth inference broke with caching off, where
// cacheWrite is 0). toolPerCall[j] = Σ resultTokens of the tools called at j.
- // 0 on legacy runs that predate the field (lever then has nothing to size).
const toolPerCall: number[] = messages.map((m) =>
m.toolUses.reduce((s, u) => s + (u.resultTokens ?? 0), 0),
);
@@ -383,14 +366,14 @@ export function buildThinkingModel(
// So `later` counts calls after j that share j's branch (parent_tool_use_id),
// NOT all M-1-j subsequent calls. On a run with no sub-agents every call is
// in the one (main) branch and this reduces to the flat M-1-j.
- const branchTotal = new Map();
+ const branchTotal = new Map();
for (const m of messages) {
branchTotal.set(
m.parentToolUseId,
(branchTotal.get(m.parentToolUseId) ?? 0) + 1,
);
}
- const branchSeen = new Map();
+ const branchSeen = new Map();
const laterInBranch = messages.map((m) => {
const seen = branchSeen.get(m.parentToolUseId) ?? 0;
branchSeen.set(m.parentToolUseId, seen + 1);
@@ -401,7 +384,7 @@ export function buildThinkingModel(
// same-branch SUCCESSORS — the later calls that actually incur its content's
// cache write (first successor) and re-reads (the rest). Drives per-message
// incidence attribution: a cache delta is booked to the row that pays it.
- const branchPositions = new Map();
+ const branchPositions = new Map();
for (let j = 0; j < calls; j++) {
const key = messages[j].parentToolUseId;
const arr = branchPositions.get(key);
diff --git a/evalboard/lib/timing.ts b/evalboard/lib/timing.ts
index 1d4982a3e..fb99f67d9 100644
--- a/evalboard/lib/timing.ts
+++ b/evalboard/lib/timing.ts
@@ -174,23 +174,16 @@ export function busyMs(
//
// A call the harness TIMED but did not BOUND contributes nothing — no
// `durationMs` fallback. That is a policy, and it is the same one
-// `coder_eval.timing.main_thread_tool_spans` has always had on the Python side:
-// its `is not None` filter drops a command with no `execution_started_at` /
-// `execution_completed_at`, so every Python surface already ignored these while
-// this function folded them in. A duration with no bounds cannot be placed on
+// `coder_eval.timing.main_thread_tool_spans` has on the Python side: its
+// `is not None` filter drops a command with no `execution_started_at` /
+// `execution_completed_at`. A duration with no bounds cannot be placed on
// the timeline, so it cannot be unioned with anything; adding it to a union
// double-books whatever it overlapped and can drive the residual negative,
// which destroys the disjointness the four-bucket identity rests on. Such time
// lands in Unaccounted instead, which is precisely what that cell is for — the
-// harness measured a duration it cannot place.
+// harness measured a duration it cannot place. The retired `delegate-sdk`
+// adapter is the one harness that recorded only bare durations.
//
-// Measured blast radius: 9336 of 12170 commands in the run history on disk are
-// unbounded, every one a codex `Bash`, ~8 h in aggregate. That population is
-// CLOSED — codex went from 0% bounded before 2026-09-10 to 100% after — so no
-// future run changes, but on historical codex runs this moves up to ~8 h out of
-// Tool exec and into Unaccounted. Going forward the only harness reporting a
-// bare duration is the out-of-tree `delegate-sdk`; see
-// docs/agents/HARNESS_PARITY.md.
// The same union, but `null` when NOTHING bounded was recorded — the direct
// twin of `result_metrics._turn_tool_union_ms`, and the one a display cell wants.
//
@@ -201,10 +194,10 @@ export function busyMs(
// list, and its caller turns an empty one into `None`), and this is that layer.
//
// The distinction is not academic. A turn that ran no tools, and a turn whose
-// tools were all TIMED BUT UNBOUNDED — the historical-codex and out-of-tree
-// `delegate-sdk` population — both produce an empty span list. Rendering `0ms`
-// there claims the tools were measured and took no time, while every Python
-// surface renders a dash for the same run.
+// tools were all TIMED BUT UNBOUNDED — the `delegate-sdk` population — both
+// produce an empty span list. Rendering `0ms` there claims the tools were
+// measured and took no time, while every Python surface renders a dash for the
+// same run.
export function measuredToolExecutionMs(messages: MessageEvent[]): number | null {
const anyBounded = messages.some((m) =>
m.toolUses.some((t) => t.execStartMs != null && t.execEndMs != null),
diff --git a/src/coder_eval/evaluation/judge_persistence.py b/src/coder_eval/evaluation/judge_persistence.py
index dcd8fbfe6..fe51184be 100644
--- a/src/coder_eval/evaluation/judge_persistence.py
+++ b/src/coder_eval/evaluation/judge_persistence.py
@@ -8,16 +8,14 @@
- ``spill_judge_transcripts``: called by the orchestrator just before it writes
``task.json``.
- ``load_judge_transcripts``: called by re-render paths after
- ``EvaluationResult.model_validate_json``. Accepts both ``.yaml`` (current) and
- ``.json`` (previous), and treats ``transcript_path is None`` as a no-op, so old
- inline records keep working.
+ ``EvaluationResult.model_validate_json``. Treats ``transcript_path is None`` as
+ a no-op.
Rationale: .claude/notes/persistence.md § Judge persistence
"""
from __future__ import annotations
-import json
import logging
from pathlib import PurePosixPath, PureWindowsPath
from typing import TYPE_CHECKING, Any
@@ -157,10 +155,7 @@ def load_judge_transcripts(result: EvaluationResult, task_dir: Path) -> int:
and attaches the parsed dict on ``transcript`` so HTML / markdown renderers
see the same shape they get during the original run.
- Missing sibling files are skipped silently and logged at debug level —
- runs predating this feature have no sibling files and render fine via
- their inline transcript (preserved through ``CriterionResult``'s
- ``extra="allow"`` round-trip).
+ Missing sibling files are skipped and logged at debug level.
Returns the count of transcripts loaded.
"""
@@ -211,9 +206,7 @@ def load_judge_transcripts(result: EvaluationResult, task_dir: Path) -> int:
sibling = resolved_sibling
try:
text = sibling.read_text(encoding="utf-8")
- # JSON for legacy siblings; everything else through PyYAML, which
- # accepts JSON as a subset anyway.
- data = json.loads(text) if path.endswith(".json") else yaml.safe_load(text)
+ data = yaml.safe_load(text)
except Exception as e:
logger.warning("Failed to read judge transcript %s: %s", sibling, e)
continue
diff --git a/src/coder_eval/formatting.py b/src/coder_eval/formatting.py
index 88e707a27..c1daa3033 100644
--- a/src/coder_eval/formatting.py
+++ b/src/coder_eval/formatting.py
@@ -169,16 +169,12 @@ def format_token_usage(usage: Any) -> str:
``in=`` shows ``uncached_input_tokens`` (the fresh, billed-at-input-rate
slice), NOT the derived ``input_tokens`` total — the total already folds in
``cache_read``, so using it here while also appending ``cached=`` would
- double-count the cached prefix. Falls back to ``input_tokens`` only for
- objects that predate the split / don't expose the uncached field.
+ double-count the cached prefix.
"""
if usage is None:
return ""
try:
- input_tokens = getattr(usage, "uncached_input_tokens", None)
- if input_tokens is None:
- input_tokens = getattr(usage, "input_tokens", 0)
- input_tokens = input_tokens or 0
+ input_tokens = getattr(usage, "uncached_input_tokens", 0) or 0
output_tokens = getattr(usage, "output_tokens", 0) or 0
if not input_tokens and not output_tokens:
return ""
diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py
index 1f52bc7a4..e665486bd 100644
--- a/src/coder_eval/models/__init__.py
+++ b/src/coder_eval/models/__init__.py
@@ -135,7 +135,6 @@
# Results
from coder_eval.models.results import (
- JUDGE_CRITERION_TYPES,
ClassificationCriterionResult,
ClassLabelStats,
ConfigLineageEntry,
@@ -369,7 +368,6 @@
"CriterionStats",
"FailedRowSummary",
"ThresholdCheck",
- "JUDGE_CRITERION_TYPES",
"JudgeCriterionResult",
"JudgeTranscript",
"JudgeTranscriptToolCall",
diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py
index c72e3ea01..ece7d4444 100644
--- a/src/coder_eval/models/experiment.py
+++ b/src/coder_eval/models/experiment.py
@@ -326,8 +326,7 @@ class ExperimentResult(BaseModel): # noqa: CE009 -- persisted result model; rou
default_factory=dict,
description=(
"Raw weighted_score per replicate, keyed variant_id → task_id → [scores]. "
- "Always populated by aggregate_results; list length equals the replicate count. "
- "Empty dict only on deserialized results from before this field existed."
+ "Always populated by aggregate_results; list length equals the replicate count."
),
)
diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py
index 1277c87be..802c00f70 100644
--- a/src/coder_eval/models/results.py
+++ b/src/coder_eval/models/results.py
@@ -87,9 +87,7 @@ class CriterionResult(BaseModel):
default="evaluated",
description=(
"Whether the criterion ran. ``not_evaluated`` is distinct from an evaluated "
- "criterion whose score is 0.0 or whose checker returned an error. Defaults to "
- "``evaluated`` so task.json files written before this field existed retain their "
- "original meaning."
+ "criterion whose score is 0.0 or whose checker returned an error."
),
)
pass_threshold: float = Field(
@@ -105,8 +103,8 @@ class CriterionResult(BaseModel):
"BaseSuccessCriterion.is_gating, i.e. weight > 0). False marks an informational "
"criterion: it is measured and reported but excluded from the score and the "
"pass/fail gate, so every display surface must render it as informational rather "
- "than failed. Defaults True so results persisted before this field existed — and "
- "any result built without a source criterion — read back as gating."
+ "than failed. Defaults True so a result built without a source criterion reads "
+ "back as gating."
),
)
@@ -235,41 +233,10 @@ class JudgeCriterionResult(CriterionResult):
)
-# Type-infers legacy ``task.json`` records that lack ``result_kind``. Listed HERE, not
-# on each criterion class: deserializing a row must not require importing every checker
-# (and ``criteria/*.py`` already imports from this module).
-#
-# CONVENTION: a new criterion type needing a non-``"basic"`` result class adds its
-# ``criterion_type`` to the matching frozenset in the same PR.
-JUDGE_CRITERION_TYPES = frozenset({"llm_judge", "agent_judge", "system_one_judge"})
-"""Criterion types whose rows deserialize as ``JudgeCriterionResult``. Also the
-rendering gate in ``reports/html.py`` — one set, so the two cannot drift."""
-_CLASSIFICATION_CRITERION_TYPES = frozenset({"classification_match", "skill_triggered"})
-
-
def _criterion_result_discriminator(v: Any) -> str:
- """Dispatch ``CriterionResult`` subclasses for serialization and validation.
-
- Prefers explicit ``result_kind`` (written by current code); falls back to
- inference from ``criterion_type`` for legacy task.json files written before
- this field existed. Unknown criterion_types fall through to ``"basic"``.
-
- Explicit ``result_kind`` wins over ``criterion_type`` inference: a dict
- carrying ``result_kind="basic"`` with ``criterion_type="llm_judge"`` is
- routed to the base class. This matters for forward-compat experiments where
- a writer deliberately downgrades a result shape.
- """
+ """Dispatch ``CriterionResult`` subclasses on ``result_kind`` for serialization and validation."""
if isinstance(v, dict):
- kind = v.get("result_kind")
- if kind:
- return str(kind)
- ct = v.get("criterion_type", "")
- if ct in JUDGE_CRITERION_TYPES:
- return "judge"
- if ct in _CLASSIFICATION_CRITERION_TYPES:
- return "classification"
- return "basic"
- # Pydantic instance — read the discriminator field.
+ return str(v.get("result_kind") or "basic")
return getattr(v, "result_kind", "basic")
@@ -480,7 +447,7 @@ class SimulationTelemetry(BaseModel):
description=(
"Resolved model that played the simulated user, captured so a persisted "
"task.json is self-describing and its cost prices from a fact rather than "
- "from the run's route. None on records written before the model was pinned."
+ "from the run's route."
),
)
@@ -623,8 +590,7 @@ class EvaluationResult(BaseModel):
"Results of all success criteria checks. Typed as the ``CriterionResultUnion`` "
"discriminated union so subclass-specific fields (``JudgeCriterionResult.findings`` / "
"``transcript``, ``ClassificationCriterionResult.observed_label``) round-trip "
- "concretely through ``model_dump_json`` → ``model_validate_json``. Legacy task.json "
- "files without ``result_kind`` are inferred from ``criterion_type``."
+ "concretely through ``model_dump_json`` → ``model_validate_json``."
),
)
post_failure_criteria_results: list[CriterionResultUnion] = Field(
@@ -1042,9 +1008,9 @@ class SkippedTask(BaseModel):
def row_cost_incomplete(row: Mapping[str, Any]) -> bool:
"""True when a task row's recorded spend is missing money.
- Reads the row's own ``cost_complete`` flag. Absent means complete: rows written
- before the field existed are read as priced rather than inferred from their
- token counts, which would be a second definition of the same predicate.
+ Reads the row's own ``cost_complete`` flag. Absent means complete: a row without
+ the flag is read as priced rather than inferred from its token counts, which
+ would be a second definition of the same predicate.
Defined here, on the row schema, so the reports and ``RunSummary`` cannot
disagree about which rows lost money.
@@ -1107,9 +1073,7 @@ def simulator_cost_usd(result: EvaluationResult) -> float | None:
"""Price an evaluation's simulator turns. ``None`` outside simulation mode.
Priced at the SIMULATOR's own model — ``SimulationConfig.model``, recorded on
- the record as ``simulator_model`` — not the subject's and not the route's. The
- two older fallbacks remain for records written before the model was pinned, when
- the simulator inherited ``BEDROCK_MODEL`` from the route.
+ the record as ``simulator_model`` — not the subject's and not the route's.
A floor. ``UserSimulator`` records only ``uncached_input_tokens`` and drops
both cache buckets, so a cached prefix is largely absent from the count.
@@ -1119,16 +1083,10 @@ def simulator_cost_usd(result: EvaluationResult) -> float | None:
sim = result.simulation
if sim is None or not (sim.simulator_input_tokens or sim.simulator_output_tokens):
return None
- route_model = (result.environment_info or {}).get("bedrock_model")
- model = sim.simulator_model or (
- # Legacy records only: the route's model, else the subject's on a non-Bedrock
- # route where the SDK picked its own default and nothing named it.
- route_model if isinstance(route_model, str) and route_model else result.model_used
- )
- if not model:
+ if not sim.simulator_model:
return None
return calculate_cost(
- model,
+ sim.simulator_model,
uncached_input_tokens=sim.simulator_input_tokens,
output_tokens=sim.simulator_output_tokens,
)
diff --git a/src/coder_eval/models/telemetry.py b/src/coder_eval/models/telemetry.py
index 210bbd308..968134208 100644
--- a/src/coder_eval/models/telemetry.py
+++ b/src/coder_eval/models/telemetry.py
@@ -5,7 +5,7 @@
from datetime import datetime
from typing import Annotated, Any, Literal
-from pydantic import BaseModel, Discriminator, Field, computed_field, model_validator
+from pydantic import BaseModel, Discriminator, Field, computed_field
class TokenUsage(BaseModel):
@@ -30,25 +30,6 @@ class TokenUsage(BaseModel):
(which already contains the cache buckets and would double-count them).
"""
- @model_validator(mode="before")
- @classmethod
- def _adopt_legacy_input_tokens(cls, data: Any) -> Any:
- """Read pre-split ``task.json`` files without silently under-counting.
-
- Before the uncached/derived split, the stored field was ``input_tokens``
- and it carried the FULL prompt. It is now a computed field, so a naive
- reload drops it and recomputes from the cache buckets only — turning a
- stored ``input_tokens=1000`` into ``350``. When the new stored field
- ``uncached_input_tokens`` is absent but a legacy ``input_tokens`` is
- present, adopt the legacy value as ``uncached_input_tokens`` so the Python
- read path agrees with the evalboard (which already falls back the same
- way). Current-format payloads carry ``uncached_input_tokens`` and are
- untouched.
- """
- if isinstance(data, dict) and "uncached_input_tokens" not in data and "input_tokens" in data:
- data = {**data, "uncached_input_tokens": data["input_tokens"]}
- return data
-
uncached_input_tokens: int = Field(
default=0,
description=(
diff --git a/src/coder_eval/reports/helpers.py b/src/coder_eval/reports/helpers.py
index bdb7a98fc..bdcb1189f 100644
--- a/src/coder_eval/reports/helpers.py
+++ b/src/coder_eval/reports/helpers.py
@@ -164,7 +164,7 @@ def paired_comparison(result: ExperimentResult, confidence: float = 0.95) -> Pai
per_rep_b = result.per_replicate_scores.get(vid_b, {})
common_tasks = sorted(t for t in set(per_rep_a) & set(per_rep_b) if per_rep_a[t] and per_rep_b[t])
if not common_tasks:
- # No shared task, or per_replicate_scores absent (results from before it existed).
+ # No task scored on both sides.
return None
# Tasks seen for at least one variant but not paired (missing or empty on the
diff --git a/src/coder_eval/reports/html.py b/src/coder_eval/reports/html.py
index eb202f6e9..03b1a1d74 100644
--- a/src/coder_eval/reports/html.py
+++ b/src/coder_eval/reports/html.py
@@ -23,7 +23,7 @@
from ..analysis import calculate_command_statistics
from ..durations import format_ms
-from ..models import JUDGE_CRITERION_TYPES, FinalStatus, eval_result_total_cost, sum_costs
+from ..models import FinalStatus, JudgeCriterionResult, eval_result_total_cost, sum_costs
from ..result_metrics import agent_wall_ms, expected_turns_overage, turn_time_buckets
from ..stats import stddev, welch_t_test
from .helpers import (
@@ -53,6 +53,7 @@
EvaluationResult,
ExperimentDefinition,
ExperimentResult,
+ JudgeTranscript,
TurnRecord,
VariantAggregate,
)
@@ -438,16 +439,12 @@ def _render_judge_section(criteria: list[CriterionResult], heading: str = "Judge
"""
cards: list[str] = []
for cr in criteria:
- # criterion_type identifies judges; details/findings/transcript may all
- # be present (typed JudgeCriterionResult) or in model_extra (round-tripped).
- if cr.criterion_type not in JUDGE_CRITERION_TYPES:
+ if not isinstance(cr, JudgeCriterionResult):
continue
- findings_raw = getattr(cr, "findings", []) or []
- findings = [str(f).strip() for f in findings_raw if str(f).strip()]
- transcript = getattr(cr, "transcript", None)
- if not findings and transcript in (None, {}, []):
+ findings = [str(f).strip() for f in cr.findings if str(f).strip()]
+ if not findings and cr.transcript is None:
continue
- cards.append(_render_judge_card(cr, findings, transcript))
+ cards.append(_render_judge_card(cr, findings))
if not cards:
return ""
@@ -471,7 +468,7 @@ def _extract_rationale(details: str | None) -> str:
return ""
-def _render_judge_card(cr: CriterionResult, findings: list[str], transcript: Any) -> str:
+def _render_judge_card(cr: JudgeCriterionResult, findings: list[str]) -> str:
"""Render one judge result as a card (type + score + rationale + findings + transcript)."""
rationale = _extract_rationale(cr.details)
rationale_html = f'{_esc(rationale)}
' if rationale else ""
@@ -487,7 +484,7 @@ def _render_judge_card(cr: CriterionResult, findings: list[str], transcript: Any
f""
)
- transcript_html = _render_judge_transcript(transcript)
+ transcript_html = _render_judge_transcript(cr.transcript)
return (
''
@@ -503,17 +500,11 @@ def _render_judge_card(cr: CriterionResult, findings: list[str], transcript: Any
)
-def _render_judge_transcript(transcript: Any) -> str:
- """Render a JudgeTranscript (or its dict round-trip form) as a disclosure block."""
+def _render_judge_transcript(transcript: JudgeTranscript | None) -> str:
+ """Render a JudgeTranscript as a disclosure block."""
if transcript is None:
return ""
- # Normalize to a plain dict so dict-form (round-tripped) and model-form work the same.
- if hasattr(transcript, "model_dump"):
- data = transcript.model_dump()
- elif isinstance(transcript, dict):
- data = transcript
- else:
- return ""
+ data = transcript.model_dump()
tool_calls = data.get("tool_calls") or []
raw_verdict = (data.get("raw_verdict") or "").strip()
@@ -526,16 +517,10 @@ def _render_judge_transcript(transcript: Any) -> str:
rows: list[str] = []
for tc in tool_calls:
- if isinstance(tc, dict):
- tool = _esc(str(tc.get("tool_name", "")))
- detail = _esc(str(tc.get("detail", "")))
- status = _esc(str(tc.get("status", "")))
- preview = _esc(str(tc.get("result_preview", "")))
- else:
- tool = _esc(getattr(tc, "tool_name", ""))
- detail = _esc(getattr(tc, "detail", ""))
- status = _esc(getattr(tc, "status", ""))
- preview = _esc(getattr(tc, "result_preview", ""))
+ tool = _esc(str(tc.get("tool_name", "")))
+ detail = _esc(str(tc.get("detail", "")))
+ status = _esc(str(tc.get("status", "")))
+ preview = _esc(str(tc.get("result_preview", "")))
rows.append(
f"
| {tool} | {status} | "
+ f"{detail} | "
diff --git a/src/coder_eval/run_record.py b/src/coder_eval/run_record.py
index 615efc077..7479de346 100644
--- a/src/coder_eval/run_record.py
+++ b/src/coder_eval/run_record.py
@@ -75,7 +75,7 @@ def eval_result_to_task_dict(
sub-dir). Repeated runs of the same task share a ``task_id``, so
without this the row is indistinguishable from its siblings and
downstream consumers (evalboard) collapse them to one. ``None`` when
- the caller doesn't track replicates (repeats disabled / legacy).
+ the caller doesn't track replicates (repeats disabled).
"""
ref_similarity: float | None = None
diff --git a/tests/lint/rules/ce060_message_id_declared.py b/tests/lint/rules/ce060_message_id_declared.py
index a2d3337e9..48fcca055 100644
--- a/tests/lint/rules/ce060_message_id_declared.py
+++ b/tests/lint/rules/ce060_message_id_declared.py
@@ -58,10 +58,8 @@ def visit_Call(self, node: ast.Call) -> None:
"so every message it builds shares one empty identity. Pass the field "
"with a real value: the CLI's own "
"id where the stream carries one, else synthesize it the way Codex does "
- "(f'{turn_id}-msg-{gen_index}'). Antigravity shipped without it: the "
- "evalboard then falls back to its SAME_EMISSION_GAP_MS wall-clock gap to "
- "group emissions, and a harness whose generation windows are contiguous "
- "has every one of a turn's generations collapse into one row. See "
- "docs/agents/HARNESS_PARITY.md.",
+ "(f'{turn_id}-msg-{gen_index}'). The evalboard joins a response's emissions "
+ "by this id only, so without it every content block renders as its own "
+ "timeline row. See docs/agents/HARNESS_PARITY.md.",
)
self.generic_visit(node)
diff --git a/tests/test_cost_accounting_paths.py b/tests/test_cost_accounting_paths.py
index a5d1c7c49..0597ab2bc 100644
--- a/tests/test_cost_accounting_paths.py
+++ b/tests/test_cost_accounting_paths.py
@@ -241,7 +241,7 @@ def test_unpriced_judge_is_skipped_not_fatal(self):
assert row["total_cost_usd"] == pytest.approx(0.12)
@staticmethod
- def _simulated(**env) -> EvaluationResult:
+ def _simulated(simulator_model: str | None) -> EvaluationResult:
return _result(
[_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))],
simulation=SimulationTelemetry(
@@ -251,26 +251,18 @@ def _simulated(**env) -> EvaluationResult:
simulator_input_tokens=1_000_000,
simulator_output_tokens=100_000,
total_turns=3,
+ simulator_model=simulator_model,
),
- environment_info=env,
)
- def test_simulator_prices_at_the_route_model_not_the_subject(self):
- """UserSimulator pins model=None, so it bills at BEDROCK_MODEL.
-
- A task that pins ``agent.model`` would otherwise mis-bill every simulated
- row. Here the subject is sonnet-5 ($2/$10) while the route is haiku-4.5
- ($1/$5), so the simulator must cost the haiku rate.
- """
- result = self._simulated(bedrock_model="claude-haiku-4-5-20251001")
+ def test_simulator_prices_at_its_own_model_not_the_subject(self):
+ """The subject is sonnet-5 ($2/$10); the simulator ran on haiku-4.5 ($1/$5)."""
+ result = self._simulated("claude-haiku-4-5-20251001")
# 1M uncached input at $1/MTok + 100K output at $5/MTok.
assert eval_result_to_task_dict(result)["simulator_cost_usd"] == pytest.approx(1.00 + 0.50)
- def test_simulator_falls_back_to_the_subject_model_off_bedrock(self):
- """A non-Bedrock route names no model on the record; the subject's is the best available."""
- result = self._simulated()
- # sonnet-5 at $2/$10 per MTok: 1M uncached input + 100K output.
- assert eval_result_to_task_dict(result)["simulator_cost_usd"] == pytest.approx(2.0 + 1.0)
+ def test_no_simulator_model_means_no_simulator_cost(self):
+ assert eval_result_to_task_dict(self._simulated(None))["simulator_cost_usd"] is None
def test_single_shot_row_has_no_simulator_cost(self):
result = _result([_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))])
@@ -278,7 +270,7 @@ def test_single_shot_row_has_no_simulator_cost(self):
def test_unpriced_simulator_route_is_skipped_not_fatal(self):
"""An unpriced simulator route leaves the rest of the row intact."""
- result = self._simulated(bedrock_model="claude-sonnet-99")
+ result = self._simulated("claude-sonnet-99")
result.total_token_usage = TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1)
row = eval_result_to_task_dict(result)
assert row["simulator_cost_usd"] is None
@@ -299,8 +291,8 @@ def test_sums_every_component(self):
simulator_input_tokens=1_000_000,
simulator_output_tokens=0,
total_turns=2,
+ simulator_model="claude-haiku-4-5-20251001",
),
- environment_info={"bedrock_model": "claude-haiku-4-5-20251001"},
)
result.total_token_usage = TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=1.0)
result.success_criteria_results = [
diff --git a/tests/test_criterion_result_round_trip.py b/tests/test_criterion_result_round_trip.py
index 26d882dc3..1a7db2f7c 100644
--- a/tests/test_criterion_result_round_trip.py
+++ b/tests/test_criterion_result_round_trip.py
@@ -89,48 +89,6 @@ def test_base_criterion_round_trip() -> None:
assert cr.score == 1.0
-def test_legacy_task_json_infers_result_kind_from_criterion_type() -> None:
- """Legacy task.json files (no ``result_kind``) infer the subclass via criterion_type."""
- legacy_payload = {
- "task_id": "t",
- "task_description": "d",
- "agent_type": "claude-code",
- "started_at": "2026-05-12T00:00:00",
- "final_status": "SUCCESS",
- "iteration_count": 1,
- "success_criteria_results": [
- {"criterion_type": "llm_judge", "description": "x", "score": 0.5, "findings": ["leg"]},
- {
- "criterion_type": "agent_judge",
- "description": "y",
- "score": 0.6,
- "findings": ["aj"],
- },
- {
- "criterion_type": "classification_match",
- "description": "c",
- "score": 1.0,
- "observed_label": "a",
- "expected_label": "a",
- },
- {
- "criterion_type": "skill_triggered",
- "description": "s",
- "score": 0.0,
- "observed_label": "(none)",
- "expected_label": "Skill",
- },
- {"criterion_type": "file_exists", "description": "f", "score": 1.0},
- ],
- }
- er = EvaluationResult.model_validate(legacy_payload)
- assert isinstance(er.success_criteria_results[0], JudgeCriterionResult)
- assert isinstance(er.success_criteria_results[1], JudgeCriterionResult)
- assert isinstance(er.success_criteria_results[2], ClassificationCriterionResult)
- assert isinstance(er.success_criteria_results[3], ClassificationCriterionResult)
- assert type(er.success_criteria_results[4]) is CriterionResult
-
-
def test_unknown_criterion_type_falls_to_basic() -> None:
"""An unknown criterion_type falls through to the base CriterionResult."""
payload = {
@@ -157,8 +115,8 @@ def test_unknown_criterion_type_falls_to_basic() -> None:
assert cr.model_extra.get("unknown_field") == 42
-def test_explicit_result_kind_overrides_inference() -> None:
- """Explicit ``result_kind`` wins over ``criterion_type`` inference."""
+def test_result_kind_alone_picks_the_class() -> None:
+ """``result_kind`` picks the class; a judge ``criterion_type`` does not."""
payload = {
"task_id": "t",
"task_description": "d",
diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py
index db114448c..c166dfd5a 100644
--- a/tests/test_custom_lint.py
+++ b/tests/test_custom_lint.py
@@ -27,8 +27,9 @@
# Rules whose defect class lives in the TEST tree, not in src/. CE048's whole
# subject is an in-process call to a Typer command, and the only place that
# happens is a test — scanning src/ alone would leave the rule permanently green
-# while the bug it exists for sat five lines away.
-_ALSO_SCAN_TESTS = {"CE048"}
+# while the bug it exists for sat five lines away. CE016's dropped kwarg makes a
+# test fixture silently build zero input, so the assertion passes on another cell.
+_ALSO_SCAN_TESTS = {"CE016", "CE048"}
@pytest.mark.lint
diff --git a/tests/test_judge_persistence.py b/tests/test_judge_persistence.py
index 0c627d962..1435200fb 100644
--- a/tests/test_judge_persistence.py
+++ b/tests/test_judge_persistence.py
@@ -512,31 +512,6 @@ def test_load_attaches_dict_when_schema_mismatch(tmp_path: Path) -> None:
assert transcript["future_field"] == "from-newer-version"
-def test_load_reads_legacy_json_sibling(tmp_path: Path) -> None:
- """Sibling files spilled before the YAML switch are ``judge-.json``.
- The loader must still read them so archived runs render."""
- judge = _make_judge_result(transcript=None)
- judge.transcript_path = "judge-0.json"
- legacy = (
- '{"raw_verdict":"legacy","judge_prompt":"legacy","tool_calls":[],'
- '"duration_seconds":0.0,"judge_system_prompt":"","truncated":false,"token_usage":null}'
- )
- (tmp_path / "judge-0.json").write_text(legacy, encoding="utf-8")
- result = _make_evaluation_result(criteria=[judge])
-
- n = load_judge_transcripts(result, tmp_path)
-
- assert n == 1
- transcript = getattr(judge, "transcript", None)
- assert transcript is not None
- # Loader prefers typed JudgeTranscript even for legacy .json siblings —
- # the on-disk format changed but the schema is the same.
- from coder_eval.models import JudgeTranscript
-
- assert isinstance(transcript, JudgeTranscript)
- assert transcript.raw_verdict == "legacy"
-
-
# --- end-to-end ---
diff --git a/tests/test_models.py b/tests/test_models.py
index 8694bfeb0..86f464ec3 100644
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -391,46 +391,6 @@ def test_evaluation_result_task_config_default_none(self):
class TestCriterionResultDiscriminator:
"""JSON round-trip must preserve ClassificationCriterionResult subclass."""
- def test_legacy_task_json_without_result_kind_migrates(self):
- """Historical task.json files (no `result_kind` tag) must still rehydrate as the right subclass."""
- import json
- from datetime import datetime
-
- from coder_eval.models import ClassificationCriterionResult, CriterionResult, EvaluationResult
-
- # Simulate a pre-discriminator task.json: a classification result
- # carries label fields but no result_kind, and a plain result has
- # neither. Without the migration validator, both would resolve to
- # base CriterionResult and the labels would be dropped.
- legacy = {
- "task_id": "t",
- "task_description": "d",
- "variant_id": "default",
- "agent_type": "claude-code",
- "started_at": datetime.now().isoformat(),
- "final_status": "SUCCESS",
- "iteration_count": 1,
- "environment_info": {},
- "success_criteria_results": [
- {"criterion_type": "file_exists", "description": "x", "score": 1.0},
- {
- "criterion_type": "classification_match",
- "description": "y",
- "score": 1.0,
- "observed_label": "POS",
- "expected_label": "POS",
- },
- ],
- }
- result = EvaluationResult.model_validate_json(json.dumps(legacy))
- kinds = [type(r).__name__ for r in result.success_criteria_results]
- assert kinds == ["CriterionResult", "ClassificationCriterionResult"]
- assert isinstance(result.success_criteria_results[0], CriterionResult)
- cls_result = result.success_criteria_results[1]
- assert isinstance(cls_result, ClassificationCriterionResult)
- assert cls_result.observed_label == "POS"
- assert cls_result.expected_label == "POS"
-
def test_mixed_results_roundtrip_preserves_subclass(self):
from datetime import datetime
diff --git a/tests/test_reports_html.py b/tests/test_reports_html.py
index 7afcd31d0..44c64cdab 100644
--- a/tests/test_reports_html.py
+++ b/tests/test_reports_html.py
@@ -160,38 +160,31 @@ def test_task_html_renders_judge_section():
assert "Raw verdict" in html
-def test_task_html_judge_section_round_trips_from_extras():
- """Reading task.json back into a base CriterionResult preserves judge fields
- via ``extra='allow'`` — the Judge Verdicts section must surface them whether
- they arrive as typed JudgeCriterionResult or as model_extra dicts."""
- base = CriterionResult(
+def test_task_html_judge_section_survives_task_json_round_trip():
+ from coder_eval.models import JudgeCriterionResult, JudgeTranscript, JudgeTranscriptToolCall
+
+ judge = JudgeCriterionResult(
criterion_type="agent_judge",
description="grade project",
score=0.6,
details="score=0.600\nrationale: round-tripped",
- # These come in as extras after a model_validate_json round-trip.
- findings=["finding via extras"], # type: ignore[call-arg]
- transcript={
- "tool_calls": [
- {
- "tool_name": "Grep",
- "detail": "pattern=foo",
- "status": "success",
- "result_preview": "match",
- }
+ findings=["finding after reload"],
+ transcript=JudgeTranscript(
+ tool_calls=[
+ JudgeTranscriptToolCall(
+ tool_name="Grep", detail="pattern=foo", status="success", result_preview="match"
+ )
],
- "duration_seconds": 1.0,
- "raw_verdict": "raw",
- "truncated": False,
- "token_usage": None,
- }, # type: ignore[call-arg]
+ duration_seconds=1.0,
+ raw_verdict="raw",
+ ),
)
- result = _make_result(criteria=[base])
- html = HTMLReportGenerator.generate_task_html(result)
+ reloaded = EvaluationResult.model_validate_json(_make_result(criteria=[judge]).model_dump_json())
+ html = HTMLReportGenerator.generate_task_html(reloaded)
assert "Judge Verdicts (1)" in html
assert "round-tripped" in html
- assert "finding via extras" in html
+ assert "finding after reload" in html
assert "pattern=foo" in html
@@ -232,7 +225,7 @@ def test_task_html_minimal_success():
commands=[
_make_command("Write", 0, parameters={"file_path": "Main.cs"}, result_summary="File created"),
],
- token_usage=TokenUsage(input_tokens=150, output_tokens=80),
+ token_usage=TokenUsage(uncached_input_tokens=150, output_tokens=80),
)
],
)
@@ -631,7 +624,7 @@ def test_variant_html_loads_rich_sections_from_run_dir(tmp_path: Path):
task_json_dir.mkdir(parents=True)
eval_result = _make_result()
eval_result.task_id = "t1"
- eval_result.total_token_usage = TokenUsage(input_tokens=100, output_tokens=50, total_cost_usd=0.02)
+ eval_result.total_token_usage = TokenUsage(uncached_input_tokens=100, output_tokens=50, total_cost_usd=0.02)
eval_result.sdk_options = {"permission_mode": "auto", "allowed_tools": ["Bash"], "model": "m"}
eval_result.environment_info = {"framework_version": "0.1"}
(task_json_dir / "task.json").write_text(eval_result.model_dump_json())
@@ -792,7 +785,7 @@ def _boom(result):
def test_task_html_renders_token_usage():
result = _make_result()
result.total_token_usage = TokenUsage(
- input_tokens=1000,
+ uncached_input_tokens=1000,
output_tokens=500,
cache_creation_input_tokens=200,
cache_read_input_tokens=300,
@@ -800,7 +793,7 @@ def test_task_html_renders_token_usage():
)
html = HTMLReportGenerator.generate_task_html(result)
assert "Token Usage" in html
- assert "1,000" in html
+ assert 'Input (uncached)
1,000
' in html
assert "500" in html
assert "$0.0123" in html
@@ -885,7 +878,7 @@ def test_task_html_omits_commands_efficiency_when_absent():
def test_task_html_renders_cost_badge_in_header():
result = _make_result()
- result.total_token_usage = TokenUsage(input_tokens=100, output_tokens=50, total_cost_usd=0.5)
+ result.total_token_usage = TokenUsage(uncached_input_tokens=100, output_tokens=50, total_cost_usd=0.5)
html = HTMLReportGenerator.generate_task_html(result)
# Header region ends at the after ".header-bar"
header = html[: html.index("") if "" in html else len(html)]
@@ -976,7 +969,7 @@ def test_task_html_no_expected_turns_badge_when_task_config_none():
def test_task_html_omits_cost_badge_when_cost_is_none():
"""Cost badge is only rendered when total_cost_usd is populated."""
result = _make_result()
- result.total_token_usage = TokenUsage(input_tokens=100, output_tokens=50, total_cost_usd=None)
+ result.total_token_usage = TokenUsage(uncached_input_tokens=100, output_tokens=50, total_cost_usd=None)
html = HTMLReportGenerator.generate_task_html(result)
header = html[: html.index("")]
assert "$" not in header
@@ -985,7 +978,7 @@ def test_task_html_omits_cost_badge_when_cost_is_none():
def test_task_html_token_usage_cost_na_when_absent():
"""When tokens are recorded but cost is None, Token Usage cell renders N/A."""
result = _make_result()
- result.total_token_usage = TokenUsage(input_tokens=100, output_tokens=50, total_cost_usd=None)
+ result.total_token_usage = TokenUsage(uncached_input_tokens=100, output_tokens=50, total_cost_usd=None)
html = HTMLReportGenerator.generate_task_html(result)
token_section = html[html.index("Token Usage
") :]
# Next heading demarcates this section
diff --git a/uv.lock b/uv.lock
index 225bd4c66..f63fadc6d 100644
--- a/uv.lock
+++ b/uv.lock
@@ -273,7 +273,7 @@ wheels = [
[[package]]
name = "azure-monitor-opentelemetry-exporter"
-version = "1.0.0b53"
+version = "1.0.0b58"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "azure-core" },
@@ -283,9 +283,9 @@ dependencies = [
{ name = "opentelemetry-sdk" },
{ name = "psutil" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/05/64/875f13849fe2e3832ceda6a218fa5422a25e72c1b86623a8514f541a8c60/azure_monitor_opentelemetry_exporter-1.0.0b53.tar.gz", hash = "sha256:1274e9008909414a25c6287185a6c5a884209705b6e651a1ffddfbdab3b76e52", size = 335614, upload-time = "2026-06-08T15:54:22.683Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/95/33/feadb708d9b8de6cc846118f287110d069ebeaee2d3abf9433695c2800ae/azure_monitor_opentelemetry_exporter-1.0.0b58.tar.gz", hash = "sha256:5152d75bd9780fef02fcc9550e0323ba3f661a8fcba017ea04b027151ae0932e", size = 362674, upload-time = "2026-10-07T17:35:54.445Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/5f/dc/6dbff881ac739999bc31f166504ed31a2e0fef45cc59a3c1b2386a4b4abe/azure_monitor_opentelemetry_exporter-1.0.0b53-py2.py3-none-any.whl", hash = "sha256:e2faadaf203369f25cb96df2a75017bd7f11655b033200c5867151f8fb2a6ba7", size = 249226, upload-time = "2026-06-08T15:54:24.413Z" },
+ { url = "https://files.pythonhosted.org/packages/55/fe/a010e943605fc017b1030159851e48e715fb5616675447f0d8c0a74de4e6/azure_monitor_opentelemetry_exporter-1.0.0b58-py2.py3-none-any.whl", hash = "sha256:cf036f7a99515d4e6626f3c33e5ac1ae633731882239542353e1f4a17285e9fc", size = 257459, upload-time = "2026-10-07T17:35:56.218Z" },
]
[[package]]
@@ -1854,20 +1854,19 @@ wheels = [
[[package]]
name = "opentelemetry-api"
-version = "1.40.0"
+version = "1.45.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "importlib-metadata" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/2c/1d/4049a9e8698361cc1a1aa03a6c59e4fa4c71e0c0f94a30f988a6876a2ae6/opentelemetry_api-1.40.0.tar.gz", hash = "sha256:159be641c0b04d11e9ecd576906462773eb97ae1b657730f0ecf64d32071569f", size = 70851, upload-time = "2026-03-04T14:17:21.555Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/2e/02/6e0ae9cc61bd3169d401077b507b3ebc344745171e1051ab430be012dcd9/opentelemetry_api-1.45.1.tar.gz", hash = "sha256:aa38ed19bcc084ba42782a73255b3582283eced7ad6dddbd6695189e69adfb75", size = 72804, upload-time = "2026-10-06T17:32:58.133Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/5f/bf/93795954016c522008da367da292adceed71cca6ee1717e1d64c83089099/opentelemetry_api-1.40.0-py3-none-any.whl", hash = "sha256:82dd69331ae74b06f6a874704be0cfaa49a1650e1537d4a813b86ecef7d0ecf9", size = 68676, upload-time = "2026-03-04T14:17:01.24Z" },
+ { url = "https://files.pythonhosted.org/packages/1e/41/f7dcf80b81ee8e71c1a2b59f14208bc723edbd89ed027a73b175abf6348e/opentelemetry_api-1.45.1-py3-none-any.whl", hash = "sha256:b31553efa588ae44bc306f863c785c5333a9ecc091248c6ee68b4b6c87fdedfb", size = 60256, upload-time = "2026-10-06T17:32:33.506Z" },
]
[[package]]
name = "opentelemetry-instrumentation"
-version = "0.61b0"
+version = "0.66b1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "opentelemetry-api" },
@@ -1875,36 +1874,36 @@ dependencies = [
{ name = "packaging" },
{ name = "wrapt" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/da/37/6bf8e66bfcee5d3c6515b79cb2ee9ad05fe573c20f7ceb288d0e7eeec28c/opentelemetry_instrumentation-0.61b0.tar.gz", hash = "sha256:cb21b48db738c9de196eba6b805b4ff9de3b7f187e4bbf9a466fa170514f1fc7", size = 32606, upload-time = "2026-03-04T14:20:16.825Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/a5/03/89e47ff8d52a4f83b343e6eb9ef1698ff45357216e5b6b2b21e0da5c5c7d/opentelemetry_instrumentation-0.66b1.tar.gz", hash = "sha256:e79a510f7d87c72d95e964ddb42193a0d9a75668c027d980eab032ea1322a5ce", size = 43308, upload-time = "2026-10-06T17:36:10.703Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/d8/3e/f6f10f178b6316de67f0dfdbbb699a24fbe8917cf1743c1595fb9dcdd461/opentelemetry_instrumentation-0.61b0-py3-none-any.whl", hash = "sha256:92a93a280e69788e8f88391247cc530fd81f16f2b011979d4d6398f805cfbc63", size = 33448, upload-time = "2026-03-04T14:19:02.447Z" },
+ { url = "https://files.pythonhosted.org/packages/da/b2/d1413681ff43e13ac9860df27e1226d3199ab0b97b352ceea41abcc660a5/opentelemetry_instrumentation-0.66b1-py3-none-any.whl", hash = "sha256:4c4aa14dc9a24a02325a9d4c42c4d0208dbb1374c2b1b8fe6c9392d59f3e1008", size = 36904, upload-time = "2026-10-06T17:35:11.663Z" },
]
[[package]]
name = "opentelemetry-sdk"
-version = "1.40.0"
+version = "1.45.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "opentelemetry-api" },
{ name = "opentelemetry-semantic-conventions" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/58/fd/3c3125b20ba18ce2155ba9ea74acb0ae5d25f8cd39cfd37455601b7955cc/opentelemetry_sdk-1.40.0.tar.gz", hash = "sha256:18e9f5ec20d859d268c7cb3c5198c8d105d073714db3de50b593b8c1345a48f2", size = 184252, upload-time = "2026-03-04T14:17:31.87Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/a1/79/7392e21a1c8f0c61d90b223e31c7e48cb9d452e91a6b820ad24cca5f23c4/opentelemetry_sdk-1.45.1.tar.gz", hash = "sha256:63d24a6ca645019a631e6a51999c73e93adcac1196ca640b8ae78a7cc4762bf3", size = 218324, upload-time = "2026-10-06T17:33:13.26Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/2c/c5/6a852903d8bfac758c6dc6e9a68b015d3c33f2f1be5e9591e0f4b69c7e0a/opentelemetry_sdk-1.40.0-py3-none-any.whl", hash = "sha256:787d2154a71f4b3d81f20524a8ce061b7db667d24e46753f32a7bc48f1c1f3f1", size = 141951, upload-time = "2026-03-04T14:17:17.961Z" },
+ { url = "https://files.pythonhosted.org/packages/95/3c/87c42b4bd6dd297536f04cd9383d212ac557ecd49f2cbdcd46da1c9ef5c8/opentelemetry_sdk-1.45.1-py3-none-any.whl", hash = "sha256:c604c11dc429810812348989115fa44bd558772a3d7442afc43d024f2c250ca4", size = 140063, upload-time = "2026-10-06T17:32:55.04Z" },
]
[[package]]
name = "opentelemetry-semantic-conventions"
-version = "0.61b0"
+version = "0.66b1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "opentelemetry-api" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/6d/c0/4ae7973f3c2cfd2b6e321f1675626f0dab0a97027cc7a297474c9c8f3d04/opentelemetry_semantic_conventions-0.61b0.tar.gz", hash = "sha256:072f65473c5d7c6dc0355b27d6c9d1a679d63b6d4b4b16a9773062cb7e31192a", size = 145755, upload-time = "2026-03-04T14:17:32.664Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/46/e4/dbbfb2a010c4db2224a5114638acede6fe563d33cc20fb1752cebcbe6298/opentelemetry_semantic_conventions-0.66b1.tar.gz", hash = "sha256:497ca63bf383723411e8eaf60c8779e9877633c936bb641080adab59d0eb6ec8", size = 150250, upload-time = "2026-10-06T17:33:14.073Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/b2/37/cc6a55e448deaa9b27377d087da8615a3416d8ad523d5960b78dbeadd02a/opentelemetry_semantic_conventions-0.61b0-py3-none-any.whl", hash = "sha256:fa530a96be229795f8cef353739b618148b0fe2b4b3f005e60e262926c4d38e2", size = 231621, upload-time = "2026-03-04T14:17:19.33Z" },
+ { url = "https://files.pythonhosted.org/packages/bc/14/67f8aa798857f8cf686f515bf93d9bb877ce952ddc8efae0fa25b45ce0d6/opentelemetry_semantic_conventions-0.66b1-py3-none-any.whl", hash = "sha256:d4cddeb4315490b35213f55e2bdc9ac54bb1e4d318927475bed62b35545e581b", size = 206279, upload-time = "2026-10-06T17:32:56.103Z" },
]
[[package]]