From dbcc398e51a3c90d3501517ec51ed6715714cbe6 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:21:15 +0000 Subject: [PATCH 1/3] Publish suite rewards as Braintrust scores --- report/publish-braintrust.ts | 20 ++++++++++++++++++-- report/results.test.ts | 3 ++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/report/publish-braintrust.ts b/report/publish-braintrust.ts index 9360f6f..19d09b3 100644 --- a/report/publish-braintrust.ts +++ b/report/publish-braintrust.ts @@ -59,6 +59,7 @@ interface BraintrustEvent { output?: unknown; expected?: unknown; error?: string; + scores?: Record; metadata?: Record; metrics?: Record; _is_merge: false; @@ -161,8 +162,22 @@ function taskInstruction(trial: BenchmarkTrial): unknown { return redactValue(userSteps.at(-1)?.message); } -// Score values ride in metadata rather than Braintrust's billed scores field; -// they stay filterable and groupable there (`metadata.scores.accuracy = 1`). +// The suite's rewards, such as ClawBench's lenient and strict judgments, are +// Braintrust scores so the experiment summary and comparisons show them. +// Derived rates stay in metadata, where they remain filterable. +function trialScores( + trial: BenchmarkTrial, +): Record | undefined { + // Infrastructure failures are not task results; metadata counts them. + if (trial.errorClass === "infra") return undefined; + const scores = Object.fromEntries( + Object.entries(trial.rewards).filter( + ([, value]) => value >= 0 && value <= 1, + ), + ); + return Object.keys(scores).length > 0 ? scores : undefined; +} + function trialMetadata(trial: BenchmarkTrial): Record { return { trialName: trial.trialName, @@ -486,6 +501,7 @@ export function buildExperimentEvents( }, expected: { reward: 1 }, error: trial.error ? redactString(trial.error, 400) : undefined, + scores: trialScores(trial), metadata: trialMetadata(trial), metrics: metricRecord(trial), _is_merge: false, diff --git a/report/results.test.ts b/report/results.test.ts index 451e7ce..b85ca92 100644 --- a/report/results.test.ts +++ b/report/results.test.ts @@ -288,7 +288,7 @@ describe("Harbor result ingestion", () => { event.span_attributes.type === "eval" && (event.output as { error?: string }).error, ); - expect(infra).not.toHaveProperty("scores"); + expect(infra?.scores).toBeUndefined(); expect(infra?.metadata?.scores).toEqual({ infra_error_rate: 1, ungraded_rate: 1, @@ -303,6 +303,7 @@ describe("Harbor result ingestion", () => { reward: 1, rewardKey: "reward_lenient", }); + expect(success?.scores).toMatchObject({ reward_lenient: 1 }); const llm = first.find((event) => event.span_attributes.type === "llm"); expect(llm?.input).toEqual([ { From ad99d0e26667a74282d382d546c5a93016dd2c27 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:13:10 +0000 Subject: [PATCH 2/3] Publish arms with infrastructure failures --- report/publish-braintrust.ts | 14 -------------- report/results.test.ts | 8 -------- 2 files changed, 22 deletions(-) diff --git a/report/publish-braintrust.ts b/report/publish-braintrust.ts index 19d09b3..a01ab3d 100644 --- a/report/publish-braintrust.ts +++ b/report/publish-braintrust.ts @@ -628,20 +628,6 @@ export async function publishBenchmark( experimentName: string, apiKey: string, ): Promise> { - const incomplete = arms - .map(summarizeArm) - .filter((summary) => !summary.complete); - if (incomplete.length > 0) { - throw new Error( - `Cannot publish incomplete benchmark arms: ${incomplete - .map( - (summary) => - `${summary.arm} (${summary.incompleteReasons.join(", ")})`, - ) - .join("; ")}`, - ); - } - const api = new BraintrustApi(apiKey); const project = await api.request("/v1/project", "POST", { name: projectName, diff --git a/report/results.test.ts b/report/results.test.ts index b85ca92..d7c6522 100644 --- a/report/results.test.ts +++ b/report/results.test.ts @@ -445,14 +445,6 @@ describe("Harbor result ingestion", () => { } }); - test("does not publish incomplete arms", async () => { - const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); - for (const trial of arm.trials) trial.rewards = {}; - await expect( - publishBenchmark([arm], "project", "experiment", "api-key"), - ).rejects.toThrow("Cannot publish incomplete benchmark arms: candidate"); - }); - test("uses the lenient reward per trial and reports incomplete arms", () => { expect(selectPrimaryReward({ reward: 0, reward_lenient: 1 })).toEqual({ key: "reward_lenient", From ebc1efab72216791d37a9ce5a815d433362ed4f6 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:01:03 +0000 Subject: [PATCH 3/3] Keep the end of trial errors in Braintrust --- report/publish-braintrust.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/report/publish-braintrust.ts b/report/publish-braintrust.ts index a01ab3d..66e1597 100644 --- a/report/publish-braintrust.ts +++ b/report/publish-braintrust.ts @@ -165,6 +165,12 @@ function taskInstruction(trial: BenchmarkTrial): unknown { // The suite's rewards, such as ClawBench's lenient and strict judgments, are // Braintrust scores so the experiment summary and comparisons show them. // Derived rates stay in metadata, where they remain filterable. +// Tracebacks put the cause last, so keep the end of long errors. +function errorTail(error: string): string { + const redacted = redactString(error, Number.MAX_SAFE_INTEGER); + return redacted.length > 2000 ? `…${redacted.slice(-2000)}` : redacted; +} + function trialScores( trial: BenchmarkTrial, ): Record | undefined { @@ -497,10 +503,10 @@ export function buildExperimentEvents( output: { reward: primaryReward?.value, rewardKey: primaryReward?.key, - error: trial.error ? redactString(trial.error, 400) : undefined, + error: trial.error ? errorTail(trial.error) : undefined, }, expected: { reward: 1 }, - error: trial.error ? redactString(trial.error, 400) : undefined, + error: trial.error ? errorTail(trial.error) : undefined, scores: trialScores(trial), metadata: trialMetadata(trial), metrics: metricRecord(trial),