diff --git a/report/publish-braintrust.ts b/report/publish-braintrust.ts index 9360f6f..66e1597 100644 --- a/report/publish-braintrust.ts +++ b/report/publish-braintrust.ts @@ -59,6 +59,7 @@ interface BraintrustEvent { output?: unknown; expected?: unknown; error?: string; + scores?: Record; metadata?: Record; metrics?: Record; _is_merge: false; @@ -161,8 +162,28 @@ function taskInstruction(trial: BenchmarkTrial): unknown { return redactValue(userSteps.at(-1)?.message); } -// Score values ride in metadata rather than Braintrust's billed scores field; -// they stay filterable and groupable there (`metadata.scores.accuracy = 1`). +// The suite's rewards, such as ClawBench's lenient and strict judgments, are +// Braintrust scores so the experiment summary and comparisons show them. +// Derived rates stay in metadata, where they remain filterable. +// Tracebacks put the cause last, so keep the end of long errors. +function errorTail(error: string): string { + const redacted = redactString(error, Number.MAX_SAFE_INTEGER); + return redacted.length > 2000 ? `…${redacted.slice(-2000)}` : redacted; +} + +function trialScores( + trial: BenchmarkTrial, +): Record | undefined { + // Infrastructure failures are not task results; metadata counts them. + if (trial.errorClass === "infra") return undefined; + const scores = Object.fromEntries( + Object.entries(trial.rewards).filter( + ([, value]) => value >= 0 && value <= 1, + ), + ); + return Object.keys(scores).length > 0 ? scores : undefined; +} + function trialMetadata(trial: BenchmarkTrial): Record { return { trialName: trial.trialName, @@ -482,10 +503,11 @@ export function buildExperimentEvents( output: { reward: primaryReward?.value, rewardKey: primaryReward?.key, - error: trial.error ? redactString(trial.error, 400) : undefined, + error: trial.error ? errorTail(trial.error) : undefined, }, expected: { reward: 1 }, - error: trial.error ? redactString(trial.error, 400) : undefined, + error: trial.error ? errorTail(trial.error) : undefined, + scores: trialScores(trial), metadata: trialMetadata(trial), metrics: metricRecord(trial), _is_merge: false, @@ -612,20 +634,6 @@ export async function publishBenchmark( experimentName: string, apiKey: string, ): Promise> { - const incomplete = arms - .map(summarizeArm) - .filter((summary) => !summary.complete); - if (incomplete.length > 0) { - throw new Error( - `Cannot publish incomplete benchmark arms: ${incomplete - .map( - (summary) => - `${summary.arm} (${summary.incompleteReasons.join(", ")})`, - ) - .join("; ")}`, - ); - } - const api = new BraintrustApi(apiKey); const project = await api.request("/v1/project", "POST", { name: projectName, diff --git a/report/results.test.ts b/report/results.test.ts index 451e7ce..d7c6522 100644 --- a/report/results.test.ts +++ b/report/results.test.ts @@ -288,7 +288,7 @@ describe("Harbor result ingestion", () => { event.span_attributes.type === "eval" && (event.output as { error?: string }).error, ); - expect(infra).not.toHaveProperty("scores"); + expect(infra?.scores).toBeUndefined(); expect(infra?.metadata?.scores).toEqual({ infra_error_rate: 1, ungraded_rate: 1, @@ -303,6 +303,7 @@ describe("Harbor result ingestion", () => { reward: 1, rewardKey: "reward_lenient", }); + expect(success?.scores).toMatchObject({ reward_lenient: 1 }); const llm = first.find((event) => event.span_attributes.type === "llm"); expect(llm?.input).toEqual([ { @@ -444,14 +445,6 @@ describe("Harbor result ingestion", () => { } }); - test("does not publish incomplete arms", async () => { - const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); - for (const trial of arm.trials) trial.rewards = {}; - await expect( - publishBenchmark([arm], "project", "experiment", "api-key"), - ).rejects.toThrow("Cannot publish incomplete benchmark arms: candidate"); - }); - test("uses the lenient reward per trial and reports incomplete arms", () => { expect(selectPrimaryReward({ reward: 0, reward_lenient: 1 })).toEqual({ key: "reward_lenient",