Publish suite rewards as Braintrust scores, including incomplete arms

report/publish-braintrust.ts

@@ -59,6 +59,7 @@ interface BraintrustEvent {
   output?: unknown;
   expected?: unknown;
   error?: string;
+  scores?: Record<string, number>;
   metadata?: Record<string, unknown>;
   metrics?: Record<string, number>;
   _is_merge: false;
@@ -161,8 +162,28 @@ function taskInstruction(trial: BenchmarkTrial): unknown {
   return redactValue(userSteps.at(-1)?.message);
 }
 
-// Score values ride in metadata rather than Braintrust's billed scores field;
-// they stay filterable and groupable there (`metadata.scores.accuracy = 1`).
+// The suite's rewards, such as ClawBench's lenient and strict judgments, are
+// Braintrust scores so the experiment summary and comparisons show them.
+// Derived rates stay in metadata, where they remain filterable.
+// Tracebacks put the cause last, so keep the end of long errors.
+function errorTail(error: string): string {
+  const redacted = redactString(error, Number.MAX_SAFE_INTEGER);
+  return redacted.length > 2000 ? `…${redacted.slice(-2000)}` : redacted;
+}
+
+function trialScores(
+  trial: BenchmarkTrial,
+): Record<string, number> | undefined {
+  // Infrastructure failures are not task results; metadata counts them.
+  if (trial.errorClass === "infra") return undefined;
+  const scores = Object.fromEntries(
+    Object.entries(trial.rewards).filter(
+      ([, value]) => value >= 0 && value <= 1,
+    ),
+  );
+  return Object.keys(scores).length > 0 ? scores : undefined;
+}
+
 function trialMetadata(trial: BenchmarkTrial): Record<string, unknown> {
   return {
     trialName: trial.trialName,
@@ -482,10 +503,11 @@ export function buildExperimentEvents(
         output: {
           reward: primaryReward?.value,
           rewardKey: primaryReward?.key,
-          error: trial.error ? redactString(trial.error, 400) : undefined,
+          error: trial.error ? errorTail(trial.error) : undefined,
         },
         expected: { reward: 1 },
-        error: trial.error ? redactString(trial.error, 400) : undefined,
+        error: trial.error ? errorTail(trial.error) : undefined,
+        scores: trialScores(trial),
         metadata: trialMetadata(trial),
         metrics: metricRecord(trial),
         _is_merge: false,
@@ -612,20 +634,6 @@ export async function publishBenchmark(
   experimentName: string,
   apiKey: string,
 ): Promise<Record<string, unknown>> {
-  const incomplete = arms
-    .map(summarizeArm)
-    .filter((summary) => !summary.complete);
-  if (incomplete.length > 0) {
-    throw new Error(
-      `Cannot publish incomplete benchmark arms: ${incomplete
-        .map(
-          (summary) =>
-            `${summary.arm} (${summary.incompleteReasons.join(", ")})`,
-        )
-        .join("; ")}`,
-    );
-  }
-
   const api = new BraintrustApi(apiKey);
   const project = await api.request<BraintrustProject>("/v1/project", "POST", {
     name: projectName,

report/results.test.ts

@@ -288,7 +288,7 @@ describe("Harbor result ingestion", () => {
         event.span_attributes.type === "eval" &&
         (event.output as { error?: string }).error,
     );
-    expect(infra).not.toHaveProperty("scores");
+    expect(infra?.scores).toBeUndefined();
     expect(infra?.metadata?.scores).toEqual({
       infra_error_rate: 1,
       ungraded_rate: 1,
@@ -303,6 +303,7 @@ describe("Harbor result ingestion", () => {
       reward: 1,
       rewardKey: "reward_lenient",
     });
+    expect(success?.scores).toMatchObject({ reward_lenient: 1 });
     const llm = first.find((event) => event.span_attributes.type === "llm");
     expect(llm?.input).toEqual([
       {
@@ -444,14 +445,6 @@ describe("Harbor result ingestion", () => {
     }
   });
 
-  test("does not publish incomplete arms", async () => {
-    const arm = readBenchmarkArm({ name: "candidate", path: fixture() });
-    for (const trial of arm.trials) trial.rewards = {};
-    await expect(
-      publishBenchmark([arm], "project", "experiment", "api-key"),
-    ).rejects.toThrow("Cannot publish incomplete benchmark arms: candidate");
-  });
-
   test("uses the lenient reward per trial and reports incomplete arms", () => {
     expect(selectPrimaryReward({ reward: 0, reward_lenient: 1 })).toEqual({
       key: "reward_lenient",