diff --git a/.github/scripts/github-app-installation-token.mjs b/.github/scripts/github-app-installation-token.mjs
new file mode 100644
index 0000000..e070184
--- /dev/null
+++ b/.github/scripts/github-app-installation-token.mjs
@@ -0,0 +1,73 @@
+#!/usr/bin/env node
+
+import { createSign } from "node:crypto";
+import { readFile, writeFile } from "node:fs/promises";
+
+const cachePath = process.env.GITHUB_APP_TOKEN_CACHE;
+
+if (cachePath) {
+  try {
+    const cached = JSON.parse(await readFile(cachePath, "utf8"));
+    if (Date.parse(cached.expiresAt) > Date.now() + 5 * 60 * 1000) {
+      process.stderr.write(`::add-mask::${cached.token}\n`);
+      process.stdout.write(cached.token);
+      process.exit(0);
+    }
+  } catch {
+    // A missing, stale, or invalid cache is replaced below.
+  }
+}
+
+const appId = process.env.GITHUB_APP_ID;
+const installationId = process.env.GITHUB_APP_INSTALLATION_ID;
+const privateKeyPath = process.env.GITHUB_APP_PRIVATE_KEY_FILE;
+const repository = process.env.GITHUB_APP_REPOSITORY;
+
+if (!appId || !installationId || !privateKeyPath) {
+  throw new Error("GitHub App ID, installation ID, and private key file are required");
+}
+
+const now = Math.floor(Date.now() / 1000);
+const encode = (value) => Buffer.from(JSON.stringify(value)).toString("base64url");
+const unsignedToken = `${encode({ alg: "RS256", typ: "JWT" })}.${encode({
+  iat: now - 60,
+  exp: now + 9 * 60,
+  iss: appId,
+})}`;
+const privateKey = (await readFile(privateKeyPath, "utf8")).replaceAll("\\n", "\n");
+const signer = createSign("RSA-SHA256");
+signer.update(unsignedToken);
+const jwt = `${unsignedToken}.${signer.sign(privateKey, "base64url")}`;
+
+const response = await fetch(
+  `https://api.github.com/app/installations/${installationId}/access_tokens`,
+  {
+    method: "POST",
+    headers: {
+      Accept: "application/vnd.github+json",
+      Authorization: `Bearer ${jwt}`,
+      "X-GitHub-Api-Version": "2022-11-28",
+      "User-Agent": "benchmarks-nightly-pinned-deps",
+    },
+    body: repository ? JSON.stringify({ repositories: [repository] }) : undefined,
+  },
+);
+
+if (!response.ok) {
+  throw new Error(`GitHub App token request failed with HTTP ${response.status}`);
+}
+
+const result = await response.json();
+if (!result.token || !result.expires_at) {
+  throw new Error("GitHub App token response was incomplete");
+}
+
+process.stderr.write(`::add-mask::${result.token}\n`);
+if (cachePath) {
+  await writeFile(
+    cachePath,
+    JSON.stringify({ token: result.token, expiresAt: result.expires_at }),
+    { mode: 0o600 },
+  );
+}
+process.stdout.write(result.token);
diff --git a/.github/workflows/nightly-pinned-deps-prompt.md b/.github/workflows/nightly-pinned-deps-prompt.md
new file mode 100644
index 0000000..65c1267
--- /dev/null
+++ b/.github/workflows/nightly-pinned-deps-prompt.md
@@ -0,0 +1,182 @@
+You are operating in a GitHub Actions runner for the Kernel benchmarks repository.
+
+The GitHub CLI is available as `gh`, and Git is authenticated as the `kernel-internal[bot]` GitHub App. Their credential helpers renew the installation token before it expires; use the normal `gh` and `git` commands, and never bypass the helpers or print credentials. You may push branches, create pull requests and merge pull requests. Pi Web Access is installed; call `web_enable` before using `web_search`, `fetch_content`, or `get_search_content` for release notes and migration guidance. Docker, uv, Bun and Node.js are installed. `OPENAI_API_KEY` and `ANTHROPIC_API_KEY` are set for the agent smoke tests.
+
+# Context
+
+- Repository: `${GITHUB_REPOSITORY}`
+- Default branch: `main`
+- Evergreen branch: `automation/nightly-pinned-deps`
+- Required outcome file: `${OUTCOME_FILE}`
+- Smoke test results: `${SMOKE_DIR}/<commit-sha>/`
+
+Read `README.md` first. The benchmarks compare agents, browser tools and browser settings on live websites. Their results are only meaningful if every arm runs the versions it declares, so every pin here is deliberate and every change to one must be traceable.
+
+# Goal
+
+Keep every third-party pin the benchmarks depend on current. Update all stale pins found in one run, including major versions. Adapt the harness, toolsets and tests to breaking changes, verify each changed integration, open or update the evergreen pull request with a record of what changed, and merge it after the latest commit is green.
+
+The inventory is:
+
+- **Agent harnesses.** `agent_version` in `arms/*.toml`: `codex` arms use the npm package `@openai/codex`, `claude-code` arms use `@anthropic-ai/claude-code`. Move every arm of an agent to that agent's latest stable version.
+- **Harbor.** `harbor[kernel]` in `pyproject.toml` and `uv.lock`. See *Harbor* below.
+- **ClawBench.** The `clawbench-eval` commit in `pyproject.toml` and `uv.lock`: move it to the head of the default branch of `TIGER-AI-Lab/ClawBench`.
+- **Kernel Python SDK.** The locked version of `kernel` in `uv.lock`. Keep the `>=` floor in `pyproject.toml` unless the code needs a newer one.
+- **Playwright MCP.** `@playwright/mcp`: `toolset_args.version` in arms that use the `playwright-mcp` toolset and the default in `toolsets/playwright-mcp/toolset.py`.
+- **Toolset images.** npm packages in `toolsets/*/image/layer.Dockerfile` (`bun`, `mcp-remote`) and the `ghcr.io/astral-sh/uv` image in `suites/*/image/Dockerfile`.
+- **Workflows.** Every action in `.github/workflows/*.yml`, including this job's own workflow, pinned to the full commit SHA of its latest release with a `# vX.Y.Z` comment, plus the uv and Bun versions passed to `setup-uv` and `setup-bun`.
+- **Report.** The exact versions in `report/package.json` and `report/bun.lock`.
+
+Keep paired pins aligned: the Bun version in workflows, `toolsets/kernel-mcp-local/image/layer.Dockerfile` and `@types/bun`; the Playwright MCP version in arms and the toolset default.
+
+Out of scope: `model`, `agent_kwargs`, `toolset`, `toolset_args` other than `version`, and `[browser]` in arms (model IDs and agent settings are experimental choices, not pins); other Python dependencies and transitive locks, except as the updates above require; Docker base images; the git refs toolsets build from (`kernel_mcp_server_ref`, `xwebmcp_ref`), which track `main` on purpose.
+
+## Harbor
+
+Harbor is pinned to a commit of the `kernel/harbor` fork, branch `kernel-environment`, until `harbor-framework/harbor#2122` is in an upstream release.
+
+1. Check whether the switch is possible: `gh pr view 2122 --repo harbor-framework/harbor --json state,mergeCommit`. If it is merged, find the latest release on PyPI (`https://pypi.org/pypi/harbor/json`) and confirm that its release tag contains the merge commit (`gh api repos/harbor-framework/harbor/compare/<tag>...<merge-sha>` reports `behind` or `identical`) and that it provides the `kernel` extra.
+2. If it does, switch to `harbor[kernel]==<that version>` and remove the fork caveat from `README.md`. Before you keep the switch, diff `src/harbor/environments/kernel.py` and the installed agents the arms use (`src/harbor/agents/installed/codex.py`, `src/harbor/agents/installed/claude_code.py`) between the fork commit and the release tag. Every parameter and behavior the harness relies on, starting with `browser_create_overrides` and multi-step tasks, must still exist. If something the harness needs is missing from the release, keep the fork pin and explain the concrete gap in the pull request body.
+3. Otherwise, move the pin to the head of `kernel-environment` (`git ls-remote https://github.com/kernel/harbor refs/heads/kernel-environment`).
+
+Once on an upstream release, keep updating to the latest release.
+
+Change `pyproject.toml`, then regenerate the lock with `uv lock --upgrade-package <name>` for each changed package. Never edit `uv.lock` or `report/bun.lock` by hand.
+
+# Rules
+
+- Use the latest stable upstream release. Do not select prereleases (alpha, beta, rc, next) unless the existing pin is already one.
+- Do not skip a major update merely because it has breaking changes. Read the release notes and adapt the code, configuration and focused tests when the migration is reasonable.
+- Revert an update only when the latest release is unusable here and no reasonable compatibility change is possible in this run. Then keep the old pin and explain the concrete blocker in the pull request body.
+- Never expose secret values in logs, commits, pull request text, or comments.
+- Never run `bench run` or anything else that creates Kernel browsers or runs live benchmark trials. `bench smoke` runs agents on a local Docker task and is allowed.
+- Do not finish while the pull request is open, checks are failing, or a smoke test fails.
+
+# Step 1: Resume or create the evergreen branch
+
+```bash
+git fetch origin main
+git fetch origin automation/nightly-pinned-deps 2>/dev/null || true
+OPEN_PR=$(gh pr list \
+  --head automation/nightly-pinned-deps \
+  --base main \
+  --state open \
+  --json number \
+  --jq '.[0].number // ""')
+```
+
+If `OPEN_PR` is non-empty, resume the remote branch and keep its work:
+
+```bash
+git checkout -B automation/nightly-pinned-deps origin/automation/nightly-pinned-deps
+git merge --no-edit origin/main
+```
+
+Then inventory every pin again so new releases join the existing pull request.
+
+If there is no open pull request, start from main:
+
+```bash
+git checkout -B automation/nightly-pinned-deps origin/main
+```
+
+A closed or merged pull request does not count. Never reuse its old branch contents.
+
+# Step 2: Inventory every pin
+
+Build a table of each pin's current and latest stable version from authoritative sources:
+
+- `npm view <package> version` and `npm view <package> time --json` for npm packages
+- `gh release view --repo <owner>/<repo>` and `gh api repos/<owner>/<repo>/commits/<tag> --jq .sha` for actions
+- `git ls-remote` for git commits, and PyPI's JSON API for Python packages
+- Pi Web Access for changelogs, release notes and migration guides
+
+Search the repository (`git grep -n '<old version>'`) for every occurrence of a pin before changing it, including `README.md`'s tables of the current arms. Leave example output tables in `README.md` alone; they record past runs.
+
+If no pin is stale and there is no open pull request, write exactly `no-updates` to `${OUTCOME_FILE}` and exit successfully. Do not create an empty commit or pull request.
+
+# Step 3: Update pins and compatibility code
+
+Update every stale pin in one coherent change. For each update with breaking or behavioral changes:
+
+1. Read its release notes and the relevant source.
+2. Find the harness, toolset, suite, report and test code that depends on the changed behavior.
+3. Make the smallest maintainable compatibility change and update its focused tests.
+
+For agent harnesses, read every release note between the old and new version and look specifically for changes that alter what a benchmark measures:
+
+- how MCP tool calls are invoked or recorded (for example, Codex nesting MCP calls inside a code-mode `exec` tool),
+- new or re-enabled built-in tools, especially web search or fetch, that let an agent answer without the browser,
+- changes to the session/rollout JSONL Harbor converts into `trajectory.json`, to CLI flags Harbor passes, or to configuration Harbor writes (`$CODEX_HOME/config.toml`, `$CLAUDE_CONFIG_DIR/.claude.json`).
+
+Validity checks and call counts read the server-side `mcp-call-log`, not the trajectory, so they survive changes in how calls are nested. Do not change arms' `agent_kwargs` yourself; report such changes as described in step 5.
+
+# Step 4: Verify what changed
+
+Run the repository checks CI runs:
+
+```bash
+uv sync --frozen
+uv run ruff check .
+uv run ruff format --check .
+uv run pytest -q
+uv run bench prepare kernel-mcp-clawbench --arms codex-kernel-mcp,claude-kernel-mcp --tasks v2-1134-chapter-finder-redcross --out "$RUNNER_TEMP/prepared"
+uv run bench prepare xwebmcp-repl --out "$RUNNER_TEMP/prepared-xwebmcp"
+(cd report && bun install --frozen-lockfile && bunx prettier --check . && bunx tsc --noEmit && bun test)
+```
+
+Then verify each changed integration with the check that covers it:
+
+- **Agent harness or Harbor.** After committing, run `smoke-harnesses HEAD` and `smoke-harnesses origin/main`. Each runs `bench smoke` for one arm per distinct agent configuration: the arm's agent, version, model and agent kwargs run under Harbor on a local Docker task whose only tool is the suite's stdio `finalize` server. It checks that the agent installs at that version, the MCP server receives the `finalize` call, the verifier scores it, and Harbor converts the session into a `trajectory.json` with tool calls, token counts and cost. Every row must pass. Compare the two tables. A change in trajectory tool names, or in the built-in tools a harness reports (`builtin_tools` in `${SMOKE_DIR}/<sha>/summary.json`), is a behavior change to report. To compare one arm across versions, run `uv run bench smoke <arm> --agent-version <version> --jobs-dir "$RUNNER_TEMP/smoke-jobs"`.
+- **ClawBench.** `uv run pytest -q tests/test_clawbench.py`, and `uv run bench prepare kernel-mcp-clawbench --arms codex-kernel-mcp --tasks all --out "$RUNNER_TEMP/prepared-all"`. Report any change in the number of tasks.
+- **Playwright MCP.** Run `npx -y @playwright/mcp@<version> --help` and confirm the flags `toolsets/playwright-mcp/image/bin/playwright-mcp-stdio` passes still exist. Send `initialize` and `tools/list` over stdio and confirm the `browser_*` tools the toolset instruction names are still listed.
+- **mcp-remote.** Confirm from its release notes and `--help` that the command line in `toolsets/kernel-mcp-local/image/bin/kernel-mcp` still works: a streamable-HTTP URL, `--header` and expansion of `${KERNEL_MCP_AUTHORIZATION}` from the environment. Exercise it: start a minimal streamable-HTTP MCP server on `127.0.0.1` that records request headers, run that exact command line through `mcp-call-log` against it, and check that `initialize`, `tools/list` and one `tools/call` work and that the expanded `Authorization` header arrives.
+- **A Dockerfile pin** (bun, mcp-remote, the uv image). Build the affected prepared task's `environment/` directory with `docker build` from the `bench prepare` output above.
+- **Workflow actions.** Read each action's release notes for removed or renamed inputs this repository uses. CI on the pull request runs the changed `ci.yml`.
+- **Report packages.** The report commands above.
+
+Record every command and the pin it covers. A version lookup alone is not a test.
+
+# Step 5: Commit, push, and create or update the pull request
+
+Review `git diff origin/main...HEAD` and remove unrelated changes. Commit in imperative mood and push:
+
+```bash
+git push --force-with-lease origin automation/nightly-pinned-deps
+```
+
+Create a ready-for-review pull request when none is open; otherwise update it. Use the title `Update pinned benchmark dependencies`.
+
+The pull request body must include, in this order:
+
+1. **Behavior changes.** Start with what can change benchmark results, so a shift in results can be traced to this change: every arm whose `agent_version`, Harbor version or Playwright MCP version changed, with old and new versions; built-in tools added or enabled for arms that do not set `disable_web_search`; changes in how tool calls are recorded; changes in the ClawBench task set. Write "None" if there are none.
+2. A table of every pin's old and new version.
+3. Compatibility changes, and any pin intentionally kept with its reason.
+4. The `bench smoke` tables for the head commit and for `origin/main`.
+5. The verification commands, with the pin each covers.
+6. Links to release notes for major or breaking updates.
+7. A statement that the branch is maintained by the nightly pinned dependency workflow.
+
+# Step 6: Wait for checks
+
+Poll `gh pr checks <pr-number>` until every check on the current head commit has completed. Fix upgrade-related failures, run the affected verification again, commit and push, and wait again. Rerun a demonstrably flaky unrelated check rather than changing unrelated code. Every push needs a new `smoke-harnesses HEAD` pass and an updated smoke table in the pull request body.
+
+# Step 7: Merge and prove the outcome
+
+When every check is green on the head commit, squash-merge with the GitHub App identity. Put the version table in the commit body so `git log` on main records which versions changed:
+
+```bash
+gh pr merge <pr-number> --squash --admin --delete-branch \
+  --subject "Update pinned benchmark dependencies (#<pr-number>)" \
+  --body "<the behavior changes and version table from the pull request body>"
+```
+
+The `gh` wrapper refuses `gh pr merge` unless the `harness` and `report` checks and every other check on the head commit are green, and it runs `smoke-harnesses` on the head commit itself. Do not bypass it with another client or the merge API.
+
+Verify with `gh pr view <pr-number> --json state` that the state is `MERGED`. Only then write the outcome:
+
+```bash
+printf 'merged:%s\n' '<pr-number>' > "${OUTCOME_FILE}"
+```
+
+Do not create a different outcome file. If the merge is blocked, keep diagnosing within this run. Do not write an outcome that claims success, and do not exit successfully with an open pull request.
diff --git a/.github/workflows/nightly-pinned-deps.yml b/.github/workflows/nightly-pinned-deps.yml
new file mode 100644
index 0000000..40c45fe
--- /dev/null
+++ b/.github/workflows/nightly-pinned-deps.yml
@@ -0,0 +1,254 @@
+name: Nightly pinned dependency updates
+
+on:
+  schedule:
+    - cron: "41 5 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+  checks: read
+  statuses: read
+
+concurrency:
+  group: nightly-pinned-dependency-updates
+  cancel-in-progress: false
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    steps:
+      - name: Generate app token
+        id: app-token
+        uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0
+        with:
+          app-id: ${{ secrets.ADMIN_APP_ID }}
+          private-key: ${{ secrets.ADMIN_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: benchmarks
+
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          fetch-depth: 0
+          token: ${{ steps.app-token.outputs.token }}
+
+      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+        with:
+          node-version: "22"
+
+      - uses: astral-sh/setup-uv@b75a909f75acd358c2196fb9a5f1299a9a8868a4 # v6.7.0
+        with:
+          version: "0.8.17"
+
+      - uses: oven-sh/setup-bun@3d267786b128fe76c2f16a390aa2448b815359f3 # v2.1.2
+        with:
+          bun-version: "1.3.3"
+
+      - name: Install Pi and web access
+        env:
+          PI_TELEMETRY: "0"
+        run: |
+          npm install -g --ignore-scripts @earendil-works/pi-coding-agent
+          pi install npm:pi-web-access@latest
+          pi --version
+          pi list
+
+      # gh and git renew the installation token through the helper. `gh pr
+      # merge` refuses to merge unless every check on the head commit is green
+      # and `bench smoke` passes for every agent configuration at that commit.
+      - name: Configure renewable GitHub App authentication
+        env:
+          APP_ID: ${{ secrets.ADMIN_APP_ID }}
+          APP_INSTALLATION_ID: ${{ steps.app-token.outputs.installation-id }}
+          APP_PRIVATE_KEY: ${{ secrets.ADMIN_APP_PRIVATE_KEY }}
+        run: |
+          auth_dir="$RUNNER_TEMP/github-app-auth"
+          bin_dir="$auth_dir/bin"
+          mkdir -p "$bin_dir"
+          chmod 700 "$auth_dir" "$bin_dir"
+
+          private_key_file="$auth_dir/private-key.pem"
+          token_helper="$auth_dir/github-app-installation-token.mjs"
+          printf '%s' "$APP_PRIVATE_KEY" > "$private_key_file"
+          cp "$GITHUB_WORKSPACE/.github/scripts/github-app-installation-token.mjs" "$token_helper"
+          chmod 600 "$private_key_file" "$token_helper"
+
+          real_gh="$(command -v gh)"
+          cat > "$bin_dir/gh" <<'EOF'
+          #!/usr/bin/env bash
+          set -euo pipefail
+          token="$(node "$GITHUB_APP_TOKEN_HELPER")"
+          if [ "${1:-}" = "pr" ] && [ "${2:-}" = "checks" ]; then
+            exec env GH_TOKEN="$GITHUB_CHECKS_TOKEN" GITHUB_TOKEN="$GITHUB_CHECKS_TOKEN" "$GITHUB_REAL_GH" "$@"
+          fi
+          if [ "${1:-}" = "pr" ] && [ "${2:-}" = "merge" ]; then
+            pr_number="${3:-}"
+            if [ -z "$pr_number" ] || [[ "$pr_number" = -* ]]; then
+              echo "A pull request number is required for the merge gate" >&2
+              exit 1
+            fi
+            head_sha="$(env GH_TOKEN="$token" GITHUB_TOKEN="$token" "$GITHUB_REAL_GH" pr view "$pr_number" --json headRefOid --jq .headRefOid)"
+            check_runs="$(env GH_TOKEN="$GITHUB_CHECKS_TOKEN" GITHUB_TOKEN="$GITHUB_CHECKS_TOKEN" "$GITHUB_REAL_GH" api "repos/$GITHUB_REPOSITORY/commits/$head_sha/check-runs?filter=latest&per_page=100")"
+            for required in harness report; do
+              conclusion="$(jq -r --arg name "$required" '[.check_runs[] | select(.name == $name and .status == "completed")] | last | .conclusion // ""' <<<"$check_runs")"
+              if [ "$conclusion" != "success" ]; then
+                echo "CI check $required is not green for $head_sha (conclusion: ${conclusion:-missing})" >&2
+                exit 1
+              fi
+            done
+            incomplete_checks="$(jq -r '[.check_runs[] | select(.status != "completed")] | map(.name) | join(", ")' <<<"$check_runs")"
+            failed_checks="$(jq -r '[.check_runs[] | select(.status == "completed" and (.conclusion | IN("success", "neutral", "skipped") | not))] | map("\(.name)=\(.conclusion)") | join(", ")' <<<"$check_runs")"
+            if [ -n "$incomplete_checks" ] || [ -n "$failed_checks" ]; then
+              echo "Pull request checks are not green for $head_sha (incomplete: ${incomplete_checks:-none}; failed: ${failed_checks:-none})" >&2
+              exit 1
+            fi
+            commit_status="$(env GH_TOKEN="$GITHUB_CHECKS_TOKEN" GITHUB_TOKEN="$GITHUB_CHECKS_TOKEN" "$GITHUB_REAL_GH" api "repos/$GITHUB_REPOSITORY/commits/$head_sha/status")"
+            status_count="$(jq -r .total_count <<<"$commit_status")"
+            status_state="$(jq -r .state <<<"$commit_status")"
+            if [ "$status_count" -gt 0 ] && [ "$status_state" != "success" ]; then
+              echo "Commit statuses are not green for $head_sha (state: $status_state)" >&2
+              exit 1
+            fi
+            if ! smoke-harnesses "$head_sha"; then
+              echo "bench smoke failed for $head_sha; see $SMOKE_DIR/$head_sha" >&2
+              exit 1
+            fi
+          fi
+          exec env GH_TOKEN="$token" GITHUB_TOKEN="$token" "$GITHUB_REAL_GH" "$@"
+          EOF
+          cat > "$bin_dir/git-credential-github-app" <<'EOF'
+          #!/usr/bin/env bash
+          set -euo pipefail
+          [ "${1:-}" = "get" ] || exit 0
+          protocol=""
+          host=""
+          while IFS='=' read -r key value; do
+            case "$key" in
+              protocol) protocol="$value" ;;
+              host) host="$value" ;;
+            esac
+          done
+          [ "$protocol" = "https" ] && [ "$host" = "github.com" ] || exit 0
+          printf 'username=x-access-token\n'
+          printf 'password=%s\n' "$(node "$GITHUB_APP_TOKEN_HELPER")"
+          EOF
+          # Runs `bench smoke` for every agent configuration in a clean
+          # worktree of the given commit. Results are kept per commit, so a
+          # pass is reused until the branch moves.
+          cat > "$bin_dir/smoke-harnesses" <<'EOF'
+          #!/usr/bin/env bash
+          set -euo pipefail
+          if [ -n "${1:-}" ]; then
+            git -C "$GITHUB_WORKSPACE" fetch -q origin "$1" 2>/dev/null || true
+          fi
+          sha="$(git -C "$GITHUB_WORKSPACE" rev-parse "${1:-HEAD}^{commit}")"
+          out="$SMOKE_DIR/$sha"
+          if [ -f "$out/passed" ]; then
+            cat "$out/summary.md"
+            exit 0
+          fi
+          rm -rf "$out"
+          mkdir -p "$out"
+          worktree="$RUNNER_TEMP/smoke-worktree"
+          git -C "$GITHUB_WORKSPACE" worktree remove --force "$worktree" 2>/dev/null || true
+          git -C "$GITHUB_WORKSPACE" worktree add --detach "$worktree" "$sha" >/dev/null
+          status=0
+          (
+            cd "$worktree"
+            uv sync --frozen --quiet
+            uv run bench smoke --jobs-dir "$out/jobs" --output "$out/summary.json" --markdown "$out/summary.md"
+          ) >"$out/smoke.log" 2>&1 || status=$?
+          git -C "$GITHUB_WORKSPACE" worktree remove --force "$worktree"
+          cat "$out/summary.md" 2>/dev/null || tail -50 "$out/smoke.log"
+          [ "$status" -eq 0 ] && touch "$out/passed"
+          exit "$status"
+          EOF
+          chmod 700 "$bin_dir/gh" "$bin_dir/git-credential-github-app" "$bin_dir/smoke-harnesses"
+
+          git config user.name "kernel-internal[bot]"
+          git config user.email "260533166+kernel-internal[bot]@users.noreply.github.com"
+          git config --local --unset-all http.https://github.com/.extraheader || true
+          git config --local --unset-all credential.helper || true
+          git config --local --unset-all credential.https://github.com.helper || true
+          git config --local credential.https://github.com.helper "$bin_dir/git-credential-github-app"
+
+          {
+            echo "GITHUB_APP_ID=$APP_ID"
+            echo "GITHUB_APP_INSTALLATION_ID=$APP_INSTALLATION_ID"
+            echo "GITHUB_APP_PRIVATE_KEY_FILE=$private_key_file"
+            echo "GITHUB_APP_REPOSITORY=benchmarks"
+            echo "GITHUB_APP_TOKEN_CACHE=$auth_dir/token.json"
+            echo "GITHUB_APP_TOKEN_HELPER=$token_helper"
+            echo "GITHUB_REAL_GH=$real_gh"
+            echo "OUTCOME_FILE=$RUNNER_TEMP/nightly-pinned-deps.outcome"
+            echo "SMOKE_DIR=$RUNNER_TEMP/smoke"
+          } >> "$GITHUB_ENV"
+          echo "$bin_dir" >> "$GITHUB_PATH"
+
+      - name: Update pinned dependencies
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          GITHUB_CHECKS_TOKEN: ${{ github.token }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          GIT_TERMINAL_PROMPT: "0"
+          PI_PREFERRED_MODEL: ${{ vars.PI_PREFERRED_MODEL }}
+          PI_SKIP_VERSION_CHECK: "1"
+          PI_TELEMETRY: "0"
+        run: |
+          : "${OPENAI_API_KEY:?Set OPENAI_API_KEY for the Codex smoke test}"
+          : "${ANTHROPIC_API_KEY:?Set ANTHROPIC_API_KEY for the Claude Code smoke test}"
+          : "${PI_PREFERRED_MODEL:?Set the PI_PREFERRED_MODEL organization variable}"
+          rendered_prompt="$RUNNER_TEMP/nightly-pinned-deps-prompt.md"
+          envsubst '${GITHUB_REPOSITORY} ${OUTCOME_FILE} ${SMOKE_DIR}' \
+            < .github/workflows/nightly-pinned-deps-prompt.md \
+            > "$rendered_prompt"
+          pi --print --no-session --approve \
+            --model "$PI_PREFERRED_MODEL" --thinking max \
+            @"$rendered_prompt"
+
+      - name: Confirm the run produced an outcome
+        run: |
+          outcome="$(tr -d '\r\n' < "$OUTCOME_FILE" 2>/dev/null || true)"
+          case "$outcome" in
+            no-updates)
+              open_pr="$(gh pr list --head automation/nightly-pinned-deps --state open --json number --jq '.[0].number // ""')"
+              if [ -n "$open_pr" ]; then
+                echo "::error::Agent reported no updates while PR #$open_pr remains open"
+                exit 1
+              fi
+              echo "All pinned dependencies are current."
+              ;;
+            merged:*)
+              pr_number="${outcome#merged:}"
+              state="$(gh pr view "$pr_number" --json state --jq .state)"
+              if [ "$state" != "MERGED" ]; then
+                echo "::error::Agent reported $outcome, but PR #$pr_number is $state"
+                exit 1
+              fi
+              echo "Merged PR #$pr_number."
+              ;;
+            *)
+              echo "::error::Agent exited without recording no-updates or merged:<pr-number> in $OUTCOME_FILE"
+              exit 1
+              ;;
+          esac
+
+      - name: Smoke test summary
+        if: always()
+        run: |
+          [ -n "${SMOKE_DIR:-}" ] || exit 0
+          shopt -s nullglob
+          for summary in "$SMOKE_DIR"/*/summary.md; do
+            sha="$(basename "$(dirname "$summary")")"
+            { echo "### bench smoke at ${sha:0:12}"; echo; cat "$summary"; echo; } >> "$GITHUB_STEP_SUMMARY"
+          done
+
+      - name: Revoke renewable app token
+        if: always()
+        run: |
+          if [ -f "$GITHUB_APP_TOKEN_CACHE" ]; then
+            gh api --method DELETE /installation/token || true
+          fi
+          rm -rf "$RUNNER_TEMP/github-app-auth"
diff --git a/README.md b/README.md
index 9a200fd..4c785ae 100644
--- a/README.md
+++ b/README.md
@@ -105,6 +105,21 @@ The job summary shows a table per arm. When Braintrust is configured, each run i
 
 GitHub stops jobs after six hours. Arms still running then are reported as incomplete. Lower the task count or raise `concurrency` for large runs.
 
+### Smoke-testing agent versions
+
+A new agent version can change how tool calls are recorded or which built-in tools the agent has, which unit tests don't see. `bench smoke` checks each agent configuration without a browser:
+
+```bash
+uv run bench smoke                                    # one arm per distinct agent, version, model and agent kwargs
+uv run bench smoke codex-repl --agent-version 0.163.0 # one arm at another version
+```
+
+Each runs the arm's agent through Harbor on a local Docker task whose only tool is a stdio MCP server with one `finalize` tool. It passes when the server logs the call, the verifier scores it, and Harbor converts the agent's session into a `trajectory.json` with tool calls, token counts and cost. The summary lists the tool names the trajectory shows, so two versions can be compared. It needs Docker and `OPENAI_API_KEY` / `ANTHROPIC_API_KEY`, and costs a few cents.
+
+### Pinned dependency updates
+
+`nightly-pinned-deps.yml` runs a [Pi](https://www.npmjs.com/package/@earendil-works/pi-coding-agent) agent every night with [the prompt next to it](.github/workflows/nightly-pinned-deps-prompt.md). It moves every pin to its latest release: agent versions in `arms/`, Harbor, ClawBench, the Kernel SDK, Playwright MCP, toolset image packages, workflow actions and report packages. It verifies each change, runs `bench smoke`, and merges one evergreen pull request when CI and the smoke tests pass. Models and agent settings in arms are never changed. The pull request and the squash commit list every version change first, so a shift in results can be traced to it.
+
 ## Current benchmarks
 
 ### `kernel-mcp-clawbench`
diff --git a/harness/cli.py b/harness/cli.py
index 2f4e647..5daa936 100644
--- a/harness/cli.py
+++ b/harness/cli.py
@@ -1,13 +1,17 @@
-"""Command line entry point: `bench prepare` and `bench run`."""
+"""Command line entry point: `bench prepare`, `bench run` and `bench smoke`."""
 
 from __future__ import annotations
 
 import argparse
+import dataclasses
+import json
 import tempfile
 from datetime import UTC, datetime
 from pathlib import Path
 
+from harness.definitions import load_arm
 from harness.run import prepare, run
+from harness.smoke import all_arms, distinct_harnesses, markdown, smoke_many
 
 
 def _split(value: str | None) -> list[str] | None:
@@ -62,7 +66,21 @@ def main(argv: list[str] | None = None) -> int:
     run_command.add_argument("--attempts", type=int, default=1, help="trials per task and arm")
     run_command.add_argument("--run-id", default=datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ"))
 
+    smoke_command = commands.add_parser(
+        "smoke", help="run arms' agents on a local Docker task with one stdio MCP tool"
+    )
+    smoke_command.add_argument(
+        "arms", nargs="*", help="arm names (default: one arm per distinct agent configuration)"
+    )
+    smoke_command.add_argument("--agent-version", help="override the agent_version of one arm")
+    smoke_command.add_argument("--jobs-dir", type=Path, default=Path("jobs"))
+    smoke_command.add_argument("--output", type=Path, help="also write the summaries as JSON")
+    smoke_command.add_argument("--markdown", type=Path, help="also write a summary table")
+
     args = parser.parse_args(argv)
+    if args.command == "smoke":
+        return _smoke(args)
+
     arms = _split(args.arms)
     tasks = _split(args.tasks)
     suite_args = _pairs(args.suite_arg)
@@ -93,3 +111,21 @@ def main(argv: list[str] | None = None) -> int:
     for arm, status in statuses.items():
         print(f"{arm}: harbor exited {status} -> {args.jobs_dir / f'{arm}-{args.run_id}'}")
     return max(statuses.values(), default=0)
+
+
+def _smoke(args: argparse.Namespace) -> int:
+    arms = [load_arm(name) for name in args.arms] if args.arms else distinct_harnesses(all_arms())
+    if args.agent_version:
+        if len(arms) != 1:
+            raise SystemExit("--agent-version needs exactly one arm")
+        arms = [dataclasses.replace(arms[0], agent_version=args.agent_version)]
+    run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
+    with tempfile.TemporaryDirectory() as work_dir:
+        summaries = smoke_many(arms, Path(work_dir), args.jobs_dir, run_id)
+    text = json.dumps(summaries, indent=2)
+    print(text)
+    if args.output:
+        args.output.write_text(text + "\n")
+    if args.markdown:
+        args.markdown.write_text(markdown(summaries))
+    return 0 if all(summary["ok"] for summary in summaries) else 1
diff --git a/harness/smoke.py b/harness/smoke.py
new file mode 100644
index 0000000..9945db5
--- /dev/null
+++ b/harness/smoke.py
@@ -0,0 +1,269 @@
+"""Smoke-test an arm's agent harness on a local Docker task, without a browser.
+
+`bench smoke <arm>` runs the arm's agent, model, version and agent kwargs
+through Harbor on one tiny task whose only tool is the xwebmcp-tasks suite's
+stdio MCP server, `bench`, logged by mcp-call-log. It checks what unit tests
+cannot: that this agent version installs and runs under Harbor, reaches a
+stdio MCP server, and still writes a session that Harbor converts into the
+trajectory the report reads. Harness upgrades can change how tool calls are
+nested or which built-in tools exist; the summary lists the tool names the
+trajectory shows so two versions can be compared.
+
+It costs one short model conversation and needs Docker and the agent's API
+key, but no Kernel browser.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+import sys
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any
+
+from harness.definitions import ROOT, Arm, load_arm
+from harness.run import RETRY_EXCEPTIONS
+
+BENCH = ROOT / "suites" / "xwebmcp-tasks" / "image" / "bench"
+CALL_LOG = ROOT / "toolsets" / "kernel-mcp-local" / "image" / "bin" / "mcp-call-log"
+TASK_NAME = "smoke"
+STEP = "run"
+
+DOCKERFILE = """\
+FROM node:22-bookworm-slim
+RUN apt-get update \\
+    && apt-get install -y --no-install-recommends ca-certificates curl git procps python3 \\
+    && rm -rf /var/lib/apt/lists/*
+COPY bench/ /opt/bench/
+RUN chmod +x /opt/bench/*
+WORKDIR /app
+"""
+
+TASK_TOML = """\
+schema_version = "1.3"
+source = "smoke"
+
+[task]
+name = "smoke/finalize"
+
+[environment]
+build_timeout_sec = 600.0
+workdir = "/app"
+
+[[environment.mcp_servers]]
+name = "bench"
+transport = "stdio"
+command = "/opt/bench/mcp-call-log"
+args = ["/logs/agent/bench-calls.jsonl", "/opt/bench/finalize-mcp"]
+
+[[steps]]
+name = "run"
+
+[steps.agent]
+timeout_sec = 300.0
+
+[steps.verifier]
+timeout_sec = 60.0
+"""
+
+INSTRUCTION = """\
+This is a smoke test of your tools. Call the `finalize` tool of the `bench` MCP server exactly
+once, with evidence "smoke test", reasoning "smoke test" and outcome "success". Do not run any
+other command or tool, then stop.
+"""
+
+TEST_SH = """\
+#!/bin/bash
+set -euo pipefail
+/opt/bench/score
+"""
+
+
+def write_task(out_dir: Path) -> Path:
+    task = out_dir / TASK_NAME
+    environment = task / "environment"
+    shutil.copytree(BENCH, environment / "bench", ignore=shutil.ignore_patterns("open-start-url"))
+    shutil.copy2(CALL_LOG, environment / "bench" / "mcp-call-log")
+    (environment / "Dockerfile").write_text(DOCKERFILE)
+    (task / "task.toml").write_text(TASK_TOML)
+    step = task / "steps" / STEP
+    (step / "tests").mkdir(parents=True)
+    (step / "instruction.md").write_text(INSTRUCTION)
+    (step / "tests" / "test.sh").write_text(TEST_SH)
+    (step / "tests" / "test.sh").chmod(0o755)
+    return task
+
+
+def harbor_command(arm: Arm, task: Path, jobs_dir: Path, job_name: str) -> list[str]:
+    command = [
+        str(Path(sys.executable).parent / "harbor"),
+        "run",
+        "--path", str(task),
+        "--agent", arm.agent,
+        "--model", arm.model,
+        "--env", "docker",
+        "--job-name", job_name,
+        "--jobs-dir", str(jobs_dir),
+        "--max-retries", "2",
+        "--delete",
+        "--yes",
+    ]  # fmt: skip
+    if arm.agent_version:
+        command += ["--agent-kwarg", f"version={arm.agent_version}"]
+    for key, value in arm.agent_kwargs.items():
+        command += ["--agent-kwarg", f"{key}={json.dumps(value)}"]
+    for exception in RETRY_EXCEPTIONS:
+        command += ["--retry-include", exception]
+    return command
+
+
+def _read_json(path: Path) -> Any:
+    try:
+        return json.loads(path.read_text())
+    except (OSError, json.JSONDecodeError):
+        return None
+
+
+def _read_jsonl(path: Path) -> list[dict[str, Any]]:
+    try:
+        lines = path.read_text().splitlines()
+    except OSError:
+        return []
+    entries = []
+    for line in lines:
+        try:
+            entry = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(entry, dict):
+            entries.append(entry)
+    return entries
+
+
+def _builtin_tools(agent_dir: Path) -> list[str] | None:
+    """Tools Claude Code reports in its stream-json init event; Codex logs none."""
+    for event in _read_jsonl(agent_dir / "claude-code.txt"):
+        if event.get("type") == "system" and event.get("subtype") == "init":
+            return sorted(str(tool) for tool in event.get("tools") or [])
+    return None
+
+
+def check_trial(trial_dir: Path, expected_version: str | None) -> dict[str, Any]:
+    """Summarize one smoke trial and check it against what the benchmarks rely on."""
+    result = _read_json(trial_dir / "result.json") or {}
+    step_dir = trial_dir / "steps" / STEP
+    agent_dir = step_dir / "agent"
+    verifier_dir = step_dir / "verifier"
+
+    calls = _read_jsonl(agent_dir / "bench-calls.jsonl")
+    finalize_calls = [call for call in calls if call.get("tool") == "finalize"]
+    rewards = _read_json(verifier_dir / "reward.json") or {}
+
+    trajectory = _read_json(agent_dir / "trajectory.json")
+    steps = trajectory.get("steps") if isinstance(trajectory, dict) else None
+    steps = steps if isinstance(steps, list) else []
+    tool_calls = [call for step in steps for call in step.get("tool_calls") or []]
+    final_metrics = (trajectory or {}).get("final_metrics") or {}
+    costs = [(step.get("metrics") or {}).get("cost_usd") for step in steps]
+    cost = sum(value for value in costs if isinstance(value, int | float))
+    agent_info = result.get("agent_info") or {}
+    version = agent_info.get("version") or ((trajectory or {}).get("agent") or {}).get("version")
+
+    checks = {
+        "trial_without_exception": not result.get("exception_info"),
+        "version_matches": expected_version is None or version == expected_version,
+        "mcp_server_got_finalize": bool(finalize_calls) and not finalize_calls[0].get("is_error"),
+        "verifier_reward_1": rewards.get("reward") == 1,
+        "trajectory_converted": bool(steps),
+        "trajectory_has_tool_calls": bool(tool_calls),
+        "trajectory_has_token_counts": bool(final_metrics.get("total_prompt_tokens")),
+        "trajectory_has_cost": cost > 0 or bool(final_metrics.get("total_cost_usd")),
+    }
+    return {
+        "ok": all(checks.values()),
+        "checks": checks,
+        "agent_version": version,
+        "server_calls": sorted({str(call.get("tool")) for call in calls}),
+        "trajectory_tools": sorted({str(call.get("function_name")) for call in tool_calls}),
+        "builtin_tools": _builtin_tools(agent_dir),
+        "steps": len(steps),
+        "tool_calls": len(tool_calls),
+        "cost_usd": round(cost, 6),
+        "trial_dir": str(trial_dir),
+    }
+
+
+def smoke(arm: Arm, work_dir: Path, jobs_dir: Path, job_name: str) -> dict[str, Any]:
+    work_dir.mkdir(parents=True, exist_ok=True)
+    task = write_task(work_dir)
+    # Harbor's progress goes to stderr so stdout is only the summary.
+    process = subprocess.run(harbor_command(arm, task, jobs_dir, job_name), stdout=sys.stderr)
+    job_dir = jobs_dir / job_name
+    trials = sorted(path for path in job_dir.iterdir() if path.is_dir()) if job_dir.is_dir() else []
+    summary = (
+        check_trial(trials[-1], arm.agent_version)
+        if trials
+        else {"ok": False, "checks": {"trial_ran": False}}
+    )
+    summary["ok"] = summary["ok"] and process.returncode == 0
+    return {
+        "arm": arm.name,
+        "agent": arm.agent,
+        "model": arm.model,
+        "requested_version": arm.agent_version,
+        "agent_kwargs": arm.agent_kwargs,
+        "harbor_exit": process.returncode,
+        **summary,
+    }
+
+
+def distinct_harnesses(arms: list[Arm]) -> list[Arm]:
+    """One arm per agent, version, model and agent kwargs; toolsets don't change the harness."""
+    seen: dict[tuple[str, str | None, str, str], Arm] = {}
+    for arm in arms:
+        key = (
+            arm.agent,
+            arm.agent_version,
+            arm.model,
+            json.dumps(arm.agent_kwargs, sort_keys=True),
+        )
+        seen.setdefault(key, arm)
+    return list(seen.values())
+
+
+def all_arms() -> list[Arm]:
+    return [load_arm(path.stem) for path in sorted((ROOT / "arms").glob("*.toml"))]
+
+
+def smoke_many(
+    arms: list[Arm], work_dir: Path, jobs_dir: Path, run_id: str
+) -> list[dict[str, Any]]:
+    def one(arm: Arm) -> dict[str, Any]:
+        job_name = f"smoke-{arm.name}-{arm.agent_version or 'latest'}-{run_id}"
+        return smoke(arm, work_dir / arm.name, jobs_dir, job_name)
+
+    with ThreadPoolExecutor(max_workers=len(arms) or 1) as pool:
+        return list(pool.map(one, arms))
+
+
+def markdown(summaries: list[dict[str, Any]]) -> str:
+    rows = [
+        "| Arm | Agent | Version | Result | Trajectory tools | Server calls | Cost |",
+        "|---|---|---|---|---|---|---:|",
+    ]
+    for summary in summaries:
+        failed = [name for name, ok in summary["checks"].items() if not ok]
+        rows.append(
+            "| {arm} | {agent} | {version} | {result} | {tools} | {calls} | ${cost:.4f} |".format(
+                arm=summary["arm"],
+                agent=summary["agent"],
+                version=summary.get("agent_version") or summary["requested_version"],
+                result="pass" if summary["ok"] else "fail: " + ", ".join(failed or ["harbor"]),
+                tools=", ".join(summary.get("trajectory_tools") or []) or "none",
+                calls=", ".join(summary.get("server_calls") or []) or "none",
+                cost=summary.get("cost_usd") or 0,
+            )
+        )
+    return "\n".join(rows) + "\n"
diff --git a/tests/test_environment.py b/tests/test_environment.py
index 557e31f..6eb11f6 100644
--- a/tests/test_environment.py
+++ b/tests/test_environment.py
@@ -1,3 +1,4 @@
+import inspect
 import json
 
 from harbor.environments.kernel import KernelEnvironment
@@ -17,3 +18,8 @@ def test_task_browser_config_becomes_create_overrides(tmp_path, monkeypatch):
     KernelTaskEnvironment(environment_dir=tmp_path, session_id="s")
 
     assert captured == {"session_id": "s", "browser_create_overrides": {"stealth": False}}
+
+
+def test_harbor_kernel_environment_takes_create_overrides():
+    # Harbor upgrades must keep the parameter the subclass passes.
+    assert "browser_create_overrides" in inspect.signature(KernelEnvironment.__init__).parameters
diff --git a/tests/test_smoke.py b/tests/test_smoke.py
new file mode 100644
index 0000000..0a8b110
--- /dev/null
+++ b/tests/test_smoke.py
@@ -0,0 +1,85 @@
+import json
+import tomllib
+from pathlib import Path
+
+from harness import smoke
+from harness.definitions import Arm
+
+
+def write_trial(trial: Path, *, tool_calls: list[dict], calls: list[dict], reward: float) -> None:
+    agent = trial / "steps" / "run" / "agent"
+    verifier = trial / "steps" / "run" / "verifier"
+    agent.mkdir(parents=True)
+    verifier.mkdir(parents=True)
+    (trial / "result.json").write_text(
+        json.dumps({"agent_info": {"version": "1.2.3"}, "exception_info": None})
+    )
+    (verifier / "reward.json").write_text(json.dumps({"reward": reward}))
+    (agent / "bench-calls.jsonl").write_text("".join(json.dumps(c) + "\n" for c in calls))
+    steps = [
+        {"step_id": 1, "source": "user", "message": "smoke"},
+        {"step_id": 2, "source": "agent", "tool_calls": tool_calls, "metrics": {"cost_usd": 0.01}},
+    ]
+    (agent / "trajectory.json").write_text(
+        json.dumps({"steps": steps, "final_metrics": {"total_prompt_tokens": 100}})
+    )
+
+
+def test_a_trial_that_called_finalize_and_converted_passes(tmp_path):
+    write_trial(
+        tmp_path,
+        tool_calls=[{"tool_call_id": "1", "function_name": "exec", "arguments": {}}],
+        calls=[{"tool": "finalize", "is_error": False}],
+        reward=1.0,
+    )
+    summary = smoke.check_trial(tmp_path, "1.2.3")
+    assert summary["ok"], summary["checks"]
+    assert summary["trajectory_tools"] == ["exec"]
+    assert summary["server_calls"] == ["finalize"]
+
+
+def test_a_trial_whose_session_did_not_convert_fails(tmp_path):
+    write_trial(tmp_path, tool_calls=[], calls=[{"tool": "finalize"}], reward=1.0)
+    (tmp_path / "steps" / "run" / "agent" / "trajectory.json").unlink()
+    summary = smoke.check_trial(tmp_path, "1.2.3")
+    assert not summary["ok"]
+    assert not summary["checks"]["trajectory_converted"]
+    assert summary["checks"]["mcp_server_got_finalize"]
+
+
+def test_a_trial_with_another_version_or_no_mcp_call_fails(tmp_path):
+    write_trial(tmp_path, tool_calls=[], calls=[], reward=0.0)
+    checks = smoke.check_trial(tmp_path, "9.9.9")["checks"]
+    assert not checks["version_matches"]
+    assert not checks["mcp_server_got_finalize"]
+    assert not checks["verifier_reward_1"]
+
+
+def test_distinct_harnesses_ignore_toolsets_and_browsers():
+    arms = [
+        Arm(name="a", agent="codex", model="m", toolset="x", agent_version="1"),
+        Arm(name="b", agent="codex", model="m", toolset="y", agent_version="1"),
+        Arm(name="c", agent="codex", model="m", toolset="x", agent_version="2"),
+        Arm(
+            name="d",
+            agent="codex",
+            model="m",
+            toolset="x",
+            agent_version="1",
+            agent_kwargs={"disable_web_search": True},
+        ),
+    ]
+    assert [arm.name for arm in smoke.distinct_harnesses(arms)] == ["a", "c", "d"]
+
+
+def test_smoke_task_wraps_the_finalize_server_in_the_call_log(tmp_path):
+    task = smoke.write_task(tmp_path)
+    config = tomllib.loads((task / "task.toml").read_text())
+    [server] = config["environment"]["mcp_servers"]
+    assert server["command"] == "/opt/bench/mcp-call-log"
+    assert server["args"][-1] == "/opt/bench/finalize-mcp"
+    bench = task / "environment" / "bench"
+    assert (bench / "finalize-mcp").is_file()
+    assert (bench / "mcp-call-log").is_file()
+    assert (bench / "score").is_file()
+    assert (task / "steps" / "run" / "instruction.md").is_file()