diff --git a/.agents/skills/bump-cli-compat/SKILL.md b/.agents/skills/bump-cli-compat/SKILL.md index 454c93e14a9..1b3f6b6e02d 100644 --- a/.agents/skills/bump-cli-compat/SKILL.md +++ b/.agents/skills/bump-cli-compat/SKILL.md @@ -64,7 +64,7 @@ If any tag doesn't exist, report the error and stop. ### Step 3: Read current manifest -Read `internal/build/cli-compat.json` and `AgentSkillsFallbackVersion` in `internal/build/clicompat.go`. Note the current versions and the list of versioned entries. The newest CLI range uses `"skills": "latest"`; the compiled constant supplies the tested fallback. Keep older released ranges pinned. Routine skills releases need no manifest bump. CLIs before `1.6.0` reject `"latest"` and use a valid cached or embedded manifest without returning a parse error; losing remote AppKit and skills compatibility updates for these CLIs is accepted. +Read `internal/build/cli-compat.json`. Note the current versions and the list of versioned entries. The newest CLI range uses `"skills": "latest"`; release builds embed the release that is latest at build time, and the newest pinned range below it is the fallback. Keep older released ranges pinned. Routine skills releases need no manifest bump. CLIs before `1.6.0` reject `"latest"` and use a valid cached or embedded manifest without returning a parse error; losing remote AppKit and skills compatibility updates for these CLIs is accepted. ### Step 4: Determine update type @@ -81,17 +81,17 @@ Ask the user: ### Step 4a: No breaking changes (update in-place) -Update the **highest versioned entry** to the new `appkit` version, keeping `"skills": "latest"`. Update `AgentSkillsFallbackVersion` in `internal/build/clicompat.go` to the tested skills version. Do NOT add new versioned keys. The manifest is range-based: updating the highest entry automatically covers all CLI versions in that range. +Update the **highest versioned entry** to the new `appkit` version, keeping `"skills": "latest"`. Do NOT add new versioned keys. The manifest is range-based: updating the highest entry automatically covers all CLI versions in that range. -Write the updated `internal/build/cli-compat.json` and `internal/build/clicompat.go`. +Write the updated `internal/build/cli-compat.json`. ### Step 4b: Breaking changes (add new entry) Ask the user for the **minimum CLI version** that supports the new features. -Add a new entry keyed to that CLI version with the new `appkit` and `"skills": "latest"`. Update `AgentSkillsFallbackVersion` in `internal/build/clicompat.go` to the tested skills version. If skills compatibility changed, replace `"latest"` in the affected older ranges with the last compatible release. Preserve unrelated AppKit and skills compatibility boundaries. +Add a new entry keyed to that CLI version with the new `appkit` and `"skills": "latest"`. If skills compatibility changed, replace `"latest"` in the affected older ranges with the last compatible release. Preserve unrelated AppKit and skills compatibility boundaries. -Write the updated `internal/build/cli-compat.json` and `internal/build/clicompat.go`. +Write the updated `internal/build/cli-compat.json`. ### Step 5: Validate @@ -107,21 +107,21 @@ If validation fails, show the errors and fix them before proceeding. ```bash # Create a new branch from the current branch (or main) -git checkout -b bump-cli-compat-appkit-{appkit_version}-skills-fallback-{skills_version} +git checkout -b bump-cli-compat-appkit-{appkit_version} # Stage and commit -git add internal/build/cli-compat.json internal/build/clicompat.go -git commit -s -m "Bump cli-compat to appkit {appkit_version}, skills fallback {skills_version}" +git add internal/build/cli-compat.json +git commit -s -m "Bump cli-compat to appkit {appkit_version}" # Push and create PR git push -u origin HEAD gh pr create \ - --title "Bump cli-compat to appkit {appkit_version}, skills fallback {skills_version}" \ + --title "Bump cli-compat to appkit {appkit_version}" \ --body "$(cat <<'EOF' ## Summary Bump `cli-compat.json` to use: - AppKit `{appkit_version}` -- Agent Skills track latest, with compiled fallback `{skills_version}` +- Agent Skills track latest ## Checklist - [ ] Evals passed with no regressions diff --git a/.github/scripts/pin-cli-compat.py b/.github/scripts/pin-cli-compat.py new file mode 100644 index 00000000000..676f4bf2194 --- /dev/null +++ b/.github/scripts/pin-cli-compat.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Resolve floating skills versions for go:embed without changing the checkout. + +GoReleaser and the local build tasks pass the resulting overlay to go build. +The GitHub CLI authenticates using GITHUB_TOKEN in release CI, or its local +credentials for developer builds. If the lookup fails, "latest" stays in the +manifest and the CLI falls back to the newest pin below that range. +""" + +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +SOURCE = Path("internal/build/cli-compat.json") +OUTPUT = Path(".databricks/cli-compat") + + +def fetch_latest_skills_release() -> str: + return subprocess.check_output( + [ + "gh", + "api", + "repos/databricks/databricks-agent-skills/releases/latest", + "--jq", + ".tag_name", + ], + text=True, + timeout=60, + ).strip() + + +def pin_latest_skills(manifest: dict, fetch_latest=fetch_latest_skills_release): + """Resolve once for every floating range, preserving pins and other fields. + + Returns a warning if the lookup fails. + + >>> manifest = { + ... "1.22.0": {"appkit": "0.84.0", "skills": "latest"}, + ... "1.23.0": {"skills": "latest"}, + ... "1.20.0": {"skills": "0.2.30"}, + ... } + >>> from unittest.mock import Mock + >>> fetch = Mock(return_value="v0.2.32") + >>> pin_latest_skills(manifest, fetch) + >>> manifest + {'1.22.0': {'appkit': '0.84.0', 'skills': '0.2.32', 'skills_resolved_from_latest': True}, '1.23.0': {'skills': '0.2.32', 'skills_resolved_from_latest': True}, '1.20.0': {'skills': '0.2.30'}} + >>> fetch.call_count + 1 + >>> pin_latest_skills(manifest, fetch) + >>> fetch.call_count + 1 + >>> pin_latest_skills({"1.22.0": {"skills": "latest"}}, lambda: "0.2.33") + + A failed lookup or an invalid tag leaves "latest" in place. + + >>> failures = [ + ... Mock(side_effect=FileNotFoundError("gh")), + ... Mock(side_effect=subprocess.CalledProcessError(1, "gh")), + ... Mock(side_effect=subprocess.TimeoutExpired("gh", 60)), + ... *(Mock(return_value=tag) for tag in ("latest", "main", "", "v1.2", "v01.2.3")), + ... ] + >>> for fetch in failures: + ... manifest = {"1.20.0": {"skills": "0.2.30"}, "1.22.0": {"appkit": "0.84.0", "skills": "latest"}} + ... assert "Could not resolve the latest skills release" in pin_latest_skills(manifest, fetch), fetch + ... assert manifest["1.22.0"] == {"appkit": "0.84.0", "skills": "latest"}, fetch + """ + floating = [entry for entry in manifest.values() if entry.get("skills") == "latest"] + if not floating: + return + + try: + version = fetch_latest().removeprefix("v") + if not re.fullmatch(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)", version): + raise ValueError(f"latest skills release must be a concrete stable version, got {version!r}") + except (OSError, subprocess.SubprocessError, ValueError) as err: + return f"Could not resolve the latest skills release ({err}). The CLI falls back to the newest pinned skills version." + for entry in floating: + entry["skills"] = version + entry["skills_resolved_from_latest"] = True + + +def build_overlay(root: Path, fetch_latest=fetch_latest_skills_release): + """Write resolved build inputs while leaving the checked-in manifest intact. + + Returns the warning from pin_latest_skills, if any. + + >>> from tempfile import TemporaryDirectory + >>> from unittest.mock import Mock + >>> with TemporaryDirectory() as tmp: + ... root = Path(tmp) + ... source = root / SOURCE + ... source.parent.mkdir(parents=True) + ... original = '{"1.22.0": {"appkit": "0.84.0", "skills": "latest"}}' + ... _ = source.write_text(original) + ... build_overlay(root, lambda: "v0.2.32") + ... overlay = json.loads((root / OUTPUT / "overlay.json").read_text()) + ... assert overlay == {"Replace": {"internal/build/cli-compat.json": ".databricks/cli-compat/cli-compat.json"}} + ... assert json.loads((root / OUTPUT / "cli-compat.json").read_text())["1.22.0"]["skills"] == "0.2.32" + ... assert source.read_text() == original + + Release CI can build tags from before cli-compat.json was introduced. + + >>> with TemporaryDirectory() as tmp: + ... root = Path(tmp) + ... build_overlay(root, Mock(side_effect=AssertionError("must not fetch"))) + ... assert json.loads((root / OUTPUT / "overlay.json").read_text()) == {"Replace": {}} + + An all-pinned manifest must build without GitHub credentials or even gh. + + >>> with TemporaryDirectory() as tmp: + ... root = Path(tmp) + ... source = root / SOURCE + ... source.parent.mkdir(parents=True) + ... _ = source.write_text('{"1.22.0": {"skills": "0.2.30"}}') + ... build_overlay(root, Mock(side_effect=AssertionError("must not fetch"))) + ... assert json.loads((root / OUTPUT / "cli-compat.json").read_text())["1.22.0"]["skills"] == "0.2.30" + """ + source = root / SOURCE + output = root / OUTPUT + output.mkdir(parents=True, exist_ok=True) + replacements = {} + warning = None + if source.exists(): + manifest = json.loads(source.read_text(encoding="utf-8")) + warning = pin_latest_skills(manifest, fetch_latest) + (output / "cli-compat.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + # Relative paths, because go build matches overlay entries against its + # own working directory, which may be a symlink that resolve() expands. + replacements[SOURCE.as_posix()] = (OUTPUT / "cli-compat.json").as_posix() + (output / "overlay.json").write_text(json.dumps({"Replace": replacements}, indent=2) + "\n", encoding="utf-8") + return warning + + +def warn(message: str) -> None: + print(f"WARNING: {message}", file=sys.stderr) + # GoReleaser shows hook output only with --verbose, so release builds + # surface the warning in the GitHub Actions job summary instead. + if summary := os.environ.get("GITHUB_STEP_SUMMARY"): + with open(summary, "a", encoding="utf-8") as f: + f.write(f"> [!WARNING]\n> {message}\n") + + +if __name__ == "__main__": + if warning := build_overlay(Path.cwd()): + warn(warning) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 197f42da958..612a62deaa8 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -3,6 +3,7 @@ version: 2 before: hooks: - go mod download + - python3 .github/scripts/pin-cli-compat.py builds: - env: @@ -17,6 +18,7 @@ builds: mod_timestamp: '{{ .CommitTimestamp }}' flags: - -trimpath + - -overlay=.databricks/cli-compat/overlay.json ldflags: - '-s -w' - -X github.com/databricks/cli/internal/build.buildProjectName={{ .ProjectName }} diff --git a/.nextchanges/cli/aitools-skills-release-fallback.md b/.nextchanges/cli/aitools-skills-release-fallback.md new file mode 100644 index 00000000000..807478ef7d0 --- /dev/null +++ b/.nextchanges/cli/aitools-skills-release-fallback.md @@ -0,0 +1 @@ +* Fix `aitools list` and plugin installation failures when GitHub's release API is unavailable or rate limited, and embed a concrete skills fallback during release builds. ([#7162](https://github.com/databricks/cli/pull/7162)) diff --git a/Taskfile.yml b/Taskfile.yml index b41178fbe61..ed03b0ee6f1 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -94,6 +94,7 @@ tasks: desc: Lint first-party Python scripts with ruff sources: - "**/*.py" + - .github/scripts/pin-cli-compat.py - ruff.toml cmds: # Lints the whole repo; fixtures and generated outputs are skipped via @@ -173,6 +174,7 @@ tasks: desc: Lint-fix and format Python files sources: - "**/*.py" + - .github/scripts/pin-cli-compat.py - ruff.toml cmds: # Auto-fix the lint issues that ruff can fix (whole repo, minus lint.exclude); @@ -345,11 +347,13 @@ tasks: - exclude: tools/** - go.mod - go.sum + - .github/scripts/pin-cli-compat.py - "{{.EMBED_SOURCES}}" generates: - cli cmds: - - "go build" + - 'uv run -p ">=3.11" --no-project python .github/scripts/pin-cli-compat.py' + - "go build -overlay=.databricks/cli-compat/overlay.json" snapshot: desc: Build snapshot binary to .databricks/databricks @@ -357,7 +361,8 @@ tasks: generates: - .databricks/databricks cmds: - - "go build -o .databricks/databricks" + - 'uv run -p ">=3.11" --no-project python .github/scripts/pin-cli-compat.py' + - "go build -overlay=.databricks/cli-compat/overlay.json -o .databricks/databricks" snapshot-release: desc: Build release binaries locally without uploading @@ -372,6 +377,7 @@ tasks: - go.mod - go.sum - .goreleaser.yaml + - .github/scripts/pin-cli-compat.py - .github/scripts/sign-windows.sh - "{{.EMBED_SOURCES}}" generates: @@ -404,6 +410,7 @@ tasks: desc: Run doctests for all Python files in the repo sources: - "**/*.py" + - .github/scripts/pin-cli-compat.py cmds: - 'uv run -p ">=3.11" --no-project python tools/run_doctests.py' diff --git a/acceptance/cmd/aitools/list-release-rate-limited/out.test.toml b/acceptance/cmd/aitools/list-release-rate-limited/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/list-release-rate-limited/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/list-release-rate-limited/output.txt b/acceptance/cmd/aitools/list-release-rate-limited/output.txt new file mode 100644 index 00000000000..d0fbd98f6ef --- /dev/null +++ b/acceptance/cmd/aitools/list-release-rate-limited/output.txt @@ -0,0 +1,23 @@ + +=== list falls back to the embedded skills version when the release lookup is rate limited + +>>> [CLI] aitools list --output json +Warn: Could not resolve latest skills release (HTTP 403 fetching latest skills release); falling back to embedded skills version 0.2.32 +{ + "release": "0.2.32", + "skills": [ + "test-stable" + ], + "agents": [ + "claude-code", + "cursor", + "codex", + "opencode", + "copilot", + "antigravity", + "pi", + "gemini", + "goose", + "kiro" + ] +} diff --git a/acceptance/cmd/aitools/list-release-rate-limited/script b/acceptance/cmd/aitools/list-release-rate-limited/script new file mode 100644 index 00000000000..3b88a45f89e --- /dev/null +++ b/acceptance/cmd/aitools/list-release-rate-limited/script @@ -0,0 +1,7 @@ +# Isolate HOME so parallel aitools tests don't race on a shared ~/.databricks. +sethome home + +title "list falls back to the embedded skills version when the release lookup is rate limited\n" +# The agent list is what editor integrations render; detection depends on the +# host, so only the names are printed. +trace $CLI aitools list --output json | jq '{release, skills: [.skills[].name], agents: [.agents[].name]}' diff --git a/acceptance/cmd/aitools/list-release-rate-limited/test.toml b/acceptance/cmd/aitools/list-release-rate-limited/test.toml new file mode 100644 index 00000000000..d853ba9235d --- /dev/null +++ b/acceptance/cmd/aitools/list-release-rate-limited/test.toml @@ -0,0 +1,31 @@ +# Mock server replaces api.github.com and raw.githubusercontent.com. No +# DATABRICKS_SKILLS_REF, so list must look up the latest skills release. +Env.DATABRICKS_SKILLS_API_URL = "$DATABRICKS_HOST" +Env.DATABRICKS_SKILLS_BASE_URL = "$DATABRICKS_HOST" +Env.DATABRICKS_FORCE_EMBEDDED_COMPAT = "true" + +Ignore = [ + "home", +] + +[[Server]] +Pattern = "GET /repos/databricks/databricks-agent-skills/releases/latest" +Response.StatusCode = 403 +Response.Body = '''{"message": "API rate limit exceeded"}''' + +[[Server]] +Pattern = "GET /{ref}/manifest.json" +Response.Body = ''' +{ + "version": "2", + "updated_at": "2026-01-01T00:00:00Z", + "skills": { + "test-stable": { + "version": "1.0.0", + "description": "Stable test skill", + "files": ["SKILL.md"], + "repo_dir": "skills" + } + } +} +''' diff --git a/cmd/aitools/install.go b/cmd/aitools/install.go index 695e58108c9..03b4c598b61 100644 --- a/cmd/aitools/install.go +++ b/cmd/aitools/install.go @@ -526,9 +526,12 @@ func executePlan(ctx context.Context, src installer.ManifestSource, plan []agent pluginCount := 0 if len(pluginItems) > 0 { + // Plugins install from the agent's marketplace, so the skills ref only + // labels the recorded version. Record it as unknown when unresolved. ref, _, err := installer.GetSkillsRef(ctx) if err != nil { - return outcomes, err + log.Debugf(ctx, "Recording plugin version as unknown: %v", err) + ref = "" } records := map[string]installer.PluginRecord{} for _, it := range pluginItems { diff --git a/cmd/aitools/install_test.go b/cmd/aitools/install_test.go index deb25154472..7488916ba19 100644 --- a/cmd/aitools/install_test.go +++ b/cmd/aitools/install_test.go @@ -6,6 +6,8 @@ import ( "context" "encoding/json" "errors" + "net/http" + "net/http/httptest" "os" "path/filepath" "runtime" @@ -269,6 +271,52 @@ func TestExecutePlanSkipBlockedPluginExit0(t *testing.T) { require.Error(t, err) } +func TestExecutePlanPluginWithUnresolvedSkillsRelease(t *testing.T) { + tmp := t.TempDir() + t.Setenv("DATABRICKS_SKILLS_REF", "") + t.Setenv("DATABRICKS_FORCE_EMBEDDED_COMPAT", "false") + t.Setenv("DATABRICKS_CACHE_ENABLED", "true") + t.Setenv("DATABRICKS_CACHE_DIR", tmp) + require.NoError(t, os.WriteFile(filepath.Join(tmp, "compat-manifest.json"), []byte(`{"0.0.0":{"appkit":"0.84.0","skills":"latest"}}`), 0o600)) + lookups := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + lookups++ + w.WriteHeader(http.StatusForbidden) + })) + t.Cleanup(srv.Close) + t.Setenv("DATABRICKS_SKILLS_API_URL", srv.URL) + + installRef, recordRef := "unset", "unset" + orig := installPluginForAgentFn + origRec := recordPluginInstallsFn + origCleanup := cleanupLegacyFn + t.Cleanup(func() { + installPluginForAgentFn = orig + recordPluginInstallsFn = origRec + cleanupLegacyFn = origCleanup + }) + cleanupLegacyFn = func(context.Context, *agents.Agent, string) error { return nil } + installPluginForAgentFn = func(_ context.Context, _ *agents.Agent, scope, ref string) (installer.PluginRecord, error) { + installRef = ref + return installer.PluginRecord{Plugin: "databricks", Scope: scope, Version: installer.DisplaySkillsVersion(ref)}, nil + } + recordPluginInstallsFn = func(_ context.Context, _ string, _ map[string]installer.PluginRecord, ref string) error { + recordRef = ref + return nil + } + + claude := testPluginAgent(agents.NameClaudeCode, "Claude Code", "claude") + plan := buildPlan([]*agents.Agent{claude}, installer.ScopeGlobal, false, true) + outcomes, err := executePlan(cmdio.MockDiscard(t.Context()), nil, plan, installer.InstallOptions{Scope: installer.ScopeGlobal}, false) + require.NoError(t, err) + require.Len(t, outcomes, 1) + assert.Equal(t, outcomeInstalled, outcomes[0].status) + assert.Equal(t, 1, lookups) + // The marketplace picks the plugin version, so a failed lookup records it as unknown. + assert.Empty(t, installRef) + assert.Empty(t, recordRef) +} + // --- RunE: skills-only path (config-dir detection, no plugin) --- func TestInstallSkillsOnlyAllAgents(t *testing.T) { diff --git a/cmd/aitools/list.go b/cmd/aitools/list.go index e6decb6a120..cbb579d584b 100644 --- a/cmd/aitools/list.go +++ b/cmd/aitools/list.go @@ -151,7 +151,7 @@ func defaultListSkills(cmd *cobra.Command, scope string) error { // returns the structured listOutput. scope=="" loads both scopes; "global" // or "project" loads only that scope. func buildListOutput(ctx context.Context, scope string) (listOutput, error) { - ref, explicit, err := installer.GetSkillsRef(ctx) + ref, explicit, err := installer.GetSkillsRefWithFallback(ctx) if err != nil { return listOutput{}, err } diff --git a/internal/build/README.md b/internal/build/README.md index 0adf8c77a22..a731bf3ea9d 100644 --- a/internal/build/README.md +++ b/internal/build/README.md @@ -8,7 +8,8 @@ Raw skills track the latest published skills release for the newest CLI range. P ```json { - "1.22.0": { "appkit": "0.84.0", "skills": "latest" }, + "1.23.0": { "appkit": "0.84.0", "skills": "latest" }, + "1.22.0": { "appkit": "0.84.0", "skills": "0.2.32" }, "1.20.0": { "appkit": "0.84.0", "skills": "0.2.30" }, "1.0.0": { "appkit": "0.76.1", "skills": "0.2.10" }, "0.299.2": { "appkit": "0.24.0", "skills": "0.1.5" } @@ -17,22 +18,23 @@ Raw skills track the latest published skills release for the newest CLI range. P Each key is a CLI version in semver format. Each entry defines a **range floor**: it applies to that CLI version and all versions above it, up to (but not including) the next entry. The manifest should be **sparse** — not every CLI version needs its own entry. Only add a new entry when a compatibility boundary changes. -`skills` is an exact pin for its CLI range or `"latest"` to install the skills repository's latest published GitHub release. When the latest-release lookup fails or the selected skills manifest is missing, ranges tracking latest use `AgentSkillsFallbackVersion` from `internal/build/clicompat.go`. Pinned ranges use their embedded manifest's skills version as the fallback. Changing either fallback takes effect in the next CLI build. Released binaries keep the fallback they shipped with. +`skills` is an exact pin for its CLI range or `"latest"` to install the skills repository's latest published GitHub release. Release builds and the `build`, `snapshot`, and `snapshot-release` tasks resolve `"latest"` at build time and embed a concrete version. The checked-in manifest remains unchanged. Released binaries keep the fallback they shipped with. -This skills fallback only applies to raw-file delivery. Version checks omit update availability when latest is unknown, and plugin operations don't substitute the fallback version. +At runtime, a live or cached manifest can still select `"latest"`. Listing skills uses the embedded version if the latest-release lookup fails, including when GitHub rate limits the request. Raw-file delivery also uses it when the selected skills manifest is missing. Plugin installation still uses the agent's marketplace commands without a version pin, and records the plugin version as unknown if the lookup fails. Version checks omit update availability when latest is unknown. Raw-skills updates and reinstalls reject automatic downgrades, including with `--force`: the fallback must be at least as new as the installed release. Existing installs without a comparable release require a successful latest-release fetch. Explicit `DATABRICKS_SKILLS_REF` overrides and compatibility pins can still select an older release. CLIs before `1.6.0` reject the whole remote manifest when any entry uses `"latest"`, even an entry they never resolve to. They use a valid cached manifest or their embedded manifest without returning a parse error. These CLIs stop receiving remote compatibility updates for both AppKit and skills. -Latest tracking starts at the CLI `1.22.0` range. CLI `1.20.x` and `1.21.x` stay pinned. +Latest tracking starts at the CLI `1.23.0` range. CLI `1.22.x` stays pinned to skills `0.2.32` because those binaries don't have the release-lookup fallback. For example, with the manifest above: - CLI `0.299.5` → uses `0.299.2` entry (appkit `0.24.0`) - CLI `1.19.0` → uses `1.0.0` entry (appkit `0.76.1`) - CLI `1.21.0` → uses `1.20.0` entry (appkit `0.84.0`, skills `0.2.30`) -- CLI `1.22.0` → uses `1.22.0` entry (appkit `0.84.0`, latest skills with compiled fallback `0.2.30`) -- CLI `0.0.0-dev+abc` → uses `1.22.0` entry (dev builds use the highest versioned entry) +- CLI `1.22.0` → uses `1.22.0` entry (appkit `0.84.0`, skills `0.2.32`) +- CLI `1.23.0` → uses `1.23.0` entry (appkit `0.84.0`, latest skills; the fallback is the release resolved at build time, or `0.2.32` if unresolved) +- CLI `0.0.0-dev+abc` → uses `1.23.0` entry (dev builds use the highest versioned entry) ## How the CLI resolves versions @@ -53,16 +55,24 @@ At runtime, the CLI resolves the manifest from four sources: Set `DATABRICKS_FORCE_EMBEDDED_COMPAT=true` to skip all tiers and use only the embedded manifest. This is useful for local development when testing with a locally compiled binary that has a modified `cli-compat.json`. +## Building the embedded manifest + +`.github/scripts/pin-cli-compat.py` resolves skills entries containing `"latest"` with one authenticated `gh api` call. Release CI supplies `GITHUB_TOKEN`; local builds use `gh auth login` credentials or `GH_TOKEN`/`GITHUB_TOKEN`. If the lookup fails or returns an invalid tag, the script leaves `"latest"` in place and prints a warning. GoReleaser hides hook output by default, so release builds also add the warning to the GitHub Actions job summary. + +The script writes the resolved manifest and a Go build overlay under `.databricks/cli-compat/`. GoReleaser and the local build tasks use this overlay for `go:embed`, leaving the source manifest unchanged. The overlay is generated once per build so all release platforms embed the same skills version. Resolved entries carry `skills_resolved_from_latest: true` so runtime resolution preserves latest tracking and automatic downgrade protection; the embedded fallback reads the concrete skills version. + +Direct `go build` invocations skip this step. Whenever the embedded manifest still contains `"latest"`, the fallback is the skills version of the newest pinned range below it, as if the `"latest"` entries were absent. Use the build tasks to embed the current release. + ## When to update The goal is to **keep the manifest sparse** — only add entries at compatibility boundaries. Routine skills releases do not require a manifest update: keep `"skills": "latest"` on the newest range. -When updating AppKit or recording a tested skills fallback: +When updating AppKit or recording a skills compatibility boundary: 1. **Run evals** on the new AppKit version. If there is no regression, proceed. 2. **Open a PR** to update `cli-compat.json`. The change depends on the type of release: - - **No breaking changes**: update the highest entry's `appkit` in-place, keeping `"skills": "latest"`. To record a tested skills fallback, update `AgentSkillsFallbackVersion` in `internal/build/clicompat.go`. Do NOT add a new versioned key. - - **Breaking changes** (the new AppKit templates require specific `apps init` features, or the new skills version requires CLI commands that older CLIs lack): add a new entry keyed to the **minimum CLI version** that supports the new features. Set `"skills": "latest"` on the new entry and update the compiled skills fallback to a tested release. If skills compatibility changed, replace `"latest"` in the affected older ranges with the last compatible release. Preserve unrelated AppKit and skills compatibility boundaries. + - **No breaking changes**: update the highest entry's `appkit` in-place, keeping `"skills": "latest"`. Do NOT add a new versioned key. + - **Breaking changes** (the new AppKit templates require specific `apps init` features, or the new skills version requires CLI commands that older CLIs lack): add a new entry keyed to the **minimum CLI version** that supports the new features. Set `"skills": "latest"` on the new entry. If skills compatibility changed, replace `"latest"` in the affected older ranges with the last compatible release. Preserve unrelated AppKit and skills compatibility boundaries. This process is manual for now but can be automated as part of the release workflow in the future. Use the `/bump-cli-compat` Claude Code skill to automate the update and PR creation. @@ -74,7 +84,7 @@ The manifest is validated by Go tests in `libs/clicompat/`: go test ./libs/clicompat/... -run TestEmbeddedManifest -v ``` -This checks: valid JSON, at least one entry, valid semver keys and AppKit versions, concrete skills versions or `"latest"`, a concrete compiled skills fallback, and latest tracking on the newest CLI range. +This checks: valid JSON, at least one entry, valid semver keys and AppKit versions, concrete skills versions or `"latest"`, and a pinned range for the latest range to fall back to. The build script's doctests also verify build-time resolution and preservation of concrete pins. ## Pruning policy diff --git a/internal/build/cli-compat.json b/internal/build/cli-compat.json index c4abac161ce..8bd95452b13 100644 --- a/internal/build/cli-compat.json +++ b/internal/build/cli-compat.json @@ -1,4 +1,5 @@ { + "1.23.0": { "appkit": "0.84.0", "skills": "latest" }, "1.22.0": { "appkit": "0.84.0", "skills": "0.2.32" }, "1.20.0": { "appkit": "0.84.0", "skills": "0.2.30" }, "1.0.0": { "appkit": "0.76.1", "skills": "0.2.10" }, diff --git a/internal/build/clicompat.go b/internal/build/clicompat.go index 3142eee1b22..c7748ddd074 100644 --- a/internal/build/clicompat.go +++ b/internal/build/clicompat.go @@ -2,11 +2,8 @@ package build import _ "embed" -// AgentSkillsFallbackVersion is the tested release used when fetching the latest -// skills release fails. Updating it requires a new CLI build. -const AgentSkillsFallbackVersion = "0.2.30" - // CLICompatManifestJSON is the cli-compat.json manifest embedded at compile time. +// Release and task builds replace skills "latest" with a concrete release. // Used as the last-resort fallback when both remote fetch and local cache fail. // //go:embed cli-compat.json diff --git a/libs/aitools/installer/installer.go b/libs/aitools/installer/installer.go index 0ca33012f0f..63d6af5eb23 100644 --- a/libs/aitools/installer/installer.go +++ b/libs/aitools/installer/installer.go @@ -105,6 +105,13 @@ func GetSkillsRef(ctx context.Context) (ref string, explicit bool, err error) { return getSkillsRef(ctx, false) } +// GetSkillsRefWithFallback resolves the skills ref, using the embedded version +// if the latest-release lookup fails. Falling back does not mark the ref as an +// explicit pin, so callers must still check for downgrades of installed skills. +func GetSkillsRefWithFallback(ctx context.Context) (ref string, explicit bool, err error) { + return getSkillsRef(ctx, true) +} + func getSkillsRef(ctx context.Context, allowFallback bool) (ref string, explicit bool, err error) { if ref := env.Get(ctx, "DATABRICKS_SKILLS_REF"); ref != "" { return ref, true, nil @@ -141,7 +148,7 @@ func resolveLatestSkillsReleaseRef(ctx context.Context, allowFallback bool) (ref } func fetchLatestSkillsReleaseRef(ctx context.Context) (string, error) { - url := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/latest", skillsRepoOwner, skillsRepoName) + url := fmt.Sprintf("%s/repos/%s/%s/releases/latest", skillsAPIURL(ctx), skillsRepoOwner, skillsRepoName) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return "", fmt.Errorf("failed to create request: %w", err) @@ -191,6 +198,15 @@ func GetSkillsBaseURL(ctx context.Context) string { return "https://raw.githubusercontent.com/" + skillsRepoOwner + "/" + skillsRepoName } +// skillsAPIURL returns the GitHub API base URL for release lookups. +// DATABRICKS_SKILLS_API_URL overrides it for acceptance tests. +func skillsAPIURL(ctx context.Context) string { + if base := env.Get(ctx, "DATABRICKS_SKILLS_API_URL"); base != "" { + return strings.TrimRight(base, "/") + } + return "https://api.github.com" +} + // Manifest describes the skills manifest fetched from the skills repo. type Manifest struct { Version string `json:"version"` diff --git a/libs/aitools/installer/installer_test.go b/libs/aitools/installer/installer_test.go index 8c395fad84a..b18ebc2d44f 100644 --- a/libs/aitools/installer/installer_test.go +++ b/libs/aitools/installer/installer_test.go @@ -1331,11 +1331,10 @@ func TestGetSkillsRefDefaultManifest(t *testing.T) { var m clicompat.Manifest require.NoError(t, json.Unmarshal(build.CLICompatManifestJSON, &m)) - // The table only records which CLI versions track latest. Pins come from - // the embedded manifest so routine skills bumps need no test changes. + // Derive expectations from the embedded manifest so routine manifest edits + // need no test changes. Entries resolved at build time still track latest. tests := []struct { version string - latest bool }{ {version: "0.299.2"}, {version: "1.0.0"}, @@ -1351,6 +1350,7 @@ func TestGetSkillsRefDefaultManifest(t *testing.T) { entry, err := clicompat.Resolve(m, tc.version) require.NoError(t, err) pinnedRef := "v" + entry.AgentSkills + latest := entry.AgentSkills == skillsLatestSentinel || entry.AgentSkillsResolvedFromLatest for _, offline := range []bool{false, true} { t.Run(fmt.Sprintf("%s/offline=%t", tc.version, offline), func(t *testing.T) { @@ -1365,21 +1365,21 @@ func TestGetSkillsRefDefaultManifest(t *testing.T) { latestErr = errors.New("HTTP 403 fetching latest skills release") } switch { - case tc.latest && offline: + case latest && offline: wantRef = "" - case tc.latest: + case latest: wantRef = "v9.9.9" } stubLatestSkillsReleaseRef(t, "v9.9.9", latestErr) ref, explicit, err := GetSkillsRef(t.Context()) - if tc.latest && offline { + if latest && offline { require.ErrorIs(t, err, latestErr) } else { require.NoError(t, err) } assert.Equal(t, wantRef, ref) - assert.Equal(t, !tc.latest, explicit) + assert.Equal(t, !latest, explicit) }) } } @@ -1398,6 +1398,70 @@ func TestGetSkillsRefLatestReleaseLookupFails(t *testing.T) { assert.False(t, explicit) } +func TestGetSkillsRefWithFallback(t *testing.T) { + for _, tc := range []struct { + name string + version string + envRef string + latestErr error + }{ + {name: "latest release available", version: skillsLatestSentinel}, + {name: "rate limited", version: skillsLatestSentinel, latestErr: errors.New("HTTP 403 fetching latest skills release")}, + {name: "network unavailable", version: skillsLatestSentinel, latestErr: errors.New("network down")}, + {name: "compatibility pin", version: "0.1.5", latestErr: errors.New("must not fetch latest")}, + {name: "environment override", version: skillsLatestSentinel, envRef: "main", latestErr: errors.New("must not fetch latest")}, + } { + t.Run(tc.name, func(t *testing.T) { + setBuildVersion(t, "1.22.0") + t.Setenv("DATABRICKS_SKILLS_REF", tc.envRef) + withCachedCompat(t, tc.version) + orig := latestSkillsReleaseRefFn + t.Cleanup(func() { latestSkillsReleaseRefFn = orig }) + called := false + latestSkillsReleaseRefFn = func(context.Context) (string, error) { + called = true + return "v9.9.9", tc.latestErr + } + + ref, explicit, err := GetSkillsRefWithFallback(cmdio.MockDiscard(t.Context())) + require.NoError(t, err) + switch { + case tc.envRef != "": + assert.Equal(t, tc.envRef, ref) + assert.True(t, explicit) + assert.False(t, called) + case tc.version != skillsLatestSentinel: + assert.Equal(t, "v"+tc.version, ref) + assert.True(t, explicit) + assert.False(t, called) + case tc.latestErr != nil: + embedded, err := clicompat.ResolveEmbeddedAgentSkillsVersion() + require.NoError(t, err) + assert.Equal(t, "v"+embedded, ref) + assert.False(t, explicit) + assert.True(t, called) + default: + assert.Equal(t, "v9.9.9", ref) + assert.False(t, explicit) + assert.True(t, called) + } + }) + } +} + +func TestGetSkillsRefBuildResolvedLatestIsNotExplicit(t *testing.T) { + setBuildVersion(t, "1.22.0") + t.Setenv("DATABRICKS_SKILLS_REF", "") + withCachedCompat(t, "0.2.32") + cachePath := filepath.Join(os.Getenv("DATABRICKS_CACHE_DIR"), "compat-manifest.json") + require.NoError(t, os.WriteFile(cachePath, []byte(`{"1.22.0":{"appkit":"0.84.0","skills":"0.2.32","skills_resolved_from_latest":true}}`), 0o600)) + stubLatestSkillsReleaseRef(t, "v9.9.9", nil) + ref, explicit, err := GetSkillsRef(t.Context()) + require.NoError(t, err) + assert.Equal(t, "v9.9.9", ref, "build-time resolution must preserve latest tracking") + assert.False(t, explicit, "build snapshots must not bypass automatic downgrade checks") +} + func TestFetchSkillsManifestWithFallback(t *testing.T) { const missingRef = "v9.9.9" setBuildVersion(t, "1.22.0") diff --git a/libs/aitools/installer/plugin.go b/libs/aitools/installer/plugin.go index 12081a171c0..e800b6a0664 100644 --- a/libs/aitools/installer/plugin.go +++ b/libs/aitools/installer/plugin.go @@ -366,7 +366,10 @@ func RecordPluginInstalls(ctx context.Context, cliScope string, records map[stri state.Plugins = make(map[string]PluginRecord, len(records)) } maps.Copy(state.Plugins, records) - state.Release = ref + // An unresolved release keeps the one recorded by earlier installs. + if ref != "" { + state.Release = ref + } state.LastUpdated = time.Now() state.Scope = cliScope return SaveState(dir, state) diff --git a/libs/aitools/installer/plugin_test.go b/libs/aitools/installer/plugin_test.go index 4b536e0e187..6d448abcec8 100644 --- a/libs/aitools/installer/plugin_test.go +++ b/libs/aitools/installer/plugin_test.go @@ -503,6 +503,25 @@ func TestUninstallPluginKeepMarketplaceFlag(t *testing.T) { } } +func TestRecordPluginInstallsUnresolvedRefKeepsRelease(t *testing.T) { + setupTestHome(t) + ctx := cmdio.MockDiscard(t.Context()) + dir, err := GlobalSkillsDir(ctx) + require.NoError(t, err) + + require.NoError(t, RecordPluginInstalls(ctx, ScopeGlobal, map[string]PluginRecord{ + agents.NameClaudeCode: {Marketplace: "databricks-agent-skills", Plugin: "databricks", Version: "0.2.6"}, + }, "v0.2.6")) + require.NoError(t, RecordPluginInstalls(ctx, ScopeGlobal, map[string]PluginRecord{ + agents.NameCodex: {Marketplace: "databricks-agent-skills", Plugin: "databricks"}, + }, "")) + + st, err := LoadState(dir) + require.NoError(t, err) + assert.Equal(t, "v0.2.6", st.Release) + assert.Contains(t, st.Plugins, agents.NameCodex) +} + func TestRecordPluginInstallsThenSkillsInstallNoPanic(t *testing.T) { tmp := setupTestHome(t) ctx := cmdio.MockDiscard(t.Context()) diff --git a/libs/clicompat/clicompat.go b/libs/clicompat/clicompat.go index 446c8e4755c..36cab4a8087 100644 --- a/libs/clicompat/clicompat.go +++ b/libs/clicompat/clicompat.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "maps" "net/http" "os" "path/filepath" @@ -56,6 +57,10 @@ const ( type Entry struct { AppKit string `json:"appkit"` AgentSkills string `json:"skills"` + + // Set by the build script when replacing latest with a concrete fallback. + // Preserve latest tracking without treating that fallback as an explicit pin. + AgentSkillsResolvedFromLatest bool `json:"skills_resolved_from_latest,omitempty"` } // Manifest is the compatibility manifest: a map of CLI version strings to entries. @@ -164,20 +169,30 @@ func ResolveEmbeddedAppKitVersion() (string, error) { } // ResolveEmbeddedAgentSkillsVersion resolves the fallback for the current CLI -// without fetching compatibility data. Ranges tracking latest use the tested -// release compiled into the binary. +// without fetching compatibility data. Build tasks resolve latest before +// embedding. If "latest" remains, the newest pin below that range applies. func ResolveEmbeddedAgentSkillsVersion() (string, error) { m, err := parseEmbeddedManifest() if err != nil { return "", fmt.Errorf("embedded manifest: %w", err) } - entry, err := Resolve(m, build.GetInfo().Version) + return resolvePinnedAgentSkillsVersion(m, build.GetInfo().Version) +} + +// resolvePinnedAgentSkillsVersion resolves the skills version as if ranges +// tracking latest were absent from the manifest. +func resolvePinnedAgentSkillsVersion(m Manifest, cliVersion string) (string, error) { + pinned := maps.Clone(m) + maps.DeleteFunc(pinned, func(_ string, entry Entry) bool { + return entry.AgentSkills == AgentSkillsLatest + }) + if len(pinned) == 0 { + return "", errors.New("embedded manifest has no pinned skills version") + } + entry, err := Resolve(pinned, cliVersion) if err != nil { return "", fmt.Errorf("embedded manifest resolve: %w", err) } - if entry.AgentSkills == AgentSkillsLatest { - return build.AgentSkillsFallbackVersion, nil - } return entry.AgentSkills, nil } @@ -284,6 +299,9 @@ func ResolveAgentSkillsVersion(ctx context.Context) (string, error) { if err != nil { return "", err } + if entry.AgentSkillsResolvedFromLatest { + return AgentSkillsLatest, nil + } return entry.AgentSkills, nil } diff --git a/libs/clicompat/clicompat_test.go b/libs/clicompat/clicompat_test.go index 644df86efdf..df6328df6b6 100644 --- a/libs/clicompat/clicompat_test.go +++ b/libs/clicompat/clicompat_test.go @@ -307,6 +307,7 @@ func TestResolveAgentSkillsVersion(t *testing.T) { }{ {name: "pinned", manifest: `{"0.0.0":{"appkit":"0.84.0","skills":"0.2.30"}}`, want: "0.2.30"}, {name: "latest sentinel", manifest: `{"0.0.0":{"appkit":"0.84.0","skills":"latest"}}`, want: AgentSkillsLatest}, + {name: "resolved at build time", manifest: `{"0.0.0":{"appkit":"0.84.0","skills":"0.2.32","skills_resolved_from_latest":true}}`, want: AgentSkillsLatest}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -340,7 +341,11 @@ func TestResolveEmbeddedAgentSkillsVersion(t *testing.T) { }{ {version: "1.0.0", want: "0.2.10"}, {version: "1.22.0", want: pinnedAgentSkillsVersion}, - {version: "0.0.0-dev+test", want: pinnedAgentSkillsVersion}, + {version: "1.22.9", want: pinnedAgentSkillsVersion}, + // The latest range falls back to the 1.22 pin, or to a newer release + // when a build overlay resolved it. + {version: "1.23.0"}, + {version: "0.0.0-dev+test"}, } { t.Run(tc.version, func(t *testing.T) { original := build.GetInfo().Version @@ -349,9 +354,48 @@ func TestResolveEmbeddedAgentSkillsVersion(t *testing.T) { v, err := ResolveEmbeddedAgentSkillsVersion() require.NoError(t, err) + if tc.want != "" { + assert.Equal(t, tc.want, v) + } else { + assert.GreaterOrEqual(t, semver.Compare("v"+v, "v"+pinnedAgentSkillsVersion), 0, "got %s", v) + } + }) + } +} + +func TestResolvePinnedAgentSkillsVersion(t *testing.T) { + tracking := Manifest{ + "1.23.0": {AppKit: "0.84.0", AgentSkills: AgentSkillsLatest}, + "1.22.0": {AppKit: "0.84.0", AgentSkills: "0.2.32"}, + "1.20.0": {AppKit: "0.84.0", AgentSkills: "0.2.30"}, + "1.0.0": {AppKit: "0.76.1", AgentSkills: "0.2.10"}, + } + resolved := Manifest{ + "1.23.0": {AppKit: "0.84.0", AgentSkills: "0.2.35", AgentSkillsResolvedFromLatest: true}, + "1.22.0": {AppKit: "0.84.0", AgentSkills: "0.2.32"}, + } + for _, tc := range []struct { + name string + manifest Manifest + version string + want string + }{ + {name: "latest range", manifest: tracking, version: "1.23.0", want: "0.2.32"}, + {name: "above latest range", manifest: tracking, version: "1.24.1", want: "0.2.32"}, + {name: "dev build", manifest: tracking, version: "0.0.0-dev+test", want: "0.2.32"}, + {name: "pinned range", manifest: tracking, version: "1.21.0", want: "0.2.30"}, + {name: "older than all ranges", manifest: tracking, version: "0.9.0", want: "0.2.10"}, + {name: "resolved at build time", manifest: resolved, version: "1.23.0", want: "0.2.35"}, + } { + t.Run(tc.name, func(t *testing.T) { + v, err := resolvePinnedAgentSkillsVersion(tc.manifest, tc.version) + require.NoError(t, err) assert.Equal(t, tc.want, v) }) } + + _, err := resolvePinnedAgentSkillsVersion(Manifest{"1.23.0": {AppKit: "0.84.0", AgentSkills: AgentSkillsLatest}}, "1.23.0") + assert.ErrorContains(t, err, "no pinned skills version") } // --- Embedded manifest validation (replaces AppKit TS validator) --- @@ -373,12 +417,9 @@ func TestEmbeddedManifest_IsWellFormed(t *testing.T) { return semver.Compare("v"+a, "v"+b) }) - // Pinning the newest range (e.g. after a breaking skills release) must - // update this assertion in the same change. - assert.Equal(t, pinnedAgentSkillsVersion, m[keys[len(keys)-1]].AgentSkills, - "the newest CLI range must stay pinned until all callers support latest-release fallback") - assert.True(t, semver.IsValid("v"+build.AgentSkillsFallbackVersion), - "compiled skills fallback must be a concrete version") + // Ranges tracking latest fall back to a pinned range below them. + _, err = resolvePinnedAgentSkillsVersion(m, "0.0.0-dev+test") + assert.NoError(t, err, "the latest range needs a pinned range to fall back to") // Keys must be in ascending semver order. for i := 1; i < len(keys); i++ {